From 2b948d3d8617da381056fb475404ba672fda3d8e Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sun, 13 Sep 2026 14:59:31 -0700 Subject: [PATCH 001/311] feat: add isolated v2 frontend phase 2 platform --- .gitignore | 1 + frontend/.gitignore | 3 + frontend/.node-version | 1 + frontend/README.md | 7 + frontend/index.html | 1 + frontend/package-lock.json | 4063 +++++++++++++++++ frontend/package.json | 8 + frontend/playwright.config.ts | 12 + frontend/scripts/check-boundaries.mjs | 3 + frontend/scripts/stage-preview.mjs | 5 + frontend/src/api/wireSchema.ts | 4 + frontend/src/app/App.svelte | 8 + frontend/src/app/routeState.ts | 6 + frontend/src/app/uiTransitions.ts | 27 + frontend/src/domain/location.ts | 2 + frontend/src/domain/publication.ts | 2 + frontend/src/export/exportModel.ts | 3 + frontend/src/export/previewExport.ts | 2 + .../src/fixtures/FixtureLocationRepository.ts | 3 + frontend/src/fixtures/locations.ts | 6 + frontend/src/fixtures/scenarios.ts | 3 + frontend/src/main.ts | 6 + frontend/src/map/LeafletMapAdapter.ts | 10 + frontend/src/map/MapAdapter.ts | 2 + frontend/src/map/mapProjection.ts | 3 + frontend/src/query/snapshotSession.ts | 37 + frontend/src/styles/base.css | 1 + frontend/src/vite-env.d.ts | 1 + frontend/svelte.config.js | 1 + frontend/tests/e2e/fixture-platform.spec.ts | 52 + frontend/tests/unit/boundary.test.ts | 2 + frontend/tests/unit/exportModel.test.ts | 2 + frontend/tests/unit/fixtureRepository.test.ts | 2 + frontend/tests/unit/routeState.test.ts | 2 + frontend/tests/unit/snapshotSession.test.ts | 3 + frontend/tests/unit/uiTransitions.test.ts | 2 + frontend/tsconfig.app.json | 1 + frontend/tsconfig.json | 1 + frontend/tsconfig.node.json | 1 + frontend/vite.config.ts | 2 + frontend/vitest.config.ts | 8 + 41 files changed, 4309 insertions(+) create mode 100644 frontend/.gitignore create mode 100644 frontend/.node-version create mode 100644 frontend/README.md create mode 100644 frontend/index.html create mode 100644 frontend/package-lock.json create mode 100644 frontend/package.json create mode 100644 frontend/playwright.config.ts create mode 100644 frontend/scripts/check-boundaries.mjs create mode 100644 frontend/scripts/stage-preview.mjs create mode 100644 frontend/src/api/wireSchema.ts create mode 100644 frontend/src/app/App.svelte create mode 100644 frontend/src/app/routeState.ts create mode 100644 frontend/src/app/uiTransitions.ts create mode 100644 frontend/src/domain/location.ts create mode 100644 frontend/src/domain/publication.ts create mode 100644 frontend/src/export/exportModel.ts create mode 100644 frontend/src/export/previewExport.ts create mode 100644 frontend/src/fixtures/FixtureLocationRepository.ts create mode 100644 frontend/src/fixtures/locations.ts create mode 100644 frontend/src/fixtures/scenarios.ts create mode 100644 frontend/src/main.ts create mode 100644 frontend/src/map/LeafletMapAdapter.ts create mode 100644 frontend/src/map/MapAdapter.ts create mode 100644 frontend/src/map/mapProjection.ts create mode 100644 frontend/src/query/snapshotSession.ts create mode 100644 frontend/src/styles/base.css create mode 100644 frontend/src/vite-env.d.ts create mode 100644 frontend/svelte.config.js create mode 100644 frontend/tests/e2e/fixture-platform.spec.ts create mode 100644 frontend/tests/unit/boundary.test.ts create mode 100644 frontend/tests/unit/exportModel.test.ts create mode 100644 frontend/tests/unit/fixtureRepository.test.ts create mode 100644 frontend/tests/unit/routeState.test.ts create mode 100644 frontend/tests/unit/snapshotSession.test.ts create mode 100644 frontend/tests/unit/uiTransitions.test.ts create mode 100644 frontend/tsconfig.app.json create mode 100644 frontend/tsconfig.json create mode 100644 frontend/tsconfig.node.json create mode 100644 frontend/vite.config.ts create mode 100644 frontend/vitest.config.ts diff --git a/.gitignore b/.gitignore index d4d7dc0..342a480 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ /target/ /node_modules/ /snapshots/ +/static/v2-preview/ /\.shuttle* /Secrets*.toml __pycache__/ diff --git a/frontend/.gitignore b/frontend/.gitignore new file mode 100644 index 0000000..0c93106 --- /dev/null +++ b/frontend/.gitignore @@ -0,0 +1,3 @@ +node_modules/ +dist/ +test-results/ diff --git a/frontend/.node-version b/frontend/.node-version new file mode 100644 index 0000000..2bd5a0a --- /dev/null +++ b/frontend/.node-version @@ -0,0 +1 @@ +22 diff --git a/frontend/README.md b/frontend/README.md new file mode 100644 index 0000000..38e72da --- /dev/null +++ b/frontend/README.md @@ -0,0 +1,7 @@ +# V2 frontend preview + +Fixture-only Svelte 5 demonstrator. Run `npm install`, then `npm run dev`. It uses no live API, external assets, analytics, or map tiles. The ethics link intentionally targets the existing `/ethics.html` page. + +Phase 2 gate notes: staging is explicit (`npm run stage`) and copies only `frontend/dist` to the resolved ignored `static/v2-preview` destination. The Leaflet adapter is isolated and uses a blank local background; no tile provider is configured. Export previews retain profile, release, limitations, source, and observation context. + +Remaining live-use blockers: the backend contract still needs canonical generated DTOs and release/revocation semantics; record-level evidence hashes, geocoder metadata, explicit review events, and scoped project approvals are not present in the current API. Human accessibility review (screen reader, 200% zoom, 320px reflow) and larger-workload performance measurements remain required before product completion. This preview must not be enabled for live publication. diff --git a/frontend/index.html b/frontend/index.html new file mode 100644 index 0000000..3f9a8c5 --- /dev/null +++ b/frontend/index.html @@ -0,0 +1 @@ +
diff --git a/frontend/package-lock.json b/frontend/package-lock.json new file mode 100644 index 0000000..1129b30 --- /dev/null +++ b/frontend/package-lock.json @@ -0,0 +1,4063 @@ +{ + "name": "until-every-cage-v2-frontend", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "until-every-cage-v2-frontend", + "dependencies": { + "@axe-core/playwright": "^4.10.2", + "@types/leaflet": "^1.9.15", + "leaflet": "^1.9.4", + "zod": "^3.24.1" + }, + "devDependencies": { + "@playwright/test": "^1.49.1", + "@sveltejs/vite-plugin-svelte": "^6.2.1", + "@types/node": "^22.10.2", + "eslint": "^9.17.0", + "svelte": "^5.19.0", + "svelte-check": "^4.1.4", + "typescript": "^5.7.2", + "vite": "^6.0.7", + "vitest": "^2.1.8" + } + }, + "node_modules/@axe-core/playwright": { + "version": "4.13.0", + "resolved": "https://registry.npmjs.org/@axe-core/playwright/-/playwright-4.13.0.tgz", + "integrity": "sha512-6YLx+kxXu5GJceG4ozFg+33a2EMTdjYwWGloJ3sb9Kta5pp+ZNS53uxGVog5JetIY8s++P5UrtX+cri+u0VAVg==", + "license": "MPL-2.0", + "dependencies": { + "axe-core": "~4.13.0" + }, + "peerDependencies": { + "playwright-core": ">= 1.0.0" + } + }, + "node_modules/@esbuild/aix-ppc64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.25.12.tgz", + "integrity": "sha512-Hhmwd6CInZ3dwpuGTF8fJG6yoWmsToE+vYgD4nytZVxcu1ulHpUQRAB1UJ8+N1Am3Mz4+xOByoQoSZf4D+CpkA==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.25.12.tgz", + "integrity": "sha512-VJ+sKvNA/GE7Ccacc9Cha7bpS8nyzVv0jdVgwNDaR4gDMC/2TTRc33Ip8qrNYUcpkOHUT5OZ0bUcNNVZQ9RLlg==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.25.12.tgz", + "integrity": "sha512-6AAmLG7zwD1Z159jCKPvAxZd4y/VTO0VkprYy+3N2FtJ8+BQWFXU+OxARIwA46c5tdD9SsKGZ/1ocqBS/gAKHg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.25.12.tgz", + "integrity": "sha512-5jbb+2hhDHx5phYR2By8GTWEzn6I9UqR11Kwf22iKbNpYrsmRB18aX/9ivc5cabcUiAT/wM+YIZ6SG9QO6a8kg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.25.12.tgz", + "integrity": "sha512-N3zl+lxHCifgIlcMUP5016ESkeQjLj/959RxxNYIthIg+CQHInujFuXeWbWMgnTo4cp5XVHqFPmpyu9J65C1Yg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.25.12.tgz", + "integrity": "sha512-HQ9ka4Kx21qHXwtlTUVbKJOAnmG1ipXhdWTmNXiPzPfWKpXqASVcWdnf2bnL73wgjNrFXAa3yYvBSd9pzfEIpA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.25.12.tgz", + "integrity": "sha512-gA0Bx759+7Jve03K1S0vkOu5Lg/85dou3EseOGUes8flVOGxbhDDh/iZaoek11Y8mtyKPGF3vP8XhnkDEAmzeg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.25.12.tgz", + "integrity": "sha512-TGbO26Yw2xsHzxtbVFGEXBFH0FRAP7gtcPE7P5yP7wGy7cXK2oO7RyOhL5NLiqTlBh47XhmIUXuGciXEqYFfBQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.25.12.tgz", + "integrity": "sha512-lPDGyC1JPDou8kGcywY0YILzWlhhnRjdof3UlcoqYmS9El818LLfJJc3PXXgZHrHCAKs/Z2SeZtDJr5MrkxtOw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.25.12.tgz", + "integrity": "sha512-8bwX7a8FghIgrupcxb4aUmYDLp8pX06rGh5HqDT7bB+8Rdells6mHvrFHHW2JAOPZUbnjUpKTLg6ECyzvas2AQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.25.12.tgz", + "integrity": "sha512-0y9KrdVnbMM2/vG8KfU0byhUN+EFCny9+8g202gYqSSVMonbsCfLjUO+rCci7pM0WBEtz+oK/PIwHkzxkyharA==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.25.12.tgz", + "integrity": "sha512-h///Lr5a9rib/v1GGqXVGzjL4TMvVTv+s1DPoxQdz7l/AYv6LDSxdIwzxkrPW438oUXiDtwM10o9PmwS/6Z0Ng==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.25.12.tgz", + "integrity": "sha512-iyRrM1Pzy9GFMDLsXn1iHUm18nhKnNMWscjmp4+hpafcZjrr2WbT//d20xaGljXDBYHqRcl8HnxbX6uaA/eGVw==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.25.12.tgz", + "integrity": "sha512-9meM/lRXxMi5PSUqEXRCtVjEZBGwB7P/D4yT8UG/mwIdze2aV4Vo6U5gD3+RsoHXKkHCfSxZKzmDssVlRj1QQA==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.25.12.tgz", + "integrity": "sha512-Zr7KR4hgKUpWAwb1f3o5ygT04MzqVrGEGXGLnj15YQDJErYu/BGg+wmFlIDOdJp0PmB0lLvxFIOXZgFRrdjR0w==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.25.12.tgz", + "integrity": "sha512-MsKncOcgTNvdtiISc/jZs/Zf8d0cl/t3gYWX8J9ubBnVOwlk65UIEEvgBORTiljloIWnBzLs4qhzPkJcitIzIg==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.25.12.tgz", + "integrity": "sha512-uqZMTLr/zR/ed4jIGnwSLkaHmPjOjJvnm6TVVitAa08SLS9Z0VM8wIRx7gWbJB5/J54YuIMInDquWyYvQLZkgw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.25.12.tgz", + "integrity": "sha512-xXwcTq4GhRM7J9A8Gv5boanHhRa/Q9KLVmcyXHCTaM4wKfIpWkdXiMog/KsnxzJ0A1+nD+zoecuzqPmCRyBGjg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.25.12.tgz", + "integrity": "sha512-Ld5pTlzPy3YwGec4OuHh1aCVCRvOXdH8DgRjfDy/oumVovmuSzWfnSJg+VtakB9Cm0gxNO9BzWkj6mtO1FMXkQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.25.12.tgz", + "integrity": "sha512-fF96T6KsBo/pkQI950FARU9apGNTSlZGsv1jZBAlcLL1MLjLNIWPBkj5NlSz8aAzYKg+eNqknrUJ24QBybeR5A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.25.12.tgz", + "integrity": "sha512-MZyXUkZHjQxUvzK7rN8DJ3SRmrVrke8ZyRusHlP+kuwqTcfWLyqMOE3sScPPyeIXN/mDJIfGXvcMqCgYKekoQw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openharmony-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.25.12.tgz", + "integrity": "sha512-rm0YWsqUSRrjncSXGA7Zv78Nbnw4XL6/dzr20cyrQf7ZmRcsovpcRBdhD43Nuk3y7XIoW2OxMVvwuRvk9XdASg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.25.12.tgz", + "integrity": "sha512-3wGSCDyuTHQUzt0nV7bocDy72r2lI33QL3gkDNGkod22EsYl04sMf0qLb8luNKTOmgF/eDEDP5BFNwoBKH441w==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.25.12.tgz", + "integrity": "sha512-rMmLrur64A7+DKlnSuwqUdRKyd3UE7oPJZmnljqEptesKM8wx9J8gx5u0+9Pq0fQQW8vqeKebwNXdfOyP+8Bsg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.25.12.tgz", + "integrity": "sha512-HkqnmmBoCbCwxUKKNPBixiWDGCpQGVsrQfJoVGYLPT41XWF8lHuE5N6WhVia2n4o5QK5M4tYr21827fNhi4byQ==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.25.12.tgz", + "integrity": "sha512-alJC0uCZpTFrSL0CCDjcgleBXPnCrEAhTBILpeAp7M/OFgoqtAetfBzX0xM00MUsVVPpVjlPuMbREqnZCXaTnA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@eslint-community/eslint-utils": { + "version": "4.10.1", + "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.10.1.tgz", + "integrity": "sha512-cuadcxVFE8sDK6iWJbs8Sn0av2Nrh2QSGQhVlBW9AaAHqHwjWsZHT8LJ4hFGPh7ASBV2deFdM7H/DPjulmh8rg==", + "dev": true, + "license": "MIT", + "dependencies": { + "eslint-visitor-keys": "^3.4.3" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + }, + "peerDependencies": { + "eslint": "^6.0.0 || ^7.0.0 || >=8.0.0" + } + }, + "node_modules/@eslint-community/eslint-utils/node_modules/eslint-visitor-keys": { + "version": "3.4.3", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.4.3.tgz", + "integrity": "sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/@eslint-community/regexpp": { + "version": "4.12.2", + "resolved": "https://registry.npmjs.org/@eslint-community/regexpp/-/regexpp-4.12.2.tgz", + "integrity": "sha512-EriSTlt5OC9/7SXkRSCAhfSxxoSUgBm33OH+IkwbdpgoqsSsUg7y3uh+IICI/Qg4BBWr3U2i39RpmycbxMq4ew==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^12.0.0 || ^14.0.0 || >=16.0.0" + } + }, + "node_modules/@eslint/config-array": { + "version": "0.21.2", + "resolved": "https://registry.npmjs.org/@eslint/config-array/-/config-array-0.21.2.tgz", + "integrity": "sha512-nJl2KGTlrf9GjLimgIru+V/mzgSK0ABCDQRvxw5BjURL7WfH5uoWmizbH7QB6MmnMBd8cIC9uceWnezL1VZWWw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@eslint/object-schema": "^2.1.7", + "debug": "^4.3.1", + "minimatch": "^3.1.5" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + } + }, + "node_modules/@eslint/config-helpers": { + "version": "0.4.2", + "resolved": "https://registry.npmjs.org/@eslint/config-helpers/-/config-helpers-0.4.2.tgz", + "integrity": "sha512-gBrxN88gOIf3R7ja5K9slwNayVcZgK6SOUORm2uBzTeIEfeVaIhOpCtTox3P6R7o2jLFwLFTLnC7kU/RGcYEgw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@eslint/core": "^0.17.0" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + } + }, + "node_modules/@eslint/core": { + "version": "0.17.0", + "resolved": "https://registry.npmjs.org/@eslint/core/-/core-0.17.0.tgz", + "integrity": "sha512-yL/sLrpmtDaFEiUj1osRP4TI2MDz1AddJL+jZ7KSqvBuliN4xqYY54IfdN8qD8Toa6g1iloph1fxQNkjOxrrpQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@types/json-schema": "^7.0.15" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + } + }, + "node_modules/@eslint/eslintrc": { + "version": "3.3.7", + "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-3.3.7.tgz", + "integrity": "sha512-F42g89Qd5oAWtp0k0nnSrjziAKza7w8SVT4mStc18LZMaRb4J1HQAHLCalEtDCxrTuksx7NU9qsmeLwpOfPqWw==", + "dev": true, + "license": "MIT", + "dependencies": { + "ajv": "^6.14.0", + "debug": "^4.3.2", + "espree": "^10.0.1", + "globals": "^14.0.0", + "ignore": "^5.2.0", + "import-fresh": "^3.2.1", + "js-yaml": "^4.3.2", + "minimatch": "^3.1.5", + "strip-json-comments": "^3.1.1" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/@eslint/js": { + "version": "9.39.5", + "resolved": "https://registry.npmjs.org/@eslint/js/-/js-9.39.5.tgz", + "integrity": "sha512-QywQuszQh77pIXCsq998c8hbhSTI/azTty1Z6N53dmAudKHhy573j3yvRLsX2BSp8YpLtoCEG8E9DJe+8zUh4A==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://eslint.org/donate" + } + }, + "node_modules/@eslint/object-schema": { + "version": "2.1.7", + "resolved": "https://registry.npmjs.org/@eslint/object-schema/-/object-schema-2.1.7.tgz", + "integrity": "sha512-VtAOaymWVfZcmZbp6E2mympDIHvyjXs/12LqWYjVw6qjrfF+VK+fyG33kChz3nnK+SU5/NeHOqrTEHS8sXO3OA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + } + }, + "node_modules/@eslint/plugin-kit": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/@eslint/plugin-kit/-/plugin-kit-0.4.1.tgz", + "integrity": "sha512-43/qtrDUokr7LJqoF2c3+RInu/t4zfrpYdoSDfYyhg52rwLV6TnOvdG4fXm7IkSB3wErkcmJS9iEhjVtOSEjjA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@eslint/core": "^0.17.0", + "levn": "^0.4.1" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + } + }, + "node_modules/@humanfs/core": { + "version": "0.19.2", + "resolved": "https://registry.npmjs.org/@humanfs/core/-/core-0.19.2.tgz", + "integrity": "sha512-UhXNm+CFMWcbChXywFwkmhqjs3PRCmcSa/hfBgLIb7oQ5HNb1wS0icWsGtSAUNgefHeI+eBrA8I1fxmbHsGdvA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@humanfs/types": "^0.15.0" + }, + "engines": { + "node": ">=18.18.0" + } + }, + "node_modules/@humanfs/node": { + "version": "0.16.8", + "resolved": "https://registry.npmjs.org/@humanfs/node/-/node-0.16.8.tgz", + "integrity": "sha512-gE1eQNZ3R++kTzFUpdGlpmy8kDZD/MLyHqDwqjkVQI0JMdI1D51sy1H958PNXYkM2rAac7e5/CnIKZrHtPh3BQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@humanfs/core": "^0.19.2", + "@humanfs/types": "^0.15.0", + "@humanwhocodes/retry": "^0.4.0" + }, + "engines": { + "node": ">=18.18.0" + } + }, + "node_modules/@humanfs/types": { + "version": "0.15.0", + "resolved": "https://registry.npmjs.org/@humanfs/types/-/types-0.15.0.tgz", + "integrity": "sha512-ZZ1w0aoQkwuUuC7Yf+7sdeaNfqQiiLcSRbfI08oAxqLtpXQr9AIVX7Ay7HLDuiLYAaFPu8oBYNq/QIi9URHJ3Q==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=18.18.0" + } + }, + "node_modules/@humanwhocodes/module-importer": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@humanwhocodes/module-importer/-/module-importer-1.0.1.tgz", + "integrity": "sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=12.22" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/nzakas" + } + }, + "node_modules/@humanwhocodes/retry": { + "version": "0.4.3", + "resolved": "https://registry.npmjs.org/@humanwhocodes/retry/-/retry-0.4.3.tgz", + "integrity": "sha512-bV0Tgo9K4hfPCek+aMAn81RppFKv2ySDQeMoSZuvTASywNTnVJCArCZE2FWqpvIatKu7VMRLWlR1EazvVhDyhQ==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=18.18" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/nzakas" + } + }, + "node_modules/@jridgewell/gen-mapping": { + "version": "0.3.13", + "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.13.tgz", + "integrity": "sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.5.0", + "@jridgewell/trace-mapping": "^0.3.24" + } + }, + "node_modules/@jridgewell/remapping": { + "version": "2.3.5", + "resolved": "https://registry.npmjs.org/@jridgewell/remapping/-/remapping-2.3.5.tgz", + "integrity": "sha512-LI9u/+laYG4Ds1TDKSJW2YPrIlcVYOwi2fUC6xB43lueCjgxV4lffOCZCtYFiH6TNOX+tQKXx97T4IKHbhyHEQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/gen-mapping": "^0.3.5", + "@jridgewell/trace-mapping": "^0.3.24" + } + }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", + "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.6.0.tgz", + "integrity": "sha512-T7jf+5zgsZHwNJ4lvQ7/aezbyk0nNX+zJVWpmHA7VYsEx7a7qr5Rg5IbtJFqkgze5Y2sruq1RUY8Q837Od7iFw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.31", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.31.tgz", + "integrity": "sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/resolve-uri": "^3.1.0", + "@jridgewell/sourcemap-codec": "^1.4.14" + } + }, + "node_modules/@napi-rs/lzma-linux-x64-gnu": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/@napi-rs/lzma-linux-x64-gnu/-/lzma-linux-x64-gnu-1.5.1.tgz", + "integrity": "sha512-oTXEIha4SsuXdTA4Iyskj0kpdx2yVXdhd75c2v3xGrHFfVMsbhTPZU/nMPL4sWKo4pBHm3aucLaqGlF696dTyQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^22.20 || ^24.12 || >=25" + } + }, + "node_modules/@playwright/test": { + "version": "1.63.0", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.63.0.tgz", + "integrity": "sha512-oxMK4vllB9RK5NQ2l1pq1IfOf2AvnEuj/vYGDj0H2nMtmtZpKtCwt/l00GEO6xjGfpBNAvjovvYdCm50dRQkpQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright": "1.63.0" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/@rollup/rollup-android-arm-eabi": { + "version": "4.63.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.63.2.tgz", + "integrity": "sha512-Xa6RDoWa+hNiX6PgsljlH6W75RaONx3y6PVlbLhkEWW+GaPQ3dP5gwbL/erAzQHWwkvW5UxdD5l87Qx2FAQ/4A==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-android-arm64": { + "version": "4.63.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.63.2.tgz", + "integrity": "sha512-vNASxsghMfQ5s+v3PrpnJd+ryL/26lxCCaGI+sDJ7VzmHiYXIrrVltsDhaawxLM1WcoMU2oYlbPHLaYQtBzhcg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-darwin-arm64": { + "version": "4.63.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.63.2.tgz", + "integrity": "sha512-0dWDjmlrpZAgjPD/aPzUDhBW8APLRjAni5bOrM76wiiZm+E+KTMVKNhAzaTBohz8UyO2fKNAl0+fygbe2HZXOA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rollup/rollup-darwin-x64": { + "version": "4.63.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.63.2.tgz", + "integrity": "sha512-N58uktcwzk3+qT4KHEuNdIxX1N01RWrkfVoml69EAbSaNDL+sbNVLx2RMl4Qd23lpA0fgPvyh5hHb4weD5WKmg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rollup/rollup-freebsd-arm64": { + "version": "4.63.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-arm64/-/rollup-freebsd-arm64-4.63.2.tgz", + "integrity": "sha512-HWF2zH8EAp2scWRpt2PGe6iUGz7zi04waXsdRr3zb4DWCk2ImIo5FZu0jjmD53nP/DGSvnW0e7/1ToCNZs2lZw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@rollup/rollup-freebsd-x64": { + "version": "4.63.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-x64/-/rollup-freebsd-x64-4.63.2.tgz", + "integrity": "sha512-MkvcwHMnzPSMOQEwB6wHnLzmc+hT8BGc5bW/Mhmjjgx3wbj6VBnlc47XsK74kD0K9MikFfXpQqyz4NUXaUW62A==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@rollup/rollup-linux-arm-gnueabihf": { + "version": "4.63.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.63.2.tgz", + "integrity": "sha512-xe1bCKPJaKsD0tfd7Rb6bGfUogJTpKbTEEthsfdb7hTfTRNJVQTdirabQx0o6ERVba/smkM720soMY+0QnrlSQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm-musleabihf": { + "version": "4.63.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.63.2.tgz", + "integrity": "sha512-yOM7LdK0p6gk6+Q773OEwtlsikT1TL3yMmYsTtRlDRPha5vV2DC5x7LqRWDr6f3cSYNMKVqxzffXv8ivxNBIFQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-gnu": { + "version": "4.63.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.63.2.tgz", + "integrity": "sha512-qiWuJJV3DybA2IfzvRimeKXGrGuVPv1zobSY/26KnP3HbV0VcNb3ECzgvtbvF3xjSMkcooou6HASXZuLdjnhpQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-musl": { + "version": "4.63.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.63.2.tgz", + "integrity": "sha512-akcZquRzCY/KpUoZAMBhGf7oi4LmXq1BzRA5CPAC3rkUf28Y/sAYV3jSL+JKd7cwEyFvR5G0XVZ0gaMedP+60A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-loong64-gnu": { + "version": "4.63.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-gnu/-/rollup-linux-loong64-gnu-4.63.2.tgz", + "integrity": "sha512-fNwYHrPyYyxauPzX/cpYw8Z7LQpp+DGA0KCoswA0aVFBpmdMil9XgjB8V3Ny64Ihu797+GKcuJqnsOKEmor7fA==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-loong64-musl": { + "version": "4.63.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-musl/-/rollup-linux-loong64-musl-4.63.2.tgz", + "integrity": "sha512-XfvsgzR7DZqREdst7K1Mj3ilSUM5xLAHJcIMDFPKdxTs9q5VHOT8aMA+a683fqBu7DQl8+Sd9HCsQYL8EMY9qA==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-ppc64-gnu": { + "version": "4.63.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-gnu/-/rollup-linux-ppc64-gnu-4.63.2.tgz", + "integrity": "sha512-Pp7gVZggEFlbcuztay+/U0gVG9S1XAh8i7I1Re/htbAzo43P5wHZHw6pTyzotISqlKohoh9RpIfnOz3RbemK1w==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-ppc64-musl": { + "version": "4.63.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-musl/-/rollup-linux-ppc64-musl-4.63.2.tgz", + "integrity": "sha512-zkgL2xff6i7u5hau/m6FGeS8gRkLEdgLw522WGmdWWlLd9btmNl3S80mcEjtGq+kvgUekQ3+BOYLLLcPlS2LIA==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-riscv64-gnu": { + "version": "4.63.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.63.2.tgz", + "integrity": "sha512-qOheJomrkVCbbHFJ7L3J97cnhfogKqguAQphv26+3ZsAQIF1L19b+dArl//s8rjJHJLz9byykyM8NBP4nmSa1g==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-riscv64-musl": { + "version": "4.63.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-musl/-/rollup-linux-riscv64-musl-4.63.2.tgz", + "integrity": "sha512-XlxLD54wQhH3FciCgMofxBw27NzUe818gJH410qWvc41UT0ZFcgxVjyX5/EK8MPTupjeVWqN5oy+9pCA9mqfCA==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-s390x-gnu": { + "version": "4.63.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.63.2.tgz", + "integrity": "sha512-vdryWeRb2bLJZf0Fv/W8se6nvsHe2PkTCxV0meheK3nQE+G90VCJcke51Miy1yQRsfm2uqIyjXOu4wmUzbTtkQ==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-x64-gnu": { + "version": "4.63.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.63.2.tgz", + "integrity": "sha512-bcq2h2pkKmH2po4cZV8VWzO4lL40STyu/nLoFpYMQp9C2tCVNTdcVv86MwSsn3D5s1FBe2Ty1atqvVAUTMimNg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-x64-musl": { + "version": "4.63.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.63.2.tgz", + "integrity": "sha512-EGoo5DMVMRkTId8fuTDaoxVlR5ZTsKULUezRjd9gCw5eeY+DjCvDpZAOlNUvKPGX+7rS1RWx6j+yOpNPx0cUgQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-openbsd-x64": { + "version": "4.63.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-openbsd-x64/-/rollup-openbsd-x64-4.63.2.tgz", + "integrity": "sha512-MErl12k7BFHZG1TI9QF/3lSSZARzq9KgNy/FjnqFMCkv+N4RSSzoUCA5h2mqHX4Mox3WaTVKblyzhQ1zRb2ZuQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ] + }, + "node_modules/@rollup/rollup-openharmony-arm64": { + "version": "4.63.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-openharmony-arm64/-/rollup-openharmony-arm64-4.63.2.tgz", + "integrity": "sha512-ILs8k07Wh4p0PsNY4wYLEaXZKMOpVhrG5QDB0yHhGhuzOfDlnyHN6sflL4El/MpUP1y8uY2lUZrv4oBS6pTT3g==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ] + }, + "node_modules/@rollup/rollup-win32-arm64-msvc": { + "version": "4.63.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.63.2.tgz", + "integrity": "sha512-hKgB3nz/TKD3Wv78XEsyXzQsNjvhOHmwKQTvXADGOyU/cIClZDO7DsoggbdmJDPGp5V80tA3Vfv61PaKTLH3LA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-ia32-msvc": { + "version": "4.63.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.63.2.tgz", + "integrity": "sha512-T4wf1mudIDxN8Q/CWIBJC1u5gQUc+r5mPvlwoSbIvNkyVTP2TAFeobEmst5AQ4gMyAz4sSByVdoTDfvTmGK/8g==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-x64-gnu": { + "version": "4.63.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-gnu/-/rollup-win32-x64-gnu-4.63.2.tgz", + "integrity": "sha512-tC3IY7qoaD9Ll3/8WJQn49j5V2f/NuI9S41NOE2iM5MPs3sPIvOkVToLcz/7Bz4pyF7PSvrtwu8I/pUrGOSecQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-x64-msvc": { + "version": "4.63.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.63.2.tgz", + "integrity": "sha512-6NHnk/K3eq2ZFYcU1X8g67s9qIJRCOTT92gwLMVBp08dB2uuuwI1/Q/empzL2Bfr2f2WRLJVwpp90RmacQyFkw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@sveltejs/acorn-typescript": { + "version": "1.0.13", + "resolved": "https://registry.npmjs.org/@sveltejs/acorn-typescript/-/acorn-typescript-1.0.13.tgz", + "integrity": "sha512-wgKggnhZVL9Bfx1OaKKTrYY9BFRk6C8UAkQNUcIv1+llzYrIqy+RZm5HPKzn0NpEBvTVhTqB4kQyllZywsRBRQ==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "acorn": "^8.9.0" + } + }, + "node_modules/@sveltejs/load-config": { + "version": "0.2.3", + "resolved": "https://registry.npmjs.org/@sveltejs/load-config/-/load-config-0.2.3.tgz", + "integrity": "sha512-VT3qmUb8pRV2QrZjd8iAmtg8lf4W0TIjZbvXtz5MKei/q96teWZgGJyyidJzOjzZzvdq616eSRVeMYIQChUTAQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 18.0.0" + } + }, + "node_modules/@sveltejs/vite-plugin-svelte": { + "version": "6.2.4", + "resolved": "https://registry.npmjs.org/@sveltejs/vite-plugin-svelte/-/vite-plugin-svelte-6.2.4.tgz", + "integrity": "sha512-ou/d51QSdTyN26D7h6dSpusAKaZkAiGM55/AKYi+9AGZw7q85hElbjK3kEyzXHhLSnRISHOYzVge6x0jRZ7DXA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@sveltejs/vite-plugin-svelte-inspector": "^5.0.0", + "deepmerge": "^4.3.1", + "magic-string": "^0.30.21", + "obug": "^2.1.0", + "vitefu": "^1.1.1" + }, + "engines": { + "node": "^20.19 || ^22.12 || >=24" + }, + "peerDependencies": { + "svelte": "^5.0.0", + "vite": "^6.3.0 || ^7.0.0" + } + }, + "node_modules/@sveltejs/vite-plugin-svelte-inspector": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/@sveltejs/vite-plugin-svelte-inspector/-/vite-plugin-svelte-inspector-5.0.2.tgz", + "integrity": "sha512-TZzRTcEtZffICSAoZGkPSl6Etsj2torOVrx6Uw0KpXxrec9Gg6jFWQ60Q3+LmNGfZSxHRCZL7vXVZIWmuV50Ig==", + "dev": true, + "license": "MIT", + "dependencies": { + "obug": "^2.1.0" + }, + "engines": { + "node": "^20.19 || ^22.12 || >=24" + }, + "peerDependencies": { + "@sveltejs/vite-plugin-svelte": "^6.0.0-next.0", + "svelte": "^5.0.0", + "vite": "^6.3.0 || ^7.0.0" + } + }, + "node_modules/@types/estree": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.9.tgz", + "integrity": "sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/geojson": { + "version": "7946.0.16", + "resolved": "https://registry.npmjs.org/@types/geojson/-/geojson-7946.0.16.tgz", + "integrity": "sha512-6C8nqWur3j98U6+lXDfTUWIfgvZU+EumvpHKcYjujKH7woYyLj2sUmff0tRhrqM7BohUw7Pz3ZB1jj2gW9Fvmg==", + "license": "MIT" + }, + "node_modules/@types/json-schema": { + "version": "7.0.15", + "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", + "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/leaflet": { + "version": "1.9.22", + "resolved": "https://registry.npmjs.org/@types/leaflet/-/leaflet-1.9.22.tgz", + "integrity": "sha512-h3lhECYEKDasG7LFHu+GiHqAvsgLuQvlJvVZzJDGONo3sEL+wUOqSFLnwkZlK0qVxnxbuGFW8iBlJNYs5wgndA==", + "license": "MIT", + "dependencies": { + "@types/geojson": "*" + } + }, + "node_modules/@types/node": { + "version": "22.20.2", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.20.2.tgz", + "integrity": "sha512-xlvWf4Vs9n1PEVYwP1n4vvG07M6y8WgvJ2t0vbrWTmijsIHp1cS+uJ2kMIRdY3nHZK0nCYKrPeD171+SzF4/zw==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~6.21.0" + } + }, + "node_modules/@vitest/expect": { + "version": "2.1.9", + "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-2.1.9.tgz", + "integrity": "sha512-UJCIkTBenHeKT1TTlKMJWy1laZewsRIzYighyYiJKZreqtdxSos/S1t+ktRMQWu2CKqaarrkeszJx1cgC5tGZw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/spy": "2.1.9", + "@vitest/utils": "2.1.9", + "chai": "^5.1.2", + "tinyrainbow": "^1.2.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/pretty-format": { + "version": "2.1.9", + "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-2.1.9.tgz", + "integrity": "sha512-KhRIdGV2U9HOUzxfiHmY8IFHTdqtOhIzCpd8WRdJiE7D/HUcZVD0EgQCVjm+Q9gkUXWgBvMmTtZgIG48wq7sOQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "tinyrainbow": "^1.2.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/runner": { + "version": "2.1.9", + "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-2.1.9.tgz", + "integrity": "sha512-ZXSSqTFIrzduD63btIfEyOmNcBmQvgOVsPNPe0jYtESiXkhd8u2erDLnMxmGrDCwHCCHE7hxwRDCT3pt0esT4g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/utils": "2.1.9", + "pathe": "^1.1.2" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/snapshot": { + "version": "2.1.9", + "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-2.1.9.tgz", + "integrity": "sha512-oBO82rEjsxLNJincVhLhaxxZdEtV0EFHMK5Kmx5sJ6H9L183dHECjiefOAdnqpIgT5eZwT04PoggUnW88vOBNQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/pretty-format": "2.1.9", + "magic-string": "^0.30.12", + "pathe": "^1.1.2" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/spy": { + "version": "2.1.9", + "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-2.1.9.tgz", + "integrity": "sha512-E1B35FwzXXTs9FHNK6bDszs7mtydNi5MIfUWpceJ8Xbfb1gBMscAnwLbEu+B44ed6W3XjL9/ehLPHR1fkf1KLQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "tinyspy": "^3.0.2" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/utils": { + "version": "2.1.9", + "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-2.1.9.tgz", + "integrity": "sha512-v0psaMSkNJ3A2NMrUEHFRzJtDPFn+/VWZ5WxImB21T9fjucJRmS7xCS3ppEnARb9y11OAzaD+P2Ps+b+BGX5iQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/pretty-format": "2.1.9", + "loupe": "^3.1.2", + "tinyrainbow": "^1.2.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/acorn": { + "version": "8.18.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.18.0.tgz", + "integrity": "sha512-lGq+9yr1/GuAWaVYIHRjvvySG5/4VfKIvC8EWxStPdcDh/Ka7FG3twP6v4d5BkravUilhIAsG4Qj83t02LWUPQ==", + "dev": true, + "license": "MIT", + "bin": { + "acorn": "bin/acorn" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/acorn-jsx": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz", + "integrity": "sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" + } + }, + "node_modules/ajv": { + "version": "6.15.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.15.0.tgz", + "integrity": "sha512-fgFx7Hfoq60ytK2c7DhnF8jIvzYgOMxfugjLOSMHjLIPgenqa7S7oaagATUq99mV6IYvN2tRmC0wnTYX6iPbMw==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", + "dev": true, + "license": "Python-2.0" + }, + "node_modules/aria-query": { + "version": "5.3.1", + "resolved": "https://registry.npmjs.org/aria-query/-/aria-query-5.3.1.tgz", + "integrity": "sha512-Z/ZeOgVl7bcSYZ/u/rh0fOpvEpq//LZmdbkXyc7syVzjPAhfOa9ebsdTSjEBDU4vs5nC98Kfduj1uFo0qyET3g==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/assertion-error": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-2.0.1.tgz", + "integrity": "sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + } + }, + "node_modules/axe-core": { + "version": "4.13.0", + "resolved": "https://registry.npmjs.org/axe-core/-/axe-core-4.13.0.tgz", + "integrity": "sha512-UzGt8zg7Ny8djbYMhxl2zuEevVa7r2gJjYY5Lwr1xM7+XU2nd6CkIWFTVcCIbAP63vSz71NaVyyuSk9lHKcy0A==", + "license": "MPL-2.0", + "engines": { + "node": ">=4" + } + }, + "node_modules/axobject-query": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/axobject-query/-/axobject-query-4.1.0.tgz", + "integrity": "sha512-qIj0G9wZbMGNLjLmg1PT6v2mE9AH2zlnADJD/2tC6E00hgmhUOfEB6greHPAfLRSufHqROIUTkw6E+M3lH0PTQ==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "dev": true, + "license": "MIT" + }, + "node_modules/brace-expansion": { + "version": "1.1.18", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.18.tgz", + "integrity": "sha512-Edep/X9fGqVNmzKBVsDYIOtD+z1tuezV70LBjdCst9Tqu76lsnvRiZ6oTic1n+/BIwX6QDGAO94PN4N2SADvtw==", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/cac": { + "version": "6.7.14", + "resolved": "https://registry.npmjs.org/cac/-/cac-6.7.14.tgz", + "integrity": "sha512-b6Ilus+c3RrdDk+JhLKUAQfzzgLEPy6wcXqS7f/xe1EETvsDP6GORG7SFuOs6cID5YkqchW/LXZbX5bc8j7ZcQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/callsites": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", + "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/chai": { + "version": "5.3.3", + "resolved": "https://registry.npmjs.org/chai/-/chai-5.3.3.tgz", + "integrity": "sha512-4zNhdJD/iOjSH0A05ea+Ke6MU5mmpQcbQsSOkgdaUMJ9zTlDTD/GYlwohmIE2u0gaxHYiVHEn1Fw9mZ/ktJWgw==", + "dev": true, + "license": "MIT", + "dependencies": { + "assertion-error": "^2.0.1", + "check-error": "^2.1.1", + "deep-eql": "^5.0.1", + "loupe": "^3.1.0", + "pathval": "^2.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/check-error": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/check-error/-/check-error-2.1.3.tgz", + "integrity": "sha512-PAJdDJusoxnwm1VwW07VWwUN1sl7smmC3OKggvndJFadxxDRyFJBX/ggnu/KE4kQAB7a3Dp8f/YXC1FlUprWmA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 16" + } + }, + "node_modules/chokidar": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-4.0.3.tgz", + "integrity": "sha512-Qgzu8kfBvo+cA4962jnP1KkS6Dop5NS6g7R5LFYJr4b8Ub94PPQXUksCw9PvXoeXPRRddRNC5C1JQUR2SMGtnA==", + "dev": true, + "license": "MIT", + "dependencies": { + "readdirp": "^4.0.1" + }, + "engines": { + "node": ">= 14.16.0" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/clsx": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/clsx/-/clsx-2.1.1.tgz", + "integrity": "sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true, + "license": "MIT" + }, + "node_modules/concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", + "dev": true, + "license": "MIT" + }, + "node_modules/cross-spawn": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "dev": true, + "license": "MIT", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/deep-eql": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/deep-eql/-/deep-eql-5.0.2.tgz", + "integrity": "sha512-h5k/5U50IJJFpzfL6nO9jaaumfjO/f2NjK/oYB2Djzm4p9L+3T9qWpZqZ2hAbLPuuYq9wrU08WQyBTL5GbPk5Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/deep-is": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", + "integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/deepmerge": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/deepmerge/-/deepmerge-4.3.1.tgz", + "integrity": "sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/devalue": { + "version": "5.9.2", + "resolved": "https://registry.npmjs.org/devalue/-/devalue-5.9.2.tgz", + "integrity": "sha512-po4PAY5c53tw5XMocSnf8A/5OHhbbUftpr93aEN6BBoAdntUmK7vu7wOATqvt7cXO7m1Cl4gMVn6p7n6n4mj0w==", + "dev": true, + "license": "MIT" + }, + "node_modules/es-module-lexer": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-1.7.0.tgz", + "integrity": "sha512-jEQoCwk8hyb2AZziIOLhDqpm5+2ww5uIE6lkO/6jcOCusfk6LhMHpXXfBLXTZ7Ydyt0j4VoUQv6uGNYbdW+kBA==", + "dev": true, + "license": "MIT" + }, + "node_modules/esbuild": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.25.12.tgz", + "integrity": "sha512-bbPBYYrtZbkt6Os6FiTLCTFxvq4tt3JKall1vRwshA3fdVztsLAatFaZobhkBC8/BrPetoa0oksYoKXoG4ryJg==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.25.12", + "@esbuild/android-arm": "0.25.12", + "@esbuild/android-arm64": "0.25.12", + "@esbuild/android-x64": "0.25.12", + "@esbuild/darwin-arm64": "0.25.12", + "@esbuild/darwin-x64": "0.25.12", + "@esbuild/freebsd-arm64": "0.25.12", + "@esbuild/freebsd-x64": "0.25.12", + "@esbuild/linux-arm": "0.25.12", + "@esbuild/linux-arm64": "0.25.12", + "@esbuild/linux-ia32": "0.25.12", + "@esbuild/linux-loong64": "0.25.12", + "@esbuild/linux-mips64el": "0.25.12", + "@esbuild/linux-ppc64": "0.25.12", + "@esbuild/linux-riscv64": "0.25.12", + "@esbuild/linux-s390x": "0.25.12", + "@esbuild/linux-x64": "0.25.12", + "@esbuild/netbsd-arm64": "0.25.12", + "@esbuild/netbsd-x64": "0.25.12", + "@esbuild/openbsd-arm64": "0.25.12", + "@esbuild/openbsd-x64": "0.25.12", + "@esbuild/openharmony-arm64": "0.25.12", + "@esbuild/sunos-x64": "0.25.12", + "@esbuild/win32-arm64": "0.25.12", + "@esbuild/win32-ia32": "0.25.12", + "@esbuild/win32-x64": "0.25.12" + } + }, + "node_modules/escape-string-regexp": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", + "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/eslint": { + "version": "9.39.5", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-9.39.5.tgz", + "integrity": "sha512-DgZS62aPLXKlnxILS/AYCoRvHaZeXceIzlXPkkGGzJWSow1aEk0lbTlxUSlyjC8jcaKxAdOnTDz+o1JFSBsyjw==", + "deprecated": "This version is no longer supported. Please see https://eslint.org/version-support for other options.", + "dev": true, + "license": "MIT", + "dependencies": { + "@eslint-community/eslint-utils": "^4.8.0", + "@eslint-community/regexpp": "^4.12.1", + "@eslint/config-array": "^0.21.2", + "@eslint/config-helpers": "^0.4.2", + "@eslint/core": "^0.17.0", + "@eslint/eslintrc": "^3.3.6", + "@eslint/js": "9.39.5", + "@eslint/plugin-kit": "^0.4.1", + "@humanfs/node": "^0.16.6", + "@humanwhocodes/module-importer": "^1.0.1", + "@humanwhocodes/retry": "^0.4.2", + "@types/estree": "^1.0.6", + "ajv": "^6.14.0", + "chalk": "^4.0.0", + "cross-spawn": "^7.0.6", + "debug": "^4.3.2", + "escape-string-regexp": "^4.0.0", + "eslint-scope": "^8.4.0", + "eslint-visitor-keys": "^4.2.1", + "espree": "^10.4.0", + "esquery": "^1.5.0", + "esutils": "^2.0.2", + "fast-deep-equal": "^3.1.3", + "file-entry-cache": "^8.0.0", + "find-up": "^5.0.0", + "glob-parent": "^6.0.2", + "ignore": "^5.2.0", + "imurmurhash": "^0.1.4", + "is-glob": "^4.0.0", + "json-stable-stringify-without-jsonify": "^1.0.1", + "lodash.merge": "^4.6.2", + "minimatch": "^3.1.5", + "natural-compare": "^1.4.0", + "optionator": "^0.9.3" + }, + "bin": { + "eslint": "bin/eslint.js" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://eslint.org/donate" + }, + "peerDependencies": { + "jiti": "*" + }, + "peerDependenciesMeta": { + "jiti": { + "optional": true + } + } + }, + "node_modules/eslint-scope": { + "version": "8.4.0", + "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-8.4.0.tgz", + "integrity": "sha512-sNXOfKCn74rt8RICKMvJS7XKV/Xk9kA7DyJr8mJik3S7Cwgy3qlkkmyS2uQB3jiJg6VNdZd/pDBJu0nvG2NlTg==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "esrecurse": "^4.3.0", + "estraverse": "^5.2.0" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/eslint-visitor-keys": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-4.2.1.tgz", + "integrity": "sha512-Uhdk5sfqcee/9H/rCOJikYz67o0a2Tw2hGRPOG2Y1R2dg7brRe1uG0yaNQDHu+TO/uQPF/5eCapvYSmHUjt7JQ==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/esm-env": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/esm-env/-/esm-env-1.2.2.tgz", + "integrity": "sha512-Epxrv+Nr/CaL4ZcFGPJIYLWFom+YeV1DqMLHJoEd9SYRxNbaFruBwfEX/kkHUJf55j2+TUbmDcmuilbP1TmXHA==", + "dev": true, + "license": "MIT" + }, + "node_modules/espree": { + "version": "10.4.0", + "resolved": "https://registry.npmjs.org/espree/-/espree-10.4.0.tgz", + "integrity": "sha512-j6PAQ2uUr79PZhBjP5C5fhl8e39FmRnOjsD5lGnWrFU8i2G776tBK7+nP8KuQUTTyAZUwfQqXAgrVH5MbH9CYQ==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "acorn": "^8.15.0", + "acorn-jsx": "^5.3.2", + "eslint-visitor-keys": "^4.2.1" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/esquery": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.7.0.tgz", + "integrity": "sha512-Ap6G0WQwcU/LHsvLwON1fAQX9Zp0A2Y6Y/cJBl9r/JbW90Zyg4/zbG6zzKa2OTALELarYHmKu0GhpM5EO+7T0g==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "estraverse": "^5.1.0" + }, + "engines": { + "node": ">=0.10" + } + }, + "node_modules/esrap": { + "version": "2.3.7", + "resolved": "https://registry.npmjs.org/esrap/-/esrap-2.3.7.tgz", + "integrity": "sha512-n2nf7fZR3c9yXf0BPEuHuXqT+KW0SJVj4cN5FMEkpCZ3scLjOQWpiccyCxVzCC2q1wubTghuEGzngJY/7Ah0Ow==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.4.15" + }, + "peerDependencies": { + "@typescript-eslint/types": "^8.2.0" + }, + "peerDependenciesMeta": { + "@typescript-eslint/types": { + "optional": true + } + } + }, + "node_modules/esrecurse": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz", + "integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "estraverse": "^5.2.0" + }, + "engines": { + "node": ">=4.0" + } + }, + "node_modules/estraverse": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", + "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", + "dev": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">=4.0" + } + }, + "node_modules/estree-walker": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-3.0.3.tgz", + "integrity": "sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0" + } + }, + "node_modules/esutils": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", + "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", + "dev": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/expect-type": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/expect-type/-/expect-type-1.4.0.tgz", + "integrity": "sha512-KfYbmpRm0VbLjEvVa9yGwCi9GI34xvi7A/HXYWQO65CSD2u3MczUJSuwXKFIxlGsgBQizV9q5J9NHj4VG0n+pA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "dev": true, + "license": "MIT" + }, + "node_modules/fast-json-stable-stringify": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", + "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==", + "dev": true, + "license": "MIT" + }, + "node_modules/fast-levenshtein": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", + "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==", + "dev": true, + "license": "MIT" + }, + "node_modules/fdir": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", + "integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12.0.0" + }, + "peerDependencies": { + "picomatch": "^3 || ^4" + }, + "peerDependenciesMeta": { + "picomatch": { + "optional": true + } + } + }, + "node_modules/file-entry-cache": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-8.0.0.tgz", + "integrity": "sha512-XXTUwCvisa5oacNGRP9SfNtYBNAMi+RPwBFmblZEF7N7swHYQS6/Zfk7SRwx4D5j3CH211YNRco1DEMNVfZCnQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "flat-cache": "^4.0.0" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/find-up": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz", + "integrity": "sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==", + "dev": true, + "license": "MIT", + "dependencies": { + "locate-path": "^6.0.0", + "path-exists": "^4.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/flat-cache": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-4.0.1.tgz", + "integrity": "sha512-f7ccFPK3SXFHpx15UIGyRJ/FJQctuKZ0zVuN3frBo4HnK3cay9VEW0R6yPYFHC0AgqhukPzKjq22t5DmAyqGyw==", + "dev": true, + "license": "MIT", + "dependencies": { + "flatted": "^3.2.9", + "keyv": "^4.5.4" + }, + "engines": { + "node": ">=16" + } + }, + "node_modules/flatted": { + "version": "3.4.4", + "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.4.4.tgz", + "integrity": "sha512-5+ybhBZANEJxaH3X5evAFatUxLfEHSr7n6kYJ+1Qd0mUqr4eu9gIf6GDbWHf8RJijHrjjO8G+la14SlL2SeS1Q==", + "dev": true, + "license": "ISC" + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/glob-parent": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", + "integrity": "sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==", + "dev": true, + "license": "ISC", + "dependencies": { + "is-glob": "^4.0.3" + }, + "engines": { + "node": ">=10.13.0" + } + }, + "node_modules/globals": { + "version": "14.0.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-14.0.0.tgz", + "integrity": "sha512-oahGvuMGQlPw/ivIYBjVSrWAfWLBeku5tpPE2fOPLi+WHffIWbuh2tCjhyQhTBPMf5E9jDEH4FOmTYgYwbKwtQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/ignore": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz", + "integrity": "sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, + "node_modules/import-fresh": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.1.tgz", + "integrity": "sha512-TR3KfrTZTYLPB6jUjfx6MF9WcWrHL9su5TObK4ZkYgBdWKPOFoSoQIdEuTuR82pmtxH2spWG9h6etwfr1pLBqQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "parent-module": "^1.0.0", + "resolve-from": "^4.0.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/imurmurhash": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", + "integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.8.19" + } + }, + "node_modules/is-extglob": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", + "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-glob": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", + "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-extglob": "^2.1.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-reference": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/is-reference/-/is-reference-3.0.3.tgz", + "integrity": "sha512-ixkJoqQvAP88E6wLydLGGqCJsrFUnqoH6HnaczB8XmDH1oaWU+xxdptvikTgaEhtZ53Ky6YXiBuUI2WXLMCwjw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.6" + } + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "dev": true, + "license": "ISC" + }, + "node_modules/js-yaml": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.3.2.tgz", + "integrity": "sha512-SFNOvSJ+Dgf/9An904Yx+CgSlIPCkIpao4qo51lpee25TIRejdH3rhR4EZMGoNx3/TP3O+wzWuiTFl4sqbltzA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/puzrin" + }, + { + "type": "github", + "url": "https://github.com/sponsors/nodeca" + } + ], + "license": "MIT", + "dependencies": { + "argparse": "^2.0.1" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, + "node_modules/json-buffer": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz", + "integrity": "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "dev": true, + "license": "MIT" + }, + "node_modules/json-stable-stringify-without-jsonify": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz", + "integrity": "sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==", + "dev": true, + "license": "MIT" + }, + "node_modules/keyv": { + "version": "4.5.4", + "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.4.tgz", + "integrity": "sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==", + "dev": true, + "license": "MIT", + "dependencies": { + "json-buffer": "3.0.1" + } + }, + "node_modules/leaflet": { + "version": "1.9.4", + "resolved": "https://registry.npmjs.org/leaflet/-/leaflet-1.9.4.tgz", + "integrity": "sha512-nxS1ynzJOmOlHp+iL3FyWqK89GtNL8U8rvlMOsQdTTssxZwCXh8N2NB3GDQOL+YR3XnWyZAxwQixURb+FA74PA==", + "license": "BSD-2-Clause" + }, + "node_modules/levn": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz", + "integrity": "sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "prelude-ls": "^1.2.1", + "type-check": "~0.4.0" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/locate-character": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/locate-character/-/locate-character-3.0.0.tgz", + "integrity": "sha512-SW13ws7BjaeJ6p7Q6CO2nchbYEc3X3J6WrmTTDto7yMPqVSZTUyY5Tjbid+Ab8gLnATtygYtiDIJGQRRn2ZOiA==", + "dev": true, + "license": "MIT" + }, + "node_modules/locate-path": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", + "integrity": "sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==", + "dev": true, + "license": "MIT", + "dependencies": { + "p-locate": "^5.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/lodash.merge": { + "version": "4.6.2", + "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", + "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/loupe": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/loupe/-/loupe-3.2.1.tgz", + "integrity": "sha512-CdzqowRJCeLU72bHvWqwRBBlLcMEtIvGrlvef74kMnV2AolS9Y8xUv1I0U/MNAWMhBlKIoyuEgoJ0t/bbwHbLQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/magic-string": { + "version": "0.30.21", + "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.21.tgz", + "integrity": "sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.5.5" + } + }, + "node_modules/minimatch": { + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.5.tgz", + "integrity": "sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w==", + "dev": true, + "license": "ISC", + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, + "node_modules/mri": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/mri/-/mri-1.2.0.tgz", + "integrity": "sha512-tzzskb3bG8LvYGFF/mDTpq3jpI6Q9wc3LEmBaghu+DdCssd1FakN7Bc0hVNmEyGq1bq3RgfkCb3cmQLpNPOroA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true, + "license": "MIT" + }, + "node_modules/nanoid": { + "version": "3.3.19", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.19.tgz", + "integrity": "sha512-Y2tUNy4ouw6tq5oDSKeQYGOyhkUBhNOcGV/02KC+6kd9eDGqdZd++mjMiIDilrBYvjEnCYvVtsuHCuP+okSfug==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, + "node_modules/natural-compare": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", + "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==", + "dev": true, + "license": "MIT" + }, + "node_modules/obug": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/obug/-/obug-2.2.1.tgz", + "integrity": "sha512-XrsrhT5sybtKI6wakr2SPOlGZWWYbUXZ7a0jT8/QOeAPau+1X/bSegNe5YR75oJmEZQbKningirmGOEJCIk61Q==", + "dev": true, + "funding": [ + "https://github.com/sponsors/sxzz", + "https://opencollective.com/debug" + ], + "license": "MIT", + "engines": { + "node": ">=12.20.0" + } + }, + "node_modules/optionator": { + "version": "0.9.4", + "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.4.tgz", + "integrity": "sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g==", + "dev": true, + "license": "MIT", + "dependencies": { + "deep-is": "^0.1.3", + "fast-levenshtein": "^2.0.6", + "levn": "^0.4.1", + "prelude-ls": "^1.2.1", + "type-check": "^0.4.0", + "word-wrap": "^1.2.5" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/p-limit": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", + "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "yocto-queue": "^0.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-locate": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-5.0.0.tgz", + "integrity": "sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==", + "dev": true, + "license": "MIT", + "dependencies": { + "p-limit": "^3.0.2" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/parent-module": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", + "integrity": "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==", + "dev": true, + "license": "MIT", + "dependencies": { + "callsites": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/path-exists": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", + "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/pathe": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/pathe/-/pathe-1.1.2.tgz", + "integrity": "sha512-whLdWMYL2TwI08hn8/ZqAbrVemu0LNaNNJZX73O6qaIdCTfXutsLhMkjdENX0qhsQ9uIimo4/aQOmXkoon2nDQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/pathval": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/pathval/-/pathval-2.0.1.tgz", + "integrity": "sha512-//nshmD55c46FuFw26xV/xFAaB5HF9Xdap7HJBBnrKdAd6/GxDBaNA1870O79+9ueg61cZLSVc+OaFlfmObYVQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 14.16" + } + }, + "node_modules/picocolors": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "dev": true, + "license": "ISC" + }, + "node_modules/picomatch": { + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.7.tgz", + "integrity": "sha512-qcJu88Q2IWqJsDD529JKMdwGm/dvInW4HvQnRwiH9JtihJvzGOscDtHE3x1pBKeUOTysQ8kVmLnJ2kJu7yhcGA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/playwright": { + "version": "1.63.0", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.63.0.tgz", + "integrity": "sha512-+7ziBLidS4NaNCdt57SUDT+wYmmd5fmiQejUic/kb+YsYSCPyOOE9sebzMjNmQrsnNpDJqd4WHvV/8lfKfUDUg==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright-core": "1.63.0" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/playwright-core": { + "version": "1.63.0", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.63.0.tgz", + "integrity": "sha512-rYCsBF/M5HjUch52bbtVONEFjv6Xu8sm8h72dNlR5bzIE1fvC/bxgspzkjSfU+MweEMmPM8KJebG6nnyxo5mCg==", + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/postcss": { + "version": "8.5.28", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.28.tgz", + "integrity": "sha512-RRuzqDtt5Y9h3quz5hWhK+TPnsmVs6WwSU6LkJMeY4HstUEDuYTG8UJSdawMRzmzAtV+KEoG8N3Qg2qLy5vM/A==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/postcss" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "nanoid": "^3.3.18", + "picocolors": "^1.1.1", + "source-map-js": "^1.2.1" + }, + "engines": { + "node": "^10 || ^12 || >=14" + } + }, + "node_modules/prelude-ls": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", + "integrity": "sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/punycode": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", + "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/readdirp": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-4.1.2.tgz", + "integrity": "sha512-GDhwkLfywWL2s6vEjyhri+eXmfH6j1L7JE27WhqLeYzoh/A3DBaYGEj2H/HFZCn/kMfim73FXxEJTw06WtxQwg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 14.18.0" + }, + "funding": { + "type": "individual", + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/resolve-from": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", + "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/rollup": { + "version": "4.63.2", + "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.63.2.tgz", + "integrity": "sha512-l5eyksV4tPBj6lJyEa37YzIOCSOV7lkZzEHUdpjWZbtD7wTcFYmEYXSgm5bT4vV+dZLb9rBG1W9GROOG4NS4Ew==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree": "1.0.9" + }, + "bin": { + "rollup": "dist/bin/rollup" + }, + "engines": { + "node": ">=18.0.0", + "npm": ">=8.0.0" + }, + "optionalDependencies": { + "@napi-rs/lzma-linux-x64-gnu": "1.5.1", + "@rollup/rollup-android-arm-eabi": "4.63.2", + "@rollup/rollup-android-arm64": "4.63.2", + "@rollup/rollup-darwin-arm64": "4.63.2", + "@rollup/rollup-darwin-x64": "4.63.2", + "@rollup/rollup-freebsd-arm64": "4.63.2", + "@rollup/rollup-freebsd-x64": "4.63.2", + "@rollup/rollup-linux-arm-gnueabihf": "4.63.2", + "@rollup/rollup-linux-arm-musleabihf": "4.63.2", + "@rollup/rollup-linux-arm64-gnu": "4.63.2", + "@rollup/rollup-linux-arm64-musl": "4.63.2", + "@rollup/rollup-linux-loong64-gnu": "4.63.2", + "@rollup/rollup-linux-loong64-musl": "4.63.2", + "@rollup/rollup-linux-ppc64-gnu": "4.63.2", + "@rollup/rollup-linux-ppc64-musl": "4.63.2", + "@rollup/rollup-linux-riscv64-gnu": "4.63.2", + "@rollup/rollup-linux-riscv64-musl": "4.63.2", + "@rollup/rollup-linux-s390x-gnu": "4.63.2", + "@rollup/rollup-linux-x64-gnu": "4.63.2", + "@rollup/rollup-linux-x64-musl": "4.63.2", + "@rollup/rollup-openbsd-x64": "4.63.2", + "@rollup/rollup-openharmony-arm64": "4.63.2", + "@rollup/rollup-win32-arm64-msvc": "4.63.2", + "@rollup/rollup-win32-ia32-msvc": "4.63.2", + "@rollup/rollup-win32-x64-gnu": "4.63.2", + "@rollup/rollup-win32-x64-msvc": "4.63.2", + "fsevents": "~2.3.2" + } + }, + "node_modules/sade": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/sade/-/sade-1.8.1.tgz", + "integrity": "sha512-xal3CZX1Xlo/k4ApwCFrHVACi9fBqJ7V+mwhBsuf/1IOKbBy098Fex+Wa/5QMubw09pSZ/u8EY8PWgevJsXp1A==", + "dev": true, + "license": "MIT", + "dependencies": { + "mri": "^1.1.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "dev": true, + "license": "MIT", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/siginfo": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/siginfo/-/siginfo-2.0.0.tgz", + "integrity": "sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==", + "dev": true, + "license": "ISC" + }, + "node_modules/source-map-js": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", + "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/stackback": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/stackback/-/stackback-0.0.2.tgz", + "integrity": "sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==", + "dev": true, + "license": "MIT" + }, + "node_modules/std-env": { + "version": "3.10.0", + "resolved": "https://registry.npmjs.org/std-env/-/std-env-3.10.0.tgz", + "integrity": "sha512-5GS12FdOZNliM5mAOxFRg7Ir0pWz8MdpYm6AY6VPkGpbA7ZzmbzNcBJQ0GPvvyWgcY7QAhCgf9Uy89I03faLkg==", + "dev": true, + "license": "MIT" + }, + "node_modules/strip-json-comments": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", + "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, + "license": "MIT", + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/svelte": { + "version": "5.57.0", + "resolved": "https://registry.npmjs.org/svelte/-/svelte-5.57.0.tgz", + "integrity": "sha512-NdbDn7fl4be1ViUG0oq/lvG6OZy3oENolV2ONjiqqsfVoeAfzaQAKUcEX3MrQod/Bebv1PgwET9rfXhgn9s4Kg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/remapping": "^2.3.4", + "@jridgewell/sourcemap-codec": "^1.5.0", + "@sveltejs/acorn-typescript": "^1.0.10", + "@types/estree": "^1.0.5", + "acorn": "^8.12.1", + "aria-query": "5.3.1", + "axobject-query": "^4.1.0", + "clsx": "^2.1.1", + "devalue": "^5.8.1", + "esm-env": "^1.2.1", + "esrap": "^2.2.12", + "is-reference": "^3.0.3", + "locate-character": "^3.0.0", + "magic-string": "^0.30.11", + "zimmerframe": "^1.1.2" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/svelte-check": { + "version": "4.7.6", + "resolved": "https://registry.npmjs.org/svelte-check/-/svelte-check-4.7.6.tgz", + "integrity": "sha512-t2scM//ZuVbSY/T2w6FSBw1v9s2NEmh/g+sy1lqtosW5ylBV5AF4wFb1Ts9Kf3MbfPDUDJDZ9L436YT0SPTdvw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/trace-mapping": "^0.3.25", + "@sveltejs/load-config": "^0.2.3", + "chokidar": "^4.0.1", + "fdir": "^6.2.0", + "picocolors": "^1.0.0", + "sade": "^1.7.4" + }, + "bin": { + "svelte-check": "bin/svelte-check" + }, + "engines": { + "node": ">= 18.0.0" + }, + "peerDependencies": { + "svelte": "^4.0.0 || ^5.0.0-next.0", + "typescript": "^5.0.0 || ^6.0.0" + } + }, + "node_modules/tinybench": { + "version": "2.9.0", + "resolved": "https://registry.npmjs.org/tinybench/-/tinybench-2.9.0.tgz", + "integrity": "sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==", + "dev": true, + "license": "MIT" + }, + "node_modules/tinyexec": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/tinyexec/-/tinyexec-0.3.2.tgz", + "integrity": "sha512-KQQR9yN7R5+OSwaK0XQoj22pwHoTlgYqmUscPYoknOoWCWfj/5/ABTMRi69FrKU5ffPVh5QcFikpWJI/P1ocHA==", + "dev": true, + "license": "MIT" + }, + "node_modules/tinyglobby": { + "version": "0.2.17", + "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.17.tgz", + "integrity": "sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g==", + "dev": true, + "license": "MIT", + "dependencies": { + "fdir": "^6.5.0", + "picomatch": "^4.0.4" + }, + "engines": { + "node": ">=12.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/SuperchupuDev" + } + }, + "node_modules/tinypool": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/tinypool/-/tinypool-1.1.1.tgz", + "integrity": "sha512-Zba82s87IFq9A9XmjiX5uZA/ARWDrB03OHlq+Vw1fSdt0I+4/Kutwy8BP4Y/y/aORMo61FQ0vIb5j44vSo5Pkg==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^18.0.0 || >=20.0.0" + } + }, + "node_modules/tinyrainbow": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/tinyrainbow/-/tinyrainbow-1.2.0.tgz", + "integrity": "sha512-weEDEq7Z5eTHPDh4xjX789+fHfF+P8boiFB+0vbWzpbnbsEr/GRaohi/uMKxg8RZMXnl1ItAi/IUHWMsjDV7kQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/tinyspy": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/tinyspy/-/tinyspy-3.0.2.tgz", + "integrity": "sha512-n1cw8k1k0x4pgA2+9XrOkFydTerNcJ1zWCO5Nn9scWHTD+5tp8dghT2x1uduQePZTZgd3Tupf+x9BxJjeJi77Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/type-check": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", + "integrity": "sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==", + "dev": true, + "license": "MIT", + "dependencies": { + "prelude-ls": "^1.2.1" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici-types": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/uri-js": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", + "integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "punycode": "^2.1.0" + } + }, + "node_modules/vite": { + "version": "6.4.3", + "resolved": "https://registry.npmjs.org/vite/-/vite-6.4.3.tgz", + "integrity": "sha512-NTKlcQjlAK7MlQoyb6LgaqHc8sso/pVyUJYWMws3jg21uTJw/LddqIFPcPqP6PzpgbIcZyKI85sFE4HBrQDA8A==", + "dev": true, + "license": "MIT", + "dependencies": { + "esbuild": "^0.25.0", + "fdir": "^6.4.4", + "picomatch": "^4.0.2", + "postcss": "^8.5.3", + "rollup": "^4.34.9", + "tinyglobby": "^0.2.13" + }, + "bin": { + "vite": "bin/vite.js" + }, + "engines": { + "node": "^18.0.0 || ^20.0.0 || >=22.0.0" + }, + "funding": { + "url": "https://github.com/vitejs/vite?sponsor=1" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + }, + "peerDependencies": { + "@types/node": "^18.0.0 || ^20.0.0 || >=22.0.0", + "jiti": ">=1.21.0", + "less": "*", + "lightningcss": "^1.21.0", + "sass": "*", + "sass-embedded": "*", + "stylus": "*", + "sugarss": "*", + "terser": "^5.16.0", + "tsx": "^4.8.1", + "yaml": "^2.4.2" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + }, + "jiti": { + "optional": true + }, + "less": { + "optional": true + }, + "lightningcss": { + "optional": true + }, + "sass": { + "optional": true + }, + "sass-embedded": { + "optional": true + }, + "stylus": { + "optional": true + }, + "sugarss": { + "optional": true + }, + "terser": { + "optional": true + }, + "tsx": { + "optional": true + }, + "yaml": { + "optional": true + } + } + }, + "node_modules/vite-node": { + "version": "2.1.9", + "resolved": "https://registry.npmjs.org/vite-node/-/vite-node-2.1.9.tgz", + "integrity": "sha512-AM9aQ/IPrW/6ENLQg3AGY4K1N2TGZdR5e4gu/MmmR2xR3Ll1+dib+nook92g4TV3PXVyeyxdWwtaCAiUL0hMxA==", + "dev": true, + "license": "MIT", + "dependencies": { + "cac": "^6.7.14", + "debug": "^4.3.7", + "es-module-lexer": "^1.5.4", + "pathe": "^1.1.2", + "vite": "^5.0.0" + }, + "bin": { + "vite-node": "vite-node.mjs" + }, + "engines": { + "node": "^18.0.0 || >=20.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/vite-node/node_modules/@esbuild/aix-ppc64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.21.5.tgz", + "integrity": "sha512-1SDgH6ZSPTlggy1yI6+Dbkiz8xzpHJEVAlF/AM1tHPLsf5STom9rwtjE4hKAF20FfXXNTFqEYXyJNWh1GiZedQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vite-node/node_modules/@esbuild/android-arm": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.21.5.tgz", + "integrity": "sha512-vCPvzSjpPHEi1siZdlvAlsPxXl7WbOVUBBAowWug4rJHb68Ox8KualB+1ocNvT5fjv6wpkX6o/iEpbDrf68zcg==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vite-node/node_modules/@esbuild/android-arm64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.21.5.tgz", + "integrity": "sha512-c0uX9VAUBQ7dTDCjq+wdyGLowMdtR/GoC2U5IYk/7D1H1JYC0qseD7+11iMP2mRLN9RcCMRcjC4YMclCzGwS/A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vite-node/node_modules/@esbuild/android-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.21.5.tgz", + "integrity": "sha512-D7aPRUUNHRBwHxzxRvp856rjUHRFW1SdQATKXH2hqA0kAZb1hKmi02OpYRacl0TxIGz/ZmXWlbZgjwWYaCakTA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vite-node/node_modules/@esbuild/darwin-arm64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.21.5.tgz", + "integrity": "sha512-DwqXqZyuk5AiWWf3UfLiRDJ5EDd49zg6O9wclZ7kUMv2WRFr4HKjXp/5t8JZ11QbQfUS6/cRCKGwYhtNAY88kQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vite-node/node_modules/@esbuild/darwin-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.21.5.tgz", + "integrity": "sha512-se/JjF8NlmKVG4kNIuyWMV/22ZaerB+qaSi5MdrXtd6R08kvs2qCN4C09miupktDitvh8jRFflwGFBQcxZRjbw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vite-node/node_modules/@esbuild/freebsd-arm64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.21.5.tgz", + "integrity": "sha512-5JcRxxRDUJLX8JXp/wcBCy3pENnCgBR9bN6JsY4OmhfUtIHe3ZW0mawA7+RDAcMLrMIZaf03NlQiX9DGyB8h4g==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vite-node/node_modules/@esbuild/freebsd-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.21.5.tgz", + "integrity": "sha512-J95kNBj1zkbMXtHVH29bBriQygMXqoVQOQYA+ISs0/2l3T9/kj42ow2mpqerRBxDJnmkUDCaQT/dfNXWX/ZZCQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vite-node/node_modules/@esbuild/linux-arm": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.21.5.tgz", + "integrity": "sha512-bPb5AHZtbeNGjCKVZ9UGqGwo8EUu4cLq68E95A53KlxAPRmUyYv2D6F0uUI65XisGOL1hBP5mTronbgo+0bFcA==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vite-node/node_modules/@esbuild/linux-arm64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.21.5.tgz", + "integrity": "sha512-ibKvmyYzKsBeX8d8I7MH/TMfWDXBF3db4qM6sy+7re0YXya+K1cem3on9XgdT2EQGMu4hQyZhan7TeQ8XkGp4Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vite-node/node_modules/@esbuild/linux-ia32": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.21.5.tgz", + "integrity": "sha512-YvjXDqLRqPDl2dvRODYmmhz4rPeVKYvppfGYKSNGdyZkA01046pLWyRKKI3ax8fbJoK5QbxblURkwK/MWY18Tg==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vite-node/node_modules/@esbuild/linux-loong64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.21.5.tgz", + "integrity": "sha512-uHf1BmMG8qEvzdrzAqg2SIG/02+4/DHB6a9Kbya0XDvwDEKCoC8ZRWI5JJvNdUjtciBGFQ5PuBlpEOXQj+JQSg==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vite-node/node_modules/@esbuild/linux-mips64el": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.21.5.tgz", + "integrity": "sha512-IajOmO+KJK23bj52dFSNCMsz1QP1DqM6cwLUv3W1QwyxkyIWecfafnI555fvSGqEKwjMXVLokcV5ygHW5b3Jbg==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vite-node/node_modules/@esbuild/linux-ppc64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.21.5.tgz", + "integrity": "sha512-1hHV/Z4OEfMwpLO8rp7CvlhBDnjsC3CttJXIhBi+5Aj5r+MBvy4egg7wCbe//hSsT+RvDAG7s81tAvpL2XAE4w==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vite-node/node_modules/@esbuild/linux-riscv64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.21.5.tgz", + "integrity": "sha512-2HdXDMd9GMgTGrPWnJzP2ALSokE/0O5HhTUvWIbD3YdjME8JwvSCnNGBnTThKGEB91OZhzrJ4qIIxk/SBmyDDA==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vite-node/node_modules/@esbuild/linux-s390x": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.21.5.tgz", + "integrity": "sha512-zus5sxzqBJD3eXxwvjN1yQkRepANgxE9lgOW2qLnmr8ikMTphkjgXu1HR01K4FJg8h1kEEDAqDcZQtbrRnB41A==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vite-node/node_modules/@esbuild/linux-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.21.5.tgz", + "integrity": "sha512-1rYdTpyv03iycF1+BhzrzQJCdOuAOtaqHTWJZCWvijKD2N5Xu0TtVC8/+1faWqcP9iBCWOmjmhoH94dH82BxPQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vite-node/node_modules/@esbuild/netbsd-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.21.5.tgz", + "integrity": "sha512-Woi2MXzXjMULccIwMnLciyZH4nCIMpWQAs049KEeMvOcNADVxo0UBIQPfSmxB3CWKedngg7sWZdLvLczpe0tLg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vite-node/node_modules/@esbuild/openbsd-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.21.5.tgz", + "integrity": "sha512-HLNNw99xsvx12lFBUwoT8EVCsSvRNDVxNpjZ7bPn947b8gJPzeHWyNVhFsaerc0n3TsbOINvRP2byTZ5LKezow==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vite-node/node_modules/@esbuild/sunos-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.21.5.tgz", + "integrity": "sha512-6+gjmFpfy0BHU5Tpptkuh8+uw3mnrvgs+dSPQXQOv3ekbordwnzTVEb4qnIvQcYXq6gzkyTnoZ9dZG+D4garKg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vite-node/node_modules/@esbuild/win32-arm64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.21.5.tgz", + "integrity": "sha512-Z0gOTd75VvXqyq7nsl93zwahcTROgqvuAcYDUr+vOv8uHhNSKROyU961kgtCD1e95IqPKSQKH7tBTslnS3tA8A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vite-node/node_modules/@esbuild/win32-ia32": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.21.5.tgz", + "integrity": "sha512-SWXFF1CL2RVNMaVs+BBClwtfZSvDgtL//G/smwAc5oVK/UPu2Gu9tIaRgFmYFFKrmg3SyAjSrElf0TiJ1v8fYA==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vite-node/node_modules/@esbuild/win32-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.21.5.tgz", + "integrity": "sha512-tQd/1efJuzPC6rCFwEvLtci/xNFcTZknmXs98FYDfGE4wP9ClFV98nyKrzJKVPMhdDnjzLhdUyMX4PsQAPjwIw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vite-node/node_modules/esbuild": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.21.5.tgz", + "integrity": "sha512-mg3OPMV4hXywwpoDxu3Qda5xCKQi+vCTZq8S9J/EpkhB2HzKXq4SNFZE3+NK93JYxc8VMSep+lOUSC/RVKaBqw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=12" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.21.5", + "@esbuild/android-arm": "0.21.5", + "@esbuild/android-arm64": "0.21.5", + "@esbuild/android-x64": "0.21.5", + "@esbuild/darwin-arm64": "0.21.5", + "@esbuild/darwin-x64": "0.21.5", + "@esbuild/freebsd-arm64": "0.21.5", + "@esbuild/freebsd-x64": "0.21.5", + "@esbuild/linux-arm": "0.21.5", + "@esbuild/linux-arm64": "0.21.5", + "@esbuild/linux-ia32": "0.21.5", + "@esbuild/linux-loong64": "0.21.5", + "@esbuild/linux-mips64el": "0.21.5", + "@esbuild/linux-ppc64": "0.21.5", + "@esbuild/linux-riscv64": "0.21.5", + "@esbuild/linux-s390x": "0.21.5", + "@esbuild/linux-x64": "0.21.5", + "@esbuild/netbsd-x64": "0.21.5", + "@esbuild/openbsd-x64": "0.21.5", + "@esbuild/sunos-x64": "0.21.5", + "@esbuild/win32-arm64": "0.21.5", + "@esbuild/win32-ia32": "0.21.5", + "@esbuild/win32-x64": "0.21.5" + } + }, + "node_modules/vite-node/node_modules/vite": { + "version": "5.4.21", + "resolved": "https://registry.npmjs.org/vite/-/vite-5.4.21.tgz", + "integrity": "sha512-o5a9xKjbtuhY6Bi5S3+HvbRERmouabWbyUcpXXUA1u+GNUKoROi9byOJ8M0nHbHYHkYICiMlqxkg1KkYmm25Sw==", + "dev": true, + "license": "MIT", + "dependencies": { + "esbuild": "^0.21.3", + "postcss": "^8.4.43", + "rollup": "^4.20.0" + }, + "bin": { + "vite": "bin/vite.js" + }, + "engines": { + "node": "^18.0.0 || >=20.0.0" + }, + "funding": { + "url": "https://github.com/vitejs/vite?sponsor=1" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + }, + "peerDependencies": { + "@types/node": "^18.0.0 || >=20.0.0", + "less": "*", + "lightningcss": "^1.21.0", + "sass": "*", + "sass-embedded": "*", + "stylus": "*", + "sugarss": "*", + "terser": "^5.4.0" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + }, + "less": { + "optional": true + }, + "lightningcss": { + "optional": true + }, + "sass": { + "optional": true + }, + "sass-embedded": { + "optional": true + }, + "stylus": { + "optional": true + }, + "sugarss": { + "optional": true + }, + "terser": { + "optional": true + } + } + }, + "node_modules/vitefu": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/vitefu/-/vitefu-1.1.3.tgz", + "integrity": "sha512-ub4okH7Z5KLjb6hDyjqrGXqWtWvoYdU3IGm/NorpgHncKoLTCfRIbvlhBm7r0YstIaQRYlp4yEbFqDcKSzXSSg==", + "dev": true, + "license": "MIT", + "workspaces": [ + "tests/deps/*", + "tests/projects/*", + "tests/projects/workspace/packages/*" + ], + "peerDependencies": { + "vite": "^3.0.0 || ^4.0.0 || ^5.0.0 || ^6.0.0 || ^7.0.0 || ^8.0.0" + }, + "peerDependenciesMeta": { + "vite": { + "optional": true + } + } + }, + "node_modules/vitest": { + "version": "2.1.9", + "resolved": "https://registry.npmjs.org/vitest/-/vitest-2.1.9.tgz", + "integrity": "sha512-MSmPM9REYqDGBI8439mA4mWhV5sKmDlBKWIYbA3lRb2PTHACE0mgKwA8yQ2xq9vxDTuk4iPrECBAEW2aoFXY0Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/expect": "2.1.9", + "@vitest/mocker": "2.1.9", + "@vitest/pretty-format": "^2.1.9", + "@vitest/runner": "2.1.9", + "@vitest/snapshot": "2.1.9", + "@vitest/spy": "2.1.9", + "@vitest/utils": "2.1.9", + "chai": "^5.1.2", + "debug": "^4.3.7", + "expect-type": "^1.1.0", + "magic-string": "^0.30.12", + "pathe": "^1.1.2", + "std-env": "^3.8.0", + "tinybench": "^2.9.0", + "tinyexec": "^0.3.1", + "tinypool": "^1.0.1", + "tinyrainbow": "^1.2.0", + "vite": "^5.0.0", + "vite-node": "2.1.9", + "why-is-node-running": "^2.3.0" + }, + "bin": { + "vitest": "vitest.mjs" + }, + "engines": { + "node": "^18.0.0 || >=20.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "@edge-runtime/vm": "*", + "@types/node": "^18.0.0 || >=20.0.0", + "@vitest/browser": "2.1.9", + "@vitest/ui": "2.1.9", + "happy-dom": "*", + "jsdom": "*" + }, + "peerDependenciesMeta": { + "@edge-runtime/vm": { + "optional": true + }, + "@types/node": { + "optional": true + }, + "@vitest/browser": { + "optional": true + }, + "@vitest/ui": { + "optional": true + }, + "happy-dom": { + "optional": true + }, + "jsdom": { + "optional": true + } + } + }, + "node_modules/vitest/node_modules/@esbuild/aix-ppc64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.21.5.tgz", + "integrity": "sha512-1SDgH6ZSPTlggy1yI6+Dbkiz8xzpHJEVAlF/AM1tHPLsf5STom9rwtjE4hKAF20FfXXNTFqEYXyJNWh1GiZedQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vitest/node_modules/@esbuild/android-arm": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.21.5.tgz", + "integrity": "sha512-vCPvzSjpPHEi1siZdlvAlsPxXl7WbOVUBBAowWug4rJHb68Ox8KualB+1ocNvT5fjv6wpkX6o/iEpbDrf68zcg==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vitest/node_modules/@esbuild/android-arm64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.21.5.tgz", + "integrity": "sha512-c0uX9VAUBQ7dTDCjq+wdyGLowMdtR/GoC2U5IYk/7D1H1JYC0qseD7+11iMP2mRLN9RcCMRcjC4YMclCzGwS/A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vitest/node_modules/@esbuild/android-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.21.5.tgz", + "integrity": "sha512-D7aPRUUNHRBwHxzxRvp856rjUHRFW1SdQATKXH2hqA0kAZb1hKmi02OpYRacl0TxIGz/ZmXWlbZgjwWYaCakTA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vitest/node_modules/@esbuild/darwin-arm64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.21.5.tgz", + "integrity": "sha512-DwqXqZyuk5AiWWf3UfLiRDJ5EDd49zg6O9wclZ7kUMv2WRFr4HKjXp/5t8JZ11QbQfUS6/cRCKGwYhtNAY88kQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vitest/node_modules/@esbuild/darwin-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.21.5.tgz", + "integrity": "sha512-se/JjF8NlmKVG4kNIuyWMV/22ZaerB+qaSi5MdrXtd6R08kvs2qCN4C09miupktDitvh8jRFflwGFBQcxZRjbw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vitest/node_modules/@esbuild/freebsd-arm64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.21.5.tgz", + "integrity": "sha512-5JcRxxRDUJLX8JXp/wcBCy3pENnCgBR9bN6JsY4OmhfUtIHe3ZW0mawA7+RDAcMLrMIZaf03NlQiX9DGyB8h4g==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vitest/node_modules/@esbuild/freebsd-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.21.5.tgz", + "integrity": "sha512-J95kNBj1zkbMXtHVH29bBriQygMXqoVQOQYA+ISs0/2l3T9/kj42ow2mpqerRBxDJnmkUDCaQT/dfNXWX/ZZCQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vitest/node_modules/@esbuild/linux-arm": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.21.5.tgz", + "integrity": "sha512-bPb5AHZtbeNGjCKVZ9UGqGwo8EUu4cLq68E95A53KlxAPRmUyYv2D6F0uUI65XisGOL1hBP5mTronbgo+0bFcA==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vitest/node_modules/@esbuild/linux-arm64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.21.5.tgz", + "integrity": "sha512-ibKvmyYzKsBeX8d8I7MH/TMfWDXBF3db4qM6sy+7re0YXya+K1cem3on9XgdT2EQGMu4hQyZhan7TeQ8XkGp4Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vitest/node_modules/@esbuild/linux-ia32": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.21.5.tgz", + "integrity": "sha512-YvjXDqLRqPDl2dvRODYmmhz4rPeVKYvppfGYKSNGdyZkA01046pLWyRKKI3ax8fbJoK5QbxblURkwK/MWY18Tg==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vitest/node_modules/@esbuild/linux-loong64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.21.5.tgz", + "integrity": "sha512-uHf1BmMG8qEvzdrzAqg2SIG/02+4/DHB6a9Kbya0XDvwDEKCoC8ZRWI5JJvNdUjtciBGFQ5PuBlpEOXQj+JQSg==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vitest/node_modules/@esbuild/linux-mips64el": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.21.5.tgz", + "integrity": "sha512-IajOmO+KJK23bj52dFSNCMsz1QP1DqM6cwLUv3W1QwyxkyIWecfafnI555fvSGqEKwjMXVLokcV5ygHW5b3Jbg==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vitest/node_modules/@esbuild/linux-ppc64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.21.5.tgz", + "integrity": "sha512-1hHV/Z4OEfMwpLO8rp7CvlhBDnjsC3CttJXIhBi+5Aj5r+MBvy4egg7wCbe//hSsT+RvDAG7s81tAvpL2XAE4w==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vitest/node_modules/@esbuild/linux-riscv64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.21.5.tgz", + "integrity": "sha512-2HdXDMd9GMgTGrPWnJzP2ALSokE/0O5HhTUvWIbD3YdjME8JwvSCnNGBnTThKGEB91OZhzrJ4qIIxk/SBmyDDA==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vitest/node_modules/@esbuild/linux-s390x": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.21.5.tgz", + "integrity": "sha512-zus5sxzqBJD3eXxwvjN1yQkRepANgxE9lgOW2qLnmr8ikMTphkjgXu1HR01K4FJg8h1kEEDAqDcZQtbrRnB41A==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vitest/node_modules/@esbuild/linux-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.21.5.tgz", + "integrity": "sha512-1rYdTpyv03iycF1+BhzrzQJCdOuAOtaqHTWJZCWvijKD2N5Xu0TtVC8/+1faWqcP9iBCWOmjmhoH94dH82BxPQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vitest/node_modules/@esbuild/netbsd-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.21.5.tgz", + "integrity": "sha512-Woi2MXzXjMULccIwMnLciyZH4nCIMpWQAs049KEeMvOcNADVxo0UBIQPfSmxB3CWKedngg7sWZdLvLczpe0tLg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vitest/node_modules/@esbuild/openbsd-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.21.5.tgz", + "integrity": "sha512-HLNNw99xsvx12lFBUwoT8EVCsSvRNDVxNpjZ7bPn947b8gJPzeHWyNVhFsaerc0n3TsbOINvRP2byTZ5LKezow==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vitest/node_modules/@esbuild/sunos-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.21.5.tgz", + "integrity": "sha512-6+gjmFpfy0BHU5Tpptkuh8+uw3mnrvgs+dSPQXQOv3ekbordwnzTVEb4qnIvQcYXq6gzkyTnoZ9dZG+D4garKg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vitest/node_modules/@esbuild/win32-arm64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.21.5.tgz", + "integrity": "sha512-Z0gOTd75VvXqyq7nsl93zwahcTROgqvuAcYDUr+vOv8uHhNSKROyU961kgtCD1e95IqPKSQKH7tBTslnS3tA8A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vitest/node_modules/@esbuild/win32-ia32": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.21.5.tgz", + "integrity": "sha512-SWXFF1CL2RVNMaVs+BBClwtfZSvDgtL//G/smwAc5oVK/UPu2Gu9tIaRgFmYFFKrmg3SyAjSrElf0TiJ1v8fYA==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vitest/node_modules/@esbuild/win32-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.21.5.tgz", + "integrity": "sha512-tQd/1efJuzPC6rCFwEvLtci/xNFcTZknmXs98FYDfGE4wP9ClFV98nyKrzJKVPMhdDnjzLhdUyMX4PsQAPjwIw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vitest/node_modules/@vitest/mocker": { + "version": "2.1.9", + "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-2.1.9.tgz", + "integrity": "sha512-tVL6uJgoUdi6icpxmdrn5YNo3g3Dxv+IHJBr0GXHaEdTcw3F+cPKnsXFhli6nO+f/6SDKPHEK1UN+k+TQv0Ehg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/spy": "2.1.9", + "estree-walker": "^3.0.3", + "magic-string": "^0.30.12" + }, + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "msw": "^2.4.9", + "vite": "^5.0.0" + }, + "peerDependenciesMeta": { + "msw": { + "optional": true + }, + "vite": { + "optional": true + } + } + }, + "node_modules/vitest/node_modules/esbuild": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.21.5.tgz", + "integrity": "sha512-mg3OPMV4hXywwpoDxu3Qda5xCKQi+vCTZq8S9J/EpkhB2HzKXq4SNFZE3+NK93JYxc8VMSep+lOUSC/RVKaBqw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=12" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.21.5", + "@esbuild/android-arm": "0.21.5", + "@esbuild/android-arm64": "0.21.5", + "@esbuild/android-x64": "0.21.5", + "@esbuild/darwin-arm64": "0.21.5", + "@esbuild/darwin-x64": "0.21.5", + "@esbuild/freebsd-arm64": "0.21.5", + "@esbuild/freebsd-x64": "0.21.5", + "@esbuild/linux-arm": "0.21.5", + "@esbuild/linux-arm64": "0.21.5", + "@esbuild/linux-ia32": "0.21.5", + "@esbuild/linux-loong64": "0.21.5", + "@esbuild/linux-mips64el": "0.21.5", + "@esbuild/linux-ppc64": "0.21.5", + "@esbuild/linux-riscv64": "0.21.5", + "@esbuild/linux-s390x": "0.21.5", + "@esbuild/linux-x64": "0.21.5", + "@esbuild/netbsd-x64": "0.21.5", + "@esbuild/openbsd-x64": "0.21.5", + "@esbuild/sunos-x64": "0.21.5", + "@esbuild/win32-arm64": "0.21.5", + "@esbuild/win32-ia32": "0.21.5", + "@esbuild/win32-x64": "0.21.5" + } + }, + "node_modules/vitest/node_modules/vite": { + "version": "5.4.21", + "resolved": "https://registry.npmjs.org/vite/-/vite-5.4.21.tgz", + "integrity": "sha512-o5a9xKjbtuhY6Bi5S3+HvbRERmouabWbyUcpXXUA1u+GNUKoROi9byOJ8M0nHbHYHkYICiMlqxkg1KkYmm25Sw==", + "dev": true, + "license": "MIT", + "dependencies": { + "esbuild": "^0.21.3", + "postcss": "^8.4.43", + "rollup": "^4.20.0" + }, + "bin": { + "vite": "bin/vite.js" + }, + "engines": { + "node": "^18.0.0 || >=20.0.0" + }, + "funding": { + "url": "https://github.com/vitejs/vite?sponsor=1" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + }, + "peerDependencies": { + "@types/node": "^18.0.0 || >=20.0.0", + "less": "*", + "lightningcss": "^1.21.0", + "sass": "*", + "sass-embedded": "*", + "stylus": "*", + "sugarss": "*", + "terser": "^5.4.0" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + }, + "less": { + "optional": true + }, + "lightningcss": { + "optional": true + }, + "sass": { + "optional": true + }, + "sass-embedded": { + "optional": true + }, + "stylus": { + "optional": true + }, + "sugarss": { + "optional": true + }, + "terser": { + "optional": true + } + } + }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "dev": true, + "license": "ISC", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/why-is-node-running": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/why-is-node-running/-/why-is-node-running-2.3.0.tgz", + "integrity": "sha512-hUrmaWBdVDcxvYqnyh09zunKzROWjbZTiNy8dBEjkS7ehEDQibXJ7XvlmtbwuTclUiIyN+CyXQD4Vmko8fNm8w==", + "dev": true, + "license": "MIT", + "dependencies": { + "siginfo": "^2.0.0", + "stackback": "0.0.2" + }, + "bin": { + "why-is-node-running": "cli.js" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/word-wrap": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.5.tgz", + "integrity": "sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/yocto-queue": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", + "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/zimmerframe": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/zimmerframe/-/zimmerframe-1.1.5.tgz", + "integrity": "sha512-msJxIvYDYcoNL+PJsu+7qmpDWsYmAxTY+2TNYXXF0hzBzBk0BMecOqDOG/EckUoKCuKwObfbugIl8QpqHDXeFA==", + "dev": true, + "license": "MIT" + }, + "node_modules/zod": { + "version": "3.25.76", + "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", + "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + } + } +} diff --git a/frontend/package.json b/frontend/package.json new file mode 100644 index 0000000..d4c3bf7 --- /dev/null +++ b/frontend/package.json @@ -0,0 +1,8 @@ +{ + "name": "until-every-cage-v2-frontend", + "private": true, + "type": "module", + "scripts": {"dev":"vite","preview":"vite preview","check":"svelte-check --tsconfig ./tsconfig.app.json && tsc -p tsconfig.node.json --noEmit","build":"npm run check && vite build","test":"vitest run","test:e2e":"playwright test","lint":"eslint .","stage":"node scripts/stage-preview.mjs","boundary":"node scripts/check-boundaries.mjs"}, + "devDependencies": {"@playwright/test":"^1.49.1","@sveltejs/vite-plugin-svelte":"^6.2.1","@types/node":"^22.10.2","eslint":"^9.17.0","svelte":"^5.19.0","svelte-check":"^4.1.4","typescript":"^5.7.2","vite":"^6.0.7","vitest":"^2.1.8"}, + "dependencies": {"@axe-core/playwright":"^4.10.2","@types/leaflet":"^1.9.15","leaflet":"^1.9.4","zod":"^3.24.1"} +} diff --git a/frontend/playwright.config.ts b/frontend/playwright.config.ts new file mode 100644 index 0000000..fe225d4 --- /dev/null +++ b/frontend/playwright.config.ts @@ -0,0 +1,12 @@ +import { defineConfig, devices } from '@playwright/test'; + +export default defineConfig({ + testDir: './tests/e2e', + fullyParallel: true, + forbidOnly: !!process.env.CI, + reporter: process.env.CI ? 'line' : 'list', + timeout: 30_000, + use: { baseURL: 'http://127.0.0.1:4173/v2-preview/', trace: 'retain-on-failure' }, + webServer: { command: 'npm run preview -- --host 127.0.0.1 --port 4173', url: 'http://127.0.0.1:4173/v2-preview/', timeout: 30_000, reuseExistingServer: true }, + projects: [{ name: 'chromium', use: { ...devices['Desktop Chrome'] } }], +}); diff --git a/frontend/scripts/check-boundaries.mjs b/frontend/scripts/check-boundaries.mjs new file mode 100644 index 0000000..eeb4015 --- /dev/null +++ b/frontend/scripts/check-boundaries.mjs @@ -0,0 +1,3 @@ +import { readFile, readdir } from 'node:fs/promises'; import { resolve } from 'node:path'; +const root=resolve(import.meta.dirname,'..'); const files=['src/domain','src/api','src/export','src/map']; const forbidden=/(static\/modules|static_data|data\/|window\.L|window\.leaflet)/; const paths=[]; +async function walk(folder){for(const entry of await readdir(resolve(root,folder),{withFileTypes:true})){const name=`${folder}/${entry.name}`;if(entry.isDirectory())await walk(name);else if(/\.(ts|svelte)$/.test(entry.name))paths.push(name);}} for(const folder of files)await walk(folder); const hits=[]; for(const file of paths){const text=await readFile(resolve(root,file),'utf8');if(forbidden.test(text))hits.push(file);} if(hits.length)throw new Error(`Forbidden boundary imports: ${hits.join(', ')}`); console.log(`Boundary check passed (${paths.length} files)`); diff --git a/frontend/scripts/stage-preview.mjs b/frontend/scripts/stage-preview.mjs new file mode 100644 index 0000000..66e28c1 --- /dev/null +++ b/frontend/scripts/stage-preview.mjs @@ -0,0 +1,5 @@ +import { cp, mkdir, rm, realpath } from 'node:fs/promises'; +import { resolve, sep } from 'node:path'; +const root=resolve(import.meta.dirname,'..','..'); const source=resolve(root,'frontend','dist'); const destination=resolve(root,'static','v2-preview'); +const safeRoot=resolve(root,'static'); if (!destination.startsWith(safeRoot+sep)) throw new Error('Unsafe staging destination'); +await realpath(source); await mkdir(safeRoot,{recursive:true}); await rm(destination,{recursive:true,force:true}); await cp(source,destination,{recursive:true}); console.log(`Staged V2 preview at ${destination}`); diff --git a/frontend/src/api/wireSchema.ts b/frontend/src/api/wireSchema.ts new file mode 100644 index 0000000..e0eb7da --- /dev/null +++ b/frontend/src/api/wireSchema.ts @@ -0,0 +1,4 @@ +import {z} from 'zod'; +export const locationSchema=z.object({id:z.string().regex(/^syn-/),name:z.string(),region:z.string(),category:z.string(),lat:z.number().nullable(),lon:z.number().nullable(),observed:z.string(),source:z.string()}); +export const envelopeSchema=z.object({data:z.array(locationSchema),meta:z.object({release:z.string(),profile:z.enum(['curated','community'])})}); +export type WireEnvelope=z.infer; diff --git a/frontend/src/app/App.svelte b/frontend/src/app/App.svelte new file mode 100644 index 0000000..6b86b71 --- /dev/null +++ b/frontend/src/app/App.svelte @@ -0,0 +1,8 @@ + +Until Every Cage · evidence desk +
UNTIL EVERY CAGE V2 / FIELD NOTEEthics & safeguards ↗
+

EVIDENCE DESK · SYNTHETIC PREVIEW

See what a record can—and cannot—tell us.

A quiet, inspectable view of animal-agriculture locations. Every fixture below is fictional, so you can examine the interface without exposing real places or people.

+
Release synthetic-2026.09
+{#if profile==='community'}
Unreviewed community claimNot verified by Until Every Cage. Privacy screening is separate from factual review.
{/if} +{#if scenario==='loading'}
Loading the fixture…
{:else if scenario==='empty'}

No eligible records

This profile has no records for the selected fixture.

{:else if scenario==='error'}{:else if scenario==='restricted'}

Preview withheld

This scenario represents restricted material. No restricted record is embedded in the demonstrator.

{:else}
{#if selected}

RECORD / {selected.id}

{selected.name}

{selected.region} · {selected.category}

OBSERVED{selected.observed}
MAP STATUS{selected.lat===null?'Not available':'Approximate display point'}

PUBLICATION CONTEXT

{profile==='community'?'Community-submitted · Community-unreviewed':'Government-sourced · Project-approved'}

{profile==='community'?'Published only in the explicitly selected community profile. This is not project approval.':'Included in the named curated release after documented project checks.'}

SOURCE NOTE

{selected.source}. Synthetic content only; no live database or external map tiles are used.

{/if}
{/if} +
diff --git a/frontend/src/app/routeState.ts b/frontend/src/app/routeState.ts new file mode 100644 index 0000000..e5c6123 --- /dev/null +++ b/frontend/src/app/routeState.ts @@ -0,0 +1,6 @@ +import type { Profile } from '../domain/publication'; +export type RouteState = Readonly<{kind:'home';profile:Profile}> | Readonly<{kind:'location';facilityId:string;profile:Profile}> | Readonly<{kind:'not-found';fragment:string}>; +const profiles: readonly Profile[]=['curated','community']; +const profileOf=(value:string|null):Profile=>profiles.includes(value as Profile)?value as Profile:'curated'; +export function parseRoute(hash:string):RouteState { const raw=hash.startsWith('#')?hash.slice(1):hash; const [pathPart,query='']=raw.split('?'); const path=pathPart ?? ''; const profile=profileOf(new URLSearchParams(query).get('profile')); if(path===''||path==='/') return {kind:'home',profile}; const match=/^\/locations\/([a-z0-9-]+)$/.exec(path); if(match?.[1]) return {kind:'location',facilityId:match[1],profile}; return {kind:'not-found',fragment:raw}; } +export function serializeRoute(route:Exclude):string { const path=route.kind==='home'?'/':`/locations/${encodeURIComponent(route.facilityId)}`; return `#${path}?profile=${route.profile}`; } diff --git a/frontend/src/app/uiTransitions.ts b/frontend/src/app/uiTransitions.ts new file mode 100644 index 0000000..7b27168 --- /dev/null +++ b/frontend/src/app/uiTransitions.ts @@ -0,0 +1,27 @@ +export type UiState = Readonly<{ + drawer: 'closed' | 'open'; + focusedId: string | null; + mapEnabled: boolean; +}>; + +export type UiAction = + | Readonly<{ type: 'drawer/open' }> + | Readonly<{ type: 'drawer/close' }> + | Readonly<{ type: 'focus/set'; id: string }> + | Readonly<{ type: 'focus/clear' }> + | Readonly<{ type: 'map/set-enabled'; enabled: boolean }>; + +function assertNever(value: never): never { throw new Error(`Unhandled UI action: ${JSON.stringify(value)}`); } + +export function reduceUi(state: UiState, action: UiAction): UiState { + switch (action.type) { + case 'drawer/open': return { ...state, drawer: 'open' }; + case 'drawer/close': return { ...state, drawer: 'closed' }; + case 'focus/set': return { ...state, focusedId: action.id }; + case 'focus/clear': return { ...state, focusedId: null }; + case 'map/set-enabled': return { ...state, mapEnabled: action.enabled }; + default: return assertNever(action); + } +} + +export const initialUiState: UiState = { drawer: 'closed', focusedId: null, mapEnabled: false }; diff --git a/frontend/src/domain/location.ts b/frontend/src/domain/location.ts new file mode 100644 index 0000000..c6ea27f --- /dev/null +++ b/frontend/src/domain/location.ts @@ -0,0 +1,2 @@ +export type LocationId = `syn-${string}`; +export type Location = Readonly<{id:LocationId,name:string,region:string,category:string,lat:number|null,lon:number|null,observed:string,source:string}>; diff --git a/frontend/src/domain/publication.ts b/frontend/src/domain/publication.ts new file mode 100644 index 0000000..8bc2b57 --- /dev/null +++ b/frontend/src/domain/publication.ts @@ -0,0 +1,2 @@ +export type Profile='curated'|'community'; +export type Publication = Readonly<{origin:'government-sourced'|'community-submitted';review:'project-approved'|'community-unreviewed';profile:Profile;published:boolean}>; diff --git a/frontend/src/export/exportModel.ts b/frontend/src/export/exportModel.ts new file mode 100644 index 0000000..c4aad5c --- /dev/null +++ b/frontend/src/export/exportModel.ts @@ -0,0 +1,3 @@ +import type { Location } from '../domain/location'; import type { Profile } from '../domain/publication'; +export type ExportModel=Readonly<{profile:Profile;release:string;limitations:readonly string[];rows:readonly Location[]}>; +export const makeExportModel=(rows:readonly Location[],profile:Profile,release='synthetic-2026.09'):ExportModel=>({profile,release,limitations:['Synthetic fixture only','Not a live database export','Coordinates may be unavailable'],rows}); diff --git a/frontend/src/export/previewExport.ts b/frontend/src/export/previewExport.ts new file mode 100644 index 0000000..bced58b --- /dev/null +++ b/frontend/src/export/previewExport.ts @@ -0,0 +1,2 @@ +import type { ExportModel } from './exportModel'; +export const previewExport=(model:ExportModel):string=>JSON.stringify({context:{profile:model.profile,release:model.release,limitations:model.limitations},rows:model.rows.map((row)=>({id:row.id,name:row.name,source:row.source,observed:row.observed}))},null,2); diff --git a/frontend/src/fixtures/FixtureLocationRepository.ts b/frontend/src/fixtures/FixtureLocationRepository.ts new file mode 100644 index 0000000..638a8ba --- /dev/null +++ b/frontend/src/fixtures/FixtureLocationRepository.ts @@ -0,0 +1,3 @@ +import type {Location} from '../domain/location'; import type {Profile} from '../domain/publication'; import {fixture,type Scenario} from './scenarios'; +export type FixtureRequest=Readonly<{profile:Profile;scenario:Scenario;signal?:AbortSignal}>; +export class FixtureLocationRepository { readonly #delayMs:number; constructor(delayMs=0){this.#delayMs=delayMs;} list(request:FixtureRequest):Promise{const {profile,scenario,signal}=request; return new Promise((resolve,reject)=>{if(signal?.aborted){reject(new DOMException('Fixture request aborted','AbortError'));return;} const timer=globalThis.setTimeout(()=>{try{const result=fixture(scenario,profile);if(result===null)throw new Error('This fixture is restricted.');resolve(result);}catch(error){reject(error);}},this.#delayMs);signal?.addEventListener('abort',()=>{globalThis.clearTimeout(timer);reject(new DOMException('Fixture request aborted','AbortError'));},{once:true});});}} diff --git a/frontend/src/fixtures/locations.ts b/frontend/src/fixtures/locations.ts new file mode 100644 index 0000000..ddc5c82 --- /dev/null +++ b/frontend/src/fixtures/locations.ts @@ -0,0 +1,6 @@ +import type {Location} from '../domain/location'; +export const locations: readonly Location[]=[ + {id:'syn-north-star',name:'North Star Cooperative',region:'North Coast',category:'Egg production',lat:54.72,lon:9.45,observed:'2026-08-14',source:'Synthetic register example'}, + {id:'syn-river-meadow',name:'River Meadow Foods',region:'Central Lowlands',category:'Dairy',lat:55.14,lon:10.21,observed:'2026-07-02',source:'Synthetic register example'}, + {id:'syn-quiet-field',name:'Quiet Field Holdings',region:'West Marches',category:'Mixed agriculture',lat:null,lon:null,observed:'2026-06-19',source:'Synthetic register example'} +]; diff --git a/frontend/src/fixtures/scenarios.ts b/frontend/src/fixtures/scenarios.ts new file mode 100644 index 0000000..e522c48 --- /dev/null +++ b/frontend/src/fixtures/scenarios.ts @@ -0,0 +1,3 @@ +import {locations} from './locations'; import type {Profile} from '../domain/publication'; +export type Scenario='ready'|'loading'|'empty'|'error'|'restricted'; +export function fixture(scenario:Scenario,profile:Profile){ if(scenario==='error') throw new Error('The fixture could not be read.'); if(scenario==='empty') return []; if(scenario==='restricted') return null; return profile==='community'?locations.slice(0,1):locations; } diff --git a/frontend/src/main.ts b/frontend/src/main.ts new file mode 100644 index 0000000..f597e29 --- /dev/null +++ b/frontend/src/main.ts @@ -0,0 +1,6 @@ +import { mount } from 'svelte'; +import './styles/base.css'; +import App from './app/App.svelte'; +const target = document.getElementById('app'); +if (!target) throw new Error('App mount missing'); +mount(App, { target }); diff --git a/frontend/src/map/LeafletMapAdapter.ts b/frontend/src/map/LeafletMapAdapter.ts new file mode 100644 index 0000000..9886ea6 --- /dev/null +++ b/frontend/src/map/LeafletMapAdapter.ts @@ -0,0 +1,10 @@ +import type { Map as LeafletMap, Marker } from 'leaflet'; +import type { MapAdapter } from './MapAdapter'; +import type { DisplayFeature } from './mapProjection'; +type MarkerFactory = (feature: DisplayFeature, map: LeafletMap) => Marker; +export class LeafletMapAdapter implements MapAdapter { + #map: LeafletMap | null = null; #markers = new Map(); #disposed = false; #makeMarker: MarkerFactory | null = null; + async mount(container: HTMLElement): Promise { const leaflet = await import('leaflet'); if (this.#disposed) return; this.#makeMarker = (feature, map) => leaflet.marker([feature.lat, feature.lon], { title: feature.label }).addTo(map); this.#map = leaflet.map(container, { attributionControl: false, zoomControl: true }).setView([55, 10], 6); this.#map.getContainer().style.background = '#ded8cc'; } + update(features: readonly DisplayFeature[], selectedId: string | null): void { if (!this.#map || !this.#makeMarker) return; const active = new Set(features.map((feature) => feature.id)); for (const [id, marker] of this.#markers) { if (!active.has(id)) { marker.remove(); this.#markers.delete(id); } } for (const feature of features) { const marker = this.#markers.get(feature.id) ?? this.#makeMarker(feature, this.#map); this.#markers.set(feature.id, marker); marker.setOpacity(selectedId === null || selectedId === feature.id ? 1 : 0.55); } } + destroy(): void { this.#disposed = true; for (const marker of this.#markers.values()) marker.remove(); this.#markers.clear(); this.#map?.remove(); this.#map = null; this.#makeMarker = null; } +} diff --git a/frontend/src/map/MapAdapter.ts b/frontend/src/map/MapAdapter.ts new file mode 100644 index 0000000..2fbf6ff --- /dev/null +++ b/frontend/src/map/MapAdapter.ts @@ -0,0 +1,2 @@ +import type { DisplayFeature } from './mapProjection'; +export interface MapAdapter { mount(container:HTMLElement):Promise; update(features:readonly DisplayFeature[],selectedId:string|null):void; destroy():void; } diff --git a/frontend/src/map/mapProjection.ts b/frontend/src/map/mapProjection.ts new file mode 100644 index 0000000..5d3e435 --- /dev/null +++ b/frontend/src/map/mapProjection.ts @@ -0,0 +1,3 @@ +import type { Location } from '../domain/location'; +export type DisplayFeature=Readonly<{id:string,label:string,lat:number,lon:number}>; +export const projectLocations=(items:readonly Location[]):readonly DisplayFeature[]=>items.flatMap((item)=>item.lat===null||item.lon===null?[]:[{id:item.id,label:item.name,lat:item.lat,lon:item.lon}]); diff --git a/frontend/src/query/snapshotSession.ts b/frontend/src/query/snapshotSession.ts new file mode 100644 index 0000000..e4d4ebe --- /dev/null +++ b/frontend/src/query/snapshotSession.ts @@ -0,0 +1,37 @@ +import type { Profile } from '../domain/publication'; + +export type SessionState = Readonly<{ + profile: Profile; + release: string | null; + generation: number; + status: 'idle' | 'loading' | 'ready' | 'error'; + locations: readonly string[]; + selectedId: string | null; + error: string | null; +}>; + +export type SnapshotResponse = Readonly<{ generation: number; profile: Profile; release: string; locationIds: readonly string[] }>; +export type SessionEvent = + | Readonly<{ type: 'query/start'; profile: Profile }> + | Readonly<{ type: 'query/success'; response: SnapshotResponse }> + | Readonly<{ type: 'query/failure'; generation: number; message: string }>; + +export const initialSession = (profile: Profile = 'curated'): SessionState => ({ profile, release: null, generation: 0, status: 'idle', locations: [], selectedId: null, error: null }); + +export function reduceSession(state: SessionState, event: SessionEvent): SessionState { + if (event.type === 'query/start') { + return { ...state, profile: event.profile, generation: state.generation + 1, status: 'loading', locations: [], selectedId: null, error: null }; + } + const generation = event.type === 'query/success' ? event.response.generation : event.generation; + if (generation !== state.generation) return state; + if (event.type === 'query/failure') return { ...state, status: 'error', locations: [], selectedId: null, error: event.message }; + if (event.response.profile !== state.profile) return state; + if (state.release !== null && event.response.release !== state.release) { + return { ...state, release: event.response.release, status: 'loading', locations: [], selectedId: null, error: 'Release changed; refreshing this profile.' }; + } + return { ...state, release: event.response.release, status: 'ready', locations: event.response.locationIds, selectedId: null, error: null }; +} + +export function acceptResponse(state: SessionState, response: SnapshotResponse): SessionState { + return reduceSession(state, { type: 'query/success', response }); +} diff --git a/frontend/src/styles/base.css b/frontend/src/styles/base.css new file mode 100644 index 0000000..f9ec993 --- /dev/null +++ b/frontend/src/styles/base.css @@ -0,0 +1 @@ +:root{font-family:ui-sans-serif,system-ui,sans-serif;color:#17283b;background:#f3eee5;font-synthesis:none}*{box-sizing:border-box}body{margin:0}main{max-width:1120px;margin:auto;padding:28px 6vw}.top,footer{display:flex;justify-content:space-between;align-items:center;border-bottom:1px solid #cfc4b2;padding-bottom:18px;font-size:.82rem}.top a,footer a{color:#a34927;text-decoration:none}.wordmark{font-weight:800;letter-spacing:.08em;color:#17283b!important}.wordmark span{display:block;color:#6b5e50;font-size:.62rem;letter-spacing:.16em;margin-top:5px}.intro{max-width:720px;padding:82px 0 42px}.eyebrow{color:#a34927;font-size:.7rem;letter-spacing:.16em;font-weight:800;margin:0 0 14px}h1{font:600 clamp(2.6rem,6vw,5.2rem)/.98 Georgia,serif;letter-spacing:-.05em;margin:0 0 24px}.lede{font-size:1.12rem;line-height:1.6;color:#526170}.toolbar{display:flex;gap:18px;align-items:end;border-top:1px solid #cfc4b2;border-bottom:1px solid #cfc4b2;padding:18px 0;margin-bottom:20px}label{font-size:.72rem;text-transform:uppercase;letter-spacing:.1em;font-weight:800}select{display:block;margin-top:7px;background:#fbf8f2;border:1px solid #b9ad9b;border-radius:3px;padding:10px 32px 10px 11px;color:#17283b;font-size:.95rem}.release{margin-left:auto;color:#5f5549;font-size:.78rem}.release strong{color:#17283b}.warning{background:#ead9c6;border-left:4px solid #a34927;padding:15px 18px;display:flex;gap:16px;margin:20px 0;line-height:1.4}.warning span{color:#624d40}.content{display:grid;grid-template-columns:35% 1fr;border:1px solid #cfc4b2;background:#f8f4ed;min-height:430px}aside{border-right:1px solid #cfc4b2}.list-head{padding:18px;display:flex;justify-content:space-between;color:#62594e;font-size:.7rem;letter-spacing:.12em}button{display:flex;width:100%;text-align:left;gap:12px;padding:17px 18px;border:0;border-top:1px solid #ded5c8;background:transparent;color:inherit;cursor:pointer}.active{background:#e9e1d5}.dot{width:9px;height:9px;background:#a34927;border-radius:50%;margin-top:5px;flex:none}button strong,button small{display:block}button small{color:#5f5549;margin-top:5px}article{padding:38px clamp(24px,5vw,62px)}h2{font:600 2.25rem Georgia,serif;margin:0 0 8px}.region{color:#5f5549}.rule{height:1px;background:#cfc4b2;margin:34px 0 25px}.facts{display:flex;gap:64px}.facts span{display:block;color:#62594e;font-size:.68rem;letter-spacing:.12em;margin-bottom:8px}.panel{border-top:1px solid #cfc4b2;margin-top:30px;padding-top:20px;line-height:1.55}.panel p{color:#4f5c69}.state{padding:55px 25px;background:#f8f4ed;border:1px solid #cfc4b2}.error{border-left:4px solid #a34927}footer{border:0;margin-top:30px;color:#62594e;font-size:.76rem}@media(max-width:680px){main{padding:20px}.intro{padding:55px 0 30px}.toolbar{flex-wrap:wrap}.release{width:100%;margin:0}.content{display:block}aside{border-right:0;border-bottom:1px solid #cfc4b2}.facts{gap:25px}.warning{display:block}.warning span{display:block;margin-top:6px}} diff --git a/frontend/src/vite-env.d.ts b/frontend/src/vite-env.d.ts new file mode 100644 index 0000000..11f02fe --- /dev/null +++ b/frontend/src/vite-env.d.ts @@ -0,0 +1 @@ +/// diff --git a/frontend/svelte.config.js b/frontend/svelte.config.js new file mode 100644 index 0000000..ff8b4c5 --- /dev/null +++ b/frontend/svelte.config.js @@ -0,0 +1 @@ +export default {}; diff --git a/frontend/tests/e2e/fixture-platform.spec.ts b/frontend/tests/e2e/fixture-platform.spec.ts new file mode 100644 index 0000000..0c8193b --- /dev/null +++ b/frontend/tests/e2e/fixture-platform.spec.ts @@ -0,0 +1,52 @@ +import { test, expect } from '@playwright/test'; +import AxeBuilder from '@axe-core/playwright'; + +test.beforeEach(async ({ page }) => { + await page.route('**/*', async (route) => { + const url = new URL(route.request().url()); + if (url.origin !== 'http://127.0.0.1:4173') throw new Error(`Unexpected external request: ${url.href}`); + await route.continue(); + }); +}); + +test('renders the synthetic evidence desk', async ({ page }) => { + await page.goto('./#/'); + await expect(page).toHaveTitle(/Until Every Cage/); + await expect(page.getByRole('heading', { name: /See what a record can/i })).toBeVisible(); + await expect(page.getByText('SYNTHETIC PREVIEW')).toBeVisible(); +}); + +test('selecting the community profile shows persistent warning context', async ({ page }) => { + await page.goto('./#/'); + await page.getByLabel('Profile').selectOption('community'); + await expect(page.getByRole('note')).toContainText('Unreviewed community claim'); + await expect(page.getByRole('note')).toContainText('Not verified by Until Every Cage'); + await expect(page.getByText('VISIBLE RECORDS')).toBeVisible(); +}); + +test('opens a direct hash detail route with a record context', async ({ page }) => { + await page.goto('./#/locations/syn-river-meadow?profile=curated'); + await expect(page.getByRole('heading', { name: 'River Meadow Foods' })).toBeVisible(); + await expect(page.getByText('RECORD / syn-river-meadow')).toBeVisible(); +}); + +test('controls are keyboard reachable with visible focus', async ({ page }) => { + await page.goto('./#/'); + await page.keyboard.press('Tab'); + await expect(page.locator(':focus')).toHaveAttribute('href', '/v2-preview/#/'); + await page.keyboard.press('Tab'); + await expect(page.locator(':focus')).toHaveAttribute('href', '/ethics.html'); + await page.keyboard.press('Tab'); + await expect(page.getByLabel('Profile')).toBeFocused(); + await page.keyboard.press('End'); + await expect(page.locator(':focus')).toBeVisible(); +}); + +test('has no obvious accessibility violations at mobile width', async ({ page }) => { + await page.setViewportSize({ width: 390, height: 844 }); + await page.goto('./#/'); + const results = await new AxeBuilder({ page }).analyze(); + expect(results.violations).toEqual([]); + await expect(page.getByRole('heading', { name: /See what a record can/i })).toBeVisible(); + expect(await page.evaluate(() => document.documentElement.scrollWidth)).toBeLessThanOrEqual(await page.evaluate(() => document.documentElement.clientWidth)); +}); diff --git a/frontend/tests/unit/boundary.test.ts b/frontend/tests/unit/boundary.test.ts new file mode 100644 index 0000000..536ab94 --- /dev/null +++ b/frontend/tests/unit/boundary.test.ts @@ -0,0 +1,2 @@ +import {describe,expect,it} from 'vitest'; import {readFileSync} from 'node:fs'; import {resolve} from 'node:path'; +describe('V1 boundary',()=>it('keeps V2 output isolated from static/',()=>{const config=readFileSync(resolve(process.cwd(),'vite.config.ts'),'utf8');expect(config).toContain("outDir:'dist'");expect(config).not.toContain("outDir:'../static'");})); diff --git a/frontend/tests/unit/exportModel.test.ts b/frontend/tests/unit/exportModel.test.ts new file mode 100644 index 0000000..ffd8a69 --- /dev/null +++ b/frontend/tests/unit/exportModel.test.ts @@ -0,0 +1,2 @@ +import {describe,expect,it} from 'vitest'; import {locations} from '../../src/fixtures/locations'; import {makeExportModel} from '../../src/export/exportModel'; import {previewExport} from '../../src/export/previewExport'; +describe('export preview',()=>it('keeps profile and provenance context attached',()=>{const result=JSON.parse(previewExport(makeExportModel(locations,'community')));expect(result.context.profile).toBe('community');expect(result.context.limitations).toContain('Synthetic fixture only');expect(result.rows[0]).toMatchObject({source:'Synthetic register example'});})); diff --git a/frontend/tests/unit/fixtureRepository.test.ts b/frontend/tests/unit/fixtureRepository.test.ts new file mode 100644 index 0000000..ad10d28 --- /dev/null +++ b/frontend/tests/unit/fixtureRepository.test.ts @@ -0,0 +1,2 @@ +import {describe,expect,it} from 'vitest'; import {FixtureLocationRepository} from '../../src/fixtures/FixtureLocationRepository'; +describe('FixtureLocationRepository',()=>{it('keeps profile counts separate',async()=>{const r=new FixtureLocationRepository();await expect(r.list({profile:'curated',scenario:'ready'})).resolves.toHaveLength(3);await expect(r.list({profile:'community',scenario:'ready'})).resolves.toHaveLength(1);});it('models error and restricted states',async()=>{const r=new FixtureLocationRepository();await expect(r.list({profile:'curated',scenario:'error'})).rejects.toThrow('could not be read');await expect(r.list({profile:'curated',scenario:'restricted'})).rejects.toThrow('restricted');});it('aborts delayed work',async()=>{const c=new AbortController();const p=new FixtureLocationRepository(30).list({profile:'curated',scenario:'ready',signal:c.signal});c.abort();await expect(p).rejects.toMatchObject({name:'AbortError'});});}); diff --git a/frontend/tests/unit/routeState.test.ts b/frontend/tests/unit/routeState.test.ts new file mode 100644 index 0000000..5db5d5e --- /dev/null +++ b/frontend/tests/unit/routeState.test.ts @@ -0,0 +1,2 @@ +import {describe,expect,it} from 'vitest'; import {parseRoute,serializeRoute} from '../../src/app/routeState'; +describe('routeState',()=>{it('parses home and defaults safely',()=>expect(parseRoute('#/')).toEqual({kind:'home',profile:'curated'}));it('parses location profile',()=>expect(parseRoute('#/locations/syn-north-star?profile=community')).toEqual({kind:'location',facilityId:'syn-north-star',profile:'community'}));it('round-trips routes',()=>{expect(serializeRoute({kind:'home',profile:'curated'})).toBe('#/?profile=curated');expect(parseRoute(serializeRoute({kind:'location',facilityId:'syn-river-meadow',profile:'community'}))).toEqual({kind:'location',facilityId:'syn-river-meadow',profile:'community'});});it('does not guess unsupported paths',()=>expect(parseRoute('#/search?q=eggs')).toEqual({kind:'not-found',fragment:'/search?q=eggs'}));}); diff --git a/frontend/tests/unit/snapshotSession.test.ts b/frontend/tests/unit/snapshotSession.test.ts new file mode 100644 index 0000000..b64c6cd --- /dev/null +++ b/frontend/tests/unit/snapshotSession.test.ts @@ -0,0 +1,3 @@ +import { describe, expect, it } from 'vitest'; import { acceptResponse, initialSession, reduceSession } from '../../src/query/snapshotSession'; +const response=(generation:number,release='synthetic-2026.09',profile:'curated'|'community'='curated')=>({generation,release,profile,locationIds:['syn-north-star'] as const}); +describe('snapshotSession',()=>{it('increments generation and clears prior data on profile change',()=>{const ready=acceptResponse({...initialSession(),generation:1,status:'loading'},response(1));const next=reduceSession(ready,{type:'query/start',profile:'community'});expect(next).toMatchObject({profile:'community',generation:2,status:'loading',locations:[],selectedId:null});});it('ignores late success and failure from an older generation',()=>{const state=reduceSession(initialSession(),{type:'query/start',profile:'curated'});expect(acceptResponse(state,response(0))).toBe(state);expect(reduceSession(state,{type:'query/failure',generation:0,message:'late'})).toBe(state);});it('rejects mismatched profile responses',()=>{const state=reduceSession(initialSession('community'),{type:'query/start',profile:'community'});expect(acceptResponse(state,response(1,'synthetic-2026.09','curated'))).toBe(state);});it('clears buffers when the release changes',()=>{const loading=reduceSession(initialSession(),{type:'query/start',profile:'curated'});const ready=acceptResponse(loading,response(1));const changed=acceptResponse(ready,response(1,'synthetic-2026.10'));expect(changed).toMatchObject({status:'loading',release:'synthetic-2026.10',locations:[],selectedId:null});});it('surfaces request failures for the current generation',()=>{const loading=reduceSession(initialSession(),{type:'query/start',profile:'curated'});expect(reduceSession(loading,{type:'query/failure',generation:1,message:'fixture failed'})).toMatchObject({status:'error',error:'fixture failed'});});}); diff --git a/frontend/tests/unit/uiTransitions.test.ts b/frontend/tests/unit/uiTransitions.test.ts new file mode 100644 index 0000000..5e42c18 --- /dev/null +++ b/frontend/tests/unit/uiTransitions.test.ts @@ -0,0 +1,2 @@ +import { describe, expect, it } from 'vitest'; import { initialUiState, reduceUi } from '../../src/app/uiTransitions'; +describe('uiTransitions', () => { it('applies drawer, focus, and map actions immutably', () => { const open=reduceUi(initialUiState,{type:'drawer/open'}); const focused=reduceUi(open,{type:'focus/set',id:'syn-north-star'}); const mapped=reduceUi(focused,{type:'map/set-enabled',enabled:true}); expect(mapped).toEqual({drawer:'open',focusedId:'syn-north-star',mapEnabled:true}); expect(initialUiState).toEqual({drawer:'closed',focusedId:null,mapEnabled:false}); }); it('clears selection explicitly', () => expect(reduceUi({...initialUiState,focusedId:'syn-x'},{type:'focus/clear'}).focusedId).toBeNull()); }); diff --git a/frontend/tsconfig.app.json b/frontend/tsconfig.app.json new file mode 100644 index 0000000..7efdc08 --- /dev/null +++ b/frontend/tsconfig.app.json @@ -0,0 +1 @@ +{"compilerOptions":{"target":"ES2022","module":"ESNext","moduleResolution":"bundler","strict":true,"noUncheckedIndexedAccess":true,"exactOptionalPropertyTypes":true,"useUnknownInCatchVariables":true,"verbatimModuleSyntax":true,"isolatedModules":true,"noEmit":true,"lib":["ES2022","DOM","DOM.Iterable"]},"include":["src"]} diff --git a/frontend/tsconfig.json b/frontend/tsconfig.json new file mode 100644 index 0000000..ff0aea3 --- /dev/null +++ b/frontend/tsconfig.json @@ -0,0 +1 @@ +{"files":[],"references":[{"path":"./tsconfig.app.json"},{"path":"./tsconfig.node.json"}]} diff --git a/frontend/tsconfig.node.json b/frontend/tsconfig.node.json new file mode 100644 index 0000000..4e4a9c3 --- /dev/null +++ b/frontend/tsconfig.node.json @@ -0,0 +1 @@ +{"compilerOptions":{"target":"ES2022","module":"ESNext","moduleResolution":"bundler","strict":true,"noUncheckedIndexedAccess":true,"exactOptionalPropertyTypes":true,"verbatimModuleSyntax":true,"isolatedModules":true,"noEmit":true,"skipLibCheck":true,"types":["node"]},"include":["vite.config.ts","svelte.config.js","vitest.config.ts"]} diff --git a/frontend/vite.config.ts b/frontend/vite.config.ts new file mode 100644 index 0000000..b6e76b3 --- /dev/null +++ b/frontend/vite.config.ts @@ -0,0 +1,2 @@ +import { defineConfig } from 'vite'; import { svelte } from '@sveltejs/vite-plugin-svelte'; +export default defineConfig({base:'/v2-preview/',plugins:[svelte()],build:{outDir:'dist',emptyOutDir:true}}); diff --git a/frontend/vitest.config.ts b/frontend/vitest.config.ts new file mode 100644 index 0000000..69dd03a --- /dev/null +++ b/frontend/vitest.config.ts @@ -0,0 +1,8 @@ +import { defineConfig } from 'vitest/config'; + +// Keep Vitest's Vite 5-compatible runtime isolated from the app's Vite 6 build. +// Component tests can add the Svelte transform once the integration suite lands; +// Phase 2's current tests exercise the framework-independent boundaries. +export default defineConfig({ + test: { environment: 'jsdom', include: ['tests/unit/**/*.test.ts', 'tests/integration/**/*.test.ts'] }, +}); From f85a8362857fb5ea1a7179a9995d2554ee2f8700 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sun, 13 Sep 2026 15:05:54 -0700 Subject: [PATCH 002/311] feat: add v2 fixture discovery filters --- frontend/src/app/App.svelte | 16 ++++++++-------- frontend/src/features/locations/filterState.ts | 4 ++++ frontend/tests/e2e/fixture-platform.spec.ts | 10 ++++++++++ frontend/tests/unit/filterState.test.ts | 2 ++ 4 files changed, 24 insertions(+), 8 deletions(-) create mode 100644 frontend/src/features/locations/filterState.ts create mode 100644 frontend/tests/unit/filterState.test.ts diff --git a/frontend/src/app/App.svelte b/frontend/src/app/App.svelte index 6b86b71..6f9e4fe 100644 --- a/frontend/src/app/App.svelte +++ b/frontend/src/app/App.svelte @@ -1,8 +1,8 @@ - -Until Every Cage · evidence desk -
UNTIL EVERY CAGE V2 / FIELD NOTEEthics & safeguards ↗
-

EVIDENCE DESK · SYNTHETIC PREVIEW

See what a record can—and cannot—tell us.

A quiet, inspectable view of animal-agriculture locations. Every fixture below is fictional, so you can examine the interface without exposing real places or people.

-
Release synthetic-2026.09
-{#if profile==='community'}
Unreviewed community claimNot verified by Until Every Cage. Privacy screening is separate from factual review.
{/if} -{#if scenario==='loading'}
Loading the fixture…
{:else if scenario==='empty'}

No eligible records

This profile has no records for the selected fixture.

{:else if scenario==='error'}{:else if scenario==='restricted'}

Preview withheld

This scenario represents restricted material. No restricted record is embedded in the demonstrator.

{:else}
{#if selected}

RECORD / {selected.id}

{selected.name}

{selected.region} · {selected.category}

OBSERVED{selected.observed}
MAP STATUS{selected.lat===null?'Not available':'Approximate display point'}

PUBLICATION CONTEXT

{profile==='community'?'Community-submitted · Community-unreviewed':'Government-sourced · Project-approved'}

{profile==='community'?'Published only in the explicitly selected community profile. This is not project approval.':'Included in the named curated release after documented project checks.'}

SOURCE NOTE

{selected.source}. Synthetic content only; no live database or external map tiles are used.

{/if}
{/if} -
+ +Until Every Cage · evidence desk
UNTIL EVERY CAGE V2 / FIELD NOTEEthics & safeguards ↗

EVIDENCE DESK · SYNTHETIC PREVIEW

See what a record can—and cannot—tell us.

A quiet, inspectable view of animal-agriculture locations. Every fixture below is fictional, so you can examine the interface without exposing real places or people.

+
Release synthetic-2026.09
+{#if profile==='community'}
Unreviewed community claimNot verified by Until Every Cage. Privacy screening is separate from factual review.
{/if}{#if scenario==='loading'}
Loading the fixture…
{:else if scenario==='empty'}

No eligible records

This profile has no records for the selected fixture.

{:else if scenario==='error'}{:else if scenario==='restricted'}

Preview withheld

This scenario represents restricted material. No restricted record is embedded in the demonstrator.

{:else}
{#if selected}

RECORD / {selected.id}

{selected.name}

{selected.region} · {selected.category}

OBSERVED{selected.observed}
MAP STATUS{selected.lat===null?'Not available':'Approximate display point'}

PUBLICATION CONTEXT

{profile==='community'?'Community-submitted · Community-unreviewed':'Government-sourced · Project-approved'}

{profile==='community'?'Published only in the explicitly selected community profile. This is not project approval.':'Included in the named curated release after documented project checks.'}

SOURCE NOTE

{selected.source}. Synthetic content only; no live database or external map tiles are used.

{/if}
{/if}
diff --git a/frontend/src/features/locations/filterState.ts b/frontend/src/features/locations/filterState.ts new file mode 100644 index 0000000..064c595 --- /dev/null +++ b/frontend/src/features/locations/filterState.ts @@ -0,0 +1,4 @@ +import type { Location } from '../../domain/location'; +export type FilterState=Readonly<{search:string;region:string;category:string}>; +export const initialFilters:FilterState={search:'',region:'all',category:'all'}; +export const filterLocations=(items:readonly Location[],filters:FilterState):readonly Location[]=>{const needle=filters.search.trim().toLocaleLowerCase();return items.filter((item)=>(!needle||`${item.name} ${item.region} ${item.category}`.toLocaleLowerCase().includes(needle))&&(filters.region==='all'||item.region===filters.region)&&(filters.category==='all'||item.category===filters.category));}; diff --git a/frontend/tests/e2e/fixture-platform.spec.ts b/frontend/tests/e2e/fixture-platform.spec.ts index 0c8193b..17f65bd 100644 --- a/frontend/tests/e2e/fixture-platform.spec.ts +++ b/frontend/tests/e2e/fixture-platform.spec.ts @@ -50,3 +50,13 @@ test('has no obvious accessibility violations at mobile width', async ({ page }) await expect(page.getByRole('heading', { name: /See what a record can/i })).toBeVisible(); expect(await page.evaluate(() => document.documentElement.scrollWidth)).toBeLessThanOrEqual(await page.evaluate(() => document.documentElement.clientWidth)); }); + +test('filters the curated list and updates the detail hash on selection', async ({ page }) => { + await page.goto('./#/'); + await page.getByLabel('Search locations').fill('dairy'); + await expect(page.getByRole('button', { name: /River Meadow Foods/ })).toBeVisible(); + await expect(page.getByRole('button', { name: /North Star Cooperative/ })).toHaveCount(0); + await page.getByRole('button', { name: /River Meadow Foods/ }).click(); + await expect(page).toHaveURL(/#\/locations\/syn-river-meadow\?profile=curated/); + await expect(page.getByRole('heading', { name: 'River Meadow Foods' })).toBeVisible(); +}); diff --git a/frontend/tests/unit/filterState.test.ts b/frontend/tests/unit/filterState.test.ts new file mode 100644 index 0000000..4376b50 --- /dev/null +++ b/frontend/tests/unit/filterState.test.ts @@ -0,0 +1,2 @@ +import {describe,expect,it} from 'vitest';import {locations} from '../../src/fixtures/locations';import {filterLocations,initialFilters} from '../../src/features/locations/filterState'; +describe('filterLocations',()=>{it('searches names, regions and categories',()=>{expect(filterLocations(locations,{...initialFilters,search:'dairy'})).toHaveLength(1);expect(filterLocations(locations,{...initialFilters,search:'north coast'})).toHaveLength(1);});it('composes explicit filters',()=>expect(filterLocations(locations,{...initialFilters,region:'West Marches'})[0]?.id).toBe('syn-quiet-field'));it('returns no match explicitly',()=>expect(filterLocations(locations,{...initialFilters,search:'unknown place'})).toEqual([]));}); From c0d123d3d817931301c373d543cfbd88c398db65 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sun, 13 Sep 2026 15:13:26 -0700 Subject: [PATCH 003/311] feat: integrate fixture map and export preview --- frontend/src/app/App.svelte | 10 ++-------- frontend/src/export/exportModel.ts | 4 +--- frontend/src/export/previewExport.ts | 3 +-- frontend/src/map/MapView.svelte | 22 +++++++++++++++++++++ frontend/tests/e2e/fixture-platform.spec.ts | 2 ++ frontend/tests/unit/mapProjection.test.ts | 1 + 6 files changed, 29 insertions(+), 13 deletions(-) create mode 100644 frontend/src/map/MapView.svelte create mode 100644 frontend/tests/unit/mapProjection.test.ts diff --git a/frontend/src/app/App.svelte b/frontend/src/app/App.svelte index 6f9e4fe..afd2d2b 100644 --- a/frontend/src/app/App.svelte +++ b/frontend/src/app/App.svelte @@ -1,8 +1,2 @@ - -Until Every Cage · evidence desk
UNTIL EVERY CAGE V2 / FIELD NOTEEthics & safeguards ↗

EVIDENCE DESK · SYNTHETIC PREVIEW

See what a record can—and cannot—tell us.

A quiet, inspectable view of animal-agriculture locations. Every fixture below is fictional, so you can examine the interface without exposing real places or people.

-
Release synthetic-2026.09
-{#if profile==='community'}
Unreviewed community claimNot verified by Until Every Cage. Privacy screening is separate from factual review.
{/if}{#if scenario==='loading'}
Loading the fixture…
{:else if scenario==='empty'}

No eligible records

This profile has no records for the selected fixture.

{:else if scenario==='error'}{:else if scenario==='restricted'}

Preview withheld

This scenario represents restricted material. No restricted record is embedded in the demonstrator.

{:else}
{#if selected}

RECORD / {selected.id}

{selected.name}

{selected.region} · {selected.category}

OBSERVED{selected.observed}
MAP STATUS{selected.lat===null?'Not available':'Approximate display point'}

PUBLICATION CONTEXT

{profile==='community'?'Community-submitted · Community-unreviewed':'Government-sourced · Project-approved'}

{profile==='community'?'Published only in the explicitly selected community profile. This is not project approval.':'Included in the named curated release after documented project checks.'}

SOURCE NOTE

{selected.source}. Synthetic content only; no live database or external map tiles are used.

{/if}
{/if}
+ +Until Every Cage · evidence desk
UNTIL EVERY CAGE V2 / FIELD NOTEEthics & safeguards ↗

EVIDENCE DESK · SYNTHETIC PREVIEW

See what a record can—and cannot—tell us.

A quiet, inspectable view of animal-agriculture locations. Every fixture below is fictional.

{#if profile==='community'}
Unreviewed community claimNot verified by Until Every Cage.
{/if}{#if scenario==='loading'}
Loading the fixture…
{:else if scenario==='empty'}

No eligible records

{:else if scenario==='error'}{:else if scenario==='restricted'}

Preview withheld

{:else}
{#if selected}

RECORD / {selected.id}

{selected.name}

{selected.region} · {selected.category}

OBSERVED{selected.observed}
MAP STATUS{selected.lat===null?'Not available':'Approximate display point'}

PUBLICATION CONTEXT

{profile==='community'?'Community-submitted · Community-unreviewed':'Government-sourced · Project-approved'}

Source context remains attached to this synthetic record.

{/if}
{/if}
{#if showMap}{/if}{#if showExport}

IN-MEMORY EXPORT PREVIEW

{profile} · synthetic-2026.09

Loaded results only. This preview is not a live or complete export.

{exportPreview}
{/if}
Method preview · no live requests
diff --git a/frontend/src/export/exportModel.ts b/frontend/src/export/exportModel.ts index c4aad5c..e3ab533 100644 --- a/frontend/src/export/exportModel.ts +++ b/frontend/src/export/exportModel.ts @@ -1,3 +1 @@ -import type { Location } from '../domain/location'; import type { Profile } from '../domain/publication'; -export type ExportModel=Readonly<{profile:Profile;release:string;limitations:readonly string[];rows:readonly Location[]}>; -export const makeExportModel=(rows:readonly Location[],profile:Profile,release='synthetic-2026.09'):ExportModel=>({profile,release,limitations:['Synthetic fixture only','Not a live database export','Coordinates may be unavailable'],rows}); +import type {Location} from '../domain/location';import type {Profile} from '../domain/publication';export type ExportModel=Readonly<{profile:Profile;release:string;limitations:readonly string[];rows:readonly Location[]}>;export const makeExportModel=(rows:readonly Location[],profile:Profile,release='synthetic-2026.09'):ExportModel=>({profile,release,limitations:['Synthetic fixture only','Loaded results only; not a complete export','Coordinates may be unavailable'],rows}); diff --git a/frontend/src/export/previewExport.ts b/frontend/src/export/previewExport.ts index bced58b..23925a3 100644 --- a/frontend/src/export/previewExport.ts +++ b/frontend/src/export/previewExport.ts @@ -1,2 +1 @@ -import type { ExportModel } from './exportModel'; -export const previewExport=(model:ExportModel):string=>JSON.stringify({context:{profile:model.profile,release:model.release,limitations:model.limitations},rows:model.rows.map((row)=>({id:row.id,name:row.name,source:row.source,observed:row.observed}))},null,2); +import type {ExportModel} from './exportModel';export const previewExport=(model:ExportModel):string=>JSON.stringify({context:{profile:model.profile,release:model.release,limitations:model.limitations},rows:model.rows.map((row)=>({id:row.id,name:row.name,source:row.source,observed:row.observed}))},null,2); diff --git a/frontend/src/map/MapView.svelte b/frontend/src/map/MapView.svelte new file mode 100644 index 0000000..9659268 --- /dev/null +++ b/frontend/src/map/MapView.svelte @@ -0,0 +1,22 @@ + + +

Blank local background · {features.length} display points · no external tiles

+ diff --git a/frontend/tests/e2e/fixture-platform.spec.ts b/frontend/tests/e2e/fixture-platform.spec.ts index 17f65bd..936a889 100644 --- a/frontend/tests/e2e/fixture-platform.spec.ts +++ b/frontend/tests/e2e/fixture-platform.spec.ts @@ -60,3 +60,5 @@ test('filters the curated list and updates the detail hash on selection', async await expect(page).toHaveURL(/#\/locations\/syn-river-meadow\?profile=curated/); await expect(page.getByRole('heading', { name: 'River Meadow Foods' })).toBeVisible(); }); + +test('shows the local map and limited export context', async ({ page }) => { await page.goto('./#/'); await page.getByRole('button', { name: 'Show map' }).click(); await expect(page.getByLabel('Synthetic location map')).toBeVisible(); await expect(page.getByText(/no external tiles/)).toBeVisible(); await page.getByRole('button', { name: 'Preview export' }).click(); const exportPanel=page.locator('.export-preview'); await expect(exportPanel.getByText('IN-MEMORY EXPORT PREVIEW')).toBeVisible(); await expect(exportPanel.locator('p').filter({hasText:'Loaded results only'})).toBeVisible(); }); diff --git a/frontend/tests/unit/mapProjection.test.ts b/frontend/tests/unit/mapProjection.test.ts new file mode 100644 index 0000000..2be4117 --- /dev/null +++ b/frontend/tests/unit/mapProjection.test.ts @@ -0,0 +1 @@ +import{describe,expect,it}from'vitest';import{locations}from'../../src/fixtures/locations';import{projectLocations}from'../../src/map/mapProjection';describe('map projection',()=>it('excludes unmapped fixtures',()=>expect(projectLocations(locations)).toHaveLength(2))); From f3936274e4cf4c69ebf75c4f84ffe782234c6c9d Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sun, 13 Sep 2026 15:25:06 -0700 Subject: [PATCH 004/311] feat: add opt-in local v2 api mode --- frontend/README.md | 15 +++++++++++++++ frontend/src/api/LocalLocationRepository.ts | 4 ++++ frontend/src/api/errors.ts | 1 + frontend/src/api/wireSchema.ts | 10 ++++++---- frontend/src/app/App.svelte | 10 ++++++++-- .../tests/unit/localLocationRepository.test.ts | 4 ++++ 6 files changed, 38 insertions(+), 6 deletions(-) create mode 100644 frontend/src/api/LocalLocationRepository.ts create mode 100644 frontend/src/api/errors.ts create mode 100644 frontend/tests/unit/localLocationRepository.test.ts diff --git a/frontend/README.md b/frontend/README.md index 38e72da..7b553c4 100644 --- a/frontend/README.md +++ b/frontend/README.md @@ -5,3 +5,18 @@ Fixture-only Svelte 5 demonstrator. Run `npm install`, then `npm run dev`. It us Phase 2 gate notes: staging is explicit (`npm run stage`) and copies only `frontend/dist` to the resolved ignored `static/v2-preview` destination. The Leaflet adapter is isolated and uses a blank local background; no tile provider is configured. Export previews retain profile, release, limitations, source, and observation context. Remaining live-use blockers: the backend contract still needs canonical generated DTOs and release/revocation semantics; record-level evidence hashes, geocoder metadata, explicit review events, and scoped project approvals are not present in the current API. Human accessibility review (screen reader, 200% zoom, 320px reflow) and larger-workload performance measurements remain required before product completion. This preview must not be enabled for live publication. + +Current-wire contract gap checklist (from the platform decision): + +- [ ] Canonical machine-readable contract tied to Rust serialization, generated frontend DTOs, and CI drift checks. +- [ ] Record/version and evidence identifiers or hashes linked to source evidence. +- [ ] Source retrieval/publication dates and explicit source-availability semantics on record responses. +- [ ] Geocoder provider, query, timestamp, precision, result, and review-state fields. +- [ ] Independent community/project review events with role, scope, date, and outcome. +- [ ] Release/profile-scoped project approval and publication metadata. +- [ ] Stable release/revocation semantics and cache invalidation signals for concurrent queries. +- [ ] Backend-supported filters, aggregates, export endpoint, and release pinning before live product controls are added. + +These are documented gaps, not frontend claims or invented DTO fields. Phase 3 continues to use only the synthetic fixture repository. + +Local V2 integration (opt-in only): run the backend with `cargo run` (port 8000), then run the frontend with `npm run dev`. The `LocalLocationRepository` targets `/api/v2/locations?profile=...` only when explicitly invoked by local development code; fixture mode remains the default and there is no V1 fallback. This slice is not production-enabled, does not proxy or rewrite requests, and fails closed on malformed, mismatched, restricted, or no-promoted-release responses. diff --git a/frontend/src/api/LocalLocationRepository.ts b/frontend/src/api/LocalLocationRepository.ts new file mode 100644 index 0000000..a679194 --- /dev/null +++ b/frontend/src/api/LocalLocationRepository.ts @@ -0,0 +1,4 @@ +import {envelopeSchema,type WireLocation} from './wireSchema';import type {ApiError} from './errors';import type {Location} from '../domain/location'; +export type FetchLike=(input:RequestInfo|URL,init?:RequestInit)=>Promise;export type LocalListResult=Readonly<{locations:readonly Location[];releaseId:string;profile:string;coverageNote:string;nextCursor:string|null}>; +const fail=(kind:ApiError['kind'],message:string,status?:number):ApiError=>status===undefined?{kind,message}:{kind,message,status};const map=(row:WireLocation):Location=>({id:row.facility_id,name:row.canonical_name,region:row.city??row.country_code,category:row.category,lat:row.latitude,lon:row.longitude,observed:row.last_observed_at??row.first_observed_at??'unknown',source:row.provenance_source_name}); +export class LocalLocationRepository{constructor(private readonly fetcher:FetchLike=globalThis.fetch){}async list(profile:'official'|'secondary'|'community'='official',signal?:AbortSignal):Promise{try{const init:RequestInit={};if(signal)init.signal=signal;const response=await this.fetcher(`/api/v2/locations?profile=${profile}`,init);if(!response.ok)throw fail('http',`Local V2 request failed with status ${response.status}`,response.status);const parsed=envelopeSchema.safeParse(await response.json());if(!parsed.success)throw fail('invalid-contract','Local V2 response did not match the current contract.');const body=parsed.data;if(body.meta.profile!==profile)throw fail('invalid-contract','Local V2 response profile did not match the request.');if(body.meta.release_id===null){if(body.data.length!==0)throw fail('invalid-contract','No-release response contained records.');throw fail('no-release',body.meta.coverage_note);}if(body.meta.ruleset_version===undefined||body.data.some((row)=>row.release_id!==body.meta.release_id||row.release_ruleset_version!==body.meta.ruleset_version))throw fail('invalid-contract','Local V2 row snapshot did not match envelope metadata.');return{locations:body.data.map(map),releaseId:body.meta.release_id,profile:body.meta.profile,coverageNote:body.meta.coverage_note,nextCursor:body.meta.next_cursor??null};}catch(error){if(error&&typeof error==='object'&&'kind'in error)throw error;if(error instanceof DOMException&&error.name==='AbortError')throw fail('aborted','Local V2 request was aborted.');if(error instanceof TypeError)throw fail('network','Local V2 request could not connect.');throw fail('invalid-contract','Local V2 response could not be read safely.');}}} diff --git a/frontend/src/api/errors.ts b/frontend/src/api/errors.ts new file mode 100644 index 0000000..2732f65 --- /dev/null +++ b/frontend/src/api/errors.ts @@ -0,0 +1 @@ +export type ApiError=Readonly<{kind:'aborted'|'network'|'http'|'invalid-contract'|'no-release';message:string;status?:number}>; diff --git a/frontend/src/api/wireSchema.ts b/frontend/src/api/wireSchema.ts index e0eb7da..f9b936d 100644 --- a/frontend/src/api/wireSchema.ts +++ b/frontend/src/api/wireSchema.ts @@ -1,4 +1,6 @@ -import {z} from 'zod'; -export const locationSchema=z.object({id:z.string().regex(/^syn-/),name:z.string(),region:z.string(),category:z.string(),lat:z.number().nullable(),lon:z.number().nullable(),observed:z.string(),source:z.string()}); -export const envelopeSchema=z.object({data:z.array(locationSchema),meta:z.object({release:z.string(),profile:z.enum(['curated','community'])})}); -export type WireEnvelope=z.infer; +import { z } from 'zod'; +const textOrNull=z.string().nullable(); +export const locationSchema=z.object({facility_id:z.string().uuid(),canonical_name:z.string().min(1),city:textOrNull,country_code:z.string().regex(/^[A-Z]{2}$/),category:z.string(),source_type:z.enum(['official','secondary','user_submitted']),publication_profile:z.enum(['official','secondary','community']),factual_review_status:z.enum(['unreviewed','reviewed','rejected']),privacy_screening_status:z.literal('passed'),project_approval:z.enum(['pending','approved']),reviewer_role:textOrNull,publication_warning:textOrNull,display_precision:z.enum(['exact','city','unmapped']),latitude:z.number().finite().nullable(),longitude:z.number().finite().nullable(),first_observed_at:textOrNull,last_observed_at:textOrNull,observation_count:z.number().int().nonnegative().nullable(),lifecycle_status:z.enum(['active_observed','explicitly_closed','not_seen_recently','status_unknown']),provenance_source_id:z.string(),provenance_source_name:z.string(),provenance_source_url:z.string().url(),provenance_retrieved_at:z.string(),release_id:z.string(),release_ruleset_version:z.string()}).superRefine((row,ctx)=>{if((row.latitude===null)!==(row.longitude===null))ctx.addIssue({code:z.ZodIssueCode.custom,message:'coordinate pair must be complete'});if(row.display_precision==='unmapped'&&(row.latitude!==null||row.longitude!==null))ctx.addIssue({code:z.ZodIssueCode.custom,message:'unmapped record cannot have coordinates'});if(row.display_precision!=='unmapped'&&(row.latitude===null||row.longitude===null))ctx.addIssue({code:z.ZodIssueCode.custom,message:'mapped record requires coordinates'});}); +export const envelopeSchema=z.object({data:z.array(locationSchema),api_version:z.literal('v2'),meta:z.object({release_id:z.string().nullable(),ruleset_version:z.string().optional(),release_created_at:z.string().optional(),profile:z.enum(['official','secondary','community']),next_cursor:z.string().nullable().optional(),coverage_note:z.string().min(1)})}); +export type WireEnvelope=z.infer;export type WireLocation=z.infer; + diff --git a/frontend/src/app/App.svelte b/frontend/src/app/App.svelte index afd2d2b..a685d2d 100644 --- a/frontend/src/app/App.svelte +++ b/frontend/src/app/App.svelte @@ -1,2 +1,8 @@ - -Until Every Cage · evidence desk
UNTIL EVERY CAGE V2 / FIELD NOTEEthics & safeguards ↗

EVIDENCE DESK · SYNTHETIC PREVIEW

See what a record can—and cannot—tell us.

A quiet, inspectable view of animal-agriculture locations. Every fixture below is fictional.

{#if profile==='community'}
Unreviewed community claimNot verified by Until Every Cage.
{/if}{#if scenario==='loading'}
Loading the fixture…
{:else if scenario==='empty'}

No eligible records

{:else if scenario==='error'}{:else if scenario==='restricted'}

Preview withheld

{:else}
{#if selected}

RECORD / {selected.id}

{selected.name}

{selected.region} · {selected.category}

OBSERVED{selected.observed}
MAP STATUS{selected.lat===null?'Not available':'Approximate display point'}

PUBLICATION CONTEXT

{profile==='community'?'Community-submitted · Community-unreviewed':'Government-sourced · Project-approved'}

Source context remains attached to this synthetic record.

{/if}
{/if}
{#if showMap}{/if}{#if showExport}

IN-MEMORY EXPORT PREVIEW

{profile} · synthetic-2026.09

Loaded results only. This preview is not a live or complete export.

{exportPreview}
{/if}
Method preview · no live requests
+ +Until Every Cage · evidence desk
UNTIL EVERY CAGE V2 / FIELD NOTEEthics & safeguards ↗

EVIDENCE DESK · {localMode?'LOCAL V2 API':'SYNTHETIC PREVIEW'}

See what a record can—and cannot—tell us.

{localMode?'Explicit local integration mode. No fixture or V1 fallback is used.':'A quiet, inspectable view of fictional locations.'}

{#if !localMode}{/if}
{#if profile==='community'}
Unreviewed community claimNot verified by Until Every Cage.
{/if}{#if localMode&&localStatus==='loading'}
Loading the local V2 release…
{:else if localMode&&(localStatus==='error'||localStatus==='no-release')}{:else if scenario==='loading'}
Loading the fixture…
{:else if scenario==='empty'}

No eligible records

{:else if scenario==='error'}{:else if scenario==='restricted'}

Preview withheld

{:else}
{#if selected}

RECORD / {selected.id}

{selected.name}

{selected.region} · {selected.category}

OBSERVED{selected.observed}
MAP STATUS{selected.lat===null?'Not available':'Approximate display point'}
{/if}
{/if}
{#if showMap}{/if}{#if showExport}

IN-MEMORY EXPORT PREVIEW

{profile} · {release}

Loaded results only. This preview is not a live or complete export.

{exportPreview}
{/if}
{localMode?'Local V2 mode · no fixture fallback':'Method preview · no live requests'}
diff --git a/frontend/tests/unit/localLocationRepository.test.ts b/frontend/tests/unit/localLocationRepository.test.ts new file mode 100644 index 0000000..521b57e --- /dev/null +++ b/frontend/tests/unit/localLocationRepository.test.ts @@ -0,0 +1,4 @@ +import{describe,expect,it,vi}from'vitest';import{LocalLocationRepository}from'../../src/api/LocalLocationRepository'; +const row={facility_id:'550e8400-e29b-41d4-a716-446655440000',canonical_name:'Local V2 Fixture',city:'North Coast',country_code:'DK',category:'dairy',source_type:'official',publication_profile:'official',factual_review_status:'reviewed',privacy_screening_status:'passed',project_approval:'approved',reviewer_role:null,publication_warning:null,display_precision:'city',latitude:55,longitude:10,first_observed_at:null,last_observed_at:'2026-01-01T00:00:00Z',observation_count:1,lifecycle_status:'active_observed',provenance_source_id:'source-1',provenance_source_name:'Synthetic local source',provenance_source_url:'https://example.test/source',provenance_retrieved_at:'2026-01-01T00:00:00Z',release_id:'rel-1',release_ruleset_version:'rules-1'}; +const response=(body:unknown,status=200)=>new Response(JSON.stringify(body),{status,headers:{'content-type':'application/json'}});const envelope=(data=[row],meta={release_id:'rel-1',ruleset_version:'rules-1',profile:'official',next_cursor:null,coverage_note:'Local promoted release.'})=>({data,api_version:'v2',meta}); +describe('LocalLocationRepository',()=>{it('maps a valid Rust-shaped envelope',async()=>{const result=await new LocalLocationRepository(vi.fn().mockResolvedValue(response(envelope()))).list();expect(result.locations[0]).toMatchObject({id:row.facility_id,name:'Local V2 Fixture',lat:55});});it('fails closed when no release is promoted',async()=>{await expect(new LocalLocationRepository(vi.fn().mockResolvedValue(response(envelope([],{release_id:null,profile:'official',coverage_note:'No promoted release.'})))).list()).rejects.toMatchObject({kind:'no-release'});});it('classifies HTTP failures',async()=>{await expect(new LocalLocationRepository(vi.fn().mockResolvedValue(response({},503))).list()).rejects.toMatchObject({kind:'http',status:503});});it('rejects malformed or restricted payloads',async()=>{await expect(new LocalLocationRepository(vi.fn().mockResolvedValue(response({...envelope(),api_version:'v1'}))).list()).rejects.toMatchObject({kind:'invalid-contract'});await expect(new LocalLocationRepository(vi.fn().mockResolvedValue(response(envelope([{...row,privacy_screening_status:'failed'}])))).list()).rejects.toMatchObject({kind:'invalid-contract'});});}); From 9281d064b2447f3aa064a1be9ba009a9bfda277d Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sun, 13 Sep 2026 15:25:21 -0700 Subject: [PATCH 005/311] test: expand local v2 browser matrix --- frontend/playwright.config.ts | 6 +++++- frontend/src/domain/location.ts | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/frontend/playwright.config.ts b/frontend/playwright.config.ts index fe225d4..12e16de 100644 --- a/frontend/playwright.config.ts +++ b/frontend/playwright.config.ts @@ -8,5 +8,9 @@ export default defineConfig({ timeout: 30_000, use: { baseURL: 'http://127.0.0.1:4173/v2-preview/', trace: 'retain-on-failure' }, webServer: { command: 'npm run preview -- --host 127.0.0.1 --port 4173', url: 'http://127.0.0.1:4173/v2-preview/', timeout: 30_000, reuseExistingServer: true }, - projects: [{ name: 'chromium', use: { ...devices['Desktop Chrome'] } }], + projects: [ + { name: 'chromium', use: { ...devices['Desktop Chrome'] } }, + { name: 'firefox', use: { ...devices['Desktop Firefox'] } }, + { name: 'webkit', use: { ...devices['Desktop Safari'] } }, + ], }); diff --git a/frontend/src/domain/location.ts b/frontend/src/domain/location.ts index c6ea27f..cdb2359 100644 --- a/frontend/src/domain/location.ts +++ b/frontend/src/domain/location.ts @@ -1,2 +1,2 @@ -export type LocationId = `syn-${string}`; +export type LocationId = string; export type Location = Readonly<{id:LocationId,name:string,region:string,category:string,lat:number|null,lon:number|null,observed:string,source:string}>; From 715f58019ba7f2a8a067b0caf6e8a3c6a2440ba1 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sun, 13 Sep 2026 15:35:44 -0700 Subject: [PATCH 006/311] feat: harden local v2 integration --- frontend/src/api/LocalLocationRepository.ts | 6 ++++-- frontend/src/api/wireSchema.ts | 2 ++ frontend/src/app/App.svelte | 11 +++++------ frontend/tests/e2e/fixture-platform.spec.ts | 2 ++ frontend/tests/unit/localDetailRepository.test.ts | 4 ++++ 5 files changed, 17 insertions(+), 8 deletions(-) create mode 100644 frontend/tests/unit/localDetailRepository.test.ts diff --git a/frontend/src/api/LocalLocationRepository.ts b/frontend/src/api/LocalLocationRepository.ts index a679194..5cb2cbf 100644 --- a/frontend/src/api/LocalLocationRepository.ts +++ b/frontend/src/api/LocalLocationRepository.ts @@ -1,4 +1,6 @@ -import {envelopeSchema,type WireLocation} from './wireSchema';import type {ApiError} from './errors';import type {Location} from '../domain/location'; +import {detailEnvelopeSchema,envelopeSchema,type WireLocation} from './wireSchema';import type {ApiError} from './errors';import type {Location} from '../domain/location'; export type FetchLike=(input:RequestInfo|URL,init?:RequestInit)=>Promise;export type LocalListResult=Readonly<{locations:readonly Location[];releaseId:string;profile:string;coverageNote:string;nextCursor:string|null}>; const fail=(kind:ApiError['kind'],message:string,status?:number):ApiError=>status===undefined?{kind,message}:{kind,message,status};const map=(row:WireLocation):Location=>({id:row.facility_id,name:row.canonical_name,region:row.city??row.country_code,category:row.category,lat:row.latitude,lon:row.longitude,observed:row.last_observed_at??row.first_observed_at??'unknown',source:row.provenance_source_name}); -export class LocalLocationRepository{constructor(private readonly fetcher:FetchLike=globalThis.fetch){}async list(profile:'official'|'secondary'|'community'='official',signal?:AbortSignal):Promise{try{const init:RequestInit={};if(signal)init.signal=signal;const response=await this.fetcher(`/api/v2/locations?profile=${profile}`,init);if(!response.ok)throw fail('http',`Local V2 request failed with status ${response.status}`,response.status);const parsed=envelopeSchema.safeParse(await response.json());if(!parsed.success)throw fail('invalid-contract','Local V2 response did not match the current contract.');const body=parsed.data;if(body.meta.profile!==profile)throw fail('invalid-contract','Local V2 response profile did not match the request.');if(body.meta.release_id===null){if(body.data.length!==0)throw fail('invalid-contract','No-release response contained records.');throw fail('no-release',body.meta.coverage_note);}if(body.meta.ruleset_version===undefined||body.data.some((row)=>row.release_id!==body.meta.release_id||row.release_ruleset_version!==body.meta.ruleset_version))throw fail('invalid-contract','Local V2 row snapshot did not match envelope metadata.');return{locations:body.data.map(map),releaseId:body.meta.release_id,profile:body.meta.profile,coverageNote:body.meta.coverage_note,nextCursor:body.meta.next_cursor??null};}catch(error){if(error&&typeof error==='object'&&'kind'in error)throw error;if(error instanceof DOMException&&error.name==='AbortError')throw fail('aborted','Local V2 request was aborted.');if(error instanceof TypeError)throw fail('network','Local V2 request could not connect.');throw fail('invalid-contract','Local V2 response could not be read safely.');}}} +export class LocalLocationRepository{constructor(private readonly fetcher:FetchLike=globalThis.fetch){}private async json(path:string,signal?:AbortSignal):Promise{const init:RequestInit={};if(signal)init.signal=signal;const response=await this.fetcher(path,init);if(!response.ok)throw fail('http',`Local V2 request failed with status ${response.status}`,response.status);try{return await response.json();}catch{throw fail('invalid-contract','Local V2 response was not valid JSON.');}} +async list(profile:'official'|'secondary'|'community'='official',signal?:AbortSignal):Promise{try{const parsed=envelopeSchema.safeParse(await this.json(`/api/v2/locations?profile=${profile}`,signal));if(!parsed.success)throw fail('invalid-contract','Local V2 list response did not match the current contract.');const body=parsed.data;if(body.meta.profile!==profile)throw fail('invalid-contract','Local V2 response profile did not match the request.');if(body.meta.release_id===null){if(body.data.length!==0)throw fail('invalid-contract','No-release response contained records.');throw fail('no-release',body.meta.coverage_note);}if(body.meta.ruleset_version===undefined||body.data.some((row)=>row.privacy_screening_status!=='passed'||row.project_approval!=='approved'||row.release_id!==body.meta.release_id||row.release_ruleset_version!==body.meta.ruleset_version))throw fail('invalid-contract','Local V2 response contained restricted or mismatched records.');return{locations:body.data.map(map),releaseId:body.meta.release_id,profile:body.meta.profile,coverageNote:body.meta.coverage_note,nextCursor:body.meta.next_cursor??null};}catch(error){if(error&&typeof error==='object'&&'kind'in error)throw error;if(error instanceof DOMException&&error.name==='AbortError')throw fail('aborted','Local V2 request was aborted.');if(error instanceof TypeError)throw fail('network','Local V2 request could not connect.');throw fail('invalid-contract','Local V2 response could not be read safely.');}} +async detail(id:string,profile:'official'|'secondary'|'community'='official',signal?:AbortSignal):Promise>{try{const parsed=detailEnvelopeSchema.safeParse(await this.json(`/api/v2/locations/${encodeURIComponent(id)}?profile=${profile}`,signal));if(!parsed.success)throw fail('invalid-contract','Local V2 detail response did not match the current contract.');const body=parsed.data;if(body.meta.profile!==profile||body.data.privacy_screening_status!=='passed'||body.data.project_approval!=='approved')throw fail('invalid-contract','Local V2 detail response was restricted or mismatched.');return{location:map(body.data),releaseId:body.meta.release_id,profile:body.meta.profile};}catch(error){if(error&&typeof error==='object'&&'kind'in error)throw error;if(error instanceof DOMException&&error.name==='AbortError')throw fail('aborted','Local V2 request was aborted.');if(error instanceof TypeError)throw fail('network','Local V2 request could not connect.');throw fail('invalid-contract','Local V2 detail response could not be read safely.');}}} diff --git a/frontend/src/api/wireSchema.ts b/frontend/src/api/wireSchema.ts index f9b936d..9f7ecfa 100644 --- a/frontend/src/api/wireSchema.ts +++ b/frontend/src/api/wireSchema.ts @@ -3,4 +3,6 @@ const textOrNull=z.string().nullable(); export const locationSchema=z.object({facility_id:z.string().uuid(),canonical_name:z.string().min(1),city:textOrNull,country_code:z.string().regex(/^[A-Z]{2}$/),category:z.string(),source_type:z.enum(['official','secondary','user_submitted']),publication_profile:z.enum(['official','secondary','community']),factual_review_status:z.enum(['unreviewed','reviewed','rejected']),privacy_screening_status:z.literal('passed'),project_approval:z.enum(['pending','approved']),reviewer_role:textOrNull,publication_warning:textOrNull,display_precision:z.enum(['exact','city','unmapped']),latitude:z.number().finite().nullable(),longitude:z.number().finite().nullable(),first_observed_at:textOrNull,last_observed_at:textOrNull,observation_count:z.number().int().nonnegative().nullable(),lifecycle_status:z.enum(['active_observed','explicitly_closed','not_seen_recently','status_unknown']),provenance_source_id:z.string(),provenance_source_name:z.string(),provenance_source_url:z.string().url(),provenance_retrieved_at:z.string(),release_id:z.string(),release_ruleset_version:z.string()}).superRefine((row,ctx)=>{if((row.latitude===null)!==(row.longitude===null))ctx.addIssue({code:z.ZodIssueCode.custom,message:'coordinate pair must be complete'});if(row.display_precision==='unmapped'&&(row.latitude!==null||row.longitude!==null))ctx.addIssue({code:z.ZodIssueCode.custom,message:'unmapped record cannot have coordinates'});if(row.display_precision!=='unmapped'&&(row.latitude===null||row.longitude===null))ctx.addIssue({code:z.ZodIssueCode.custom,message:'mapped record requires coordinates'});}); export const envelopeSchema=z.object({data:z.array(locationSchema),api_version:z.literal('v2'),meta:z.object({release_id:z.string().nullable(),ruleset_version:z.string().optional(),release_created_at:z.string().optional(),profile:z.enum(['official','secondary','community']),next_cursor:z.string().nullable().optional(),coverage_note:z.string().min(1)})}); export type WireEnvelope=z.infer;export type WireLocation=z.infer; +export const detailEnvelopeSchema=z.object({data:locationSchema,api_version:z.literal('v2'),meta:z.object({release_id:z.string(),ruleset_version:z.string(),release_created_at:z.string(),profile:z.enum(['official','secondary','community'])})}); +export type DetailEnvelope=z.infer; diff --git a/frontend/src/app/App.svelte b/frontend/src/app/App.svelte index a685d2d..60b785f 100644 --- a/frontend/src/app/App.svelte +++ b/frontend/src/app/App.svelte @@ -1,8 +1,7 @@ -Until Every Cage · evidence desk
UNTIL EVERY CAGE V2 / FIELD NOTEEthics & safeguards ↗

EVIDENCE DESK · {localMode?'LOCAL V2 API':'SYNTHETIC PREVIEW'}

See what a record can—and cannot—tell us.

{localMode?'Explicit local integration mode. No fixture or V1 fallback is used.':'A quiet, inspectable view of fictional locations.'}

{#if !localMode}{/if}
{#if profile==='community'}
Unreviewed community claimNot verified by Until Every Cage.
{/if}{#if localMode&&localStatus==='loading'}
Loading the local V2 release…
{:else if localMode&&(localStatus==='error'||localStatus==='no-release')}{:else if scenario==='loading'}
Loading the fixture…
{:else if scenario==='empty'}

No eligible records

{:else if scenario==='error'}{:else if scenario==='restricted'}

Preview withheld

{:else}
{#if selected}

RECORD / {selected.id}

{selected.name}

{selected.region} · {selected.category}

OBSERVED{selected.observed}
MAP STATUS{selected.lat===null?'Not available':'Approximate display point'}
{/if}
{/if}
{#if showMap}{/if}{#if showExport}

IN-MEMORY EXPORT PREVIEW

{profile} · {release}

Loaded results only. This preview is not a live or complete export.

{exportPreview}
{/if}
{localMode?'Local V2 mode · no fixture fallback':'Method preview · no live requests'}
+Until Every Cage · evidence desk
UNTIL EVERY CAGE V2 / FIELD NOTEEthics & safeguards ↗

EVIDENCE DESK · {localMode?'LOCAL V2 API':'SYNTHETIC PREVIEW'}

See what a record can—and cannot—tell us.

{localMode?'Explicit local integration mode. No fixture or V1 fallback is used.':'A quiet, inspectable view of fictional locations.'}

{#if profile==='community'}
Unreviewed community claimNot verified by Until Every Cage.
{/if}{#if localMode&&localStatus==='loading'}
Loading the local V2 release…
{:else if localMode&&(localStatus==='error'||localStatus==='no-release')}{:else if localMode&&detailStatus==='loading'}
Loading the selected local record…
{:else if localMode&&detailStatus==='error'}{:else if scenario==='loading'}
Loading the fixture…
{:else if scenario==='empty'}

No eligible records

{:else if scenario==='error'}{:else if scenario==='restricted'}

Preview withheld

{:else}
{#if selected}

RECORD / {selected.id}

{selected.name}

{selected.region} · {selected.category}

OBSERVED{selected.observed}
MAP STATUS{selected.lat===null?'Not available':'Approximate display point'}
{/if}
{/if}
{#if showMap}{/if}{#if showExport}

IN-MEMORY EXPORT PREVIEW

{profile} · {release}

Loaded results only. This preview is not a live or complete export.

{exportPreview}
{/if}
{localMode?'Local V2 mode · no fixture fallback':'Method preview · no live requests'}
diff --git a/frontend/tests/e2e/fixture-platform.spec.ts b/frontend/tests/e2e/fixture-platform.spec.ts index 936a889..aebf806 100644 --- a/frontend/tests/e2e/fixture-platform.spec.ts +++ b/frontend/tests/e2e/fixture-platform.spec.ts @@ -62,3 +62,5 @@ test('filters the curated list and updates the detail hash on selection', async }); test('shows the local map and limited export context', async ({ page }) => { await page.goto('./#/'); await page.getByRole('button', { name: 'Show map' }).click(); await expect(page.getByLabel('Synthetic location map')).toBeVisible(); await expect(page.getByText(/no external tiles/)).toBeVisible(); await page.getByRole('button', { name: 'Preview export' }).click(); const exportPanel=page.locator('.export-preview'); await expect(exportPanel.getByText('IN-MEMORY EXPORT PREVIEW')).toBeVisible(); await expect(exportPanel.locator('p').filter({hasText:'Loaded results only'})).toBeVisible(); }); + +test('local mode fails safely on a mocked 503 without fixture fallback', async ({ page }) => { await page.route('**/api/v2/locations*', (route) => route.fulfill({ status: 503, body: 'unavailable' })); await page.goto('./?mode=local-v2#/'); await expect(page.getByRole('alert')).toContainText('Could not load local V2 data'); await expect(page.getByText('Local V2 mode · no fixture fallback')).toBeVisible(); await expect(page.getByRole('button', { name: /North Star Cooperative/ })).toHaveCount(0); }); diff --git a/frontend/tests/unit/localDetailRepository.test.ts b/frontend/tests/unit/localDetailRepository.test.ts new file mode 100644 index 0000000..5522fa5 --- /dev/null +++ b/frontend/tests/unit/localDetailRepository.test.ts @@ -0,0 +1,4 @@ +import{describe,expect,it,vi}from'vitest';import{LocalLocationRepository}from'../../src/api/LocalLocationRepository'; +const row={facility_id:'550e8400-e29b-41d4-a716-446655440000',canonical_name:'Detail Local Fixture',city:'North Coast',country_code:'DK',category:'dairy',source_type:'official',publication_profile:'official',factual_review_status:'reviewed',privacy_screening_status:'passed',project_approval:'approved',reviewer_role:null,publication_warning:null,display_precision:'city',latitude:55,longitude:10,first_observed_at:null,last_observed_at:'2026-01-01T00:00:00Z',observation_count:1,lifecycle_status:'active_observed',provenance_source_id:'source-1',provenance_source_name:'Synthetic local source',provenance_source_url:'https://example.test/source',provenance_retrieved_at:'2026-01-01T00:00:00Z',release_id:'rel-1',release_ruleset_version:'rules-1'}; +const body=(data=row,meta={release_id:'rel-1',ruleset_version:'rules-1',release_created_at:'2026-01-01T00:00:00Z',profile:'official'})=>({data,api_version:'v2',meta}); +describe('LocalLocationRepository detail',()=>{it('maps a valid detail envelope',async()=>{const fetcher=vi.fn().mockResolvedValue(new Response(JSON.stringify(body()),{status:200}));const result=await new LocalLocationRepository(fetcher).detail(row.facility_id);expect(result).toMatchObject({releaseId:'rel-1',profile:'official',location:{id:row.facility_id,name:'Detail Local Fixture'}});expect(fetcher).toHaveBeenCalledWith(`/api/v2/locations/${row.facility_id}?profile=official`,expect.any(Object));});it('rejects wrong profile and aborts',async()=>{const wrong=vi.fn().mockResolvedValue(new Response(JSON.stringify(body(row,{...body().meta,profile:'community'}))));await expect(new LocalLocationRepository(wrong).detail(row.facility_id)).rejects.toMatchObject({kind:'invalid-contract'});const controller=new AbortController();const fetcher=vi.fn().mockRejectedValue(new DOMException('aborted','AbortError'));await expect(new LocalLocationRepository(fetcher).detail(row.facility_id,'official',controller.signal)).rejects.toMatchObject({kind:'aborted'});});}); From 6bc12e4e6e8a1b0ce8abbd27897356d5ea5126f9 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sun, 13 Sep 2026 15:42:40 -0700 Subject: [PATCH 007/311] feat: connect frontend to loopback v2 api --- frontend/src/api/LocalLocationRepository.ts | 7 +------ frontend/src/app/App.svelte | 4 ++-- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/frontend/src/api/LocalLocationRepository.ts b/frontend/src/api/LocalLocationRepository.ts index 5cb2cbf..ad396aa 100644 --- a/frontend/src/api/LocalLocationRepository.ts +++ b/frontend/src/api/LocalLocationRepository.ts @@ -1,6 +1 @@ -import {detailEnvelopeSchema,envelopeSchema,type WireLocation} from './wireSchema';import type {ApiError} from './errors';import type {Location} from '../domain/location'; -export type FetchLike=(input:RequestInfo|URL,init?:RequestInit)=>Promise;export type LocalListResult=Readonly<{locations:readonly Location[];releaseId:string;profile:string;coverageNote:string;nextCursor:string|null}>; -const fail=(kind:ApiError['kind'],message:string,status?:number):ApiError=>status===undefined?{kind,message}:{kind,message,status};const map=(row:WireLocation):Location=>({id:row.facility_id,name:row.canonical_name,region:row.city??row.country_code,category:row.category,lat:row.latitude,lon:row.longitude,observed:row.last_observed_at??row.first_observed_at??'unknown',source:row.provenance_source_name}); -export class LocalLocationRepository{constructor(private readonly fetcher:FetchLike=globalThis.fetch){}private async json(path:string,signal?:AbortSignal):Promise{const init:RequestInit={};if(signal)init.signal=signal;const response=await this.fetcher(path,init);if(!response.ok)throw fail('http',`Local V2 request failed with status ${response.status}`,response.status);try{return await response.json();}catch{throw fail('invalid-contract','Local V2 response was not valid JSON.');}} -async list(profile:'official'|'secondary'|'community'='official',signal?:AbortSignal):Promise{try{const parsed=envelopeSchema.safeParse(await this.json(`/api/v2/locations?profile=${profile}`,signal));if(!parsed.success)throw fail('invalid-contract','Local V2 list response did not match the current contract.');const body=parsed.data;if(body.meta.profile!==profile)throw fail('invalid-contract','Local V2 response profile did not match the request.');if(body.meta.release_id===null){if(body.data.length!==0)throw fail('invalid-contract','No-release response contained records.');throw fail('no-release',body.meta.coverage_note);}if(body.meta.ruleset_version===undefined||body.data.some((row)=>row.privacy_screening_status!=='passed'||row.project_approval!=='approved'||row.release_id!==body.meta.release_id||row.release_ruleset_version!==body.meta.ruleset_version))throw fail('invalid-contract','Local V2 response contained restricted or mismatched records.');return{locations:body.data.map(map),releaseId:body.meta.release_id,profile:body.meta.profile,coverageNote:body.meta.coverage_note,nextCursor:body.meta.next_cursor??null};}catch(error){if(error&&typeof error==='object'&&'kind'in error)throw error;if(error instanceof DOMException&&error.name==='AbortError')throw fail('aborted','Local V2 request was aborted.');if(error instanceof TypeError)throw fail('network','Local V2 request could not connect.');throw fail('invalid-contract','Local V2 response could not be read safely.');}} -async detail(id:string,profile:'official'|'secondary'|'community'='official',signal?:AbortSignal):Promise>{try{const parsed=detailEnvelopeSchema.safeParse(await this.json(`/api/v2/locations/${encodeURIComponent(id)}?profile=${profile}`,signal));if(!parsed.success)throw fail('invalid-contract','Local V2 detail response did not match the current contract.');const body=parsed.data;if(body.meta.profile!==profile||body.data.privacy_screening_status!=='passed'||body.data.project_approval!=='approved')throw fail('invalid-contract','Local V2 detail response was restricted or mismatched.');return{location:map(body.data),releaseId:body.meta.release_id,profile:body.meta.profile};}catch(error){if(error&&typeof error==='object'&&'kind'in error)throw error;if(error instanceof DOMException&&error.name==='AbortError')throw fail('aborted','Local V2 request was aborted.');if(error instanceof TypeError)throw fail('network','Local V2 request could not connect.');throw fail('invalid-contract','Local V2 detail response could not be read safely.');}}} +import{detailEnvelopeSchema,envelopeSchema,type WireLocation}from'./wireSchema';import type{ApiError}from'./errors';import type{Location}from'../domain/location';export type FetchLike=(input:RequestInfo|URL,init?:RequestInit)=>Promise;export const localOrigin=(value:string|undefined):string|undefined=>{if(!value)return undefined;const url=new URL(value);if(url.protocol!=='http:'||!['127.0.0.1','localhost','::1'].includes(url.hostname))throw new Error('Local API origin must be loopback HTTP.');return url.origin;};const fail=(kind:ApiError['kind'],message:string,status?:number):ApiError=>status===undefined?{kind,message}:{kind,message,status};const map=(r:WireLocation):Location=>({id:r.facility_id,name:r.canonical_name,region:r.city??r.country_code,category:r.category,lat:r.latitude,lon:r.longitude,observed:r.last_observed_at??r.first_observed_at??'unknown',source:r.provenance_source_name});export class LocalLocationRepository{readonly#base:string|undefined;constructor(private readonly fetcher:FetchLike=globalThis.fetch,baseUrl?:string){this.#base=localOrigin(baseUrl);}private async json(path:string,signal?:AbortSignal){const init:RequestInit={};if(signal)init.signal=signal;const response=await this.fetcher(`${this.#base??''}${path}`,init);if(!response.ok)throw fail('http',`Local V2 request failed with status ${response.status}`,response.status);try{return await response.json();}catch{throw fail('invalid-contract','Local V2 response was not valid JSON.');}}async list(profile:'official'|'secondary'|'community'='official'){try{const b=envelopeSchema.safeParse(await this.json(`/api/v2/locations?profile=${profile}`));if(!b.success||b.data.meta.profile!==profile)throw fail('invalid-contract','Local V2 list response was rejected.');if(b.data.meta.release_id===null)throw fail('no-release',b.data.meta.coverage_note);if(b.data.meta.ruleset_version===undefined||b.data.data.some(r=>r.privacy_screening_status!=='passed'||r.project_approval!=='approved'||r.release_id!==b.data.meta.release_id||r.release_ruleset_version!==b.data.meta.ruleset_version))throw fail('invalid-contract','Local V2 list snapshot was rejected.');return{locations:b.data.data.map(map),releaseId:b.data.meta.release_id,profile:b.data.meta.profile,coverageNote:b.data.meta.coverage_note,nextCursor:b.data.meta.next_cursor??null};}catch(e){if(e&&typeof e==='object'&&'kind'in e)throw e;if(e instanceof DOMException&&e.name==='AbortError')throw fail('aborted','Local V2 request was aborted.');if(e instanceof TypeError)throw fail('network','Local V2 request could not connect.');throw fail('invalid-contract','Local V2 response could not be read safely.');}}async detail(id:string,profile:'official'|'secondary'|'community'='official'){try{const b=detailEnvelopeSchema.safeParse(await this.json(`/api/v2/locations/${encodeURIComponent(id)}?profile=${profile}`));if(!b.success||b.data.meta.profile!==profile||b.data.data.privacy_screening_status!=='passed'||b.data.data.project_approval!=='approved')throw fail('invalid-contract','Local V2 detail response was rejected.');return{location:map(b.data.data),releaseId:b.data.meta.release_id,profile:b.data.meta.profile};}catch(e){if(e&&typeof e==='object'&&'kind'in e)throw e;if(e instanceof DOMException&&e.name==='AbortError')throw fail('aborted','Local V2 request was aborted.');if(e instanceof TypeError)throw fail('network','Local V2 request could not connect.');throw fail('invalid-contract','Local V2 detail response could not be read safely.');}}} diff --git a/frontend/src/app/App.svelte b/frontend/src/app/App.svelte index 60b785f..b1f163e 100644 --- a/frontend/src/app/App.svelte +++ b/frontend/src/app/App.svelte @@ -1,7 +1,7 @@ Until Every Cage · evidence desk
UNTIL EVERY CAGE V2 / FIELD NOTEEthics & safeguards ↗

EVIDENCE DESK · {localMode?'LOCAL V2 API':'SYNTHETIC PREVIEW'}

See what a record can—and cannot—tell us.

{localMode?'Explicit local integration mode. No fixture or V1 fallback is used.':'A quiet, inspectable view of fictional locations.'}

{#if profile==='community'}
Unreviewed community claimNot verified by Until Every Cage.
{/if}{#if localMode&&localStatus==='loading'}
Loading the local V2 release…
{:else if localMode&&(localStatus==='error'||localStatus==='no-release')}{:else if localMode&&detailStatus==='loading'}
Loading the selected local record…
{:else if localMode&&detailStatus==='error'}{:else if scenario==='loading'}
Loading the fixture…
{:else if scenario==='empty'}

No eligible records

{:else if scenario==='error'}{:else if scenario==='restricted'}

Preview withheld

{:else}
{#if selected}

RECORD / {selected.id}

{selected.name}

{selected.region} · {selected.category}

OBSERVED{selected.observed}
MAP STATUS{selected.lat===null?'Not available':'Approximate display point'}
{/if}
{/if}
{#if showMap}{/if}{#if showExport}

IN-MEMORY EXPORT PREVIEW

{profile} · {release}

Loaded results only. This preview is not a live or complete export.

{exportPreview}
{/if}
{localMode?'Local V2 mode · no fixture fallback':'Method preview · no live requests'}
From 37f7327efb27e85e1f7f5b31d828e64cd0e194c1 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sun, 13 Sep 2026 15:55:21 -0700 Subject: [PATCH 008/311] feat: add persistent local v2 workflow --- frontend/README.md | 2 ++ frontend/scripts/probe-local-v2.mjs | 1 + pipeline/scripts/maintenance/local-v2.ps1 | 3 +++ 3 files changed, 6 insertions(+) create mode 100644 frontend/scripts/probe-local-v2.mjs create mode 100644 pipeline/scripts/maintenance/local-v2.ps1 diff --git a/frontend/README.md b/frontend/README.md index 7b553c4..2c40575 100644 --- a/frontend/README.md +++ b/frontend/README.md @@ -20,3 +20,5 @@ Current-wire contract gap checklist (from the platform decision): These are documented gaps, not frontend claims or invented DTO fields. Phase 3 continues to use only the synthetic fixture repository. Local V2 integration (opt-in only): run the backend with `cargo run` (port 8000), then run the frontend with `npm run dev`. The `LocalLocationRepository` targets `/api/v2/locations?profile=...` only when explicitly invoked by local development code; fixture mode remains the default and there is no V1 fallback. This slice is not production-enabled, does not proxy or rewrite requests, and fails closed on malformed, mismatched, restricted, or no-promoted-release responses. + +Persistent local two-port workflow (never uses `down -v`): from the repository root run `powershell -ExecutionPolicy Bypass -File pipeline/scripts/maintenance/local-v2.ps1 start`, then `$env:UEC_DATABASE_URL='postgresql://uec:uec-local-development-only@127.0.0.1:5433/uec?sslmode=disable'; cargo run` in another terminal, and finally `npm --prefix frontend run dev`. Check with `... local-v2.ps1 status`; probe list/detail with `node frontend/scripts/probe-local-v2.mjs`; stop containers with `... local-v2.ps1 stop`. The seed is the existing synthetic contract fixture and is marker-gated for repeatability. Promotion is not automated by this helper; existing validation/promotion gates must authorize it. diff --git a/frontend/scripts/probe-local-v2.mjs b/frontend/scripts/probe-local-v2.mjs new file mode 100644 index 0000000..4aad5a4 --- /dev/null +++ b/frontend/scripts/probe-local-v2.mjs @@ -0,0 +1 @@ +const base=process.argv[2]??'http://127.0.0.1:8000';const list=await fetch(`${base}/api/v2/locations?profile=official&limit=1`);if(!list.ok)throw new Error(`list probe failed: ${list.status}`);const body=await list.json();if(body.api_version!=='v2'||!body.meta)throw new Error('list contract probe failed');console.log(`list probe passed: release=${body.meta.release_id}`);if(Array.isArray(body.data)&&body.data[0]?.facility_id){const id=encodeURIComponent(body.data[0].facility_id);const detail=await fetch(`${base}/api/v2/locations/${id}?profile=official`);if(!detail.ok)throw new Error(`detail probe failed: ${detail.status}`);const detailBody=await detail.json();if(detailBody.api_version!=='v2'||!detailBody.data)throw new Error('detail contract probe failed');console.log('detail probe passed');}else console.log('detail probe skipped: no public promoted record'); diff --git a/pipeline/scripts/maintenance/local-v2.ps1 b/pipeline/scripts/maintenance/local-v2.ps1 new file mode 100644 index 0000000..abea777 --- /dev/null +++ b/pipeline/scripts/maintenance/local-v2.ps1 @@ -0,0 +1,3 @@ +param([ValidateSet('start','status','stop','probe')][string]$Command='status') +$ErrorActionPreference='Stop';$root=(Resolve-Path (Join-Path $PSScriptRoot '..\..\..')).Path;$compose=Join-Path $root 'docker-compose.pipeline.yml';$project='uec-local-v2';$port=5433;$db="postgresql://uec:uec-local-development-only@127.0.0.1:$port/uec";$env:UEC_PIPELINE_DB_PORT="$port";$env:UEC_DATABASE_URL="${db}?sslmode=disable" +Push-Location $root;try{switch($Command){'start'{& docker compose -p $project -f $compose up -d --wait;if($LASTEXITCODE){throw 'Postgres startup failed.'};python pipeline/scripts/maintenance/apply-migrations.py;if($LASTEXITCODE){throw 'Migration application failed.'};$seeded=& docker compose -p $project -f $compose exec -T postgres psql -U uec -d uec -Atc "SELECT to_regclass('uec.local_v2_fixture_seed') IS NOT NULL";if($seeded.Trim() -ne 't'){Get-Content pipeline/tests/standard_contract_seed.sql -Raw|& docker compose -p $project -f $compose exec -T postgres psql -v ON_ERROR_STOP=1 -U uec -d uec;if($LASTEXITCODE){throw 'Synthetic seed failed.'};& docker compose -p $project -f $compose exec -T postgres psql -v ON_ERROR_STOP=1 -U uec -d uec -c "CREATE TABLE IF NOT EXISTS uec.local_v2_fixture_seed (seed_name text primary key, seeded_at timestamptz not null default now()); INSERT INTO uec.local_v2_fixture_seed(seed_name) VALUES ('standard-contract') ON CONFLICT DO NOTHING;"};Write-Host "Local Postgres ready on $port. Start Axum separately with: `$env:UEC_DATABASE_URL='${db}?sslmode=disable'; cargo run"}'status'{& docker compose -p $project -f $compose ps;if($LASTEXITCODE){throw 'Status failed.'}}'stop'{& docker compose -p $project -f $compose stop;if($LASTEXITCODE){throw 'Stop failed.'}}'probe'{& curl.exe --fail-with-body "http://127.0.0.1:8000/api/v2/locations?profile=official&limit=1";if($LASTEXITCODE){throw 'List probe failed.'}}}}finally{Pop-Location} From b31c1749cafcac98229785f241b067572d167540 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sun, 13 Sep 2026 16:01:38 -0700 Subject: [PATCH 009/311] feat: manage local v2 backend lifecycle --- pipeline/scripts/maintenance/local-v2.ps1 | 52 ++++++++++++++++++++++- 1 file changed, 50 insertions(+), 2 deletions(-) diff --git a/pipeline/scripts/maintenance/local-v2.ps1 b/pipeline/scripts/maintenance/local-v2.ps1 index abea777..7625983 100644 --- a/pipeline/scripts/maintenance/local-v2.ps1 +++ b/pipeline/scripts/maintenance/local-v2.ps1 @@ -1,3 +1,51 @@ param([ValidateSet('start','status','stop','probe')][string]$Command='status') -$ErrorActionPreference='Stop';$root=(Resolve-Path (Join-Path $PSScriptRoot '..\..\..')).Path;$compose=Join-Path $root 'docker-compose.pipeline.yml';$project='uec-local-v2';$port=5433;$db="postgresql://uec:uec-local-development-only@127.0.0.1:$port/uec";$env:UEC_PIPELINE_DB_PORT="$port";$env:UEC_DATABASE_URL="${db}?sslmode=disable" -Push-Location $root;try{switch($Command){'start'{& docker compose -p $project -f $compose up -d --wait;if($LASTEXITCODE){throw 'Postgres startup failed.'};python pipeline/scripts/maintenance/apply-migrations.py;if($LASTEXITCODE){throw 'Migration application failed.'};$seeded=& docker compose -p $project -f $compose exec -T postgres psql -U uec -d uec -Atc "SELECT to_regclass('uec.local_v2_fixture_seed') IS NOT NULL";if($seeded.Trim() -ne 't'){Get-Content pipeline/tests/standard_contract_seed.sql -Raw|& docker compose -p $project -f $compose exec -T postgres psql -v ON_ERROR_STOP=1 -U uec -d uec;if($LASTEXITCODE){throw 'Synthetic seed failed.'};& docker compose -p $project -f $compose exec -T postgres psql -v ON_ERROR_STOP=1 -U uec -d uec -c "CREATE TABLE IF NOT EXISTS uec.local_v2_fixture_seed (seed_name text primary key, seeded_at timestamptz not null default now()); INSERT INTO uec.local_v2_fixture_seed(seed_name) VALUES ('standard-contract') ON CONFLICT DO NOTHING;"};Write-Host "Local Postgres ready on $port. Start Axum separately with: `$env:UEC_DATABASE_URL='${db}?sslmode=disable'; cargo run"}'status'{& docker compose -p $project -f $compose ps;if($LASTEXITCODE){throw 'Status failed.'}}'stop'{& docker compose -p $project -f $compose stop;if($LASTEXITCODE){throw 'Stop failed.'}}'probe'{& curl.exe --fail-with-body "http://127.0.0.1:8000/api/v2/locations?profile=official&limit=1";if($LASTEXITCODE){throw 'List probe failed.'}}}}finally{Pop-Location} +$ErrorActionPreference='Stop' +$root=(Resolve-Path (Join-Path $PSScriptRoot '..\..\..')).Path +$compose=Join-Path $root 'docker-compose.pipeline.yml' +$project='uec-local-v2'; $dbPort=5433; $apiPort=8000 +$db="postgresql://uec:uec-local-development-only@127.0.0.1:$dbPort/uec?sslmode=disable" +$stateDir=Join-Path $root 'target\local-v2'; $pidFile=Join-Path $stateDir 'uec-api.pid'; $logFile=Join-Path $stateDir 'uec-api.log'; $errorFile=Join-Path $stateDir 'uec-api-error.log' +$env:UEC_PIPELINE_DB_PORT="$dbPort"; $env:UEC_DATABASE_URL=$db; $env:PORT="$apiPort" + +function Get-OwnedApiProcess { + if (!(Test-Path $pidFile)) { return $null } + $processId=[int](Get-Content $pidFile -Raw).Trim(); $process=Get-Process -Id $processId -ErrorAction SilentlyContinue + if ($process -and $process.ProcessName -eq 'uec-api' -and $process.Path -eq (Join-Path $root 'target\debug\uec-api.exe')) { return $process } + return $null +} + +Push-Location $root +try { + switch ($Command) { + 'start' { + New-Item -ItemType Directory -Force -Path $stateDir | Out-Null + & docker compose -p $project -f $compose up -d --wait + if ($LASTEXITCODE) { throw 'Postgres startup failed.' } + python pipeline/scripts/maintenance/apply-migrations.py + if ($LASTEXITCODE) { throw 'Migration application failed.' } + $seeded=& docker compose -p $project -f $compose exec -T postgres psql -U uec -d uec -Atc "SELECT to_regclass('uec.local_v2_fixture_seed') IS NOT NULL" + if ($seeded.Trim() -ne 't') { + Get-Content pipeline/tests/standard_contract_seed.sql -Raw | & docker compose -p $project -f $compose exec -T postgres psql -v ON_ERROR_STOP=1 -U uec -d uec + if ($LASTEXITCODE) { throw 'Synthetic seed failed.' } + & docker compose -p $project -f $compose exec -T postgres psql -v ON_ERROR_STOP=1 -U uec -d uec -c "CREATE TABLE IF NOT EXISTS uec.local_v2_fixture_seed (seed_name text primary key, seeded_at timestamptz not null default now()); INSERT INTO uec.local_v2_fixture_seed(seed_name) VALUES ('standard-contract') ON CONFLICT DO NOTHING;" + } + if (!(Get-OwnedApiProcess)) { + $api=Join-Path $root 'target\debug\uec-api.exe'; if (!(Test-Path $api)) { cargo build --bin uec-api --quiet } + $process=Start-Process -FilePath $api -WorkingDirectory $root -WindowStyle Hidden -RedirectStandardOutput $logFile -RedirectStandardError $errorFile -PassThru + Set-Content -Path $pidFile -Value $process.Id -NoNewline + } + Write-Host "Local V2 ready: http://127.0.0.1:$apiPort (database $dbPort)." + } + 'status' { + & docker compose -p $project -f $compose ps + $api=Get-OwnedApiProcess; if ($api) { Write-Host "Axum running: PID $($api.Id), port $apiPort" } else { Write-Host 'Axum not managed by local-v2.ps1.' } + } + 'stop' { + $api=Get-OwnedApiProcess; if ($api) { Stop-Process -Id $api.Id -Force; Remove-Item $pidFile -Force } + & docker compose -p $project -f $compose stop + if ($LASTEXITCODE) { throw 'Postgres stop failed.' } + Write-Host 'Local V2 Axum and Postgres stopped; data volume preserved.' + } + 'probe' { & node frontend/scripts/probe-local-v2.mjs "http://127.0.0.1:$apiPort"; if ($LASTEXITCODE) { throw 'Local V2 probe failed.' } } + } +} finally { Pop-Location } From a30ac11d3f6774e6b089f182981212e64d447896 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sun, 13 Sep 2026 16:19:54 -0700 Subject: [PATCH 010/311] feat(api): add versioned v2 contract and stable errors --- docs/api/v2-contract.json | 12 +++ docs/api/v2-contract.md | 9 +++ src/lib.rs | 161 +++++++++++++++++++++++++++++--------- src/main.rs | 2 +- 4 files changed, 144 insertions(+), 40 deletions(-) create mode 100644 docs/api/v2-contract.json create mode 100644 docs/api/v2-contract.md diff --git a/docs/api/v2-contract.json b/docs/api/v2-contract.json new file mode 100644 index 0000000..4301868 --- /dev/null +++ b/docs/api/v2-contract.json @@ -0,0 +1,12 @@ +{ + "version": "v2", + "profiles": ["official", "secondary", "community"], + "endpoints": { + "GET /health/live": {"success": {"status": "ok", "service": "uec-api"}}, + "GET /health/ready": {"success": {"status": "ready", "database": "ok"}, "unavailable_status": 503}, + "GET /api/v2/locations": {"success": {"api_version": "v2", "data": [], "meta": {"profile": "official", "next_cursor": null}}, "no_release": 200}, + "GET /api/v2/locations/{facility_id}": {"success": {"api_version": "v2", "data": {}, "meta": {}}, "not_found": 404} + }, + "error": {"shape": {"api_version": "v2", "error": {"code": "string", "message": "string"}}, "codes": ["invalid_filter", "invalid_profile", "invalid_limit", "invalid_offset", "invalid_cursor", "invalid_pagination", "database_not_configured", "database_pool_unavailable", "database_transaction_unavailable", "release_query_failed", "location_query_failed", "location_not_found", "rate_limited"]}, + "privacy": "Public responses contain reviewed projection fields only; restricted records and raw evidence are never returned." +} diff --git a/docs/api/v2-contract.md b/docs/api/v2-contract.md new file mode 100644 index 0000000..002b5c3 --- /dev/null +++ b/docs/api/v2-contract.md @@ -0,0 +1,9 @@ +# V2 API contract + +The machine-readable contract is [v2-contract.json](v2-contract.json). Successful list/detail response shapes remain unchanged. Errors use one additive, stable envelope: + +```json +{"api_version":"v2","error":{"code":"invalid_profile","message":"profile is unsupported"}} +``` + +Frontend clients should branch on HTTP status and `error.code`, display `message` only as user-safe text, and treat unknown codes as generic failures. A list request with no promoted eligible release is a successful empty response; an unavailable database is `503`; an absent or suppressed detail is `404`; rate limiting is `429` with `Retry-After`. diff --git a/src/lib.rs b/src/lib.rs index fa2d36a..39ed249 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -16,7 +16,11 @@ // Contact the developer directly at untileverycageproject@protonmail.com use axum::extract::{Path, Query, State}; -use axum::{Json, http::StatusCode, response::IntoResponse}; +use axum::{ + Json, + http::{Response, StatusCode}, + response::IntoResponse, +}; use deadpool_postgres::Pool; use include_dir::{Dir, include_dir}; use once_cell::sync::Lazy; @@ -26,6 +30,18 @@ use std::error::Error; use std::time::{Duration, Instant}; use tokio::sync::Mutex; +pub fn v2_error( + status: StatusCode, + code: &'static str, + message: &'static str, +) -> Response { + ( + status, + Json(json!({"api_version":"v2", "error": {"code": code, "message": message}})), + ) + .into_response() +} + #[derive(Clone)] pub struct ApiState { pub database: Option, @@ -128,14 +144,22 @@ pub async fn get_v2_locations_handler( .as_deref() .is_some_and(|v| !PRECISIONS.contains(&v)) { - return (StatusCode::BAD_REQUEST, "invalid display_precision").into_response(); + return v2_error( + StatusCode::BAD_REQUEST, + "invalid_display_precision", + "display_precision is unsupported", + ); } if params .lifecycle_status .as_deref() .is_some_and(|v| !LIFECYCLES.contains(&v)) { - return (StatusCode::BAD_REQUEST, "invalid lifecycle_status").into_response(); + return v2_error( + StatusCode::BAD_REQUEST, + "invalid_lifecycle_status", + "lifecycle_status is unsupported", + ); } if params .category @@ -143,36 +167,60 @@ pub async fn get_v2_locations_handler( .is_some_and(|v| v.trim().is_empty()) || params.country_code.as_deref().is_some_and(|v| v.len() != 2) { - return (StatusCode::BAD_REQUEST, "invalid filter").into_response(); + return v2_error( + StatusCode::BAD_REQUEST, + "invalid_filter", + "filter is invalid", + ); } if params .source_type .as_deref() .is_some_and(|v| !["official", "secondary", "user_submitted"].contains(&v)) { - return (StatusCode::BAD_REQUEST, "invalid source_type").into_response(); + return v2_error( + StatusCode::BAD_REQUEST, + "invalid_source_type", + "source_type is unsupported", + ); } if params .profile .as_deref() .is_some_and(|v| !["official", "secondary", "community"].contains(&v)) { - return (StatusCode::BAD_REQUEST, "invalid profile").into_response(); + return v2_error( + StatusCode::BAD_REQUEST, + "invalid_profile", + "profile is unsupported", + ); } let limit = match params.limit.as_deref().map(str::parse::).transpose() { Ok(value) => value.unwrap_or(100).clamp(1, 1000), - Err(_) => return (StatusCode::BAD_REQUEST, "limit must be an integer").into_response(), + Err(_) => { + return v2_error( + StatusCode::BAD_REQUEST, + "invalid_limit", + "limit must be an integer", + ); + } }; let offset = match params.offset.as_deref().map(str::parse::).transpose() { Ok(value) => value.unwrap_or(0).clamp(0, 1_000_000), - Err(_) => return (StatusCode::BAD_REQUEST, "offset must be an integer").into_response(), + Err(_) => { + return v2_error( + StatusCode::BAD_REQUEST, + "invalid_offset", + "offset must be an integer", + ); + } }; if params.cursor.is_some() && params.offset.is_some() { - return ( + return v2_error( StatusCode::BAD_REQUEST, + "invalid_pagination", "cursor and offset cannot be combined", - ) - .into_response(); + ); } let cursor = match params .cursor @@ -182,7 +230,11 @@ pub async fn get_v2_locations_handler( { Ok(cursor) => cursor, Err(_) => { - return (StatusCode::BAD_REQUEST, "cursor must be a facility UUID").into_response(); + return v2_error( + StatusCode::BAD_REQUEST, + "invalid_cursor", + "cursor must be a facility UUID", + ); } }; let effective_offset = if cursor.is_some() { 0 } else { offset }; @@ -190,16 +242,19 @@ pub async fn get_v2_locations_handler( Some(pool) => match pool.get().await { Ok(client) => client, Err(_) => { - return (StatusCode::SERVICE_UNAVAILABLE, "Database pool unavailable") - .into_response(); + return v2_error( + StatusCode::SERVICE_UNAVAILABLE, + "database_pool_unavailable", + "database pool unavailable", + ); } }, None => { - return ( + return v2_error( StatusCode::SERVICE_UNAVAILABLE, + "database_not_configured", "V2 database is not configured", - ) - .into_response(); + ); } }; let transaction = match client @@ -211,11 +266,11 @@ pub async fn get_v2_locations_handler( { Ok(transaction) => transaction, Err(_) => { - return ( + return v2_error( StatusCode::SERVICE_UNAVAILABLE, + "database_transaction_unavailable", "V2 database transaction unavailable", - ) - .into_response(); + ); } }; let requested_profile = params.profile.as_deref().unwrap_or("official"); @@ -223,7 +278,11 @@ pub async fn get_v2_locations_handler( let release = match release { Ok(release) => release, Err(_) => { - return (StatusCode::INTERNAL_SERVER_ERROR, "V2 release query failed").into_response(); + return v2_error( + StatusCode::INTERNAL_SERVER_ERROR, + "release_query_failed", + "V2 release query failed", + ); } }; let Some(release) = release else { @@ -254,7 +313,7 @@ pub async fn get_v2_locations_handler( ORDER BY facility_id LIMIT $8 OFFSET $9 "#, &[&promoted_release_id, &cursor, ¶ms.country_code, ¶ms.category, ¶ms.display_precision, ¶ms.lifecycle_status, ¶ms.source_type, &query_limit, &effective_offset]).await { Ok(rows) => rows, - Err(_) => return (StatusCode::INTERNAL_SERVER_ERROR, "V2 location query failed").into_response(), + Err(_) => return v2_error(StatusCode::INTERNAL_SERVER_ERROR, "location_query_failed", "V2 location query failed"), }; let has_next = rows.len() as i64 > limit; let data = rows @@ -327,16 +386,19 @@ pub async fn get_v2_location_detail_handler( Some(pool) => match pool.get().await { Ok(client) => client, Err(_) => { - return (StatusCode::SERVICE_UNAVAILABLE, "Database pool unavailable") - .into_response(); + return v2_error( + StatusCode::SERVICE_UNAVAILABLE, + "database_pool_unavailable", + "database pool unavailable", + ); } }, None => { - return ( + return v2_error( StatusCode::SERVICE_UNAVAILABLE, + "database_not_configured", "V2 database is not configured", - ) - .into_response(); + ); } }; let transaction = match client @@ -348,24 +410,32 @@ pub async fn get_v2_location_detail_handler( { Ok(transaction) => transaction, Err(_) => { - return ( + return v2_error( StatusCode::SERVICE_UNAVAILABLE, + "database_transaction_unavailable", "V2 database transaction unavailable", - ) - .into_response(); + ); } }; let requested_profile = params.profile.as_deref().unwrap_or("official"); if !["official", "secondary", "community"].contains(&requested_profile) { - return (StatusCode::BAD_REQUEST, "invalid profile").into_response(); + return v2_error( + StatusCode::BAD_REQUEST, + "invalid_profile", + "profile is unsupported", + ); } let release = match transaction.query_opt("SELECT release_id, ruleset_version, created_at, profile FROM uec.releases WHERE status = 'promoted' AND profile = $1 ORDER BY created_at DESC, release_id DESC LIMIT 1", &[&requested_profile]).await { Ok(release) => release, - Err(_) => return (StatusCode::INTERNAL_SERVER_ERROR, "V2 release query failed").into_response(), + Err(_) => return v2_error(StatusCode::INTERNAL_SERVER_ERROR, "release_query_failed", "V2 release query failed"), }; let Some(release) = release else { let _ = transaction.commit().await; - return (StatusCode::NOT_FOUND, "location not found").into_response(); + return v2_error( + StatusCode::NOT_FOUND, + "location_not_found", + "location not found", + ); }; let release_id: String = release.get(0); let ruleset: String = release.get(1); @@ -383,11 +453,15 @@ pub async fn get_v2_location_detail_handler( WHERE facility_id = $1 AND release_id = $2 "#, &[&facility_id, &release_id]).await { Ok(row) => row, - Err(_) => return (StatusCode::INTERNAL_SERVER_ERROR, "V2 location query failed").into_response(), + Err(_) => return v2_error(StatusCode::INTERNAL_SERVER_ERROR, "location_query_failed", "V2 location query failed"), }; let Some(row) = row else { let _ = transaction.commit().await; - return (StatusCode::NOT_FOUND, "location not found").into_response(); + return v2_error( + StatusCode::NOT_FOUND, + "location_not_found", + "location not found", + ); }; let item = V2Location { facility_id: row.get(0), @@ -461,6 +535,15 @@ mod v2_api_tests { } } + #[test] + fn versioned_contract_lists_supported_profiles_and_error_shape() { + let contract: serde_json::Value = + serde_json::from_str(include_str!("../docs/api/v2-contract.json")).unwrap(); + assert_eq!(contract["version"], "v2"); + assert_eq!(contract["profiles"].as_array().unwrap().len(), 3); + assert_eq!(contract["error"]["shape"]["api_version"], "v2"); + } + #[tokio::test] async fn v2_response_is_json_when_database_is_configured() { let url = std::env::var("UEC_DATABASE_URL").unwrap_or_else(|_| { @@ -546,10 +629,10 @@ mod v2_api_tests { std::env::set_var("UEC_DATABASE_URL", url); } for uri in [ - "/api/v2/locations", - "/api/v2/locations?category=logistics_and_storage", - "/api/v2/locations?category=retail_and_prepared_food&display_precision=exact", - "/api/v2/locations?lifecycle_status=explicitly_closed", + "/api/v2/locations?country_code=ZZ", + "/api/v2/locations?country_code=ZZ&category=logistics_and_storage", + "/api/v2/locations?country_code=ZZ&category=retail_and_prepared_food&display_precision=exact", + "/api/v2/locations?country_code=ZZ&lifecycle_status=explicitly_closed", ] { let response = Router::new() .route( diff --git a/src/main.rs b/src/main.rs index 0af152b..02169ff 100644 --- a/src/main.rs +++ b/src/main.rs @@ -109,7 +109,7 @@ async fn rate_limit( .header(header::RETRY_AFTER, RATE_WINDOW.as_secs().to_string()) .header(header::CONTENT_TYPE, "application/json") .body(axum::body::Body::from( - r#"{"status":"rate_limited","reason":"request_rate_limit"}"#, + r#"{"api_version":"v2","error":{"code":"rate_limited","message":"request rate limit exceeded"}}"#, )) .unwrap(); } From 6eaf1aa2365dd10d73b2d165f9fdc32a1eb59603 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sun, 13 Sep 2026 16:25:35 -0700 Subject: [PATCH 011/311] feat(api): expose verified release manifests --- Cargo.lock | 84 +++++++++++++++-- Cargo.toml | 1 + pipeline/migrations/020_release_manifests.sql | 12 +++ pipeline/scripts/stages/promote-release.py | 15 ++- pipeline/tests/e2e/backup-restore.ps1 | 4 +- pipeline/tests/e2e/backup_restore_seed.sql | 2 + src/lib.rs | 92 +++++++++++++++++++ src/main.rs | 4 + 8 files changed, 202 insertions(+), 12 deletions(-) create mode 100644 pipeline/migrations/020_release_manifests.sql diff --git a/Cargo.lock b/Cargo.lock index 0f93ec7..193a163 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -187,6 +187,15 @@ version = "2.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967" +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + [[package]] name = "block-buffer" version = "0.12.1" @@ -266,7 +275,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.3.1", "rand_core 0.10.1", ] @@ -318,6 +327,15 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + [[package]] name = "cpufeatures" version = "0.3.1" @@ -327,6 +345,16 @@ dependencies = [ "libc", ] +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + [[package]] name = "crypto-common" version = "0.2.2" @@ -424,15 +452,25 @@ dependencies = [ "syn 2.0.103", ] +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer 0.10.4", + "crypto-common 0.1.7", +] + [[package]] name = "digest" version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" dependencies = [ - "block-buffer", + "block-buffer 0.12.1", "const-oid 0.10.2", - "crypto-common", + "crypto-common 0.2.2", "ctutils", ] @@ -546,6 +584,16 @@ dependencies = [ "slab", ] +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + [[package]] name = "getrandom" version = "0.2.17" @@ -603,7 +651,7 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" dependencies = [ - "digest", + "digest 0.11.3", ] [[package]] @@ -980,7 +1028,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69b6441f590336821bb897fb28fc622898ccceb1d6cea3fde5ea86b090c4de98" dependencies = [ "cfg-if", - "digest", + "digest 0.11.3", ] [[package]] @@ -1144,7 +1192,7 @@ dependencies = [ "md-5", "memchr", "rand 0.10.2", - "sha2", + "sha2 0.11.0", "stringprep", ] @@ -1514,6 +1562,17 @@ dependencies = [ "serde", ] +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest 0.10.7", +] + [[package]] name = "sha2" version = "0.11.0" @@ -1521,8 +1580,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" dependencies = [ "cfg-if", - "cpufeatures", - "digest", + "cpufeatures 0.3.1", + "digest 0.11.3", ] [[package]] @@ -1791,7 +1850,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4c2ad44aa0ae96db89c4742212ed41645b2f597311ff6e1945542a4d9fadc2fb" dependencies = [ "rustls", - "sha2", + "sha2 0.11.0", "tokio", "tokio-postgres", "tokio-rustls", @@ -1939,6 +1998,7 @@ dependencies = [ "serde", "serde-xml-rs", "serde_json", + "sha2 0.10.9", "tokio", "tokio-postgres", "tokio-postgres-rustls", @@ -2014,6 +2074,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + [[package]] name = "want" version = "0.3.1" diff --git a/Cargo.toml b/Cargo.toml index 6a6ab22..3142b94 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,6 +22,7 @@ tower = { version = "=0.5.2", features = ["util"] } reqwest = { version = "=0.12.24", features = ["json", "rustls-tls"], default-features = false } serde_json = "=1.0.140" +sha2 = "=0.10.9" # Shuttle dependencies serde-xml-rs = "0.8.1" diff --git a/pipeline/migrations/020_release_manifests.sql b/pipeline/migrations/020_release_manifests.sql new file mode 100644 index 0000000..2e68987 --- /dev/null +++ b/pipeline/migrations/020_release_manifests.sql @@ -0,0 +1,12 @@ +-- Safe, immutable release-verification summaries. Never store raw artifacts, +-- credentials, addresses, coordinates, or suppression payloads here. +CREATE TABLE uec.release_manifests ( + release_id TEXT PRIMARY KEY REFERENCES uec.releases(release_id), + manifest JSONB NOT NULL, + manifest_sha256 CHAR(64) NOT NULL CHECK (manifest_sha256 ~ '^[0-9a-f]{64}$'), + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE TRIGGER release_manifests_append_only + BEFORE UPDATE OR DELETE ON uec.release_manifests + FOR EACH ROW EXECUTE FUNCTION uec.reject_evidence_mutation(); diff --git a/pipeline/scripts/stages/promote-release.py b/pipeline/scripts/stages/promote-release.py index 215af83..ac24bc3 100644 --- a/pipeline/scripts/stages/promote-release.py +++ b/pipeline/scripts/stages/promote-release.py @@ -2,6 +2,7 @@ """Promote a validated release to the active public release state.""" import argparse +import hashlib import json import os import sys @@ -17,7 +18,7 @@ def can_promote(status: str) -> bool: def promote(database_url: str, release_id: str) -> dict: with psycopg.connect(database_url) as connection: with connection.transaction(): - target = connection.execute("SELECT status, profile FROM uec.releases WHERE release_id = %s FOR UPDATE", (release_id,)).fetchone() + target = connection.execute("SELECT status, profile, ruleset_version FROM uec.releases WHERE release_id = %s FOR UPDATE", (release_id,)).fetchone() if not target: raise ValueError(f"release not found: {release_id}") if not can_promote(target[0]): @@ -37,6 +38,18 @@ def promote(database_url: str, release_id: str) -> dict: """, (release_id,)).fetchone() if any(unsafe): raise ValueError(f"release safety gates failed: coordinate_not_ready={unsafe[0]}, review_required={unsafe[1]}, publication_not_approved={unsafe[2]}, active_suppression={unsafe[3]}") + summary = connection.execute(""" + SELECT count(*), coalesce(array_agg(DISTINCT sr.source_id ORDER BY sr.source_id), ARRAY[]::text[]) + FROM uec.release_members m JOIN uec.observations o ON o.observation_id=m.observation_id + JOIN uec.source_records sr ON sr.source_record_id=o.source_record_id + WHERE m.release_id=%s AND m.default_visible + AND NOT EXISTS (SELECT 1 FROM uec.public_access_restricted x WHERE x.source_record_id=sr.source_record_id) + """, (release_id,)).fetchone() + manifest = {"manifest_version": "v1", "release_id": release_id, "profile": target[1], "ruleset_version": target[2], "eligible_record_count": summary[0], "source_ids": summary[1]} + # Python's sorted-key JSON is the canonical representation shared by consumers. + canonical = json.dumps(manifest, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + digest = hashlib.sha256(canonical.encode()).hexdigest() + connection.execute("INSERT INTO uec.release_manifests (release_id,manifest,manifest_sha256) VALUES (%s,%s,%s)", (release_id, json.dumps(manifest, ensure_ascii=False), digest)) previous = connection.execute("SELECT release_id FROM uec.releases WHERE status = 'promoted' AND profile = %s AND release_id <> %s", (target[1], release_id)).fetchall() connection.execute("UPDATE uec.releases SET status = 'validated' WHERE status = 'promoted' AND profile = %s AND release_id <> %s", (target[1], release_id)) connection.execute("UPDATE uec.releases SET status = 'promoted' WHERE release_id = %s", (release_id,)) diff --git a/pipeline/tests/e2e/backup-restore.ps1 b/pipeline/tests/e2e/backup-restore.ps1 index 7e3bde4..cbb9450 100644 --- a/pipeline/tests/e2e/backup-restore.ps1 +++ b/pipeline/tests/e2e/backup-restore.ps1 @@ -40,8 +40,8 @@ try { if ($LASTEXITCODE -ne 0) { throw "Backup extraction failed (exit $LASTEXITCODE)." } & docker compose -p $project -f $compose exec -T postgres pg_restore -U uec -d uec --clean --if-exists /tmp/uec.dump if ($LASTEXITCODE -ne 0) { throw "Restore failed (exit $LASTEXITCODE)." } - $checks = & docker compose -p $project -f $compose exec -T postgres psql -U uec -d uec -At -c "SELECT count(*) FROM uec.releases WHERE release_id='e2e-promoted' AND status='promoted'; SELECT count(*) FROM uec.public_access_restricted WHERE source_record_id='00000000-0000-0000-0000-000000000002'; SELECT count(*) FROM uec.map_facilities_display WHERE source_record_id='00000000-0000-0000-0000-000000000002'; SELECT count(*) FROM uec.map_facilities_display_history WHERE source_record_id='00000000-0000-0000-0000-000000000002';" - if (($checks | Where-Object { $_ -eq '1' }).Count -ne 2 -or ($checks | Where-Object { $_ -eq '0' }).Count -ne 2) { throw "Backup/restore invariant failed (expected 1,1,0,0): $checks" } + $checks = & docker compose -p $project -f $compose exec -T postgres psql -U uec -d uec -At -c "SELECT count(*) FROM uec.releases WHERE release_id='e2e-promoted' AND status='promoted'; SELECT count(*) FROM uec.release_manifests WHERE release_id='e2e-promoted' AND manifest_sha256='cabe8641a05beb76c9517006a8ec4cdd60b3bad58aa5b0fc29335fee1ac7d5dd'; SELECT count(*) FROM uec.public_access_restricted WHERE source_record_id='00000000-0000-0000-0000-000000000002'; SELECT count(*) FROM uec.map_facilities_display WHERE source_record_id='00000000-0000-0000-0000-000000000002'; SELECT count(*) FROM uec.map_facilities_display_history WHERE source_record_id='00000000-0000-0000-0000-000000000002';" + if (($checks | Where-Object { $_ -eq '1' }).Count -ne 3 -or ($checks | Where-Object { $_ -eq '0' }).Count -ne 2) { throw "Backup/restore invariant failed (expected 1,1,1,0,0): $checks" } Write-Host 'PASS: promoted synthetic release restored; restricted source is excluded from both public projections.' } finally { $savedPreference = $ErrorActionPreference diff --git a/pipeline/tests/e2e/backup_restore_seed.sql b/pipeline/tests/e2e/backup_restore_seed.sql index fc05e3a..5084757 100644 --- a/pipeline/tests/e2e/backup_restore_seed.sql +++ b/pipeline/tests/e2e/backup_restore_seed.sql @@ -3,6 +3,8 @@ INSERT INTO uec.sources (source_id,country_code,name,official_url,access_method) VALUES ('e2e.backup','DK','Synthetic backup source','https://example.invalid/backup','fixture'); INSERT INTO uec.releases (release_id,status,ruleset_version,profile,summary) VALUES ('e2e-promoted','promoted','synthetic-v1','official','{}'); +INSERT INTO uec.release_manifests (release_id,manifest,manifest_sha256) +VALUES ('e2e-promoted','{"eligible_record_count":1,"manifest_version":"v1","profile":"official","release_id":"e2e-promoted","ruleset_version":"synthetic-v1","source_ids":["e2e.backup"]}', 'cabe8641a05beb76c9517006a8ec4cdd60b3bad58aa5b0fc29335fee1ac7d5dd'); INSERT INTO uec.raw_artifacts (artifact_id,storage_key,sha256,byte_size,retrieved_at) VALUES ('00000000-0000-0000-0000-000000000001','e2e/backup/restricted',repeat('a',64),0,now()); INSERT INTO uec.source_records (source_record_id,source_id,source_record_key,artifact_id,raw_fields,parsed_at) diff --git a/src/lib.rs b/src/lib.rs index 39ed249..19a61ed 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -26,6 +26,7 @@ use include_dir::{Dir, include_dir}; use once_cell::sync::Lazy; use serde::{Deserialize, Serialize}; use serde_json::{Value, json}; +use sha2::{Digest, Sha256}; use std::error::Error; use std::time::{Duration, Instant}; use tokio::sync::Mutex; @@ -42,6 +43,97 @@ pub fn v2_error( .into_response() } +fn canonical_json(value: &Value) -> String { + match value { + Value::Object(map) => { + let mut keys: Vec<_> = map.keys().collect(); + keys.sort(); + format!( + "{{{}}}", + keys.into_iter() + .map(|k| format!( + "{}:{}", + serde_json::to_string(k).unwrap(), + canonical_json(&map[k]) + )) + .collect::>() + .join(",") + ) + } + Value::Array(items) => format!( + "[{}]", + items + .iter() + .map(canonical_json) + .collect::>() + .join(",") + ), + _ => value.to_string(), + } +} + +pub async fn get_v2_release_manifest_handler( + State(state): State, + Query(params): Query, +) -> impl IntoResponse { + let profile = params.profile.as_deref().unwrap_or("official"); + if !["official", "secondary", "community"].contains(&profile) { + return v2_error( + StatusCode::BAD_REQUEST, + "invalid_profile", + "profile is unsupported", + ); + } + let Some(pool) = state.database else { + return v2_error( + StatusCode::SERVICE_UNAVAILABLE, + "database_not_configured", + "V2 database is not configured", + ); + }; + let client = match pool.get().await { + Ok(c) => c, + Err(_) => { + return v2_error( + StatusCode::SERVICE_UNAVAILABLE, + "database_pool_unavailable", + "database pool unavailable", + ); + } + }; + let row = match client.query_opt("SELECT r.release_id, r.profile, m.manifest::text, m.manifest_sha256 FROM uec.releases r JOIN uec.release_manifests m ON m.release_id=r.release_id WHERE r.status='promoted' AND r.profile=$1 ORDER BY r.created_at DESC, r.release_id DESC LIMIT 1", &[&profile]).await { + Ok(row) => row, Err(_) => return v2_error(StatusCode::SERVICE_UNAVAILABLE, "release_manifest_unavailable", "release manifest unavailable") + }; + let Some(row) = row else { + return v2_error( + StatusCode::NOT_FOUND, + "release_not_found", + "no promoted eligible release", + ); + }; + let manifest_text: String = row.get(2); + let manifest: Value = match serde_json::from_str(&manifest_text) { + Ok(value) => value, + Err(_) => { + return v2_error( + StatusCode::SERVICE_UNAVAILABLE, + "release_manifest_invalid", + "release manifest integrity check failed", + ); + } + }; + let digest: String = row.get(3); + let actual = format!("{:x}", Sha256::digest(canonical_json(&manifest).as_bytes())); + if actual != digest { + return v2_error( + StatusCode::SERVICE_UNAVAILABLE, + "release_manifest_invalid", + "release manifest integrity check failed", + ); + } + Json(json!({"api_version":"v2", "data": {"release_id": row.get::<_,String>(0), "profile": row.get::<_,String>(1), "manifest": manifest, "manifest_sha256": digest}})).into_response() +} + #[derive(Clone)] pub struct ApiState { pub database: Option, diff --git a/src/main.rs b/src/main.rs index 02169ff..edc45f6 100644 --- a/src/main.rs +++ b/src/main.rs @@ -44,6 +44,10 @@ pub fn app(state: uec_api::ApiState) -> Router { .route("/health/ready", get(readiness)) .route("/api/locations", get(uec_api::get_locations_handler)) .route("/api/v2/locations", get(uec_api::get_v2_locations_handler)) + .route( + "/api/v2/releases/manifest", + get(uec_api::get_v2_release_manifest_handler), + ) .route( "/api/v2/locations/{facility_id}", get(uec_api::get_v2_location_detail_handler), From 5403e15592f58ef01e863d6673228143ac2d3716 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sun, 13 Sep 2026 16:29:26 -0700 Subject: [PATCH 012/311] feat(api): add bounded v2 research export --- docs/api/v2-contract.json | 3 +- docs/api/v2-contract.md | 2 + pipeline/tests/e2e/fixture.py | 1 + pipeline/tests/e2e/test_seeded_api.py | 10 ++ src/lib.rs | 129 ++++++++++++++++++++++++++ src/main.rs | 4 + 6 files changed, 148 insertions(+), 1 deletion(-) diff --git a/docs/api/v2-contract.json b/docs/api/v2-contract.json index 4301868..fdff7b8 100644 --- a/docs/api/v2-contract.json +++ b/docs/api/v2-contract.json @@ -5,7 +5,8 @@ "GET /health/live": {"success": {"status": "ok", "service": "uec-api"}}, "GET /health/ready": {"success": {"status": "ready", "database": "ok"}, "unavailable_status": 503}, "GET /api/v2/locations": {"success": {"api_version": "v2", "data": [], "meta": {"profile": "official", "next_cursor": null}}, "no_release": 200}, - "GET /api/v2/locations/{facility_id}": {"success": {"api_version": "v2", "data": {}, "meta": {}}, "not_found": 404} + "GET /api/v2/locations/{facility_id}": {"success": {"api_version": "v2", "data": {}, "meta": {}}, "not_found": 404}, + "GET /api/v2/locations.csv": {"success_content_type": "text/csv", "bounded_rows": 1000, "profile_required_for_nonofficial": true, "not_found_without_promoted_manifest": 404} }, "error": {"shape": {"api_version": "v2", "error": {"code": "string", "message": "string"}}, "codes": ["invalid_filter", "invalid_profile", "invalid_limit", "invalid_offset", "invalid_cursor", "invalid_pagination", "database_not_configured", "database_pool_unavailable", "database_transaction_unavailable", "release_query_failed", "location_query_failed", "location_not_found", "rate_limited"]}, "privacy": "Public responses contain reviewed projection fields only; restricted records and raw evidence are never returned." diff --git a/docs/api/v2-contract.md b/docs/api/v2-contract.md index 002b5c3..f9f50f2 100644 --- a/docs/api/v2-contract.md +++ b/docs/api/v2-contract.md @@ -7,3 +7,5 @@ The machine-readable contract is [v2-contract.json](v2-contract.json). Successfu ``` Frontend clients should branch on HTTP status and `error.code`, display `message` only as user-safe text, and treat unknown codes as generic failures. A list request with no promoted eligible release is a successful empty response; an unavailable database is `503`; an absent or suppressed detail is `404`; rate limiting is `429` with `Retry-After`. + +Researchers may request `GET /api/v2/locations.csv?profile=official` (or another explicit supported profile). The export is bounded to 1,000 rows, uses deterministic CSV columns and escaping, contains only the public reviewed projection, and includes `release_profile`, `release_id`, and `manifest_sha256` on every row plus matching response headers. It is unavailable when no promoted release with a manifest exists; it never exposes raw evidence or restricted records. diff --git a/pipeline/tests/e2e/fixture.py b/pipeline/tests/e2e/fixture.py index e30fb88..e4369a4 100644 --- a/pipeline/tests/e2e/fixture.py +++ b/pipeline/tests/e2e/fixture.py @@ -110,6 +110,7 @@ def seed_official_scenario(self): db.execute("INSERT INTO uec.sources (source_id,country_code,name,official_url,access_method) VALUES ('e2e.official','DK','Synthetic official source','https://example.invalid/official','fixture')") release = 'e2e-promoted' db.execute("INSERT INTO uec.releases (release_id,status,ruleset_version,summary) VALUES (%s,'promoted','e2e-v1','{}')", (release,)) + db.execute("INSERT INTO uec.release_manifests (release_id,manifest,manifest_sha256) VALUES ('e2e-promoted','{\"eligible_record_count\":3,\"manifest_version\":\"v1\",\"profile\":\"official\",\"release_id\":\"e2e-promoted\",\"ruleset_version\":\"e2e-v1\",\"source_ids\":[\"e2e.official\"]}', 'dcf1cb50c078057cac2527936332e35892c2c13ecdcf2f545176acd17897cde7')") city = 'Testby' db.execute("INSERT INTO uec.city_reference_points (country_code,city_name,reference_location,reference_source,source_retrieved_at,source_reference_id) VALUES ('DK',%s,ST_SetSRID(ST_MakePoint(10,55),4326)::geography,'https://example.invalid/cities',%s,'e2e-city')", (city, now)) cases = [('exact','slaughter', 'accepted', True, True), ('city','fish_processing','review_required', False, True), ('unmapped','logistics_and_storage','unresolved', False, True), ('restricted','slaughter','accepted', True, True), ('unapproved','retail_and_prepared_food','accepted', True, False)] diff --git a/pipeline/tests/e2e/test_seeded_api.py b/pipeline/tests/e2e/test_seeded_api.py index 882848d..461f035 100644 --- a/pipeline/tests/e2e/test_seeded_api.py +++ b/pipeline/tests/e2e/test_seeded_api.py @@ -62,6 +62,16 @@ def test_detail_endpoint_matches_public_list_contract(self): self.assertEqual(detail['data']['release_id'], detail['meta']['release_id']) self.assertEqual(detail['data']['provenance_source_id'], 'e2e.official') + def test_csv_export_is_escaped_bounded_and_manifest_bound(self): + request = urllib.request.Request(f"http://localhost:{self.env.api_port}/api/v2/locations.csv?profile=official") + with urllib.request.urlopen(request, timeout=10) as response: + self.assertEqual(response.headers['Content-Type'], 'text/csv; charset=utf-8') + self.assertEqual(response.headers['X-Uec-Manifest-Sha256'], 'dcf1cb50c078057cac2527936332e35892c2c13ecdcf2f545176acd17897cde7') + body = response.read().decode() + self.assertIn('release_profile', body) + self.assertIn('manifest_sha256', body) + self.assertNotIn('E2E restricted', body) + def test_provenance_and_precision_are_returned_for_each_public_record(self): response = self.get('/api/v2/locations?limit=100') self.assertEqual(response['meta']['release_id'], 'e2e-promoted') diff --git a/src/lib.rs b/src/lib.rs index 19a61ed..68a3e21 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -134,6 +134,135 @@ pub async fn get_v2_release_manifest_handler( Json(json!({"api_version":"v2", "data": {"release_id": row.get::<_,String>(0), "profile": row.get::<_,String>(1), "manifest": manifest, "manifest_sha256": digest}})).into_response() } +#[derive(Serialize)] +struct V2ExportRow { + facility_id: uuid::Uuid, + canonical_name: Option, + country_code: String, + city: Option, + category: String, + display_precision: String, + factual_review_status: String, + privacy_screening_status: String, + project_approval: String, + reviewer_role: Option, + source_type: String, + provenance_source_id: String, + provenance_source_name: String, + provenance_source_url: String, + provenance_retrieved_at: chrono::DateTime, + release_id: String, + release_profile: String, + manifest_sha256: String, +} + +pub async fn get_v2_locations_export_handler( + State(state): State, + Query(params): Query, +) -> impl IntoResponse { + let profile = params.profile.as_deref().unwrap_or("official"); + if !["official", "secondary", "community"].contains(&profile) { + return v2_error( + StatusCode::BAD_REQUEST, + "invalid_profile", + "profile is unsupported", + ); + } + let Some(pool) = state.database else { + return v2_error( + StatusCode::SERVICE_UNAVAILABLE, + "database_not_configured", + "V2 database is not configured", + ); + }; + let client = match pool.get().await { + Ok(c) => c, + Err(_) => { + return v2_error( + StatusCode::SERVICE_UNAVAILABLE, + "database_pool_unavailable", + "database pool unavailable", + ); + } + }; + let release = match client.query_opt("SELECT r.release_id, m.manifest_sha256 FROM uec.releases r JOIN uec.release_manifests m ON m.release_id=r.release_id WHERE r.status='promoted' AND r.profile=$1 ORDER BY r.created_at DESC, r.release_id DESC LIMIT 1", &[&profile]).await { + Ok(row) => row, Err(_) => return v2_error(StatusCode::SERVICE_UNAVAILABLE, "release_query_failed", "release query failed") + }; + let Some(release) = release else { + return v2_error( + StatusCode::NOT_FOUND, + "release_not_found", + "no promoted eligible release", + ); + }; + let release_id: String = release.get(0); + let manifest_sha256: String = release.get(1); + let rows = match client.query("SELECT h.facility_id, h.canonical_name, h.country_code, h.city, h.classification_category, h.display_precision, r.factual_review_status, r.privacy_screening_status, r.maintainer_approval, r.reviewer_role, h.provenance_origin_type, h.provenance_source_id, h.provenance_source_name, h.provenance_source_url, h.provenance_retrieved_at, h.release_id FROM uec.map_facilities_display_history h JOIN uec.publication_review_current r ON r.source_record_id=h.source_record_id WHERE h.release_id=$1 AND r.publication_eligible=true AND r.privacy_screening_status='passed' AND r.maintainer_approval='approved' ORDER BY h.facility_id LIMIT 1001", &[&release_id]).await { + Ok(rows) => rows, Err(_) => return v2_error(StatusCode::SERVICE_UNAVAILABLE, "export_query_failed", "public export unavailable") + }; + if rows.len() > 1000 { + return v2_error( + StatusCode::BAD_REQUEST, + "export_too_large", + "export exceeds the bounded limit", + ); + } + let mut writer = csv::Writer::from_writer(Vec::new()); + for row in rows { + if writer + .serialize(V2ExportRow { + facility_id: row.get(0), + canonical_name: row.get(1), + country_code: row.get(2), + city: row.get(3), + category: row.get(4), + display_precision: row.get(5), + factual_review_status: row.get(6), + privacy_screening_status: row.get(7), + project_approval: row.get(8), + reviewer_role: row.get(9), + source_type: row.get(10), + provenance_source_id: row.get(11), + provenance_source_name: row.get(12), + provenance_source_url: row.get(13), + provenance_retrieved_at: row.get(14), + release_id: row.get(15), + release_profile: profile.to_string(), + manifest_sha256: manifest_sha256.clone(), + }) + .is_err() + { + return v2_error( + StatusCode::SERVICE_UNAVAILABLE, + "export_encoding_failed", + "public export unavailable", + ); + } + } + let body = match writer.into_inner() { + Ok(body) => body, + Err(_) => { + return v2_error( + StatusCode::SERVICE_UNAVAILABLE, + "export_encoding_failed", + "public export unavailable", + ); + } + }; + Response::builder() + .status(StatusCode::OK) + .header("content-type", "text/csv; charset=utf-8") + .header( + "content-disposition", + "attachment; filename=uec-v2-locations.csv", + ) + .header("x-uec-release-id", release_id) + .header("x-uec-manifest-sha256", manifest_sha256) + .body(axum::body::Body::from(body)) + .unwrap() + .into_response() +} + #[derive(Clone)] pub struct ApiState { pub database: Option, diff --git a/src/main.rs b/src/main.rs index edc45f6..55bd7f4 100644 --- a/src/main.rs +++ b/src/main.rs @@ -48,6 +48,10 @@ pub fn app(state: uec_api::ApiState) -> Router { "/api/v2/releases/manifest", get(uec_api::get_v2_release_manifest_handler), ) + .route( + "/api/v2/locations.csv", + get(uec_api::get_v2_locations_export_handler), + ) .route( "/api/v2/locations/{facility_id}", get(uec_api::get_v2_location_detail_handler), From aa3b50cb60c77b3539e9cda9024fd30be50ff404 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sun, 13 Sep 2026 16:29:43 -0700 Subject: [PATCH 013/311] feat: complete local v2 frontend integration workflow --- frontend/README.md | 4 ++-- frontend/package.json | 2 +- frontend/playwright.config.ts | 2 +- frontend/src/api/LocalLocationRepository.ts | 2 +- frontend/src/app/App.svelte | 3 ++- frontend/tests/e2e/fixture-platform.spec.ts | 3 ++- frontend/tests/e2e/local-backend.spec.ts | 19 +++++++++++++++++++ frontend/vite.config.ts | 3 ++- pipeline/scripts/maintenance/local-v2.ps1 | 13 +++++++++++++ 9 files changed, 43 insertions(+), 8 deletions(-) create mode 100644 frontend/tests/e2e/local-backend.spec.ts diff --git a/frontend/README.md b/frontend/README.md index 2c40575..7f93a1e 100644 --- a/frontend/README.md +++ b/frontend/README.md @@ -19,6 +19,6 @@ Current-wire contract gap checklist (from the platform decision): These are documented gaps, not frontend claims or invented DTO fields. Phase 3 continues to use only the synthetic fixture repository. -Local V2 integration (opt-in only): run the backend with `cargo run` (port 8000), then run the frontend with `npm run dev`. The `LocalLocationRepository` targets `/api/v2/locations?profile=...` only when explicitly invoked by local development code; fixture mode remains the default and there is no V1 fallback. This slice is not production-enabled, does not proxy or rewrite requests, and fails closed on malformed, mismatched, restricted, or no-promoted-release responses. +Local V2 integration (opt-in only): run the backend with `cargo run` (port 8000), then run the frontend with `npm run dev` (port 5173). In development/preview, Vite proxies `/api` to `http://127.0.0.1:8000`; use `http://127.0.0.1:5173/v2-preview/?mode=local-v2#/` to opt in. The `LocalLocationRepository` still targets `/api/v2/locations?profile=...` only when explicitly invoked; fixture mode remains the default and there is no V1 fallback. The proxy is development-only configuration and production builds do not enable a backend connection. -Persistent local two-port workflow (never uses `down -v`): from the repository root run `powershell -ExecutionPolicy Bypass -File pipeline/scripts/maintenance/local-v2.ps1 start`, then `$env:UEC_DATABASE_URL='postgresql://uec:uec-local-development-only@127.0.0.1:5433/uec?sslmode=disable'; cargo run` in another terminal, and finally `npm --prefix frontend run dev`. Check with `... local-v2.ps1 status`; probe list/detail with `node frontend/scripts/probe-local-v2.mjs`; stop containers with `... local-v2.ps1 stop`. The seed is the existing synthetic contract fixture and is marker-gated for repeatability. Promotion is not automated by this helper; existing validation/promotion gates must authorize it. +Persistent local two-port workflow (never uses `down -v`): from the repository root run `powershell -ExecutionPolicy Bypass -File pipeline/scripts/maintenance/local-v2.ps1 start`. It starts the named Postgres stack on `5433`, applies migrations, seeds and promotes the synthetic contract release, and starts Axum on `8000`. Run `npm --prefix frontend run dev` in another terminal and open `http://127.0.0.1:5173/v2-preview/?mode=local-v2#/`. Check ownership and health with `... local-v2.ps1 status`; probe list/detail with `... local-v2.ps1 probe`; stop both services with `... local-v2.ps1 stop`. The helper is local-only and does not alter V1, production, or unrelated data. diff --git a/frontend/package.json b/frontend/package.json index d4c3bf7..f287f9e 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -2,7 +2,7 @@ "name": "until-every-cage-v2-frontend", "private": true, "type": "module", - "scripts": {"dev":"vite","preview":"vite preview","check":"svelte-check --tsconfig ./tsconfig.app.json && tsc -p tsconfig.node.json --noEmit","build":"npm run check && vite build","test":"vitest run","test:e2e":"playwright test","lint":"eslint .","stage":"node scripts/stage-preview.mjs","boundary":"node scripts/check-boundaries.mjs"}, + "scripts": {"dev":"vite","preview":"vite preview","check":"svelte-check --tsconfig ./tsconfig.app.json && tsc -p tsconfig.node.json --noEmit","build":"npm run check && vite build","test":"vitest run","test:e2e":"playwright test","test:e2e:local":"set LOCAL_V2_E2E=1&& playwright test tests/e2e/local-backend.spec.ts","lint":"eslint .","stage":"node scripts/stage-preview.mjs","boundary":"node scripts/check-boundaries.mjs"}, "devDependencies": {"@playwright/test":"^1.49.1","@sveltejs/vite-plugin-svelte":"^6.2.1","@types/node":"^22.10.2","eslint":"^9.17.0","svelte":"^5.19.0","svelte-check":"^4.1.4","typescript":"^5.7.2","vite":"^6.0.7","vitest":"^2.1.8"}, "dependencies": {"@axe-core/playwright":"^4.10.2","@types/leaflet":"^1.9.15","leaflet":"^1.9.4","zod":"^3.24.1"} } diff --git a/frontend/playwright.config.ts b/frontend/playwright.config.ts index 12e16de..e39bfc4 100644 --- a/frontend/playwright.config.ts +++ b/frontend/playwright.config.ts @@ -7,7 +7,7 @@ export default defineConfig({ reporter: process.env.CI ? 'line' : 'list', timeout: 30_000, use: { baseURL: 'http://127.0.0.1:4173/v2-preview/', trace: 'retain-on-failure' }, - webServer: { command: 'npm run preview -- --host 127.0.0.1 --port 4173', url: 'http://127.0.0.1:4173/v2-preview/', timeout: 30_000, reuseExistingServer: true }, + webServer: { command: 'npm run dev -- --host 127.0.0.1 --port 4173', url: 'http://127.0.0.1:4173/v2-preview/', timeout: 30_000, reuseExistingServer: false }, projects: [ { name: 'chromium', use: { ...devices['Desktop Chrome'] } }, { name: 'firefox', use: { ...devices['Desktop Firefox'] } }, diff --git a/frontend/src/api/LocalLocationRepository.ts b/frontend/src/api/LocalLocationRepository.ts index ad396aa..f182394 100644 --- a/frontend/src/api/LocalLocationRepository.ts +++ b/frontend/src/api/LocalLocationRepository.ts @@ -1 +1 @@ -import{detailEnvelopeSchema,envelopeSchema,type WireLocation}from'./wireSchema';import type{ApiError}from'./errors';import type{Location}from'../domain/location';export type FetchLike=(input:RequestInfo|URL,init?:RequestInit)=>Promise;export const localOrigin=(value:string|undefined):string|undefined=>{if(!value)return undefined;const url=new URL(value);if(url.protocol!=='http:'||!['127.0.0.1','localhost','::1'].includes(url.hostname))throw new Error('Local API origin must be loopback HTTP.');return url.origin;};const fail=(kind:ApiError['kind'],message:string,status?:number):ApiError=>status===undefined?{kind,message}:{kind,message,status};const map=(r:WireLocation):Location=>({id:r.facility_id,name:r.canonical_name,region:r.city??r.country_code,category:r.category,lat:r.latitude,lon:r.longitude,observed:r.last_observed_at??r.first_observed_at??'unknown',source:r.provenance_source_name});export class LocalLocationRepository{readonly#base:string|undefined;constructor(private readonly fetcher:FetchLike=globalThis.fetch,baseUrl?:string){this.#base=localOrigin(baseUrl);}private async json(path:string,signal?:AbortSignal){const init:RequestInit={};if(signal)init.signal=signal;const response=await this.fetcher(`${this.#base??''}${path}`,init);if(!response.ok)throw fail('http',`Local V2 request failed with status ${response.status}`,response.status);try{return await response.json();}catch{throw fail('invalid-contract','Local V2 response was not valid JSON.');}}async list(profile:'official'|'secondary'|'community'='official'){try{const b=envelopeSchema.safeParse(await this.json(`/api/v2/locations?profile=${profile}`));if(!b.success||b.data.meta.profile!==profile)throw fail('invalid-contract','Local V2 list response was rejected.');if(b.data.meta.release_id===null)throw fail('no-release',b.data.meta.coverage_note);if(b.data.meta.ruleset_version===undefined||b.data.data.some(r=>r.privacy_screening_status!=='passed'||r.project_approval!=='approved'||r.release_id!==b.data.meta.release_id||r.release_ruleset_version!==b.data.meta.ruleset_version))throw fail('invalid-contract','Local V2 list snapshot was rejected.');return{locations:b.data.data.map(map),releaseId:b.data.meta.release_id,profile:b.data.meta.profile,coverageNote:b.data.meta.coverage_note,nextCursor:b.data.meta.next_cursor??null};}catch(e){if(e&&typeof e==='object'&&'kind'in e)throw e;if(e instanceof DOMException&&e.name==='AbortError')throw fail('aborted','Local V2 request was aborted.');if(e instanceof TypeError)throw fail('network','Local V2 request could not connect.');throw fail('invalid-contract','Local V2 response could not be read safely.');}}async detail(id:string,profile:'official'|'secondary'|'community'='official'){try{const b=detailEnvelopeSchema.safeParse(await this.json(`/api/v2/locations/${encodeURIComponent(id)}?profile=${profile}`));if(!b.success||b.data.meta.profile!==profile||b.data.data.privacy_screening_status!=='passed'||b.data.data.project_approval!=='approved')throw fail('invalid-contract','Local V2 detail response was rejected.');return{location:map(b.data.data),releaseId:b.data.meta.release_id,profile:b.data.meta.profile};}catch(e){if(e&&typeof e==='object'&&'kind'in e)throw e;if(e instanceof DOMException&&e.name==='AbortError')throw fail('aborted','Local V2 request was aborted.');if(e instanceof TypeError)throw fail('network','Local V2 request could not connect.');throw fail('invalid-contract','Local V2 detail response could not be read safely.');}}} +import{detailEnvelopeSchema,envelopeSchema,type WireLocation}from'./wireSchema';import type{ApiError}from'./errors';import type{Location}from'../domain/location';export type FetchLike=(input:RequestInfo|URL,init?:RequestInit)=>Promise;export const localOrigin=(value:string|undefined):string|undefined=>{if(!value)return undefined;const url=new URL(value);if(url.protocol!=='http:'||!['127.0.0.1','localhost','::1'].includes(url.hostname))throw new Error('Local API origin must be loopback HTTP.');return url.origin;};const fail=(kind:ApiError['kind'],message:string,status?:number):ApiError=>Object.assign(new Error(message),status===undefined?{kind}:{kind,status});const map=(r:WireLocation):Location=>({id:r.facility_id,name:r.canonical_name,region:r.city??r.country_code,category:r.category,lat:r.latitude,lon:r.longitude,observed:r.last_observed_at??r.first_observed_at??'unknown',source:r.provenance_source_name});export class LocalLocationRepository{readonly#base:string|undefined;constructor(private readonly fetcher:FetchLike=globalThis.fetch,baseUrl?:string){this.#base=localOrigin(baseUrl);}private async json(path:string,signal?:AbortSignal){const init:RequestInit={};if(signal)init.signal=signal;const response=await this.fetcher.call(globalThis,`${this.#base??''}${path}`,init);if(!response.ok)throw fail('http',`Local V2 request failed with status ${response.status}`,response.status);try{return await response.json();}catch{throw fail('invalid-contract','Local V2 response was not valid JSON.');}}async list(profile:'official'|'secondary'|'community'='official'){try{const b=envelopeSchema.safeParse(await this.json(`/api/v2/locations?profile=${profile}`));if(!b.success||b.data.meta.profile!==profile)throw fail('invalid-contract','Local V2 list response was rejected.');if(b.data.meta.release_id===null)throw fail('no-release',b.data.meta.coverage_note);if(b.data.meta.ruleset_version===undefined||b.data.data.some(r=>r.privacy_screening_status!=='passed'||r.project_approval!=='approved'||r.release_id!==b.data.meta.release_id||r.release_ruleset_version!==b.data.meta.ruleset_version))throw fail('invalid-contract','Local V2 list snapshot was rejected.');return{locations:b.data.data.map(map),releaseId:b.data.meta.release_id,profile:b.data.meta.profile,coverageNote:b.data.meta.coverage_note,nextCursor:b.data.meta.next_cursor??null};}catch(e){if(e&&typeof e==='object'&&'kind'in e)throw e;if(e instanceof DOMException&&e.name==='AbortError')throw fail('aborted','Local V2 request was aborted.');if(e instanceof TypeError)throw fail('network','Local V2 request could not connect.');throw fail('invalid-contract','Local V2 response could not be read safely.');}}async detail(id:string,profile:'official'|'secondary'|'community'='official'){try{const b=detailEnvelopeSchema.safeParse(await this.json(`/api/v2/locations/${encodeURIComponent(id)}?profile=${profile}`));if(!b.success||b.data.meta.profile!==profile||b.data.data.privacy_screening_status!=='passed'||b.data.data.project_approval!=='approved')throw fail('invalid-contract','Local V2 detail response was rejected.');return{location:map(b.data.data),releaseId:b.data.meta.release_id,profile:b.data.meta.profile};}catch(e){if(e&&typeof e==='object'&&'kind'in e)throw e;if(e instanceof DOMException&&e.name==='AbortError')throw fail('aborted','Local V2 request was aborted.');if(e instanceof TypeError)throw fail('network','Local V2 request could not connect.');throw fail('invalid-contract','Local V2 response could not be read safely.');}}} diff --git a/frontend/src/app/App.svelte b/frontend/src/app/App.svelte index b1f163e..db89bc6 100644 --- a/frontend/src/app/App.svelte +++ b/frontend/src/app/App.svelte @@ -2,6 +2,7 @@ import{onMount}from'svelte';import{locations}from'../fixtures/locations';import type{Location}from'../domain/location';import type{Profile}from'../domain/publication';import{parseRoute}from'./routeState';import{filterLocations,initialFilters,type FilterState}from'../features/locations/filterState';import MapView from'../map/MapView.svelte';import{makeExportModel}from'../export/exportModel';import{previewExport}from'../export/previewExport';import{LocalLocationRepository}from'../api/LocalLocationRepository'; let profile:Profile='curated';let scenario='ready';let selected:Location|undefined=locations[0];let search='';let region='all';let filters:FilterState=initialFilters;let showMap=false;let showExport=false;let localMode=false;let localStatus:'idle'|'loading'|'ready'|'error'|'no-release'='idle';let detailStatus:'idle'|'loading'|'error'='idle';let localError='';let loaded:readonly Location[]=[];let release='synthetic-2026.09';let repo=new LocalLocationRepository();$:filters={search,region,category:'all'};$:source=localMode?loaded:(profile==='community'?locations.slice(0,1):locations);$:visibleLocations=filterLocations(source,filters);$:exportPreview=previewExport(makeExportModel(visibleLocations,profile,release)); const syncRoute=async()=>{const r=parseRoute(window.location.hash);if(r.kind==='location'&&localMode){detailStatus='loading';try{const result=await repo.detail(r.facilityId,profile==='community'?'community':'official');selected=result.location;release=result.releaseId;detailStatus='idle';}catch(error){selected=undefined;detailStatus='error';localError=error instanceof Error?error.message:'The local detail response was rejected safely.';}}else if(r.kind==='location')selected=source.find(x=>x.id===r.facilityId)??selected;else if(r.kind==='home')selected=source[0];}; -const loadLocal=async()=>{localStatus='loading';localError='';try{const result=await repo.list(profile==='community'?'community':'official');loaded=result.locations;release=result.releaseId;localStatus='ready';await syncRoute();}catch(error){const kind=error&&typeof error==='object'&&'kind'in error?(error as {kind:string}).kind:'error';localStatus=kind==='no-release'?'no-release':'error';localError=error instanceof Error?error.message:'Local V2 response was rejected safely.';loaded=[];selected=undefined;}};const select=(id:string)=>{selected=source.find(x=>x.id===id)??selected;window.location.hash=`/locations/${id}?profile=${profile}`};onMount(()=>{const params=new URLSearchParams(window.location.search);localMode=params.get('mode')==='local-v2';if(localMode){try{repo=new LocalLocationRepository(globalThis.fetch,params.get('api')??undefined);}catch(error){localStatus='error';localError=error instanceof Error?error.message:'Local API origin was rejected safely.';}}if(localMode&&localStatus!=='error')void loadLocal();else if(!localMode)void syncRoute();const onHashChange=()=>void syncRoute();window.addEventListener('hashchange',onHashChange);return()=>window.removeEventListener('hashchange',onHashChange)}); +const loadLocal=async()=>{localStatus='loading';localError='';try{const result=await repo.list(profile==='community'?'community':'official');loaded=result.locations;selected=result.locations[0];release=result.releaseId;localStatus='ready';await syncRoute();}catch(error){const kind=error&&typeof error==='object'&&'kind'in error?(error as {kind:string}).kind:'error';localStatus=kind==='no-release'?'no-release':'error';localError=error instanceof Error?error.message:'Local V2 response was rejected safely.';loaded=[];selected=undefined;}};const select=(id:string)=>{selected=source.find(x=>x.id===id)??selected;window.location.hash=`/locations/${id}?profile=${profile}`};onMount(()=>{const params=new URLSearchParams(window.location.search);localMode=params.get('mode')==='local-v2';if(localMode){try{repo=new LocalLocationRepository(globalThis.fetch,params.get('api')??undefined);}catch(error){localStatus='error';localError=error instanceof Error?error.message:'Local API origin was rejected safely.';}}if(localMode&&localStatus!=='error')void loadLocal();else if(!localMode)void syncRoute();const onHashChange=()=>void syncRoute();window.addEventListener('hashchange',onHashChange);return()=>window.removeEventListener('hashchange',onHashChange)}); +$: if(localMode&&localStatus==='ready'&&!selected)selected=loaded[0]; Until Every Cage · evidence desk
UNTIL EVERY CAGE V2 / FIELD NOTEEthics & safeguards ↗

EVIDENCE DESK · {localMode?'LOCAL V2 API':'SYNTHETIC PREVIEW'}

See what a record can—and cannot—tell us.

{localMode?'Explicit local integration mode. No fixture or V1 fallback is used.':'A quiet, inspectable view of fictional locations.'}

{#if profile==='community'}
Unreviewed community claimNot verified by Until Every Cage.
{/if}{#if localMode&&localStatus==='loading'}
Loading the local V2 release…
{:else if localMode&&(localStatus==='error'||localStatus==='no-release')}{:else if localMode&&detailStatus==='loading'}
Loading the selected local record…
{:else if localMode&&detailStatus==='error'}{:else if scenario==='loading'}
Loading the fixture…
{:else if scenario==='empty'}

No eligible records

{:else if scenario==='error'}{:else if scenario==='restricted'}

Preview withheld

{:else}
{#if selected}

RECORD / {selected.id}

{selected.name}

{selected.region} · {selected.category}

OBSERVED{selected.observed}
MAP STATUS{selected.lat===null?'Not available':'Approximate display point'}
{/if}
{/if}
{#if showMap}{/if}{#if showExport}

IN-MEMORY EXPORT PREVIEW

{profile} · {release}

Loaded results only. This preview is not a live or complete export.

{exportPreview}
{/if}
{localMode?'Local V2 mode · no fixture fallback':'Method preview · no live requests'}
diff --git a/frontend/tests/e2e/fixture-platform.spec.ts b/frontend/tests/e2e/fixture-platform.spec.ts index aebf806..57dbde4 100644 --- a/frontend/tests/e2e/fixture-platform.spec.ts +++ b/frontend/tests/e2e/fixture-platform.spec.ts @@ -4,7 +4,8 @@ import AxeBuilder from '@axe-core/playwright'; test.beforeEach(async ({ page }) => { await page.route('**/*', async (route) => { const url = new URL(route.request().url()); - if (url.origin !== 'http://127.0.0.1:4173') throw new Error(`Unexpected external request: ${url.href}`); + const localApiAllowed = process.env.LOCAL_V2_E2E === '1' && url.origin === 'http://127.0.0.1:8000'; + if (url.origin !== 'http://127.0.0.1:4173' && !localApiAllowed) throw new Error(`Unexpected external request: ${url.href}`); await route.continue(); }); }); diff --git a/frontend/tests/e2e/local-backend.spec.ts b/frontend/tests/e2e/local-backend.spec.ts new file mode 100644 index 0000000..172da47 --- /dev/null +++ b/frontend/tests/e2e/local-backend.spec.ts @@ -0,0 +1,19 @@ +import { test, expect } from '@playwright/test'; + +test.skip(process.env.LOCAL_V2_E2E !== '1', 'Set LOCAL_V2_E2E=1 to run against the real local backend'); + +test('renders the real seeded local V2 record and opens its detail route', async ({ page }) => { + const fixtureNames = ['North Star Cooperative', 'River Meadow Foods', 'Quiet Field Holdings']; + let list: { data?: Array<{ facility_id: string; canonical_name: string }> } = {}; + const response = await fetch('http://127.0.0.1:8000/api/v2/locations?profile=official&limit=1'); + expect(response.ok).toBeTruthy(); + list = await response.json() as typeof list; + const record = list.data?.[0]; + expect(record?.facility_id).toBeTruthy(); + expect(record?.canonical_name).toBeTruthy(); + await page.goto('./?mode=local-v2&api=http%3A%2F%2F127.0.0.1%3A8000#/'); + await expect(page.getByRole('heading', { name: record?.canonical_name ?? '' })).toBeVisible(); + for (const name of fixtureNames) await expect(page.getByText(name, { exact: true })).toHaveCount(0); + await page.goto(`./?mode=local-v2&api=http%3A%2F%2F127.0.0.1%3A8000#/locations/${record?.facility_id}?profile=curated`); + await expect(page.getByRole('heading', { name: record?.canonical_name ?? '' })).toBeVisible(); +}); diff --git a/frontend/vite.config.ts b/frontend/vite.config.ts index b6e76b3..c6d8827 100644 --- a/frontend/vite.config.ts +++ b/frontend/vite.config.ts @@ -1,2 +1,3 @@ import { defineConfig } from 'vite'; import { svelte } from '@sveltejs/vite-plugin-svelte'; -export default defineConfig({base:'/v2-preview/',plugins:[svelte()],build:{outDir:'dist',emptyOutDir:true}}); +const localApiProxy={ '/api': { target:'http://127.0.0.1:8000', changeOrigin:false } }; +export default defineConfig({base:'/v2-preview/',plugins:[svelte()],server:{proxy:localApiProxy},preview:{proxy:localApiProxy},build:{outDir:'dist',emptyOutDir:true}}); diff --git a/pipeline/scripts/maintenance/local-v2.ps1 b/pipeline/scripts/maintenance/local-v2.ps1 index 7625983..6601824 100644 --- a/pipeline/scripts/maintenance/local-v2.ps1 +++ b/pipeline/scripts/maintenance/local-v2.ps1 @@ -6,6 +6,7 @@ $project='uec-local-v2'; $dbPort=5433; $apiPort=8000 $db="postgresql://uec:uec-local-development-only@127.0.0.1:$dbPort/uec?sslmode=disable" $stateDir=Join-Path $root 'target\local-v2'; $pidFile=Join-Path $stateDir 'uec-api.pid'; $logFile=Join-Path $stateDir 'uec-api.log'; $errorFile=Join-Path $stateDir 'uec-api-error.log' $env:UEC_PIPELINE_DB_PORT="$dbPort"; $env:UEC_DATABASE_URL=$db; $env:PORT="$apiPort" +$env:UEC_CORS_ORIGIN="http://127.0.0.1:4173" function Get-OwnedApiProcess { if (!(Test-Path $pidFile)) { return $null } @@ -29,6 +30,18 @@ try { if ($LASTEXITCODE) { throw 'Synthetic seed failed.' } & docker compose -p $project -f $compose exec -T postgres psql -v ON_ERROR_STOP=1 -U uec -d uec -c "CREATE TABLE IF NOT EXISTS uec.local_v2_fixture_seed (seed_name text primary key, seeded_at timestamptz not null default now()); INSERT INTO uec.local_v2_fixture_seed(seed_name) VALUES ('standard-contract') ON CONFLICT DO NOTHING;" } + $releaseStatus=(& docker compose -p $project -f $compose exec -T postgres psql -U uec -d uec -Atc "SELECT status FROM uec.releases WHERE release_id = 'standard-candidate'") + if ($releaseStatus.Trim() -eq 'candidate') { + python pipeline/scripts/stages/validate-release.py standard-candidate --expected-records 1 --mark-validated + if ($LASTEXITCODE) { throw 'Synthetic release validation failed.' } + $releaseStatus='validated' + } + if ($releaseStatus.Trim() -eq 'validated') { + python pipeline/scripts/stages/promote-release.py standard-candidate + if ($LASTEXITCODE) { throw 'Synthetic release promotion failed.' } + } elseif ($releaseStatus.Trim() -ne 'promoted') { + throw "Synthetic release has unexpected status: $($releaseStatus.Trim())" + } if (!(Get-OwnedApiProcess)) { $api=Join-Path $root 'target\debug\uec-api.exe'; if (!(Test-Path $api)) { cargo build --bin uec-api --quiet } $process=Start-Process -FilePath $api -WorkingDirectory $root -WindowStyle Hidden -RedirectStandardOutput $logFile -RedirectStandardError $errorFile -PassThru From 76374e85ab0e1da3964108820cd772a181441afb Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sun, 13 Sep 2026 16:47:34 -0700 Subject: [PATCH 014/311] feat: add safe v2 release context and csv export --- frontend/src/api/LocalCsvExportRepository.ts | 30 ++++++ frontend/src/app/App.svelte | 101 ++++++++++++++++-- frontend/src/ui/ExportControl.svelte | 12 +++ frontend/src/ui/ReleaseContext.svelte | 14 +++ .../unit/localCsvExportRepository.test.ts | 17 +++ 5 files changed, 168 insertions(+), 6 deletions(-) create mode 100644 frontend/src/api/LocalCsvExportRepository.ts create mode 100644 frontend/src/ui/ExportControl.svelte create mode 100644 frontend/src/ui/ReleaseContext.svelte create mode 100644 frontend/tests/unit/localCsvExportRepository.test.ts diff --git a/frontend/src/api/LocalCsvExportRepository.ts b/frontend/src/api/LocalCsvExportRepository.ts new file mode 100644 index 0000000..798e9a9 --- /dev/null +++ b/frontend/src/api/LocalCsvExportRepository.ts @@ -0,0 +1,30 @@ +import type { FetchLike } from './LocalLocationRepository'; + +export type CsvExport = Readonly<{ + body: string; + releaseId: string; + profile: string; + manifestSha256?: string; +}>; + +export class LocalCsvExportRepository { + constructor(private readonly fetcher: FetchLike = globalThis.fetch, private readonly baseUrl = '') {} + + async download(profile: 'official' | 'secondary' | 'community' = 'official'): Promise { + const response = await this.fetcher.call(globalThis, `${this.baseUrl}/api/v2/locations.csv?profile=${profile}`, { cache: 'no-store' }); + if (!response.ok) { + let message = `Local V2 export request failed with status ${response.status}.`; + try { + const payload = await response.json() as { error?: { message?: string } }; + if (payload.error?.message) message = payload.error.message; + } catch { /* Preserve the status-based safe message. */ } + throw Object.assign(new Error(message), { status: response.status }); + } + const body = await response.text(); + const releaseId = response.headers.get('x-uec-release-id'); + const responseProfile = profile; + if (!releaseId || !body.trim()) throw new Error('The local V2 export did not include eligible release context.'); + const manifestSha256 = response.headers.get('x-uec-manifest-sha256'); + return manifestSha256 ? { body, releaseId, profile: responseProfile, manifestSha256 } : { body, releaseId, profile: responseProfile }; + } +} diff --git a/frontend/src/app/App.svelte b/frontend/src/app/App.svelte index db89bc6..c4491d9 100644 --- a/frontend/src/app/App.svelte +++ b/frontend/src/app/App.svelte @@ -1,8 +1,97 @@ -Until Every Cage · evidence desk
UNTIL EVERY CAGE V2 / FIELD NOTEEthics & safeguards ↗

EVIDENCE DESK · {localMode?'LOCAL V2 API':'SYNTHETIC PREVIEW'}

See what a record can—and cannot—tell us.

{localMode?'Explicit local integration mode. No fixture or V1 fallback is used.':'A quiet, inspectable view of fictional locations.'}

{#if profile==='community'}
Unreviewed community claimNot verified by Until Every Cage.
{/if}{#if localMode&&localStatus==='loading'}
Loading the local V2 release…
{:else if localMode&&(localStatus==='error'||localStatus==='no-release')}{:else if localMode&&detailStatus==='loading'}
Loading the selected local record…
{:else if localMode&&detailStatus==='error'}{:else if scenario==='loading'}
Loading the fixture…
{:else if scenario==='empty'}

No eligible records

{:else if scenario==='error'}{:else if scenario==='restricted'}

Preview withheld

{:else}
{#if selected}

RECORD / {selected.id}

{selected.name}

{selected.region} · {selected.category}

OBSERVED{selected.observed}
MAP STATUS{selected.lat===null?'Not available':'Approximate display point'}
{/if}
{/if}
{#if showMap}{/if}{#if showExport}

IN-MEMORY EXPORT PREVIEW

{profile} · {release}

Loaded results only. This preview is not a live or complete export.

{exportPreview}
{/if}
{localMode?'Local V2 mode · no fixture fallback':'Method preview · no live requests'}
+ +Until Every Cage · evidence desk +
UNTIL EVERY CAGE V2 / FIELD NOTEEthics & safeguards ↗
+

EVIDENCE DESK · {localMode ? 'LOCAL V2 API' : 'SYNTHETIC PREVIEW'}

See what a record can—and cannot—tell us.

{localMode ? 'Explicit local integration mode. No fixture or V1 fallback is used.' : 'A quiet, inspectable view of fictional locations.'}

+
+{#if profile === 'community'}
Unreviewed community claimNot verified by Until Every Cage.
{/if} +{#if localMode && localStatus === 'loading'}
Loading the local V2 release…
{:else if localMode && (localStatus === 'error' || localStatus === 'no-release')}{:else if localMode && detailStatus === 'loading'}
Loading the selected local record…
{:else if localMode && detailStatus === 'error'}{:else}
{#if selected}

RECORD / {selected.id}

{selected.name}

{selected.region} · {selected.category}

OBSERVED{selected.observed}
MAP STATUS{selected.lat === null ? 'Not available' : 'Approximate display point'}
{/if}
{/if} +{#if localMode && localStatus === 'ready'}{/if} +
{#if showMap}{/if}{#if showExport}

IN-MEMORY EXPORT PREVIEW

{profile} · {release}

Loaded results only. This preview is not a live or complete export.

{exportPreview}
{/if}
{localMode ? 'Local V2 mode · no fixture fallback' : 'Method preview · no live requests'}
diff --git a/frontend/src/ui/ExportControl.svelte b/frontend/src/ui/ExportControl.svelte new file mode 100644 index 0000000..581fea4 --- /dev/null +++ b/frontend/src/ui/ExportControl.svelte @@ -0,0 +1,12 @@ + + +
+ + {#if !enabled}

CSV export is available only for an explicit official profile with a selected eligible release.

{/if} + {#if error}{/if} +
diff --git a/frontend/src/ui/ReleaseContext.svelte b/frontend/src/ui/ReleaseContext.svelte new file mode 100644 index 0000000..14a3b6b --- /dev/null +++ b/frontend/src/ui/ReleaseContext.svelte @@ -0,0 +1,14 @@ + + +
+
PROFILE{profile}
+
RELEASE{releaseId}
+ {#if ruleset}
RULESET{ruleset}
{/if} + {#if manifestSha256}
MANIFEST DIGEST{manifestSha256}
{/if} +

Release context supports traceability and integrity checks; it is not human approval or cryptographic signing.

+
diff --git a/frontend/tests/unit/localCsvExportRepository.test.ts b/frontend/tests/unit/localCsvExportRepository.test.ts new file mode 100644 index 0000000..e949988 --- /dev/null +++ b/frontend/tests/unit/localCsvExportRepository.test.ts @@ -0,0 +1,17 @@ +import { describe, expect, it, vi } from 'vitest'; +import { LocalCsvExportRepository } from '../../src/api/LocalCsvExportRepository'; + +const response = (status = 200, body = 'facility_id\nabc\n') => new Response(body, { status, headers: { 'x-uec-release-id': 'release-1', 'x-uec-manifest-sha256': 'digest-1' } }); + +describe('LocalCsvExportRepository', () => { + it('returns bounded CSV and release manifest context', async () => { + const result = await new LocalCsvExportRepository(vi.fn().mockResolvedValue(response())).download('official'); + expect(result).toMatchObject({ releaseId: 'release-1', profile: 'official', manifestSha256: 'digest-1' }); + }); + it.each([400, 404, 429, 503])('surfaces HTTP status %s', async (status) => { + await expect(new LocalCsvExportRepository(vi.fn().mockResolvedValue(response(status))).download('official')).rejects.toMatchObject({ status }); + }); + it('rejects an empty response or missing release context', async () => { + await expect(new LocalCsvExportRepository(vi.fn().mockResolvedValue(response(200, ''))).download('official')).rejects.toThrow(/eligible release context/); + }); +}); From 4e7e5fac459f8cc18d7cdbba2251cda9ea734c3d Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sun, 13 Sep 2026 16:56:14 -0700 Subject: [PATCH 015/311] feat(api): add versioned discovery filter metadata --- docs/api/v2-contract.json | 3 +- docs/api/v2-contract.md | 2 + pipeline/tests/e2e/test_public_api.py | 7 ++ src/lib.rs | 55 ++++++++++---- src/main.rs | 101 +++++++++++++++++++++++--- 5 files changed, 144 insertions(+), 24 deletions(-) diff --git a/docs/api/v2-contract.json b/docs/api/v2-contract.json index fdff7b8..3365572 100644 --- a/docs/api/v2-contract.json +++ b/docs/api/v2-contract.json @@ -6,7 +6,8 @@ "GET /health/ready": {"success": {"status": "ready", "database": "ok"}, "unavailable_status": 503}, "GET /api/v2/locations": {"success": {"api_version": "v2", "data": [], "meta": {"profile": "official", "next_cursor": null}}, "no_release": 200}, "GET /api/v2/locations/{facility_id}": {"success": {"api_version": "v2", "data": {}, "meta": {}}, "not_found": 404}, - "GET /api/v2/locations.csv": {"success_content_type": "text/csv", "bounded_rows": 1000, "profile_required_for_nonofficial": true, "not_found_without_promoted_manifest": 404} + "GET /api/v2/locations.csv": {"success_content_type": "text/csv", "bounded_rows": 1000, "profile_required_for_nonofficial": true, "not_found_without_promoted_manifest": 404}, + "GET /api/v2/discovery/filters": {"success": {"api_version": "v2", "contract_version": "v1", "dimensions": {"country_code": "allowlist", "category": "allowlist", "source_type": "allowlist", "profile": "allowlist", "display_precision": "allowlist", "lifecycle_status": "allowlist"}}} }, "error": {"shape": {"api_version": "v2", "error": {"code": "string", "message": "string"}}, "codes": ["invalid_filter", "invalid_profile", "invalid_limit", "invalid_offset", "invalid_cursor", "invalid_pagination", "database_not_configured", "database_pool_unavailable", "database_transaction_unavailable", "release_query_failed", "location_query_failed", "location_not_found", "rate_limited"]}, "privacy": "Public responses contain reviewed projection fields only; restricted records and raw evidence are never returned." diff --git a/docs/api/v2-contract.md b/docs/api/v2-contract.md index f9f50f2..2404ffb 100644 --- a/docs/api/v2-contract.md +++ b/docs/api/v2-contract.md @@ -9,3 +9,5 @@ The machine-readable contract is [v2-contract.json](v2-contract.json). Successfu Frontend clients should branch on HTTP status and `error.code`, display `message` only as user-safe text, and treat unknown codes as generic failures. A list request with no promoted eligible release is a successful empty response; an unavailable database is `503`; an absent or suppressed detail is `404`; rate limiting is `429` with `Retry-After`. Researchers may request `GET /api/v2/locations.csv?profile=official` (or another explicit supported profile). The export is bounded to 1,000 rows, uses deterministic CSV columns and escaping, contains only the public reviewed projection, and includes `release_profile`, `release_id`, and `manifest_sha256` on every row plus matching response headers. It is unavailable when no promoted release with a manifest exists; it never exposes raw evidence or restricted records. + +Clients can discover the current controlled vocabularies at `GET /api/v2/discovery/filters`. Filter values are allowlisted and versioned; clients must not invent country/category/status values or send arbitrary free-text search. New country adapters should register capabilities and vocabularies in the contract before becoming public. diff --git a/pipeline/tests/e2e/test_public_api.py b/pipeline/tests/e2e/test_public_api.py index c144cea..01ce5f1 100644 --- a/pipeline/tests/e2e/test_public_api.py +++ b/pipeline/tests/e2e/test_public_api.py @@ -52,5 +52,12 @@ def test_cursor_and_offset_cannot_be_combined(self): self.get("/api/v2/locations?cursor=00000000-0000-0000-0000-000000000000&offset=1") self.assertEqual(error.exception.code, 400) + def test_filter_metadata_is_versioned_and_allowlisted(self): + status, body = self.get("/api/v2/discovery/filters") + self.assertEqual(status, 200) + self.assertEqual(body["api_version"], "v2") + self.assertIn("community", body["dimensions"]["profile"]["values"]) + self.assertNotIn("address", body["dimensions"]) + if __name__ == "__main__": unittest.main() diff --git a/src/lib.rs b/src/lib.rs index 68a3e21..bd8027b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -306,6 +306,31 @@ pub async fn get_locations_handler(Query(params): Query) -> impl } } +const V2_CATEGORIES: &[&str] = &[ + "slaughter", + "fish_processing", + "logistics_and_storage", + "retail_and_prepared_food", +]; +const V2_COUNTRIES: &[&str] = &["DK"]; +const V2_SOURCE_TYPES: &[&str] = &["official", "secondary", "user_submitted"]; +const V2_PROFILES: &[&str] = &["official", "secondary", "community"]; +const V2_PRECISIONS: &[&str] = &["exact", "city", "unmapped"]; +const V2_LIFECYCLES: &[&str] = &[ + "active_observed", + "explicitly_closed", + "not_seen_recently", + "status_unknown", +]; + +pub async fn get_v2_filter_metadata_handler() -> impl IntoResponse { + Json(json!({"api_version":"v2", "contract_version":"v1", "dimensions": { + "country_code":{"values":V2_COUNTRIES}, "category":{"values":V2_CATEGORIES}, + "source_type":{"values":V2_SOURCE_TYPES}, "profile":{"values":V2_PROFILES,"default":"official"}, + "display_precision":{"values":V2_PRECISIONS}, "lifecycle_status":{"values":V2_LIFECYCLES} + }, "pagination":{"limit_max":1000,"cursor":"facility_id"}, "privacy":"Filters operate only on eligible records in the selected promoted release; filters never override suppression or publication review."})).into_response() +} + #[derive(Deserialize)] pub struct V2LocationParams { pub country_code: Option, @@ -353,17 +378,10 @@ pub async fn get_v2_locations_handler( State(state): State, Query(params): Query, ) -> impl IntoResponse { - const PRECISIONS: &[&str] = &["exact", "city", "unmapped"]; - const LIFECYCLES: &[&str] = &[ - "active_observed", - "explicitly_closed", - "not_seen_recently", - "status_unknown", - ]; if params .display_precision .as_deref() - .is_some_and(|v| !PRECISIONS.contains(&v)) + .is_some_and(|v| !V2_PRECISIONS.contains(&v)) { return v2_error( StatusCode::BAD_REQUEST, @@ -374,7 +392,7 @@ pub async fn get_v2_locations_handler( if params .lifecycle_status .as_deref() - .is_some_and(|v| !LIFECYCLES.contains(&v)) + .is_some_and(|v| !V2_LIFECYCLES.contains(&v)) { return v2_error( StatusCode::BAD_REQUEST, @@ -385,8 +403,11 @@ pub async fn get_v2_locations_handler( if params .category .as_deref() - .is_some_and(|v| v.trim().is_empty()) - || params.country_code.as_deref().is_some_and(|v| v.len() != 2) + .is_some_and(|v| !V2_CATEGORIES.contains(&v)) + || params + .country_code + .as_deref() + .is_some_and(|v| v.len() != 2 || !v.chars().all(|c| c.is_ascii_uppercase())) { return v2_error( StatusCode::BAD_REQUEST, @@ -397,7 +418,7 @@ pub async fn get_v2_locations_handler( if params .source_type .as_deref() - .is_some_and(|v| !["official", "secondary", "user_submitted"].contains(&v)) + .is_some_and(|v| !V2_SOURCE_TYPES.contains(&v)) { return v2_error( StatusCode::BAD_REQUEST, @@ -408,7 +429,7 @@ pub async fn get_v2_locations_handler( if params .profile .as_deref() - .is_some_and(|v| !["official", "secondary", "community"].contains(&v)) + .is_some_and(|v| !V2_PROFILES.contains(&v)) { return v2_error( StatusCode::BAD_REQUEST, @@ -765,6 +786,14 @@ mod v2_api_tests { assert_eq!(contract["error"]["shape"]["api_version"], "v2"); } + #[tokio::test] + async fn filter_metadata_is_versioned_and_allowlisted() { + let response = get_v2_filter_metadata_handler().await.into_response(); + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(V2_CATEGORIES.len(), 4); + assert!(!V2_COUNTRIES.contains(&"ZZ")); + } + #[tokio::test] async fn v2_response_is_json_when_database_is_configured() { let url = std::env::var("UEC_DATABASE_URL").unwrap_or_else(|_| { diff --git a/src/main.rs b/src/main.rs index 55bd7f4..59c6184 100644 --- a/src/main.rs +++ b/src/main.rs @@ -23,7 +23,7 @@ use std::collections::HashMap; use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant}; use tower_http::compression::CompressionLayer; -use tower_http::cors::CorsLayer; +use tower_http::cors::{AllowOrigin, CorsLayer}; use deadpool_postgres::{Config, ManagerConfig, RecyclingMethod, Runtime}; use tokio_postgres::NoTls; @@ -31,14 +31,7 @@ use tokio_postgres_rustls::MakeRustlsConnect; use tower_http::services::ServeDir; pub fn app(state: uec_api::ApiState) -> Router { - let origin = - std::env::var("UEC_CORS_ORIGIN").unwrap_or_else(|_| "http://localhost:3000".to_string()); - let origin = origin - .parse::() - .expect("UEC_CORS_ORIGIN must be a valid origin"); - let cors = CorsLayer::new() - .allow_origin(origin) - .allow_methods([Method::GET]); + let cors = cors_layer().expect("CORS configuration must be validated before app startup"); Router::new() .route("/health/live", get(liveness)) .route("/health/ready", get(readiness)) @@ -52,6 +45,10 @@ pub fn app(state: uec_api::ApiState) -> Router { "/api/v2/locations.csv", get(uec_api::get_v2_locations_export_handler), ) + .route( + "/api/v2/discovery/filters", + get(uec_api::get_v2_filter_metadata_handler), + ) .route( "/api/v2/locations/{facility_id}", get(uec_api::get_v2_location_detail_handler), @@ -72,6 +69,59 @@ pub fn app(state: uec_api::ApiState) -> Router { .with_state(state) } +fn parse_cors_origins( + mode: &str, + configured: Option<&str>, + legacy: Option<&str>, +) -> Result, &'static str> { + let value = configured.or(legacy).unwrap_or(if mode == "development" { + "http://localhost:3000" + } else { + "" + }); + if mode == "production" && value.trim().is_empty() { + return Err("UEC_CORS_ORIGINS is required in production"); + } + let mut origins = Vec::new(); + for raw in value.split(',').map(str::trim).filter(|v| !v.is_empty()) { + if raw == "*" || raw.contains('*') { + return Err("UEC_CORS_ORIGINS must not contain wildcards"); + } + let uri = raw + .parse::() + .map_err(|_| "UEC_CORS_ORIGINS contains a malformed origin")?; + if !matches!(uri.scheme_str(), Some("http") | Some("https")) + || uri.host().is_none() + || !uri.path().is_empty() && uri.path() != "/" + || uri.query().is_some() + || raw.contains('#') + { + return Err("UEC_CORS_ORIGINS must contain bare http(s) origins"); + } + origins.push( + raw.parse::() + .map_err(|_| "UEC_CORS_ORIGINS contains an invalid header value")?, + ); + } + if origins.is_empty() { + return Err("UEC_CORS_ORIGINS must contain at least one origin"); + } + Ok(origins) +} + +fn cors_layer() -> Result { + let mode = std::env::var("UEC_RUNTIME_MODE").unwrap_or_else(|_| "development".into()); + let origins = parse_cors_origins( + mode.as_str(), + std::env::var("UEC_CORS_ORIGINS").ok().as_deref(), + std::env::var("UEC_CORS_ORIGIN").ok().as_deref(), + )?; + Ok(CorsLayer::new() + .allow_origin(AllowOrigin::list(origins)) + .allow_methods([Method::GET, Method::OPTIONS]) + .allow_headers([axum::http::header::CONTENT_TYPE, axum::http::header::ACCEPT])) +} + const RATE_WINDOW: Duration = Duration::from_secs(60); const RATE_LIMIT: u32 = 60; static GLOBAL_LIMITER: once_cell::sync::Lazy = @@ -184,6 +234,17 @@ async fn main() { ); std::process::exit(2) }); + if let Err(error) = parse_cors_origins( + mode.as_str(), + std::env::var("UEC_CORS_ORIGINS").ok().as_deref(), + std::env::var("UEC_CORS_ORIGIN").ok().as_deref(), + ) { + eprintln!( + "{{\"event\":\"configuration_error\",\"reason\":\"{}\"}}", + error + ); + std::process::exit(2); + } let database = database_url.and_then(|url| { let mut config = Config::new(); config.url = Some(url); @@ -230,7 +291,7 @@ async fn main() { #[cfg(test)] mod config_tests { - use super::validate_runtime; + use super::{parse_cors_origins, validate_runtime}; #[test] fn development_allows_local_defaults() { assert_eq!(validate_runtime("development", None, "8000"), Ok(8000)); @@ -248,6 +309,26 @@ mod config_tests { assert!(validate_runtime("production", Some("redacted"), "bad").is_err()); assert!(validate_runtime("production", Some("redacted"), "0").is_err()); } + #[test] + fn cors_requires_narrow_production_allowlist() { + assert!(parse_cors_origins("production", None, None).is_err()); + assert!(parse_cors_origins("production", Some("*"), None).is_err()); + assert!(parse_cors_origins("production", Some("https://example.test/path"), None).is_err()); + assert_eq!( + parse_cors_origins( + "production", + Some("https://example.test,https://research.test"), + None + ) + .unwrap() + .len(), + 2 + ); + assert_eq!( + parse_cors_origins("development", None, None).unwrap().len(), + 1 + ); + } } #[cfg(test)] From cb8167491ab2ddba0caa6ba88076fd913abc5743 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sun, 13 Sep 2026 16:58:44 -0700 Subject: [PATCH 016/311] feat(api): apply versioned discovery filters --- docs/api/v2-contract.md | 2 +- pipeline/tests/e2e/test_public_api.py | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/docs/api/v2-contract.md b/docs/api/v2-contract.md index 2404ffb..e5d435f 100644 --- a/docs/api/v2-contract.md +++ b/docs/api/v2-contract.md @@ -10,4 +10,4 @@ Frontend clients should branch on HTTP status and `error.code`, display `message Researchers may request `GET /api/v2/locations.csv?profile=official` (or another explicit supported profile). The export is bounded to 1,000 rows, uses deterministic CSV columns and escaping, contains only the public reviewed projection, and includes `release_profile`, `release_id`, and `manifest_sha256` on every row plus matching response headers. It is unavailable when no promoted release with a manifest exists; it never exposes raw evidence or restricted records. -Clients can discover the current controlled vocabularies at `GET /api/v2/discovery/filters`. Filter values are allowlisted and versioned; clients must not invent country/category/status values or send arbitrary free-text search. New country adapters should register capabilities and vocabularies in the contract before becoming public. +Clients can discover the current controlled vocabularies at `GET /api/v2/discovery/filters`. Filter values are allowlisted and versioned; clients must not invent category/source/profile/status values or send arbitrary free-text search. `country_code` is validated as an uppercase ISO alpha-2 code and returns zero rows when the project has no capability/source coverage for that country. New country adapters should register capabilities and vocabularies in the contract before becoming public. diff --git a/pipeline/tests/e2e/test_public_api.py b/pipeline/tests/e2e/test_public_api.py index 01ce5f1..8f4a8e0 100644 --- a/pipeline/tests/e2e/test_public_api.py +++ b/pipeline/tests/e2e/test_public_api.py @@ -59,5 +59,18 @@ def test_filter_metadata_is_versioned_and_allowlisted(self): self.assertIn("community", body["dimensions"]["profile"]["values"]) self.assertNotIn("address", body["dimensions"]) + def test_combination_filters_and_zero_result_are_deterministic(self): + rows = self.get('/api/v2/locations?country_code=DK&category=slaughter&display_precision=exact&limit=10')['data'] + self.assertEqual(len(rows), 1) + restricted = self.get('/api/v2/locations?country_code=DK&category=retail_and_prepared_food&limit=10')['data'] + self.assertEqual(restricted, []) + empty = self.get('/api/v2/locations?country_code=ZZ&limit=10') + self.assertEqual(empty['data'], []) + + def test_unknown_controlled_filter_is_rejected(self): + with self.assertRaises(urllib.error.HTTPError) as error: + self.get('/api/v2/locations?category=arbitrary') + self.assertEqual(error.exception.code, 400) + if __name__ == "__main__": unittest.main() From f0dcff27c1be94310a3689857758ac83d8b1d1d7 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sun, 13 Sep 2026 16:58:58 -0700 Subject: [PATCH 017/311] feat: establish accessible v2 research workflow --- frontend/src/app/App.svelte | 89 ++++++++++++-------------------- frontend/src/main.ts | 1 + frontend/src/styles/research.css | 1 + 3 files changed, 35 insertions(+), 56 deletions(-) create mode 100644 frontend/src/styles/research.css diff --git a/frontend/src/app/App.svelte b/frontend/src/app/App.svelte index c4491d9..5067ba9 100644 --- a/frontend/src/app/App.svelte +++ b/frontend/src/app/App.svelte @@ -15,83 +15,60 @@ let profile: Profile = 'curated'; let selected: Location | undefined = locations[0]; - let search = ''; - let region = 'all'; + let search = ''; let region = 'all'; let category = 'all'; let filters: FilterState = initialFilters; - let showMap = false; - let showExport = false; + let showMap = false; let showExport = false; let showGuidance = false; let localMode = false; let localStatus: 'idle' | 'loading' | 'ready' | 'error' | 'no-release' = 'idle'; let detailStatus: 'idle' | 'loading' | 'error' = 'idle'; - let localError = ''; - let exportError = ''; - let exportBusy = false; - let loaded: readonly Location[] = []; - let release = 'synthetic-2026.09'; - let ruleset: string | undefined; - let manifestSha256: string | undefined; - let repo = new LocalLocationRepository(); - let csvRepo = new LocalCsvExportRepository(); - $: filters = { search, region, category: 'all' }; + let localError = ''; let exportError = ''; let exportBusy = false; + let loaded: readonly Location[] = []; let release = 'synthetic-2026.09'; + let ruleset: string | undefined; let manifestSha256: string | undefined; + let repo = new LocalLocationRepository(); let csvRepo = new LocalCsvExportRepository(); + $: filters = { search, region, category }; $: source = localMode ? loaded : (profile === 'community' ? locations.slice(0, 1) : locations); $: visibleLocations = filterLocations(source, filters); $: exportPreview = previewExport(makeExportModel(visibleLocations, profile, release)); $: eligibleExport = localMode && profile === 'curated' && localStatus === 'ready' && Boolean(release); + $: profileLabel = profile === 'curated' ? 'Curated release' : 'Community claims'; const syncRoute = async () => { const route = parseRoute(window.location.hash); if (route.kind === 'location' && localMode) { detailStatus = 'loading'; - try { - const result = await repo.detail(route.facilityId, profile === 'community' ? 'community' : 'official'); - selected = result.location; release = result.releaseId; detailStatus = 'idle'; - } catch (error) { - selected = undefined; detailStatus = 'error'; localError = error instanceof Error ? error.message : 'The local detail response was rejected safely.'; - } + try { const result = await repo.detail(route.facilityId, profile === 'community' ? 'community' : 'official'); selected = result.location; release = result.releaseId; detailStatus = 'idle'; } + catch (error) { selected = undefined; detailStatus = 'error'; localError = error instanceof Error ? error.message : 'The local detail response was rejected safely.'; } } else if (route.kind === 'location') selected = source.find((item) => item.id === route.facilityId) ?? selected; else if (route.kind === 'home') selected = localMode ? loaded[0] : source[0]; }; - const loadLocal = async () => { - localStatus = 'loading'; localError = ''; - try { - const result = await repo.list(profile === 'community' ? 'community' : 'official'); - loaded = result.locations; selected = result.locations[0]; release = result.releaseId; localStatus = 'ready'; - await syncRoute(); - } catch (error) { - const kind = error && typeof error === 'object' && 'kind' in error ? (error as { kind: string }).kind : 'error'; - localStatus = kind === 'no-release' ? 'no-release' : 'error'; localError = error instanceof Error ? error.message : 'Local V2 response was rejected safely.'; loaded = []; selected = undefined; - } + localStatus = 'loading'; localError = ''; manifestSha256 = undefined; + try { const result = await repo.list(profile === 'community' ? 'community' : 'official'); loaded = result.locations; selected = result.locations[0]; release = result.releaseId; localStatus = 'ready'; await syncRoute(); } + catch (error) { const kind = error && typeof error === 'object' && 'kind' in error ? (error as { kind: string }).kind : 'error'; localStatus = kind === 'no-release' ? 'no-release' : 'error'; localError = error instanceof Error ? error.message : 'Local V2 response was rejected safely.'; loaded = []; selected = undefined; } }; - const downloadCsv = async () => { - if (!eligibleExport || exportBusy) return; - exportBusy = true; exportError = ''; - try { - const result = await csvRepo.download('official'); - release = result.releaseId; manifestSha256 = result.manifestSha256; - const url = URL.createObjectURL(new Blob([result.body], { type: 'text/csv;charset=utf-8' })); - const anchor = document.createElement('a'); anchor.href = url; anchor.download = `uec-v2-${result.releaseId}.csv`; anchor.click(); URL.revokeObjectURL(url); - } catch (error) { - const status = error && typeof error === 'object' && 'status' in error ? (error as { status: number }).status : undefined; - exportError = status === 400 ? 'Choose an explicit supported profile before exporting.' : status === 404 ? 'No eligible promoted release with a manifest is available.' : status === 429 ? 'Export is temporarily rate-limited; try again later.' : error instanceof Error ? error.message : 'The CSV export could not be prepared safely.'; - } finally { exportBusy = false; } + if (!eligibleExport || exportBusy) return; exportBusy = true; exportError = ''; + try { const result = await csvRepo.download('official'); release = result.releaseId; manifestSha256 = result.manifestSha256; const url = URL.createObjectURL(new Blob([result.body], { type: 'text/csv;charset=utf-8' })); const anchor = document.createElement('a'); anchor.href = url; anchor.download = `uec-v2-${result.releaseId}.csv`; anchor.click(); URL.revokeObjectURL(url); } + catch (error) { const status = error && typeof error === 'object' && 'status' in error ? (error as { status: number }).status : undefined; exportError = status === 400 ? 'Choose an explicit supported profile before exporting.' : status === 404 ? 'No eligible promoted release with a manifest is available.' : status === 429 ? 'Export is temporarily rate-limited; try again later.' : error instanceof Error ? error.message : 'The CSV export could not be prepared safely.'; } + finally { exportBusy = false; } }; - const select = (id: string) => { selected = source.find((item) => item.id === id) ?? selected; window.location.hash = `/locations/${id}?profile=${profile}`; }; - onMount(() => { - const params = new URLSearchParams(window.location.search); localMode = params.get('mode') === 'local-v2'; - if (localMode) { try { repo = new LocalLocationRepository(globalThis.fetch, params.get('api') ?? undefined); csvRepo = new LocalCsvExportRepository(globalThis.fetch, params.get('api') ?? ''); } catch (error) { localStatus = 'error'; localError = error instanceof Error ? error.message : 'Local API origin was rejected safely.'; } } - if (localMode && localStatus !== 'error') void loadLocal(); else if (!localMode) void syncRoute(); - const onHashChange = () => void syncRoute(); window.addEventListener('hashchange', onHashChange); return () => window.removeEventListener('hashchange', onHashChange); - }); + const profileChanged = () => { if (localMode) void loadLocal(); }; + onMount(() => { const params = new URLSearchParams(window.location.search); localMode = params.get('mode') === 'local-v2'; if (localMode) { try { const api = params.get('api') ?? undefined; repo = new LocalLocationRepository(globalThis.fetch, api); csvRepo = new LocalCsvExportRepository(globalThis.fetch, api ?? ''); } catch (error) { localStatus = 'error'; localError = error instanceof Error ? error.message : 'Local API origin was rejected safely.'; } } if (localMode && localStatus !== 'error') void loadLocal(); else if (!localMode) void syncRoute(); const onHashChange = () => void syncRoute(); window.addEventListener('hashchange', onHashChange); return () => window.removeEventListener('hashchange', onHashChange); }); Until Every Cage · evidence desk -
UNTIL EVERY CAGE V2 / FIELD NOTEEthics & safeguards ↗
-

EVIDENCE DESK · {localMode ? 'LOCAL V2 API' : 'SYNTHETIC PREVIEW'}

See what a record can—and cannot—tell us.

{localMode ? 'Explicit local integration mode. No fixture or V1 fallback is used.' : 'A quiet, inspectable view of fictional locations.'}

-
-{#if profile === 'community'}
Unreviewed community claimNot verified by Until Every Cage.
{/if} -{#if localMode && localStatus === 'loading'}
Loading the local V2 release…
{:else if localMode && (localStatus === 'error' || localStatus === 'no-release')}{:else if localMode && detailStatus === 'loading'}
Loading the selected local record…
{:else if localMode && detailStatus === 'error'}{:else}
{#if selected}

RECORD / {selected.id}

{selected.name}

{selected.region} · {selected.category}

OBSERVED{selected.observed}
MAP STATUS{selected.lat === null ? 'Not available' : 'Approximate display point'}
{/if}
{/if} -{#if localMode && localStatus === 'ready'}{/if} -
{#if showMap}{/if}{#if showExport}

IN-MEMORY EXPORT PREVIEW

{profile} · {release}

Loaded results only. This preview is not a live or complete export.

{exportPreview}
{/if}
{localMode ? 'Local V2 mode · no fixture fallback' : 'Method preview · no live requests'}
+
+
UNTIL EVERY CAGE V2 / FIELD NOTE
+

EVIDENCE DESK · {localMode ? 'LOCAL V2 API' : 'SYNTHETIC PREVIEW'}

See what a record can—and cannot—tell us.

Start with the source profile, narrow the visible evidence, then inspect what a record can—and cannot—tell us.

+

01 / DISCOVER

Choose the evidence lane

Profiles stay separate. A community claim is never silently promoted into a curated result.

+ {#if profile === 'community'}
Unreviewed community claimNot verified by Until Every Cage. Review this profile before relying on it.
{/if} + {#if localMode && localStatus === 'loading'}
Loading the {profileLabel.toLowerCase()}…
{:else if localMode && (localStatus === 'error' || localStatus === 'no-release')}{:else if localMode && detailStatus === 'loading'}
Loading the selected local record…
{:else if localMode && detailStatus === 'error'}{:else} +

02 / FILTER & COMPARE

Results {visibleLocations.length}

{localMode ? 'Current eligible response' : 'Fictional demonstration data'} · {profileLabel}

+
{#if selected}

03 / RECORD DETAIL

{selected.name}

{selected.region} · {selected.category}

OBSERVED{selected.observed}
MAP STATUS{selected.lat === null ? 'No publishable map location' : 'Approximate display point'}

READ WITH CARE

This is an observation in a particular release, not a guarantee of current operation. Source origin, factual review, privacy screening, and project approval are separate signals.

{/if}
+ {#if selected}

RECORD / {selected.id}

{/if} + {/if} + {#if localMode && localStatus === 'ready'}{/if} +
{#if showGuidance}

Do not infer closure, identity, or permission from a map point. For a correction, privacy concern, or suppression request, preserve the record ID and contact the project maintainer through the reporting channel on the ethics page. Do not include sensitive personal details in a public issue.

Read reporting guidance ↗
{/if}
+
{#if showMap}{/if}{#if showExport}

IN-MEMORY EXPORT PREVIEW

{profile} · {release}

Loaded results only. This preview is not a live or complete export.

{exportPreview}
{/if}
{localMode ? 'Local V2 mode · no fixture fallback' : 'Method preview · no live requests'}
+
diff --git a/frontend/src/main.ts b/frontend/src/main.ts index f597e29..7f399a4 100644 --- a/frontend/src/main.ts +++ b/frontend/src/main.ts @@ -1,5 +1,6 @@ import { mount } from 'svelte'; import './styles/base.css'; +import './styles/research.css'; import App from './app/App.svelte'; const target = document.getElementById('app'); if (!target) throw new Error('App mount missing'); diff --git a/frontend/src/styles/research.css b/frontend/src/styles/research.css new file mode 100644 index 0000000..66e98ed --- /dev/null +++ b/frontend/src/styles/research.css @@ -0,0 +1 @@ +.research-bar{display:grid;grid-template-columns:minmax(220px,.8fr) 1.6fr;gap:30px;border-top:1px solid #cfc4b2;border-bottom:1px solid #cfc4b2;padding:24px 0;margin-bottom:22px}.research-bar h2,.results-head h2{font:600 1.8rem Georgia,serif;margin:0 0 8px}.research-bar p:not(.eyebrow){color:#5f5549;line-height:1.5;margin:0}.research-bar .toolbar{border:0;padding:0;margin:0;display:grid;grid-template-columns:repeat(2,minmax(120px,1fr));gap:12px}.research-bar .toolbar label{min-width:0}.research-bar input,.research-bar select{display:block;width:100%;margin-top:7px;background:#fbf8f2;border:1px solid #b9ad9b;border-radius:3px;padding:10px 11px;color:#17283b;font-size:.95rem}.results-head{display:flex;justify-content:space-between;align-items:end;margin:26px 0 12px}.results-head h2 span{font:700 .85rem ui-sans-serif;color:#a34927;vertical-align:middle}.scope{color:#62594e;font-size:.8rem}.release-panel{display:flex;flex-wrap:wrap;gap:18px;border:1px solid #cfc4b2;background:#ece4d8;padding:18px;margin-top:22px}.release-panel div{min-width:130px}.release-panel span{display:block;color:#62594e;font-size:.65rem;letter-spacing:.12em;margin-bottom:5px}.release-panel strong{font-size:.88rem;overflow-wrap:anywhere}.release-panel .digest{font-family:ui-monospace,monospace;font-size:.72rem}.release-panel p{width:100%;margin:0;color:#62594e;font-size:.78rem;line-height:1.45}.export-control{display:flex;align-items:center;gap:18px;padding:15px 0;border-bottom:1px solid #cfc4b2}.export-control button,.state button{width:auto;background:#a34927;color:#fff;padding:11px 15px;border-radius:3px;font-weight:700}.export-control button:disabled{background:#b9ad9b;cursor:not-allowed}.export-control p{margin:0;color:#62594e;font-size:.78rem;line-height:1.4}.export-control .export-error{color:#a34927}.empty{padding:24px 18px;color:#62594e;line-height:1.5}.guidance{margin-top:28px;border-top:1px solid #cfc4b2;border-bottom:1px solid #cfc4b2}.guidance-toggle{align-items:center;justify-content:space-between;padding:18px 0;border:0}.guidance-toggle>span:first-child{display:flex;flex-direction:column;gap:4px}.guidance-toggle .eyebrow{margin:0}.guidance-toggle>span:last-child{color:#a34927;font-size:.8rem;font-weight:700}.guidance-body{max-width:680px;padding:0 0 20px;line-height:1.55;color:#4f5c69}.guidance-body a{color:#a34927;font-weight:700}@media(max-width:680px){.research-bar{display:block}.research-bar .toolbar{margin-top:22px}.results-head{display:block}.scope{margin-top:10px}.export-control{display:block}.export-control p{margin-top:10px}.release-panel{display:block}.release-panel div{margin-bottom:13px}} From fba1ddcde6d2cf80390acaa39edcc4ee643030e9 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sun, 13 Sep 2026 17:06:05 -0700 Subject: [PATCH 018/311] feat(api): add privacy-safe discovery facets --- docs/api/v2-contract.json | 1 + docs/api/v2-contract.md | 2 + pipeline/tests/e2e/test_public_api.py | 10 +++ src/lib.rs | 109 ++++++++++++++++++++++++++ src/main.rs | 4 + 5 files changed, 126 insertions(+) diff --git a/docs/api/v2-contract.json b/docs/api/v2-contract.json index 3365572..29c7928 100644 --- a/docs/api/v2-contract.json +++ b/docs/api/v2-contract.json @@ -8,6 +8,7 @@ "GET /api/v2/locations/{facility_id}": {"success": {"api_version": "v2", "data": {}, "meta": {}}, "not_found": 404}, "GET /api/v2/locations.csv": {"success_content_type": "text/csv", "bounded_rows": 1000, "profile_required_for_nonofficial": true, "not_found_without_promoted_manifest": 404}, "GET /api/v2/discovery/filters": {"success": {"api_version": "v2", "contract_version": "v1", "dimensions": {"country_code": "allowlist", "category": "allowlist", "source_type": "allowlist", "profile": "allowlist", "display_precision": "allowlist", "lifecycle_status": "allowlist"}}} + ,"GET /api/v2/discovery/facets": {"success": {"api_version": "v2", "meta": {"profile": "official", "release_id": "string", "filters": {}}, "dimensions": {}}, "max_values_per_dimension": 20, "counts_are_from": "selected eligible promoted public projection"} }, "error": {"shape": {"api_version": "v2", "error": {"code": "string", "message": "string"}}, "codes": ["invalid_filter", "invalid_profile", "invalid_limit", "invalid_offset", "invalid_cursor", "invalid_pagination", "database_not_configured", "database_pool_unavailable", "database_transaction_unavailable", "release_query_failed", "location_query_failed", "location_not_found", "rate_limited"]}, "privacy": "Public responses contain reviewed projection fields only; restricted records and raw evidence are never returned." diff --git a/docs/api/v2-contract.md b/docs/api/v2-contract.md index e5d435f..fc2d107 100644 --- a/docs/api/v2-contract.md +++ b/docs/api/v2-contract.md @@ -11,3 +11,5 @@ Frontend clients should branch on HTTP status and `error.code`, display `message Researchers may request `GET /api/v2/locations.csv?profile=official` (or another explicit supported profile). The export is bounded to 1,000 rows, uses deterministic CSV columns and escaping, contains only the public reviewed projection, and includes `release_profile`, `release_id`, and `manifest_sha256` on every row plus matching response headers. It is unavailable when no promoted release with a manifest exists; it never exposes raw evidence or restricted records. Clients can discover the current controlled vocabularies at `GET /api/v2/discovery/filters`. Filter values are allowlisted and versioned; clients must not invent category/source/profile/status values or send arbitrary free-text search. `country_code` is validated as an uppercase ISO alpha-2 code and returns zero rows when the project has no capability/source coverage for that country. New country adapters should register capabilities and vocabularies in the contract before becoming public. + +`GET /api/v2/discovery/facets` returns deterministic value/count pairs for the same controlled dimensions, scoped to the selected promoted profile and current public projection. It applies the supplied filters before counting, caps each dimension at 20 values, and returns no addresses, queries, raw payloads, inactive releases, or restricted records. diff --git a/pipeline/tests/e2e/test_public_api.py b/pipeline/tests/e2e/test_public_api.py index 8f4a8e0..4756172 100644 --- a/pipeline/tests/e2e/test_public_api.py +++ b/pipeline/tests/e2e/test_public_api.py @@ -59,6 +59,16 @@ def test_filter_metadata_is_versioned_and_allowlisted(self): self.assertIn("community", body["dimensions"]["profile"]["values"]) self.assertNotIn("address", body["dimensions"]) + def test_facets_apply_filters_and_never_include_restricted_record(self): + status, body = self.get('/api/v2/discovery/facets?profile=official&category=slaughter') + self.assertEqual(status, 200) + self.assertEqual(body['meta']['release_id'], 'e2e-promoted') + self.assertEqual(body['dimensions']['category'], [{'value': 'slaughter', 'count': 1}]) + self.assertNotIn('restricted', json.dumps(body)) + status, empty = self.get('/api/v2/discovery/facets?country_code=ZZ') + self.assertEqual(status, 200) + self.assertEqual(empty['dimensions']['category'], []) + def test_combination_filters_and_zero_result_are_deterministic(self): rows = self.get('/api/v2/locations?country_code=DK&category=slaughter&display_precision=exact&limit=10')['data'] self.assertEqual(len(rows), 1) diff --git a/src/lib.rs b/src/lib.rs index bd8027b..fd24d38 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -331,6 +331,115 @@ pub async fn get_v2_filter_metadata_handler() -> impl IntoResponse { }, "pagination":{"limit_max":1000,"cursor":"facility_id"}, "privacy":"Filters operate only on eligible records in the selected promoted release; filters never override suppression or publication review."})).into_response() } +pub async fn get_v2_facets_handler( + State(state): State, + Query(params): Query, +) -> impl IntoResponse { + let profile = params.profile.as_deref().unwrap_or("official"); + if !V2_PROFILES.contains(&profile) { + return v2_error( + StatusCode::BAD_REQUEST, + "invalid_profile", + "profile is unsupported", + ); + } + if params + .category + .as_deref() + .is_some_and(|v| !V2_CATEGORIES.contains(&v)) + || params + .source_type + .as_deref() + .is_some_and(|v| !V2_SOURCE_TYPES.contains(&v)) + || params + .display_precision + .as_deref() + .is_some_and(|v| !V2_PRECISIONS.contains(&v)) + || params + .lifecycle_status + .as_deref() + .is_some_and(|v| !V2_LIFECYCLES.contains(&v)) + || params + .country_code + .as_deref() + .is_some_and(|v| v.len() != 2 || !v.chars().all(|c| c.is_ascii_uppercase())) + { + return v2_error( + StatusCode::BAD_REQUEST, + "invalid_filter", + "filter is invalid", + ); + } + let Some(pool) = state.database else { + return v2_error( + StatusCode::SERVICE_UNAVAILABLE, + "database_not_configured", + "V2 database is not configured", + ); + }; + let client = match pool.get().await { + Ok(c) => c, + Err(_) => { + return v2_error( + StatusCode::SERVICE_UNAVAILABLE, + "database_pool_unavailable", + "database pool unavailable", + ); + } + }; + let release = match client.query_opt("SELECT release_id FROM uec.releases WHERE status='promoted' AND profile=$1 ORDER BY created_at DESC, release_id DESC LIMIT 1", &[&profile]).await { Ok(row) => row, Err(_) => return v2_error(StatusCode::SERVICE_UNAVAILABLE, "release_query_failed", "release query failed") }; + let Some(release) = release else { + return v2_error( + StatusCode::NOT_FOUND, + "release_not_found", + "no promoted eligible release", + ); + }; + let release_id: String = release.get(0); + let rows = match client.query("SELECT country_code, classification_category, display_precision, lifecycle_status, provenance_origin_type FROM uec.map_facilities_display_history WHERE release_id=$1 AND ($2::text IS NULL OR country_code=$2) AND ($3::text IS NULL OR classification_category=$3) AND ($4::text IS NULL OR provenance_origin_type=$4) AND ($5::text IS NULL OR display_precision=$5) AND ($6::text IS NULL OR lifecycle_status=$6)", &[&release_id, ¶ms.country_code, ¶ms.category, ¶ms.source_type, ¶ms.display_precision, ¶ms.lifecycle_status]).await { Ok(rows) => rows, Err(_) => return v2_error(StatusCode::SERVICE_UNAVAILABLE, "facets_query_failed", "public facets unavailable") }; + let mut dimensions = serde_json::Map::new(); + for (name, values) in [ + ( + "country_code", + rows.iter() + .map(|r| r.get::<_, String>(0)) + .collect::>(), + ), + ( + "category", + rows.iter().map(|r| r.get::<_, String>(1)).collect(), + ), + ( + "display_precision", + rows.iter().map(|r| r.get::<_, String>(2)).collect(), + ), + ( + "lifecycle_status", + rows.iter().map(|r| r.get::<_, String>(3)).collect(), + ), + ( + "source_type", + rows.iter().map(|r| r.get::<_, String>(4)).collect(), + ), + ] { + let mut counts = std::collections::BTreeMap::::new(); + for value in values { + *counts.entry(value).or_default() += 1; + } + dimensions.insert( + name.into(), + json!( + counts + .into_iter() + .take(20) + .map(|(value, count)| json!({"value":value,"count":count})) + .collect::>() + ), + ); + } + Json(json!({"api_version":"v2", "meta":{"profile":profile,"release_id":release_id,"filters":{"country_code":params.country_code,"category":params.category,"source_type":params.source_type,"display_precision":params.display_precision,"lifecycle_status":params.lifecycle_status}}, "dimensions":dimensions})).into_response() +} + #[derive(Deserialize)] pub struct V2LocationParams { pub country_code: Option, diff --git a/src/main.rs b/src/main.rs index 59c6184..5d2f9ff 100644 --- a/src/main.rs +++ b/src/main.rs @@ -49,6 +49,10 @@ pub fn app(state: uec_api::ApiState) -> Router { "/api/v2/discovery/filters", get(uec_api::get_v2_filter_metadata_handler), ) + .route( + "/api/v2/discovery/facets", + get(uec_api::get_v2_facets_handler), + ) .route( "/api/v2/locations/{facility_id}", get(uec_api::get_v2_location_detail_handler), From e5478d2dcdaf9559a3fa61fecfe9824760b5b778 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sun, 13 Sep 2026 17:11:03 -0700 Subject: [PATCH 019/311] feat: wire v2 discovery filter metadata --- frontend/src/api/FilterMetadataRepository.ts | 10 ++++++++++ frontend/src/api/LocalLocationRepository.ts | 20 ++++++++++++++++++- frontend/src/app/App.svelte | 14 ++++++++++--- .../unit/filterMetadataRepository.test.ts | 10 ++++++++++ 4 files changed, 50 insertions(+), 4 deletions(-) create mode 100644 frontend/src/api/FilterMetadataRepository.ts create mode 100644 frontend/tests/unit/filterMetadataRepository.test.ts diff --git a/frontend/src/api/FilterMetadataRepository.ts b/frontend/src/api/FilterMetadataRepository.ts new file mode 100644 index 0000000..b2c6462 --- /dev/null +++ b/frontend/src/api/FilterMetadataRepository.ts @@ -0,0 +1,10 @@ +import { z } from 'zod'; +import type { FetchLike } from './LocalLocationRepository'; + +const dimension = z.object({ values: z.array(z.string()), default: z.string().optional() }); +const metadataSchema = z.object({ api_version: z.literal('v2'), contract_version: z.string(), dimensions: z.object({ country_code: dimension, category: dimension, source_type: dimension, profile: dimension, display_precision: dimension, lifecycle_status: dimension }), pagination: z.object({ limit_max: z.number().int().positive(), cursor: z.string() }), privacy: z.string().min(1) }); +export type FilterMetadata = z.infer; +export class FilterMetadataRepository { + constructor(private readonly fetcher: FetchLike = globalThis.fetch, private readonly baseUrl = '') {} + async get(): Promise { const response = await this.fetcher.call(globalThis, `${this.baseUrl}/api/v2/discovery/filters`, { cache: 'no-store' }); if (!response.ok) throw Object.assign(new Error(`Filter metadata unavailable (${response.status}).`), { status: response.status }); const result = metadataSchema.safeParse(await response.json()); if (!result.success) throw new Error('Filter metadata was rejected safely.'); return result.data; } +} diff --git a/frontend/src/api/LocalLocationRepository.ts b/frontend/src/api/LocalLocationRepository.ts index f182394..ece5696 100644 --- a/frontend/src/api/LocalLocationRepository.ts +++ b/frontend/src/api/LocalLocationRepository.ts @@ -1 +1,19 @@ -import{detailEnvelopeSchema,envelopeSchema,type WireLocation}from'./wireSchema';import type{ApiError}from'./errors';import type{Location}from'../domain/location';export type FetchLike=(input:RequestInfo|URL,init?:RequestInit)=>Promise;export const localOrigin=(value:string|undefined):string|undefined=>{if(!value)return undefined;const url=new URL(value);if(url.protocol!=='http:'||!['127.0.0.1','localhost','::1'].includes(url.hostname))throw new Error('Local API origin must be loopback HTTP.');return url.origin;};const fail=(kind:ApiError['kind'],message:string,status?:number):ApiError=>Object.assign(new Error(message),status===undefined?{kind}:{kind,status});const map=(r:WireLocation):Location=>({id:r.facility_id,name:r.canonical_name,region:r.city??r.country_code,category:r.category,lat:r.latitude,lon:r.longitude,observed:r.last_observed_at??r.first_observed_at??'unknown',source:r.provenance_source_name});export class LocalLocationRepository{readonly#base:string|undefined;constructor(private readonly fetcher:FetchLike=globalThis.fetch,baseUrl?:string){this.#base=localOrigin(baseUrl);}private async json(path:string,signal?:AbortSignal){const init:RequestInit={};if(signal)init.signal=signal;const response=await this.fetcher.call(globalThis,`${this.#base??''}${path}`,init);if(!response.ok)throw fail('http',`Local V2 request failed with status ${response.status}`,response.status);try{return await response.json();}catch{throw fail('invalid-contract','Local V2 response was not valid JSON.');}}async list(profile:'official'|'secondary'|'community'='official'){try{const b=envelopeSchema.safeParse(await this.json(`/api/v2/locations?profile=${profile}`));if(!b.success||b.data.meta.profile!==profile)throw fail('invalid-contract','Local V2 list response was rejected.');if(b.data.meta.release_id===null)throw fail('no-release',b.data.meta.coverage_note);if(b.data.meta.ruleset_version===undefined||b.data.data.some(r=>r.privacy_screening_status!=='passed'||r.project_approval!=='approved'||r.release_id!==b.data.meta.release_id||r.release_ruleset_version!==b.data.meta.ruleset_version))throw fail('invalid-contract','Local V2 list snapshot was rejected.');return{locations:b.data.data.map(map),releaseId:b.data.meta.release_id,profile:b.data.meta.profile,coverageNote:b.data.meta.coverage_note,nextCursor:b.data.meta.next_cursor??null};}catch(e){if(e&&typeof e==='object'&&'kind'in e)throw e;if(e instanceof DOMException&&e.name==='AbortError')throw fail('aborted','Local V2 request was aborted.');if(e instanceof TypeError)throw fail('network','Local V2 request could not connect.');throw fail('invalid-contract','Local V2 response could not be read safely.');}}async detail(id:string,profile:'official'|'secondary'|'community'='official'){try{const b=detailEnvelopeSchema.safeParse(await this.json(`/api/v2/locations/${encodeURIComponent(id)}?profile=${profile}`));if(!b.success||b.data.meta.profile!==profile||b.data.data.privacy_screening_status!=='passed'||b.data.data.project_approval!=='approved')throw fail('invalid-contract','Local V2 detail response was rejected.');return{location:map(b.data.data),releaseId:b.data.meta.release_id,profile:b.data.meta.profile};}catch(e){if(e&&typeof e==='object'&&'kind'in e)throw e;if(e instanceof DOMException&&e.name==='AbortError')throw fail('aborted','Local V2 request was aborted.');if(e instanceof TypeError)throw fail('network','Local V2 request could not connect.');throw fail('invalid-contract','Local V2 response could not be read safely.');}}} +import { detailEnvelopeSchema, envelopeSchema, type WireLocation } from './wireSchema'; +import type { ApiError } from './errors'; +import type { Location } from '../domain/location'; + +export type FetchLike = (input: RequestInfo | URL, init?: RequestInit) => Promise; +export type LocationFilters = Readonly<{ country_code?: string | undefined; category?: string | undefined; source_type?: string | undefined; display_precision?: string | undefined; lifecycle_status?: string | undefined; cursor?: string | undefined }>; +export type LocalListResult = Readonly<{ locations: readonly Location[]; releaseId: string; profile: string; coverageNote: string; nextCursor: string | null; ruleset?: string }>; +export const localOrigin = (value: string | undefined): string | undefined => { if (!value) return undefined; const url = new URL(value); if (url.protocol !== 'http:' || !['127.0.0.1', 'localhost', '::1'].includes(url.hostname)) throw new Error('Local API origin must be loopback HTTP.'); return url.origin; }; +const fail = (kind: ApiError['kind'], message: string, status?: number): ApiError => Object.assign(new Error(message), status === undefined ? { kind } : { kind, status }); +const map = (r: WireLocation): Location => ({ id: r.facility_id, name: r.canonical_name, region: r.city ?? r.country_code, category: r.category, lat: r.latitude, lon: r.longitude, observed: r.last_observed_at ?? r.first_observed_at ?? 'unknown', source: r.provenance_source_name }); +const query = (profile: string, filters: LocationFilters) => { const params = new URLSearchParams({ profile }); for (const [key, value] of Object.entries(filters)) if (value) params.set(key, value); return `/api/v2/locations?${params}`; }; + +export class LocalLocationRepository { + readonly #base: string | undefined; + constructor(private readonly fetcher: FetchLike = globalThis.fetch, baseUrl?: string) { this.#base = localOrigin(baseUrl); } + private async json(path: string, signal?: AbortSignal) { const init: RequestInit = { cache: 'no-store' }; if (signal) init.signal = signal; const response = await this.fetcher.call(globalThis, `${this.#base ?? ''}${path}`, init); if (!response.ok) throw fail('http', `Local V2 request failed with status ${response.status}`, response.status); try { return await response.json(); } catch { throw fail('invalid-contract', 'Local V2 response was not valid JSON.'); } } + async list(profile: 'official' | 'secondary' | 'community' = 'official', filters: LocationFilters = {}): Promise { try { const b = envelopeSchema.safeParse(await this.json(query(profile, filters))); if (!b.success || b.data.meta.profile !== profile) throw fail('invalid-contract', 'Local V2 list response was rejected.'); if (b.data.meta.release_id === null) throw fail('no-release', b.data.meta.coverage_note); if (b.data.meta.ruleset_version === undefined || b.data.data.some(r => r.privacy_screening_status !== 'passed' || r.project_approval !== 'approved' || r.release_id !== b.data.meta.release_id || r.release_ruleset_version !== b.data.meta.ruleset_version)) throw fail('invalid-contract', 'Local V2 list snapshot was rejected.'); return { locations: b.data.data.map(map), releaseId: b.data.meta.release_id, profile: b.data.meta.profile, coverageNote: b.data.meta.coverage_note, nextCursor: b.data.meta.next_cursor ?? null, ruleset: b.data.meta.ruleset_version }; } catch (e) { if (e && typeof e === 'object' && 'kind' in e) throw e; if (e instanceof DOMException && e.name === 'AbortError') throw fail('aborted', 'Local V2 request was aborted.'); if (e instanceof TypeError) throw fail('network', 'Local V2 request could not connect.'); throw fail('invalid-contract', 'Local V2 response could not be read safely.'); } } + async detail(id: string, profile: 'official' | 'secondary' | 'community' = 'official') { try { const b = detailEnvelopeSchema.safeParse(await this.json(`/api/v2/locations/${encodeURIComponent(id)}?profile=${profile}`)); if (!b.success || b.data.meta.profile !== profile || b.data.data.privacy_screening_status !== 'passed' || b.data.data.project_approval !== 'approved') throw fail('invalid-contract', 'Local V2 detail response was rejected.'); return { location: map(b.data.data), releaseId: b.data.meta.release_id, profile: b.data.meta.profile }; } catch (e) { if (e && typeof e === 'object' && 'kind' in e) throw e; if (e instanceof DOMException && e.name === 'AbortError') throw fail('aborted', 'Local V2 request was aborted.'); if (e instanceof TypeError) throw fail('network', 'Local V2 request could not connect.'); throw fail('invalid-contract', 'Local V2 response could not be read safely.'); } } +} diff --git a/frontend/src/app/App.svelte b/frontend/src/app/App.svelte index 5067ba9..a2f588b 100644 --- a/frontend/src/app/App.svelte +++ b/frontend/src/app/App.svelte @@ -10,6 +10,7 @@ import { previewExport } from '../export/previewExport'; import { LocalLocationRepository } from '../api/LocalLocationRepository'; import { LocalCsvExportRepository } from '../api/LocalCsvExportRepository'; + import { FilterMetadataRepository, type FilterMetadata } from '../api/FilterMetadataRepository'; import ReleaseContext from '../ui/ReleaseContext.svelte'; import ExportControl from '../ui/ExportControl.svelte'; @@ -25,12 +26,16 @@ let loaded: readonly Location[] = []; let release = 'synthetic-2026.09'; let ruleset: string | undefined; let manifestSha256: string | undefined; let repo = new LocalLocationRepository(); let csvRepo = new LocalCsvExportRepository(); + let metadata: FilterMetadata | undefined; let metadataStatus: 'idle' | 'loading' | 'ready' | 'error' = 'idle'; $: filters = { search, region, category }; $: source = localMode ? loaded : (profile === 'community' ? locations.slice(0, 1) : locations); $: visibleLocations = filterLocations(source, filters); $: exportPreview = previewExport(makeExportModel(visibleLocations, profile, release)); $: eligibleExport = localMode && profile === 'curated' && localStatus === 'ready' && Boolean(release); $: profileLabel = profile === 'curated' ? 'Curated release' : 'Community claims'; + let lastRemoteQuery = ''; + $: remoteQuery = `${profile}|${region}|${category}|${search}`; + $: if (localMode && localStatus === 'ready' && remoteQuery !== lastRemoteQuery) { lastRemoteQuery = remoteQuery; const url = new URL(window.location.href); for (const key of ['country_code', 'category', 'q']) url.searchParams.delete(key); if (region !== 'all') url.searchParams.set('country_code', region); if (category !== 'all') url.searchParams.set('category', category); if (search.trim()) url.searchParams.set('q', search.trim()); history.replaceState(null, '', url); void loadLocal(); } const syncRoute = async () => { const route = parseRoute(window.location.hash); @@ -43,7 +48,7 @@ }; const loadLocal = async () => { localStatus = 'loading'; localError = ''; manifestSha256 = undefined; - try { const result = await repo.list(profile === 'community' ? 'community' : 'official'); loaded = result.locations; selected = result.locations[0]; release = result.releaseId; localStatus = 'ready'; await syncRoute(); } + try { const result = await repo.list(profile === 'community' ? 'community' : 'official', { country_code: region === 'all' ? undefined : region, category: category === 'all' ? undefined : category }); loaded = result.locations; selected = result.locations[0]; release = result.releaseId; ruleset = result.ruleset; localStatus = 'ready'; await syncRoute(); } catch (error) { const kind = error && typeof error === 'object' && 'kind' in error ? (error as { kind: string }).kind : 'error'; localStatus = kind === 'no-release' ? 'no-release' : 'error'; localError = error instanceof Error ? error.message : 'Local V2 response was rejected safely.'; loaded = []; selected = undefined; } }; const downloadCsv = async () => { @@ -54,14 +59,17 @@ }; const select = (id: string) => { selected = source.find((item) => item.id === id) ?? selected; window.location.hash = `/locations/${id}?profile=${profile}`; }; const profileChanged = () => { if (localMode) void loadLocal(); }; - onMount(() => { const params = new URLSearchParams(window.location.search); localMode = params.get('mode') === 'local-v2'; if (localMode) { try { const api = params.get('api') ?? undefined; repo = new LocalLocationRepository(globalThis.fetch, api); csvRepo = new LocalCsvExportRepository(globalThis.fetch, api ?? ''); } catch (error) { localStatus = 'error'; localError = error instanceof Error ? error.message : 'Local API origin was rejected safely.'; } } if (localMode && localStatus !== 'error') void loadLocal(); else if (!localMode) void syncRoute(); const onHashChange = () => void syncRoute(); window.addEventListener('hashchange', onHashChange); return () => window.removeEventListener('hashchange', onHashChange); }); + const clearFilters = () => { search = ''; region = 'all'; category = 'all'; }; + onMount(() => { const params = new URLSearchParams(window.location.search); localMode = params.get('mode') === 'local-v2'; if (localMode) { try { const api = params.get('api') ?? undefined; repo = new LocalLocationRepository(globalThis.fetch, api); csvRepo = new LocalCsvExportRepository(globalThis.fetch, api ?? ''); metadataStatus = 'loading'; void new FilterMetadataRepository(globalThis.fetch, api ?? '').get().then((value) => { metadata = value; metadataStatus = 'ready'; }).catch(() => { metadataStatus = 'error'; }); } catch (error) { localStatus = 'error'; localError = error instanceof Error ? error.message : 'Local API origin was rejected safely.'; } } if (localMode && localStatus !== 'error') void loadLocal(); else if (!localMode) void syncRoute(); const onHashChange = () => void syncRoute(); window.addEventListener('hashchange', onHashChange); return () => window.removeEventListener('hashchange', onHashChange); }); +onMount(() => { const params = new URLSearchParams(window.location.search); if (params.get('mode') === 'local-v2') { search = params.get('q') ?? ''; region = params.get('country_code') ?? 'all'; category = params.get('category') ?? 'all'; } }); Until Every Cage · evidence desk
UNTIL EVERY CAGE V2 / FIELD NOTE

EVIDENCE DESK · {localMode ? 'LOCAL V2 API' : 'SYNTHETIC PREVIEW'}

See what a record can—and cannot—tell us.

Start with the source profile, narrow the visible evidence, then inspect what a record can—and cannot—tell us.

-

01 / DISCOVER

Choose the evidence lane

Profiles stay separate. A community claim is never silently promoted into a curated result.

+

01 / DISCOVER

Choose the evidence lane

Profiles stay separate. A community claim is never silently promoted into a curated result.

{#if localMode && metadataStatus === 'loading'}

Loading available filter values…

{:else if localMode && metadataStatus === 'error'}

Filter metadata is unavailable; only unscoped discovery is shown.

{/if}
+
{search || region !== 'all' || category !== 'all' ? `Filters applied: ${[search && `name “${search}”`, region !== 'all' && region, category !== 'all' && category].filter(Boolean).join(' · ')}` : 'No additional filters applied'}
{#if profile === 'community'}
Unreviewed community claimNot verified by Until Every Cage. Review this profile before relying on it.
{/if} {#if localMode && localStatus === 'loading'}
Loading the {profileLabel.toLowerCase()}…
{:else if localMode && (localStatus === 'error' || localStatus === 'no-release')}{:else if localMode && detailStatus === 'loading'}
Loading the selected local record…
{:else if localMode && detailStatus === 'error'}{:else}

02 / FILTER & COMPARE

Results {visibleLocations.length}

{localMode ? 'Current eligible response' : 'Fictional demonstration data'} · {profileLabel}

diff --git a/frontend/tests/unit/filterMetadataRepository.test.ts b/frontend/tests/unit/filterMetadataRepository.test.ts new file mode 100644 index 0000000..f715c35 --- /dev/null +++ b/frontend/tests/unit/filterMetadataRepository.test.ts @@ -0,0 +1,10 @@ +import { describe, expect, it, vi } from 'vitest'; +import { FilterMetadataRepository } from '../../src/api/FilterMetadataRepository'; + +const valid = { api_version: 'v2', contract_version: 'v1', dimensions: { country_code: { values: ['DK'] }, category: { values: ['dairy'] }, source_type: { values: ['official'] }, profile: { values: ['official'], default: 'official' }, display_precision: { values: ['city'] }, lifecycle_status: { values: ['active_observed'] } }, pagination: { limit_max: 1000, cursor: 'facility_id' }, privacy: 'eligible only' }; +const response = (body: unknown, status = 200) => new Response(JSON.stringify(body), { status }); + +describe('FilterMetadataRepository', () => { + it('accepts the versioned allowlisted dimensions', async () => { await expect(new FilterMetadataRepository(vi.fn().mockResolvedValue(response(valid))).get()).resolves.toMatchObject({ dimensions: { category: { values: ['dairy'] } } }); }); + it('rejects malformed and unavailable metadata safely', async () => { await expect(new FilterMetadataRepository(vi.fn().mockResolvedValue(response({ api_version: 'v1' }))).get()).rejects.toThrow(/rejected safely/); await expect(new FilterMetadataRepository(vi.fn().mockResolvedValue(response({}, 503))).get()).rejects.toThrow(/503/); }); +}); From 310afadafa759a01836a3449efef80dbfa9d2dd6 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sun, 13 Sep 2026 17:19:24 -0700 Subject: [PATCH 020/311] feat: preserve v2 exploration history state --- frontend/src/app/App.svelte | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/frontend/src/app/App.svelte b/frontend/src/app/App.svelte index a2f588b..fc3fcac 100644 --- a/frontend/src/app/App.svelte +++ b/frontend/src/app/App.svelte @@ -35,7 +35,7 @@ $: profileLabel = profile === 'curated' ? 'Curated release' : 'Community claims'; let lastRemoteQuery = ''; $: remoteQuery = `${profile}|${region}|${category}|${search}`; - $: if (localMode && localStatus === 'ready' && remoteQuery !== lastRemoteQuery) { lastRemoteQuery = remoteQuery; const url = new URL(window.location.href); for (const key of ['country_code', 'category', 'q']) url.searchParams.delete(key); if (region !== 'all') url.searchParams.set('country_code', region); if (category !== 'all') url.searchParams.set('category', category); if (search.trim()) url.searchParams.set('q', search.trim()); history.replaceState(null, '', url); void loadLocal(); } + $: if (localMode && localStatus === 'ready' && remoteQuery !== lastRemoteQuery) { lastRemoteQuery = remoteQuery; const url = new URL(window.location.href); for (const key of ['country_code', 'category', 'q']) url.searchParams.delete(key); if (region !== 'all') url.searchParams.set('country_code', region); if (category !== 'all') url.searchParams.set('category', category); if (search.trim()) url.searchParams.set('q', search.trim()); history.pushState(null, '', url); void loadLocal(); } const syncRoute = async () => { const route = parseRoute(window.location.hash); @@ -61,7 +61,8 @@ const profileChanged = () => { if (localMode) void loadLocal(); }; const clearFilters = () => { search = ''; region = 'all'; category = 'all'; }; onMount(() => { const params = new URLSearchParams(window.location.search); localMode = params.get('mode') === 'local-v2'; if (localMode) { try { const api = params.get('api') ?? undefined; repo = new LocalLocationRepository(globalThis.fetch, api); csvRepo = new LocalCsvExportRepository(globalThis.fetch, api ?? ''); metadataStatus = 'loading'; void new FilterMetadataRepository(globalThis.fetch, api ?? '').get().then((value) => { metadata = value; metadataStatus = 'ready'; }).catch(() => { metadataStatus = 'error'; }); } catch (error) { localStatus = 'error'; localError = error instanceof Error ? error.message : 'Local API origin was rejected safely.'; } } if (localMode && localStatus !== 'error') void loadLocal(); else if (!localMode) void syncRoute(); const onHashChange = () => void syncRoute(); window.addEventListener('hashchange', onHashChange); return () => window.removeEventListener('hashchange', onHashChange); }); -onMount(() => { const params = new URLSearchParams(window.location.search); if (params.get('mode') === 'local-v2') { search = params.get('q') ?? ''; region = params.get('country_code') ?? 'all'; category = params.get('category') ?? 'all'; } }); + onMount(() => { const params = new URLSearchParams(window.location.search); if (params.get('mode') === 'local-v2') { search = params.get('q') ?? ''; region = params.get('country_code') ?? 'all'; category = params.get('category') ?? 'all'; } }); + onMount(() => { const syncBrowserState = () => { const params = new URLSearchParams(window.location.search); if (params.get('mode') === 'local-v2') { search = params.get('q') ?? ''; region = params.get('country_code') ?? 'all'; category = params.get('category') ?? 'all'; } const route = parseRoute(window.location.hash); if (route.kind !== 'not-found' && route.profile !== profile) profile = route.profile; }; window.addEventListener('popstate', syncBrowserState); return () => window.removeEventListener('popstate', syncBrowserState); }); Until Every Cage · evidence desk From a5f7b7fecb05273e1a73cf0ef8a4ea4723cd0984 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sun, 13 Sep 2026 17:23:03 -0700 Subject: [PATCH 021/311] feat(pipeline): integrate country adapter foundations --- pipeline/__init__.py | 1 + pipeline/adapter-capabilities.json | 7 + pipeline/common/__init__.py | 1 + pipeline/common/adapter_registry.py | 20 ++ pipeline/common/orchestrator.py | 65 +++++++ pipeline/common/test_orchestrator.py | 31 ++++ pipeline/common/test_registry.py | 17 ++ pipeline/contracts/__init__.py | 1 + pipeline/contracts/adapter_contract.py | 16 ++ pipeline/germany/README.md | 29 +++ pipeline/germany/adapter.py | 173 ++++++++++++++++++ .../germany/fixtures/synthetic_source.csv | 5 + pipeline/germany/orchestrator.py | 19 ++ pipeline/germany/source-registry.json | 14 ++ pipeline/germany/test_adapter.py | 70 +++++++ pipeline/germany/test_orchestrator.py | 73 ++++++++ pipeline/sources/__init__.py | 1 + pipeline/sources/uk/__init__.py | 1 + pipeline/sources/uk/approved/README.md | 21 +++ pipeline/sources/uk/approved/__init__.py | 3 + pipeline/sources/uk/approved/compose.py | 133 ++++++++++++++ pipeline/sources/uk/approved/test_compose.py | 93 ++++++++++ pipeline/sources/uk/fsa_approved/README.md | 25 +++ pipeline/sources/uk/fsa_approved/__init__.py | 3 + pipeline/sources/uk/fsa_approved/adapter.py | 152 +++++++++++++++ pipeline/sources/uk/fsa_approved/config.json | 19 ++ .../uk/fsa_approved/fixtures/quarantine.csv | 9 + .../uk/fsa_approved/fixtures/schema_drift.csv | 2 + .../uk/fsa_approved/fixtures/valid.csv | 4 + .../sources/uk/fsa_approved/test_adapter.py | 47 +++++ pipeline/sources/uk/fss_approved/README.md | 19 ++ pipeline/sources/uk/fss_approved/__init__.py | 3 + pipeline/sources/uk/fss_approved/adapter.py | 149 +++++++++++++++ pipeline/sources/uk/fss_approved/config.json | 16 ++ .../uk/fss_approved/fixtures/quarantine.csv | 5 + .../uk/fss_approved/fixtures/schema_drift.csv | 2 + .../uk/fss_approved/fixtures/valid.csv | 3 + .../sources/uk/fss_approved/test_adapter.py | 56 ++++++ pipeline/sources/uk/registry.json | 1 + 39 files changed, 1309 insertions(+) create mode 100644 pipeline/__init__.py create mode 100644 pipeline/adapter-capabilities.json create mode 100644 pipeline/common/__init__.py create mode 100644 pipeline/common/adapter_registry.py create mode 100644 pipeline/common/orchestrator.py create mode 100644 pipeline/common/test_orchestrator.py create mode 100644 pipeline/common/test_registry.py create mode 100644 pipeline/contracts/__init__.py create mode 100644 pipeline/contracts/adapter_contract.py create mode 100644 pipeline/germany/README.md create mode 100644 pipeline/germany/adapter.py create mode 100644 pipeline/germany/fixtures/synthetic_source.csv create mode 100644 pipeline/germany/orchestrator.py create mode 100644 pipeline/germany/source-registry.json create mode 100644 pipeline/germany/test_adapter.py create mode 100644 pipeline/germany/test_orchestrator.py create mode 100644 pipeline/sources/__init__.py create mode 100644 pipeline/sources/uk/__init__.py create mode 100644 pipeline/sources/uk/approved/README.md create mode 100644 pipeline/sources/uk/approved/__init__.py create mode 100644 pipeline/sources/uk/approved/compose.py create mode 100644 pipeline/sources/uk/approved/test_compose.py create mode 100644 pipeline/sources/uk/fsa_approved/README.md create mode 100644 pipeline/sources/uk/fsa_approved/__init__.py create mode 100644 pipeline/sources/uk/fsa_approved/adapter.py create mode 100644 pipeline/sources/uk/fsa_approved/config.json create mode 100644 pipeline/sources/uk/fsa_approved/fixtures/quarantine.csv create mode 100644 pipeline/sources/uk/fsa_approved/fixtures/schema_drift.csv create mode 100644 pipeline/sources/uk/fsa_approved/fixtures/valid.csv create mode 100644 pipeline/sources/uk/fsa_approved/test_adapter.py create mode 100644 pipeline/sources/uk/fss_approved/README.md create mode 100644 pipeline/sources/uk/fss_approved/__init__.py create mode 100644 pipeline/sources/uk/fss_approved/adapter.py create mode 100644 pipeline/sources/uk/fss_approved/config.json create mode 100644 pipeline/sources/uk/fss_approved/fixtures/quarantine.csv create mode 100644 pipeline/sources/uk/fss_approved/fixtures/schema_drift.csv create mode 100644 pipeline/sources/uk/fss_approved/fixtures/valid.csv create mode 100644 pipeline/sources/uk/fss_approved/test_adapter.py create mode 100644 pipeline/sources/uk/registry.json diff --git a/pipeline/__init__.py b/pipeline/__init__.py new file mode 100644 index 0000000..2510a78 --- /dev/null +++ b/pipeline/__init__.py @@ -0,0 +1 @@ +"""Versioned, source-first data pipelines.""" diff --git a/pipeline/adapter-capabilities.json b/pipeline/adapter-capabilities.json new file mode 100644 index 0000000..d734e33 --- /dev/null +++ b/pipeline/adapter-capabilities.json @@ -0,0 +1,7 @@ +{ + "schema_version": "adapter-capabilities-v1", + "adapters": [ + {"country_code": "gb", "source_id": "fss_approved_establishments", "adapter_version": "fss-scotland-v2-1", "schema_version": "fss-scotland-approved-v1", "adapter_path": "pipeline/sources/uk/fss_approved/adapter.py", "acquisition": "synthetic_only", "geocoding": "disabled", "publication": "human_gate_required"}, + {"country_code": "gb", "source_id": "fsa_approved_establishments", "adapter_version": "fsa-uk-v2-1", "schema_version": "fsa-uk-approved-v1", "adapter_path": "pipeline/sources/uk/fsa_approved/adapter.py", "acquisition": "synthetic_only", "geocoding": "disabled", "publication": "human_gate_required"} + ] +} diff --git a/pipeline/common/__init__.py b/pipeline/common/__init__.py new file mode 100644 index 0000000..b91ca55 --- /dev/null +++ b/pipeline/common/__init__.py @@ -0,0 +1 @@ +"""Reusable orchestration and registration primitives.""" diff --git a/pipeline/common/adapter_registry.py b/pipeline/common/adapter_registry.py new file mode 100644 index 0000000..18c8766 --- /dev/null +++ b/pipeline/common/adapter_registry.py @@ -0,0 +1,20 @@ +"""Load the versioned, source-agnostic adapter capability registry.""" +from __future__ import annotations + +import json +from pathlib import Path + + +def load(path: str | Path) -> dict: + registry = json.loads(Path(path).read_text(encoding="utf-8")) + if registry.get("schema_version") != "adapter-capabilities-v1": + raise ValueError("unsupported adapter capability schema") + seen: set[str] = set() + for entry in registry.get("adapters", []): + for field in ("country_code", "adapter_version", "schema_version", "adapter_path", "source_id"): + if not entry.get(field): + raise ValueError(f"adapter entry missing {field}") + if entry["source_id"] in seen: + raise ValueError(f"duplicate registered source: {entry['source_id']}") + seen.add(entry["source_id"]) + return registry diff --git a/pipeline/common/orchestrator.py b/pipeline/common/orchestrator.py new file mode 100644 index 0000000..7804992 --- /dev/null +++ b/pipeline/common/orchestrator.py @@ -0,0 +1,65 @@ +"""Shared restricted run coordination; it does not fetch or publish sources.""" +from __future__ import annotations + +import hashlib +import json +import os +import tempfile +from pathlib import Path +from typing import Any, Callable + +ORCHESTRATOR_VERSION = "v2-orchestrator-1" + + +def _atomic(path: Path, payload: bytes) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + fd, name = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent) + try: + with os.fdopen(fd, "wb") as handle: + handle.write(payload) + os.replace(name, path) + except Exception: + try: + os.unlink(name) + except FileNotFoundError: + pass + raise + + +def register_input(raw: bytes, staging_dir: str | Path, config: dict[str, Any]) -> tuple[Path, dict[str, Any]]: + """Register caller-supplied bytes by hash. This function never downloads.""" + staging = Path(staging_dir) + digest = hashlib.sha256(raw).hexdigest() + artifact = staging / "raw" / f"{digest}.artifact" + if not artifact.exists(): + _atomic(artifact, raw) + metadata = {**config, "checksum_sha256": digest, "byte_size": len(raw), + "orchestrator_version": ORCHESTRATOR_VERSION, "raw_artifact": str(artifact)} + _atomic(staging / "raw" / f"{digest}.manifest.json", + (json.dumps(metadata, ensure_ascii=False, sort_keys=True, indent=2) + "\n").encode()) + return artifact, metadata + + +def run_registered_input(raw_path: str | Path, runs_dir: str | Path, config: dict[str, Any], + adapter_runner: Callable[..., dict[str, Any]], + suppressed_ids: set[str] | None = None, + prior_eligible_release: dict[str, Any] | None = None) -> dict[str, Any]: + """Run an adapter to a human-gated candidate, preserving prior release on failure.""" + raw = Path(raw_path) + run_dir = Path(runs_dir) / hashlib.sha256(raw.read_bytes()).hexdigest()[:16] + try: + manifest = adapter_runner(raw, run_dir, config) + records = [json.loads(line) for line in (run_dir / "normalized" / "records.jsonl").read_text(encoding="utf-8").splitlines() if line] + suppressed = suppressed_ids or set() + candidate = [r for r in records if r.get("source_id") not in suppressed] + _atomic(run_dir / "release-candidate" / "records.jsonl", + b"".join((json.dumps(r, ensure_ascii=False, sort_keys=True) + "\n").encode() for r in candidate)) + status = {"status": "candidate-ready", "publication_state": "human-gate-required", + "release_promoted": False, "suppressed_count": len(records) - len(candidate), + "manifest": manifest, "prior_eligible_release": prior_eligible_release} + except Exception as exc: + status = {"status": "failed", "publication_state": "unchanged", "release_promoted": False, + "error_type": type(exc).__name__, "error": str(exc), + "prior_eligible_release": prior_eligible_release} + _atomic(run_dir / "run-status.json", (json.dumps(status, ensure_ascii=False, sort_keys=True, indent=2) + "\n").encode()) + return status diff --git a/pipeline/common/test_orchestrator.py b/pipeline/common/test_orchestrator.py new file mode 100644 index 0000000..9af3313 --- /dev/null +++ b/pipeline/common/test_orchestrator.py @@ -0,0 +1,31 @@ +import json +import tempfile +import unittest +from pathlib import Path + +from .adapter_registry import load +from .orchestrator import register_input, run_registered_input +from pipeline.sources.uk.fss_approved.adapter import FssApprovedEstablishmentsAdapter + + +class SharedPipelineTests(unittest.TestCase): + def test_registry_and_suppression_are_shared(self): + root = Path(__file__).parents[1] + registry = load(root / "adapter-capabilities.json") + self.assertEqual(registry["adapters"][0]["source_id"], "fss_approved_establishments") + adapter = FssApprovedEstablishmentsAdapter() + with tempfile.TemporaryDirectory() as directory: + staging = Path(directory) / "staging" + raw = (root / "sources/uk/fss_approved/fixtures/valid.csv").read_bytes() + artifact, metadata = register_input(raw, staging, {"source_id": adapter.source_id}) + self.assertEqual(artifact.read_bytes(), raw) + status = run_registered_input(artifact, Path(directory) / "runs", metadata, adapter.run, + suppressed_ids={adapter.source_id}) + self.assertEqual(status["status"], "candidate-ready") + self.assertEqual(status["suppressed_count"], 2) + self.assertEqual((status["manifest"]["release_state"]), "not-created") + self.assertEqual((status["prior_eligible_release"]), None) + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/common/test_registry.py b/pipeline/common/test_registry.py new file mode 100644 index 0000000..d7fd50d --- /dev/null +++ b/pipeline/common/test_registry.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python3 +import unittest +from pathlib import Path + +from .adapter_registry import load + + +class RegistryTests(unittest.TestCase): + def test_registered_adapters_have_versioned_capabilities(self): + registry = load(Path(__file__).parents[1] / "adapter-capabilities.json") + self.assertEqual({entry["country_code"] for entry in registry["adapters"]}, {"de", "nl"}) + self.assertTrue(all(entry["geocoding"] == "disabled" for entry in registry["adapters"])) + self.assertTrue(all(entry["publication"] == "human_gate_required" for entry in registry["adapters"])) + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/contracts/__init__.py b/pipeline/contracts/__init__.py new file mode 100644 index 0000000..9f4c4ce --- /dev/null +++ b/pipeline/contracts/__init__.py @@ -0,0 +1 @@ +"""Shared adapter contract assertions.""" diff --git a/pipeline/contracts/adapter_contract.py b/pipeline/contracts/adapter_contract.py new file mode 100644 index 0000000..f330acb --- /dev/null +++ b/pipeline/contracts/adapter_contract.py @@ -0,0 +1,16 @@ +from __future__ import annotations +import hashlib +from pathlib import Path +import json + + +def assert_manifest(manifest: dict, raw: bytes, schema_version: str) -> None: + assert manifest["checksum_sha256"] == hashlib.sha256(raw).hexdigest() + assert manifest["byte_size"] == len(raw) + assert manifest["schema_version"] == schema_version + assert manifest["input_rows"] == manifest["normalized_rows"] + manifest["quarantined_rows"] + assert manifest["release_state"] == "not-created" + + +def read_jsonl(path: Path) -> list[dict]: + return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line] diff --git a/pipeline/germany/README.md b/pipeline/germany/README.md new file mode 100644 index 0000000..9e35bcc --- /dev/null +++ b/pipeline/germany/README.md @@ -0,0 +1,29 @@ +# Germany V2 adapter foundation + +This is a local, synthetic-only foundation. It accepts a previously acquired raw +CSV path; it does not scrape, download, geocode, publish, or promote records. The +adapter writes separate `parsed/`, `normalized/`, and `quarantined/` outputs and +creates an empty `released/` state to make the publication boundary explicit. + +Run the acceptance tests from this directory: + +```text +python -m unittest -v test_adapter.py +``` + +The future acquisition runner must populate the registry fields in +[`source-registry.json`](source-registry.json), preserve the raw artifact outside +the repository where required, and record URL, retrieval time, checksum, byte size, +and source publication date. Terms, privacy/safety, suppression, legal, and project +publication approval remain named human gates. No technical pass is approval. + +The source classification mapping is intentionally narrow: `CP` and `GME` map to +`Meat Processing`, while `SH` maps to `Meat Slaughter`. Unknown codes quarantine +with their original source values. Missing coordinates remain explicitly unresolved; +the adapter never geocodes or guesses. `orchestrator.py` now provides hash-addressed +input registration, schema/version manifests, isolated runs, failure-safe candidate +handoff, and suppression application. It deliberately leaves the prior eligible +release reference unchanged on failure and never promotes or publishes a release. +Before any real data is considered, add source-specific acquisition, dependency +capture, and suppression checks across map/API/export/cache/history surfaces; terms, +privacy/safety, legal, review, and publication approval remain human gates. diff --git a/pipeline/germany/adapter.py b/pipeline/germany/adapter.py new file mode 100644 index 0000000..f6923e9 --- /dev/null +++ b/pipeline/germany/adapter.py @@ -0,0 +1,173 @@ +#!/usr/bin/env python3 +"""Synthetic-only foundation for the Germany V2 adapter. + +The adapter accepts an already-acquired local artifact. It does not fetch, geocode, +publish, or promote records. Source values are retained with every parsed and +normalized record so later review can distinguish evidence from interpretation. +""" + +from __future__ import annotations + +import csv +import hashlib +import io +import json +import os +import tempfile +from pathlib import Path +from typing import Any + +ADAPTER_VERSION = "de-v2-foundation-1" +SCHEMA_VERSION = "location-v2-foundation-1" +REQUIRED_METADATA = { + "source_url", + "retrieval_timestamp", + "checksum_sha256", + "byte_size", + "source_publication_date", +} +REQUIRED_COLUMNS = { + "source_id", + "name", + "activity_code", + "species_codes", + "street", + "city", + "zip", + "latitude", + "longitude", +} +ACTIVITY_MAP = { + "CP": "Meat Processing", + "GME": "Meat Processing", + "SH": "Meat Slaughter", +} + + +def source_metadata(raw: bytes, config: dict[str, Any]) -> dict[str, Any]: + """Build a deterministic manifest fragment for an acquired artifact.""" + return { + **config, + "checksum_sha256": hashlib.sha256(raw).hexdigest(), + "byte_size": len(raw), + "adapter_version": ADAPTER_VERSION, + "schema_version": SCHEMA_VERSION, + } + + +def _validate_metadata(metadata: dict[str, Any]) -> None: + missing = REQUIRED_METADATA - metadata.keys() + if missing: + raise ValueError(f"missing provenance fields: {', '.join(sorted(missing))}") + if not metadata.get("source_url"): + raise ValueError("source_url must be non-empty") + + +def parse(raw: bytes, metadata: dict[str, Any]) -> list[dict[str, Any]]: + """Parse CSV into evidence-preserving records; do not classify or geocode.""" + _validate_metadata(metadata) + reader = csv.DictReader(io.StringIO(raw.decode("utf-8-sig", errors="strict"))) + if not reader.fieldnames: + raise ValueError("missing source header") + missing = REQUIRED_COLUMNS - set(reader.fieldnames) + if missing: + raise ValueError(f"missing source columns: {', '.join(sorted(missing))}") + records = [] + for row_number, row in enumerate(reader, start=2): + records.append( + { + "source_id": (row["source_id"] or "").strip(), + "source_values": dict(row), + "source_row": row_number, + "provenance": metadata, + } + ) + return records + + +def normalize(records: list[dict[str, Any]]) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: + """Return normalized records and explicit quarantine records.""" + normalized: list[dict[str, Any]] = [] + quarantine: list[dict[str, Any]] = [] + for record in records: + source = record["source_values"] + activity_code = (source.get("activity_code") or "").strip() + activity = ACTIVITY_MAP.get(activity_code) + if not record["source_id"] or not (source.get("name") or "").strip(): + quarantine.append({**record, "quarantine_reason": "missing identity"}) + continue + if activity is None: + quarantine.append({**record, "quarantine_reason": "unknown activity code"}) + continue + lat_text, lon_text = (source.get("latitude") or "").strip(), (source.get("longitude") or "").strip() + coordinate_status = "source" + lat: float | None + lon: float | None + try: + lat, lon = float(lat_text), float(lon_text) + except ValueError: + lat, lon, coordinate_status = None, None, "unresolved" + if lat == 0.0 and lon == 0.0: + lat, lon, coordinate_status = None, None, "unresolved" + if lat is not None and lon is not None and not (47.0 <= lat <= 55.2 and 5.8 <= lon <= 15.1): + quarantine.append({**record, "quarantine_reason": "coordinates outside configured Germany bounds"}) + continue + normalized.append( + { + "schema_version": SCHEMA_VERSION, + "source_id": record["source_id"], + "establishment_id": record["source_id"], + "establishment_name": source["name"].strip(), + "type": activity, + "street": (source.get("street") or "").strip(), + "city": (source.get("city") or "").strip(), + "zip": (source.get("zip") or "").strip(), + "latitude": lat, + "longitude": lon, + "coordinate_status": coordinate_status, + "source_values": source, + "provenance": record["provenance"], + } + ) + return normalized, quarantine + + +def _write_jsonl_atomic(path: Path, rows: list[dict[str, Any]]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + fd, temp_name = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent) + try: + with os.fdopen(fd, "w", encoding="utf-8", newline="\n") as handle: + for row in rows: + handle.write(json.dumps(row, ensure_ascii=False, sort_keys=True) + "\n") + os.replace(temp_name, path) + except Exception: + try: + os.unlink(temp_name) + except FileNotFoundError: + pass + raise + + +def run(raw_path: Path, output_dir: Path, config: dict[str, Any]) -> dict[str, Any]: + """Process a local artifact into parsed/normalized/quarantined states only.""" + raw = raw_path.read_bytes() + metadata = source_metadata(raw, config) + if config.get("checksum_sha256") and config["checksum_sha256"] != metadata["checksum_sha256"]: + raise ValueError("source checksum does not match configured checksum") + parsed = parse(raw, metadata) + normalized, quarantine = normalize(parsed) + _write_jsonl_atomic(output_dir / "parsed" / "records.jsonl", parsed) + _write_jsonl_atomic(output_dir / "normalized" / "records.jsonl", normalized) + _write_jsonl_atomic(output_dir / "quarantined" / "records.jsonl", quarantine) + manifest = { + **metadata, + "input_rows": len(parsed), + "normalized_rows": len(normalized), + "quarantined_rows": len(quarantine), + "release_state": "not-created", + } + (output_dir / "released").mkdir(parents=True, exist_ok=True) + manifest_path = output_dir / "run-manifest.json" + manifest_path.parent.mkdir(parents=True, exist_ok=True) + manifest_path.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8") + return manifest diff --git a/pipeline/germany/fixtures/synthetic_source.csv b/pipeline/germany/fixtures/synthetic_source.csv new file mode 100644 index 0000000..c3c53fc --- /dev/null +++ b/pipeline/germany/fixtures/synthetic_source.csv @@ -0,0 +1,5 @@ +source_id,name,activity_code,species_codes,street,city,zip,latitude,longitude +DE-SYN-001,Synthetic Cutting Plant,CP,B,Pretendstrasse 1,Berlin,10115,52.5200,13.4050 +DE-SYN-002,Synthetic Slaughter House,SH,B|P,Musterweg 2,Hamburg,20095,53.5511,9.9937 +DE-SYN-003,Synthetic Unknown Facility,XX,B,Unknownweg 3,Munich,80331,48.1351,11.5820 +DE-SYN-004,Synthetic Unresolved Facility,CP,P,Unresolvedweg 4,Cologne,50667,, diff --git a/pipeline/germany/orchestrator.py b/pipeline/germany/orchestrator.py new file mode 100644 index 0000000..ab462df --- /dev/null +++ b/pipeline/germany/orchestrator.py @@ -0,0 +1,19 @@ +"""Compatibility wrapper; shared coordination lives in ``pipeline/common``.""" + +from __future__ import annotations + +try: + from .adapter import run as _germany_adapter + from common.orchestrator import register_input + from common.orchestrator import run_registered_input as _run_registered_input +except ImportError: # direct test invocation from this directory + from adapter import run as _germany_adapter + from pathlib import Path + import sys + sys.path.insert(0, str(Path(__file__).parents[1])) + from common.orchestrator import register_input + from common.orchestrator import run_registered_input as _run_registered_input + + +def run_registered_input(raw_path, runs_dir, config, prior_eligible_release=None, suppressed_ids=None, adapter_runner=None): + return _run_registered_input(raw_path, runs_dir, config, prior_eligible_release, suppressed_ids, adapter_runner or _germany_adapter) diff --git a/pipeline/germany/source-registry.json b/pipeline/germany/source-registry.json new file mode 100644 index 0000000..7d5b89f --- /dev/null +++ b/pipeline/germany/source-registry.json @@ -0,0 +1,14 @@ +{ + "country_code": "de", + "adapter_version": "de-v2-foundation-1", + "schema_version": "location-v2-foundation-1", + "source_url": "https://gis.bvl.bund.de/datenportal/", + "retrieval_timestamp": "NOT-ACQUIRED", + "source_publication_date": "UNKNOWN-UNTIL-EXPORT", + "checksum_sha256": "COMPUTED-FROM-RAW-ARTIFACT", + "byte_size": "COMPUTED-FROM-RAW-ARTIFACT", + "terms_review": "BLOCKED-PENDING-HUMAN-CONFIRMATION", + "acquisition_status": "restricted_pending_terms", + "privacy_review": "REQUIRED-HUMAN-GATE", + "publication_approval": "REQUIRED-HUMAN-GATE" +} diff --git a/pipeline/germany/test_adapter.py b/pipeline/germany/test_adapter.py new file mode 100644 index 0000000..b4bbe4f --- /dev/null +++ b/pipeline/germany/test_adapter.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 +import hashlib +import json +import tempfile +import unittest +from pathlib import Path + +from adapter import parse, normalize, run, source_metadata + + +ROOT = Path(__file__).parent +FIXTURE = ROOT / "fixtures" / "synthetic_source.csv" +CONFIG = { + "source_url": "https://example.invalid/synthetic-germany.csv", + "retrieval_timestamp": "2026-09-13T00:00:00Z", + "source_publication_date": "2026-09-12", +} + + +class GermanyAdapterTests(unittest.TestCase): + def test_provenance_and_source_values_are_preserved(self): + raw = FIXTURE.read_bytes() + metadata = source_metadata(raw, CONFIG) + records = parse(raw, metadata) + self.assertEqual(metadata["checksum_sha256"], hashlib.sha256(raw).hexdigest()) + self.assertEqual(records[0]["source_id"], "DE-SYN-001") + self.assertEqual(records[0]["source_values"]["activity_code"], "CP") + self.assertIn("source_url", records[0]["provenance"]) + + def test_classification_anomaly_is_quarantined_and_unknown_is_explicit(self): + normalized, quarantine = normalize(parse(FIXTURE.read_bytes(), source_metadata(FIXTURE.read_bytes(), CONFIG))) + self.assertEqual([row["type"] for row in normalized], ["Meat Processing", "Meat Slaughter", "Meat Processing"]) + self.assertEqual(len(quarantine), 1) + self.assertEqual(quarantine[0]["quarantine_reason"], "unknown activity code") + unresolved = next(row for row in normalized if row["source_id"] == "DE-SYN-004") + self.assertEqual(unresolved["coordinate_status"], "unresolved") + self.assertIsNone(unresolved["latitude"]) + + def test_rerun_is_deterministic(self): + with tempfile.TemporaryDirectory() as directory: + first = run(FIXTURE, Path(directory) / "one", CONFIG) + second = run(FIXTURE, Path(directory) / "two", CONFIG) + self.assertEqual(first, second) + for state in ("parsed", "normalized", "quarantined"): + one = (Path(directory) / "one" / state / "records.jsonl").read_bytes() + two = (Path(directory) / "two" / state / "records.jsonl").read_bytes() + self.assertEqual(one, two) + + def test_failed_input_does_not_replace_previous_outputs(self): + with tempfile.TemporaryDirectory() as directory: + output = Path(directory) / "out" + run(FIXTURE, output, CONFIG) + before = (output / "normalized" / "records.jsonl").read_bytes() + bad = Path(directory) / "bad.csv" + bad.write_text("source_id,name\nBROKEN,row\n", encoding="utf-8") + with self.assertRaises(ValueError): + run(bad, output, CONFIG) + self.assertEqual(before, (output / "normalized" / "records.jsonl").read_bytes()) + + def test_no_release_is_created_by_adapter(self): + with tempfile.TemporaryDirectory() as directory: + output = Path(directory) / "out" + manifest = run(FIXTURE, output, CONFIG) + self.assertEqual(manifest["release_state"], "not-created") + self.assertFalse((output / "released" / "records.jsonl").exists()) + self.assertEqual(json.loads((output / "run-manifest.json").read_text())["quarantined_rows"], 1) + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/germany/test_orchestrator.py b/pipeline/germany/test_orchestrator.py new file mode 100644 index 0000000..def01d3 --- /dev/null +++ b/pipeline/germany/test_orchestrator.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +import json +import tempfile +import unittest +from pathlib import Path + +from orchestrator import register_input, run_registered_input + + +ROOT = Path(__file__).parent +FIXTURE = ROOT / "fixtures" / "synthetic_source.csv" +CONFIG = { + "source_url": "https://example.invalid/synthetic-germany.csv", + "retrieval_timestamp": "2026-09-13T00:00:00Z", + "source_publication_date": "2026-09-12", +} + + +class OrchestratorTests(unittest.TestCase): + def test_hash_addressed_registration_is_idempotent(self): + with tempfile.TemporaryDirectory() as directory: + staging = Path(directory) + raw = FIXTURE.read_bytes() + first_path, first = register_input(raw, staging, CONFIG) + second_path, second = register_input(raw, staging, CONFIG) + self.assertEqual(first_path, second_path) + self.assertEqual(first, second) + self.assertEqual(len(list((staging / "raw").glob("*.artifact"))), 1) + + def test_suppression_survives_rerun_and_candidate_handoff(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + raw_path, _ = register_input(FIXTURE.read_bytes(), root, CONFIG) + prior = {"release_id": "de-release-previous", "eligible": True} + suppressed = {"DE-SYN-002"} + first = run_registered_input(raw_path, root / "runs-one", CONFIG, prior, suppressed) + second = run_registered_input(raw_path, root / "runs-two", CONFIG, prior, suppressed) + self.assertEqual(first["status"], "candidate-ready") + self.assertEqual(first["publication_state"], "human-gate-required") + self.assertFalse(first["release_promoted"]) + for run_root in (root / "runs-one", root / "runs-two"): + run_dir = next(run_root.iterdir()) + rows = [json.loads(line) for line in (run_dir / "release-candidate" / "records.jsonl").read_text().splitlines()] + self.assertNotIn("DE-SYN-002", {row["source_id"] for row in rows}) + self.assertEqual(first["manifest"], second["manifest"]) + + def test_failure_preserves_prior_reference_and_emits_no_candidate(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + bad = root / "bad.csv" + bad.write_text("source_id,name\nBROKEN,row\n", encoding="utf-8") + prior = {"release_id": "de-release-previous", "eligible": True} + result = run_registered_input(bad, root / "runs", CONFIG, prior) + self.assertEqual(result["status"], "failed") + self.assertEqual(result["publication_state"], "unchanged") + self.assertEqual(result["prior_eligible_release"], prior) + run_dir = next((root / "runs").iterdir()) + self.assertFalse((run_dir / "release-candidate").exists()) + + def test_pending_terms_can_stage_but_cannot_create_candidate(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + raw_path, _ = register_input(FIXTURE.read_bytes(), root, CONFIG) + restricted = {**CONFIG, "terms_status": "pending_confirmation", "acquisition_status": "restricted_pending_terms"} + result = run_registered_input(raw_path, root / "runs", restricted) + self.assertEqual(result["status"], "staged-restricted") + self.assertFalse(result["candidate_created"]) + run_dir = next((root / "runs").iterdir()) + self.assertFalse((run_dir / "release-candidate").exists()) + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/sources/__init__.py b/pipeline/sources/__init__.py new file mode 100644 index 0000000..2dd46b4 --- /dev/null +++ b/pipeline/sources/__init__.py @@ -0,0 +1 @@ +"""Source adapters.""" diff --git a/pipeline/sources/uk/__init__.py b/pipeline/sources/uk/__init__.py new file mode 100644 index 0000000..22fa2b4 --- /dev/null +++ b/pipeline/sources/uk/__init__.py @@ -0,0 +1 @@ +"""United Kingdom source adapters.""" diff --git a/pipeline/sources/uk/approved/README.md b/pipeline/sources/uk/approved/README.md new file mode 100644 index 0000000..13c8d0f --- /dev/null +++ b/pipeline/sources/uk/approved/README.md @@ -0,0 +1,21 @@ +# UK approved-establishments country composition + +This layer consumes restricted normalized outputs from the distinct FSS +Scotland and FSA England/Wales/Northern Ireland adapters. It keeps the source +record and its complete source run manifest nested under each composed row; +source IDs, nation, authority, source-specific lifecycle, terms state and +review state are never flattened or overwritten. + +No records are automatically merged across FSS/FSA or across nations. A +possible-match review signal is emitted only for an exact normalized trading +name and postcode match across different sources and nations, and carries +references to both records rather than combining them. Suppression is keyed +by `(source_id, source_record_id)` and therefore survives reimport without +silently suppressing a similarly numbered record from another jurisdiction. + +The composition manifest is always `release_state: not-created` and +`candidate_created: false`. Unconfirmed terms or incomplete project review +block any candidate/release state. Missing or failed source outputs and +incompatible adapter schema versions fail closed before replacing a prior +view. This is restricted review infrastructure only: no downloads, farms, +geocoding, public API/export, promotion, or release is performed. diff --git a/pipeline/sources/uk/approved/__init__.py b/pipeline/sources/uk/approved/__init__.py new file mode 100644 index 0000000..e6f20d8 --- /dev/null +++ b/pipeline/sources/uk/approved/__init__.py @@ -0,0 +1,3 @@ +from .compose import CompositionError, compose_sources + +__all__ = ["CompositionError", "compose_sources"] diff --git a/pipeline/sources/uk/approved/compose.py b/pipeline/sources/uk/approved/compose.py new file mode 100644 index 0000000..f48ca43 --- /dev/null +++ b/pipeline/sources/uk/approved/compose.py @@ -0,0 +1,133 @@ +"""Restricted country view over the distinct UK FSS and FSA source outputs.""" +from __future__ import annotations + +import hashlib +import json +import os +import re +from pathlib import Path +from typing import Any + +from pipeline.common.adapter_registry import load as load_registry +from pipeline.common.orchestrator import ORCHESTRATOR_VERSION + +ROOT = Path(__file__).parents[3] +REGISTRY_PATH = ROOT / "adapter-capabilities.json" +EXPECTED_SCHEMA = { + "fss_approved_establishments": "fss-scotland-approved-v1", + "fsa_approved_establishments": "fsa-uk-approved-v1", +} + + +class CompositionError(ValueError): + """Inputs cannot safely form a country review view.""" + + +def _atomic(path: Path, payload: bytes) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_name(path.name + ".tmp") + temporary.write_bytes(payload) + os.replace(temporary, path) + + +def _read_jsonl(path: Path) -> list[dict[str, Any]]: + if not path.exists(): + raise CompositionError(f"source output unavailable: {path}") + return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line] + + +def _key(value: str | None) -> str: + return re.sub(r"\s+", " ", (value or "").strip()).casefold() + + +def _source_record_id(source_id: str, record: dict[str, Any]) -> str | None: + normalized = record.get("normalized", {}) + field = "approval_number" if source_id == "fss_approved_establishments" else "establishment_id" + value = normalized.get(field) + return value if isinstance(value, str) else None + + +def _possible_match(left: dict[str, Any], right: dict[str, Any]) -> bool: + a, b = left["source_record"], right["source_record"] + na, nb = a.get("normalized", {}), b.get("normalized", {}) + if left["source_id"] == right["source_id"] or na.get("nation") == nb.get("nation"): + return False + return bool(_key(na.get("trading_name")) and _key(na.get("trading_name")) == _key(nb.get("trading_name")) + and _key(na.get("postcode")) and _key(na.get("postcode")) == _key(nb.get("postcode"))) + + +def compose_sources(inputs: list[dict[str, Any]], output_dir: str | Path, + suppressed: set[tuple[str, str]] | None = None, + prior_view: dict[str, Any] | None = None) -> dict[str, Any]: + """Compose source outputs without merging them; output remains non-release.""" + if not inputs: + raise CompositionError("at least one source output is required") + registry = load_registry(REGISTRY_PATH) + registered = {entry["source_id"]: entry for entry in registry["adapters"]} + seen_sources: set[str] = set() + items: list[dict[str, Any]] = [] + try: + for item in inputs: + source_id = item.get("source_id") + manifest = item.get("manifest") or {} + if source_id in seen_sources: + raise CompositionError(f"duplicate source input: {source_id}") + if source_id not in EXPECTED_SCHEMA or source_id not in registered: + raise CompositionError(f"unregistered source: {source_id}") + if manifest.get("schema_version") != EXPECTED_SCHEMA[source_id]: + raise CompositionError(f"incompatible schema/version for {source_id}") + if manifest.get("release_state") != "not-created": + raise CompositionError(f"source is not restricted/reviewable: {source_id}") + records = _read_jsonl(Path(item["normalized_path"])) + seen_sources.add(source_id) + source_state = { + "terms_state": item.get("terms_state", "unresolved"), + "review_state": item.get("review_state", "human-review-required"), + "acquisition_state": item.get("acquisition_state", "synthetic-only"), + "manifest": manifest, + } + for record in records: + record_id = _source_record_id(source_id, record) + if not record_id: + raise CompositionError(f"source record lacks stable identifier: {source_id}") + if (source_id, record_id) in (suppressed or set()): + continue + items.append({"source_id": source_id, "source_record_id": record_id, + "source_record": record, "source_state": source_state}) + except (KeyError, TypeError, json.JSONDecodeError) as exc: + raise CompositionError("invalid source output") from exc + + items.sort(key=lambda row: (row["source_id"], row["source_record_id"], row["source_record"]["source_row"])) + signals = [] + for index, left in enumerate(items): + for right in items[index + 1:]: + if _possible_match(left, right): + signals.append({"type": "possible_match_review", "record_refs": [ + {"source_id": left["source_id"], "source_record_id": left["source_record_id"]}, + {"source_id": right["source_id"], "source_record_id": right["source_record_id"]}], + "basis": "exact normalized trading name and postcode; no automatic merge"}) + blockers = sorted({f"{row['source_id']}:{row['source_state']['terms_state']}" for row in items + if row["source_state"]["terms_state"] != "confirmed"}) + blockers.extend(sorted({f"{row['source_id']}:review-required" for row in items + if row["source_state"]["review_state"] != "project-approved"})) + output = Path(output_dir) + _write_jsonl(output / "reviewable" / "records.jsonl", items) + _write_jsonl(output / "reviewable" / "possible-match-signals.jsonl", signals) + _write_jsonl(output / "quarantined" / "records.jsonl", []) + (output / "released").mkdir(parents=True, exist_ok=True) + manifest = {"composition_id": hashlib.sha256(json.dumps(items, sort_keys=True, default=list).encode()).hexdigest(), + "orchestrator_version": ORCHESTRATOR_VERSION, "source_ids": sorted(seen_sources), + "input_rows": len(items), "possible_match_signals": len(signals), + "terms_state": "blocked" if blockers else "confirmed", + "review_state": "blocked" if blockers else "project-approved", + "release_state": "not-created", "candidate_created": False, + "publication_state": "human-gate-required", "blockers": blockers} + _atomic(output / "manifest.json", (json.dumps(manifest, ensure_ascii=False, sort_keys=True, indent=2) + "\n").encode()) + _atomic(output / "status.json", (json.dumps({"status": "reviewable-restricted", "manifest": manifest, + "prior_view": prior_view}, ensure_ascii=False, + sort_keys=True, indent=2) + "\n").encode()) + return manifest + + +def _write_jsonl(path: Path, rows: list[dict[str, Any]]) -> None: + _atomic(path, b"".join((json.dumps(row, ensure_ascii=False, sort_keys=True, default=list) + "\n").encode() for row in rows)) diff --git a/pipeline/sources/uk/approved/test_compose.py b/pipeline/sources/uk/approved/test_compose.py new file mode 100644 index 0000000..bb3d4a5 --- /dev/null +++ b/pipeline/sources/uk/approved/test_compose.py @@ -0,0 +1,93 @@ +import json +import tempfile +import unittest +from pathlib import Path + +from ..fsa_approved.adapter import FsaApprovedEstablishmentsAdapter +from ..fss_approved.adapter import FssApprovedEstablishmentsAdapter +from .compose import CompositionError, compose_sources + +BASE = Path(__file__).parents[4] / "pipeline/sources/uk" +FSS_FIXTURE = BASE / "fss_approved/fixtures/valid.csv" +FSA_FIXTURE = BASE / "fsa_approved/fixtures/valid.csv" + + +class UkCompositionTests(unittest.TestCase): + def _runs(self, root): + fss_dir, fsa_dir = root / "fss", root / "fsa" + fss = FssApprovedEstablishmentsAdapter() + fsa = FsaApprovedEstablishmentsAdapter() + fss_manifest = fss.run(FSS_FIXTURE, fss_dir) + fsa_manifest = fsa.run(FSA_FIXTURE, fsa_dir) + return [ + {"source_id": fss.source_id, "manifest": fss_manifest, "normalized_path": fss_dir / "normalized/records.jsonl", "terms_state": "unresolved", "review_state": "human-review-required"}, + {"source_id": fsa.source_id, "manifest": fsa_manifest, "normalized_path": fsa_dir / "normalized/records.jsonl", "terms_state": "unresolved", "review_state": "human-review-required"}, + ] + + def test_mixed_sources_preserve_provenance_and_duplicate_ids_by_jurisdiction(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + inputs = self._runs(root) + manifest = compose_sources(inputs, root / "country") + rows = [json.loads(line) for line in (root / "country/reviewable/records.jsonl").read_text().splitlines()] + self.assertEqual(manifest["source_ids"], ["fsa_approved_establishments", "fss_approved_establishments"]) + self.assertEqual(len(rows), 5) + fsa_rows = [r for r in rows if r["source_id"] == "fsa_approved_establishments"] + self.assertEqual({r["source_record_id"] for r in fsa_rows}, {"00017", "NI-004"}) + self.assertEqual({r["source_record"]["normalized"]["nation"] for r in fsa_rows}, {"England", "Wales", "Northern Ireland"}) + self.assertTrue(all("manifest" in row["source_state"] for row in rows)) + self.assertFalse((root / "country/released/records.jsonl").exists()) + + def test_source_specific_suppression_survives_reimport(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + inputs = self._runs(root) + suppressed = {("fsa_approved_establishments", "00017")} + for suffix in ("one", "reimport"): + compose_sources(inputs, root / suffix, suppressed=suppressed) + rows = [json.loads(line) for line in (root / suffix / "reviewable/records.jsonl").read_text().splitlines()] + self.assertFalse(any(r["source_id"] == "fsa_approved_establishments" and r["source_record_id"] == "00017" for r in rows)) + self.assertTrue(any(r["source_id"] == "fss_approved_establishments" for r in rows)) + + def test_exact_cross_source_name_postcode_is_signal_not_merge(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + inputs = self._runs(root) + fss_path = inputs[0]["normalized_path"] + fss_record = json.loads(fss_path.read_text().splitlines()[0]) + fss_record["normalized"]["trading_name"] = "East March Foods" + fss_record["normalized"]["postcode"] = "PE1 2AB" + fss_path.write_text(json.dumps(fss_record) + "\n") + manifest = compose_sources(inputs, root / "country") + signals = [json.loads(line) for line in (root / "country/reviewable/possible-match-signals.jsonl").read_text().splitlines()] + self.assertEqual(manifest["possible_match_signals"], 1) + self.assertEqual(len(signals[0]["record_refs"]), 2) + self.assertFalse(any("merged" in row for row in signals)) + + def test_outage_and_incompatible_version_do_not_replace_prior_view(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + inputs = self._runs(root) + prior = root / "country/manifest.json" + prior.parent.mkdir(parents=True) + prior.write_text("previous-view\n") + missing = [dict(inputs[0]), dict(inputs[1], normalized_path=root / "missing.jsonl")] + with self.assertRaises(CompositionError): + compose_sources(missing, root / "country", prior_view={"manifest": "previous"}) + self.assertEqual(prior.read_text(), "previous-view\n") + incompatible = [dict(inputs[0]), dict(inputs[1], manifest={**inputs[1]["manifest"], "schema_version": "wrong"})] + with self.assertRaises(CompositionError): + compose_sources(incompatible, root / "other") + + def test_terms_and_review_gates_block_candidate_and_release(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + manifest = compose_sources(self._runs(root), root / "country") + self.assertFalse(manifest["candidate_created"]) + self.assertEqual(manifest["release_state"], "not-created") + self.assertIn("fsa_approved_establishments:unresolved", manifest["blockers"]) + self.assertEqual(manifest["publication_state"], "human-gate-required") + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/sources/uk/fsa_approved/README.md b/pipeline/sources/uk/fsa_approved/README.md new file mode 100644 index 0000000..c38cb1a --- /dev/null +++ b/pipeline/sources/uk/fsa_approved/README.md @@ -0,0 +1,25 @@ +# FSA England, Wales and Northern Ireland approved establishments + +This adapter is synthetic-fixture-only. Its CSV columns and authority/nation +mapping are explicit test assumptions, not an inferred live FSA artifact +schema. FSS Scotland remains a separate source and adapter; no FSS records or +configuration are merged into this capability. + +The adapter preserves every source cell and identifier string, including +leading zeroes, and emits no guessed coordinates. Establishment IDs are +unique only within a nation, allowing authority datasets to be isolated even +when identifiers repeat across nations. Authority/nation mismatches, schema +drift, duplicate or missing IDs, unknown activities/statuses, malformed rows, +remarks and privacy-risk addresses are quarantined or fail closed. + +## FSA-specific acquisition gates + +Before any acquisition, a maintainer must verify separately for England, +Wales and Northern Ireland: the current artifact URL and format, publication +and effective dates, FSA/department ownership, terms/licence, attribution +requirements, update automation/rate limits, raw-artifact retention and +removal rules, and whether the source permits redistribution. No live schema, +download, automation, geocoding, release, public API/export, or external +contact is authorized by this fixture contract. Privacy/suppression review, +human factual review, project approval and publication remain independent +gates. diff --git a/pipeline/sources/uk/fsa_approved/__init__.py b/pipeline/sources/uk/fsa_approved/__init__.py new file mode 100644 index 0000000..27f6bfe --- /dev/null +++ b/pipeline/sources/uk/fsa_approved/__init__.py @@ -0,0 +1,3 @@ +from .adapter import FsaApprovedEstablishmentsAdapter, FsaContractError + +__all__ = ["FsaApprovedEstablishmentsAdapter", "FsaContractError"] diff --git a/pipeline/sources/uk/fsa_approved/adapter.py b/pipeline/sources/uk/fsa_approved/adapter.py new file mode 100644 index 0000000..a53a500 --- /dev/null +++ b/pipeline/sources/uk/fsa_approved/adapter.py @@ -0,0 +1,152 @@ +"""Synthetic-only FSA England/Wales/Northern Ireland adapter. + +The CSV shape is a pinned test contract, not a claim about a live FSA file. +""" +from __future__ import annotations +import csv +import hashlib +import json +import os +import re +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Any + +ROOT = Path(__file__).parent +CONFIG = json.loads((ROOT / "config.json").read_text(encoding="utf-8")) +REQUIRED_COLUMNS = tuple(CONFIG["required_columns"]) +ALLOWED_ACTIVITIES = frozenset(CONFIG["allowed_activities"]) +ALLOWED_STATUSES = frozenset(CONFIG["allowed_statuses"]) +AUTHORITY_BY_NATION = CONFIG["authority_by_nation"] +ADDRESS_RISK = re.compile(r"\b(flat|apartment|house|home|residential|c/o|care of|caravan|lodge)\b", re.I) + + +class FsaContractError(ValueError): + """The supplied artifact cannot be interpreted under the assumed contract.""" + + +@dataclass(frozen=True) +class ValidationResult: + accepted: tuple[dict[str, Any], ...] + quarantined: tuple[dict[str, Any], ...] + source_sha256: str + contract_version: str = CONFIG["contract_version"] + release_allowed: bool = False + + def as_dict(self) -> dict[str, Any]: + return asdict(self) + + +def _clean(value: str | None) -> str | None: + if value is None: + return None + value = value.strip() + return value or None + + +def _split(value: str | None) -> tuple[str, ...]: + return tuple(part for part in (_clean(item) for item in (value or "").split(";")) if part) + + +def _record(row: dict[str, str], line: int) -> dict[str, Any]: + nation = _clean(row.get("nation")) + return {"source_id": CONFIG["source_id"], "source_row": line, + "source_values": dict(row), "normalized": { + "establishment_id": _clean(row.get("establishment_id")), + "trading_name": _clean(row.get("trading_name")), + "address_lines": tuple(_clean(row.get(f"address_line_{n}")) for n in range(1, 4)), + "postcode": _clean(row.get("postcode")), "activities": _split(row.get("activities")), + "species": _clean(row.get("species")), + "competent_authority": _clean(row.get("competent_authority")), + "nation": nation, "authority_nation_key": nation, + "status": _clean(row.get("status")), "remarks": _clean(row.get("remarks")), + "published_date": _clean(row.get("published_date")), "coordinates": None}} + + +class FsaApprovedEstablishmentsAdapter: + source_id = CONFIG["source_id"] + schema_version = CONFIG["contract_version"] + adapter_version = CONFIG["adapter_version"] + + def parse_bytes(self, content: bytes) -> ValidationResult: + digest = hashlib.sha256(content).hexdigest() + try: + text = content.decode("utf-8-sig") + reader = csv.DictReader(text.splitlines(), strict=True) + if tuple(reader.fieldnames or ()) != REQUIRED_COLUMNS: + raise FsaContractError("schema drift: expected pinned synthetic FSA columns in exact order") + rows = list(reader) + except UnicodeDecodeError as exc: + raise FsaContractError("source is not UTF-8 CSV") from exc + except csv.Error as exc: + raise FsaContractError("malformed CSV") from exc + if any(None in row for row in rows): + raise FsaContractError("schema drift: a row has extra columns") + keys = [(_clean(row.get("nation")), _clean(row.get("establishment_id"))) for row in rows] + duplicates = {key for key in keys if key[0] and key[1] and keys.count(key) > 1} + accepted: list[dict[str, Any]] = [] + quarantined: list[dict[str, Any]] = [] + for line, row in enumerate(rows, 2): + reasons: list[str] = [] + nation = _clean(row.get("nation")) + identifier = _clean(row.get("establishment_id")) + if any(value is None for value in row.values()): + reasons.append("malformed_row") + if not identifier: + reasons.append("missing_establishment_id") + if (nation, identifier) in duplicates: + reasons.append("duplicate_id_within_nation") + if nation not in CONFIG["covered_nations"]: + reasons.append("unknown_nation") + authority = _clean(row.get("competent_authority")) + if nation in AUTHORITY_BY_NATION and authority != AUTHORITY_BY_NATION[nation]: + reasons.append("authority_nation_mismatch") + activities = _split(row.get("activities")) + if not activities: + reasons.append("missing_activity") + elif any(activity not in ALLOWED_ACTIVITIES for activity in activities): + reasons.append("unknown_activity") + status = _clean(row.get("status")) + if status and status.lower() not in ALLOWED_STATUSES: + reasons.append("unknown_status") + if _clean(row.get("remarks")): + reasons.append("remarks_present") + address = " ".join(_clean(row.get(f"address_line_{n}")) or "" for n in range(1, 4)) + if ADDRESS_RISK.search(address): + reasons.append("address_privacy_risk") + record = _record(row, line) + (quarantined if reasons else accepted).append({"reasons": tuple(reasons), "record": record} if reasons else record) + return ValidationResult(tuple(accepted), tuple(quarantined), digest) + + def parse_file(self, path: str | Path) -> ValidationResult: + return self.parse_bytes(Path(path).read_bytes()) + + def run(self, raw_path: str | Path, run_dir: str | Path, config: dict[str, Any] | None = None) -> dict[str, Any]: + raw = Path(raw_path).read_bytes() + result = self.parse_bytes(raw) + root = Path(run_dir) + _write_jsonl(root / "parsed" / "records.jsonl", list(result.accepted) + [item["record"] for item in result.quarantined]) + _write_jsonl(root / "normalized" / "records.jsonl", list(result.accepted)) + _write_jsonl(root / "quarantined" / "records.jsonl", list(result.quarantined)) + (root / "released").mkdir(parents=True, exist_ok=True) + manifest = {"source_id": self.source_id, "adapter_version": self.adapter_version, + "schema_version": self.schema_version, "schema_status": CONFIG["schema_status"], + "checksum_sha256": result.source_sha256, "byte_size": len(raw), + "input_rows": len(result.accepted) + len(result.quarantined), + "normalized_rows": len(result.accepted), "quarantined_rows": len(result.quarantined), + "release_state": "not-created", "publication_state": "human-gate-required", + "acquisition": CONFIG["acquisition"], "source_url": (config or {}).get("source_url"), + "retrieved_at": (config or {}).get("retrieved_at")} + _atomic(root / "manifest.json", (json.dumps(manifest, ensure_ascii=False, sort_keys=True, indent=2) + "\n").encode()) + return manifest + + +def _write_jsonl(path: Path, rows: list[dict[str, Any]]) -> None: + _atomic(path, b"".join((json.dumps(row, ensure_ascii=False, sort_keys=True, default=list) + "\n").encode() for row in rows)) + + +def _atomic(path: Path, payload: bytes) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_name(path.name + ".tmp") + temporary.write_bytes(payload) + os.replace(temporary, path) diff --git a/pipeline/sources/uk/fsa_approved/config.json b/pipeline/sources/uk/fsa_approved/config.json new file mode 100644 index 0000000..62cfcf7 --- /dev/null +++ b/pipeline/sources/uk/fsa_approved/config.json @@ -0,0 +1,19 @@ +{ + "contract_version": "fsa-uk-approved-v1", + "adapter_version": "fsa-uk-v2-1", + "source_id": "fsa_approved_establishments", + "authority": "Food Standards Agency", + "covered_nations": ["England", "Wales", "Northern Ireland"], + "authority_by_nation": { + "England": "Food Standards Agency", + "Wales": "Food Standards Agency", + "Northern Ireland": "Food Standards Agency Northern Ireland" + }, + "format": "csv", + "schema_status": "synthetic-assumption-pending-live-verification", + "acquisition": "synthetic-fixture-only", + "release_allowed_by_default": false, + "allowed_activities": ["slaughter", "cutting", "processing", "storage"], + "allowed_statuses": ["active", "inactive", "suspended", "closed"], + "required_columns": ["establishment_id", "trading_name", "address_line_1", "address_line_2", "address_line_3", "postcode", "activities", "species", "competent_authority", "nation", "status", "remarks", "published_date", "source_url", "source_licence"] +} diff --git a/pipeline/sources/uk/fsa_approved/fixtures/quarantine.csv b/pipeline/sources/uk/fsa_approved/fixtures/quarantine.csv new file mode 100644 index 0000000..a4a9bf4 --- /dev/null +++ b/pipeline/sources/uk/fsa_approved/fixtures/quarantine.csv @@ -0,0 +1,9 @@ +establishment_id,trading_name,address_line_1,address_line_2,address_line_3,postcode,activities,species,competent_authority,nation,status,remarks,published_date,source_url,source_licence +00017,Duplicate England,Works,,,PE1 2AB,processing,pig,Food Standards Agency,England,,,2026-09-01,https://example.invalid/fsa,terms-pending +00017,Duplicate England Again,Works,,,PE1 2AB,processing,pig,Food Standards Agency,England,,,2026-09-01,https://example.invalid/fsa,terms-pending +00017,Same Identifier Wales,Works,,,CF1 2CD,processing,pig,Food Standards Agency,Wales,,,2026-09-01,https://example.invalid/fsa,terms-pending +,Missing Identifier,Works,,,BT1 2EF,slaughter,pig,Food Standards Agency Northern Ireland,Northern Ireland,,,2026-09-01,https://example.invalid/fsa,terms-pending +NI-005,Unknown Activity,Works,,,BT1 2EF,rendering,pig,Food Standards Agency Northern Ireland,Northern Ireland,,,2026-09-01,https://example.invalid/fsa,terms-pending +NI-006,Unknown Status,Works,,,BT1 2EF,storage,pig,Food Standards Agency Northern Ireland,Northern Ireland,operating,,2026-09-01,https://example.invalid/fsa,terms-pending +NI-007,Private Location,Flat 2,Home Road,,BT1 2EF,storage,pig,Food Standards Agency Northern Ireland,Northern Ireland,,,2026-09-01,https://example.invalid/fsa,terms-pending +NI-008,Wrong Authority,Works,,,BT1 2EF,storage,pig,Food Standards Agency,Northern Ireland,,,2026-09-01,https://example.invalid/fsa,terms-pending diff --git a/pipeline/sources/uk/fsa_approved/fixtures/schema_drift.csv b/pipeline/sources/uk/fsa_approved/fixtures/schema_drift.csv new file mode 100644 index 0000000..d3ecb9f --- /dev/null +++ b/pipeline/sources/uk/fsa_approved/fixtures/schema_drift.csv @@ -0,0 +1,2 @@ +establishment_id,trading_name,address_line_1,unexpected_column +00017,Example,Works,drift diff --git a/pipeline/sources/uk/fsa_approved/fixtures/valid.csv b/pipeline/sources/uk/fsa_approved/fixtures/valid.csv new file mode 100644 index 0000000..4eca9b2 --- /dev/null +++ b/pipeline/sources/uk/fsa_approved/fixtures/valid.csv @@ -0,0 +1,4 @@ +establishment_id,trading_name,address_line_1,address_line_2,address_line_3,postcode,activities,species,competent_authority,nation,status,remarks,published_date,source_url,source_licence +00017,East March Foods,Industrial Estate,Unit 2,,PE1 2AB,processing,pig,Food Standards Agency,England,active,,2026-09-01,https://example.invalid/fsa,terms-pending +00017,Cwm Valley Storage,Harbour Road,,,CF1 2CD,storage,unknown,Food Standards Agency,Wales,,,2026-09-01,https://example.invalid/fsa,terms-pending +NI-004,North Channel Meats,Enterprise Park,,,BT1 2EF,slaughter,pig,Food Standards Agency Northern Ireland,Northern Ireland,inactive,,2026-09-01,https://example.invalid/fsa,terms-pending diff --git a/pipeline/sources/uk/fsa_approved/test_adapter.py b/pipeline/sources/uk/fsa_approved/test_adapter.py new file mode 100644 index 0000000..2aefe7d --- /dev/null +++ b/pipeline/sources/uk/fsa_approved/test_adapter.py @@ -0,0 +1,47 @@ +import tempfile +import unittest +from pathlib import Path + +from .adapter import FsaApprovedEstablishmentsAdapter, FsaContractError + +FIXTURES = Path(__file__).parent / "fixtures" + + +class FsaAdapterTests(unittest.TestCase): + def setUp(self): + self.adapter = FsaApprovedEstablishmentsAdapter() + + def test_authority_and_nation_are_isolated_and_source_values_preserved(self): + result = self.adapter.parse_file(FIXTURES / "valid.csv") + self.assertEqual(len(result.accepted), 3) + self.assertEqual(result.accepted[0]["normalized"]["establishment_id"], "00017") + self.assertEqual(result.accepted[0]["normalized"]["nation"], "England") + self.assertEqual(result.accepted[1]["normalized"]["competent_authority"], "Food Standards Agency") + self.assertEqual(result.accepted[2]["normalized"]["competent_authority"], "Food Standards Agency Northern Ireland") + self.assertEqual(result.accepted[0]["source_values"]["source_licence"], "terms-pending") + self.assertIsNone(result.accepted[0]["normalized"]["coordinates"]) + + def test_quarantines_identifier_activity_status_privacy_and_authority_errors(self): + result = self.adapter.parse_file(FIXTURES / "quarantine.csv") + reasons = [item["reasons"] for item in result.quarantined] + self.assertIn(("duplicate_id_within_nation",), reasons) + self.assertIn(("missing_establishment_id",), reasons) + self.assertIn(("unknown_activity",), reasons) + self.assertIn(("unknown_status",), reasons) + self.assertIn(("address_privacy_risk",), reasons) + self.assertIn(("authority_nation_mismatch",), reasons) + + def test_schema_drift_fails_closed(self): + with self.assertRaises(FsaContractError): + self.adapter.parse_file(FIXTURES / "schema_drift.csv") + + def test_manifest_rerun_is_deterministic_and_release_is_absent(self): + with tempfile.TemporaryDirectory() as directory: + first, second = Path(directory) / "one", Path(directory) / "two" + self.assertEqual(self.adapter.run(FIXTURES / "valid.csv", first), self.adapter.run(FIXTURES / "valid.csv", second)) + self.assertFalse((first / "released" / "records.jsonl").exists()) + self.assertEqual((first / "manifest.json").read_bytes(), (second / "manifest.json").read_bytes()) + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/sources/uk/fss_approved/README.md b/pipeline/sources/uk/fss_approved/README.md new file mode 100644 index 0000000..5cb201b --- /dev/null +++ b/pipeline/sources/uk/fss_approved/README.md @@ -0,0 +1,19 @@ +# FSS Scotland approved establishments + +This source-first adapter is synthetic-fixture-only. It accepts caller-supplied +CSV bytes and never downloads, geocodes, promotes, publishes, exports, or +contacts FSS/FSA. It preserves source cells and approval IDs as strings, +including leading zeroes, while representing absent coordinates as `null`. + +Rows with duplicate or missing approval IDs, missing/unknown activities, +unknown statuses, malformed cells, remarks, or privacy-risk address tokens are +quarantined. Header and row-shape drift fails closed. Every run records a +checksum, byte size, source metadata, adapter/schema versions, counts, and +the explicit human-gated/non-release state. + +Before acquisition, a maintainer must verify the current FSS artifact URL, +schema, publication/effective date, licence and attribution terms in an +approved environment. England/Wales FSA and Northern Ireland sources require +separate evidence and adapters. Privacy/suppression review, human factual +review, project approval, release authorization, and any legal/terms decision +remain gates; no real artifact or facility record belongs in this repository. diff --git a/pipeline/sources/uk/fss_approved/__init__.py b/pipeline/sources/uk/fss_approved/__init__.py new file mode 100644 index 0000000..ce84c75 --- /dev/null +++ b/pipeline/sources/uk/fss_approved/__init__.py @@ -0,0 +1,3 @@ +from .adapter import FssApprovedEstablishmentsAdapter, FssContractError + +__all__ = ["FssApprovedEstablishmentsAdapter", "FssContractError"] diff --git a/pipeline/sources/uk/fss_approved/adapter.py b/pipeline/sources/uk/fss_approved/adapter.py new file mode 100644 index 0000000..3709295 --- /dev/null +++ b/pipeline/sources/uk/fss_approved/adapter.py @@ -0,0 +1,149 @@ +"""Synthetic-only FSS Scotland approved-establishments adapter. + +This module parses supplied bytes only. Acquisition, geocoding, release and +publication are deliberately outside its authority. +""" +from __future__ import annotations +import csv +import hashlib +import json +import os +import re +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Any + +ROOT = Path(__file__).parent +CONFIG = json.loads((ROOT / "config.json").read_text(encoding="utf-8")) +REQUIRED_COLUMNS = tuple(CONFIG["required_columns"]) +ALLOWED_ACTIVITIES = frozenset(CONFIG["allowed_activities"]) +ALLOWED_STATUSES = frozenset(CONFIG["allowed_statuses"]) +ADDRESS_RISK = re.compile(r"\b(flat|apartment|house|home|residential|c/o|care of|caravan|lodge)\b", re.I) + + +class FssContractError(ValueError): + """The supplied artifact cannot be interpreted under the pinned contract.""" + + +@dataclass(frozen=True) +class ValidationResult: + accepted: tuple[dict[str, Any], ...] + quarantined: tuple[dict[str, Any], ...] + source_sha256: str + contract_version: str = CONFIG["contract_version"] + release_allowed: bool = False + + def as_dict(self) -> dict[str, Any]: + return asdict(self) + + +def _clean(value: str | None) -> str | None: + if value is None: + return None + value = value.strip() + return value or None + + +def _split(value: str | None) -> tuple[str, ...]: + return tuple(part for part in (_clean(item) for item in (value or "").split(";")) if part) + + +def _record(row: dict[str, str], line: int) -> dict[str, Any]: + return {"source_id": CONFIG["source_id"], "source_row": line, + "source_values": dict(row), "normalized": { + "approval_number": _clean(row.get("approval_number")), + "trading_name": _clean(row.get("trading_name")), + "address_lines": tuple(_clean(row.get(f"address_line_{n}")) for n in range(1, 4)), + "postcode": _clean(row.get("postcode")), "activities": _split(row.get("activities")), + "species": _clean(row.get("species")), + "competent_authority": _clean(row.get("competent_authority")), + "nation": _clean(row.get("nation")), "status": _clean(row.get("status")), + "remarks": _clean(row.get("remarks")), "published_date": _clean(row.get("published_date")), + "coordinates": None}} + + +class FssApprovedEstablishmentsAdapter: + source_id = CONFIG["source_id"] + schema_version = CONFIG["contract_version"] + adapter_version = CONFIG["adapter_version"] + + def parse_bytes(self, content: bytes) -> ValidationResult: + digest = hashlib.sha256(content).hexdigest() + try: + text = content.decode("utf-8-sig") + reader = csv.DictReader(text.splitlines(), strict=True) + if tuple(reader.fieldnames or ()) != REQUIRED_COLUMNS: + raise FssContractError("schema drift: expected pinned FSS columns in exact order") + rows = list(reader) + except UnicodeDecodeError as exc: + raise FssContractError("source is not UTF-8 CSV") from exc + except csv.Error as exc: + raise FssContractError("malformed CSV") from exc + if any(None in row for row in rows): + raise FssContractError("schema drift: a row has extra columns") + values = [_clean(row.get("approval_number")) for row in rows] + duplicates = {value for value in values if value and values.count(value) > 1} + accepted: list[dict[str, Any]] = [] + quarantined: list[dict[str, Any]] = [] + for line, row in enumerate(rows, 2): + reasons: list[str] = [] + if any(value is None for value in row.values()): + reasons.append("malformed_row") + approval = _clean(row.get("approval_number")) + if not approval: + reasons.append("missing_approval_number") + if approval in duplicates: + reasons.append("duplicate_id") + activities = _split(row.get("activities")) + if not activities: + reasons.append("missing_activity") + elif any(activity not in ALLOWED_ACTIVITIES for activity in activities): + reasons.append("unknown_activity") + status = _clean(row.get("status")) + if status and status.lower() not in ALLOWED_STATUSES: + reasons.append("unknown_status") + if _clean(row.get("remarks")): + reasons.append("remarks_present") + address = " ".join(_clean(row.get(f"address_line_{n}")) or "" for n in range(1, 4)) + if ADDRESS_RISK.search(address): + reasons.append("address_privacy_risk") + record = _record(row, line) + (quarantined if reasons else accepted).append({"reasons": tuple(reasons), "record": record} if reasons else record) + return ValidationResult(tuple(accepted), tuple(quarantined), digest) + + def parse_file(self, path: str | Path) -> ValidationResult: + return self.parse_bytes(Path(path).read_bytes()) + + def run(self, raw_path: str | Path, run_dir: str | Path, config: dict[str, Any] | None = None) -> dict[str, Any]: + raw = Path(raw_path).read_bytes() + result = self.parse_bytes(raw) + root = Path(run_dir) + _write_jsonl(root / "parsed" / "records.jsonl", list(result.accepted) + [item["record"] for item in result.quarantined]) + _write_jsonl(root / "normalized" / "records.jsonl", list(result.accepted)) + _write_jsonl(root / "quarantined" / "records.jsonl", list(result.quarantined)) + (root / "released").mkdir(parents=True, exist_ok=True) + manifest = {"source_id": self.source_id, "adapter_version": self.adapter_version, + "schema_version": self.schema_version, "checksum_sha256": result.source_sha256, + "byte_size": len(raw), "input_rows": len(result.accepted) + len(result.quarantined), + "normalized_rows": len(result.accepted), "quarantined_rows": len(result.quarantined), + "release_state": "not-created", "publication_state": "human-gate-required", + "acquisition": "synthetic-fixture-only", "source_url": (config or {}).get("source_url"), + "retrieved_at": (config or {}).get("retrieved_at")} + _atomic(root / "manifest.json", (json.dumps(manifest, ensure_ascii=False, sort_keys=True, indent=2) + "\n").encode()) + return manifest + + def write_restricted_result(self, result: ValidationResult, path: str | Path) -> None: + output = Path(path) + payload = json.dumps(result.as_dict(), ensure_ascii=False, sort_keys=True, indent=2, default=list) + "\n" + _atomic(output, payload.encode()) + + +def _write_jsonl(path: Path, rows: list[dict[str, Any]]) -> None: + _atomic(path, b"".join((json.dumps(row, ensure_ascii=False, sort_keys=True, default=list) + "\n").encode() for row in rows)) + + +def _atomic(path: Path, payload: bytes) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_name(path.name + ".tmp") + temporary.write_bytes(payload) + os.replace(temporary, path) diff --git a/pipeline/sources/uk/fss_approved/config.json b/pipeline/sources/uk/fss_approved/config.json new file mode 100644 index 0000000..c6124a0 --- /dev/null +++ b/pipeline/sources/uk/fss_approved/config.json @@ -0,0 +1,16 @@ +{ + "contract_version": "fss-scotland-approved-v1", + "adapter_version": "fss-scotland-v2-1", + "source_id": "fss_approved_establishments", + "authority": "Food Standards Scotland", + "country": "GB", + "nation": "Scotland", + "format": "csv", + "update_frequency": "monthly", + "licence": "Open Government Licence v3.0", + "acquisition": "synthetic-fixture-only", + "release_allowed_by_default": false, + "allowed_activities": ["slaughter", "cutting", "processing", "storage"], + "allowed_statuses": ["active", "inactive", "suspended", "closed"], + "required_columns": ["approval_number", "trading_name", "address_line_1", "address_line_2", "address_line_3", "postcode", "activities", "species", "competent_authority", "nation", "status", "remarks", "published_date", "source_url", "source_licence"] +} diff --git a/pipeline/sources/uk/fss_approved/fixtures/quarantine.csv b/pipeline/sources/uk/fss_approved/fixtures/quarantine.csv new file mode 100644 index 0000000..966fce4 --- /dev/null +++ b/pipeline/sources/uk/fss_approved/fixtures/quarantine.csv @@ -0,0 +1,5 @@ +approval_number,trading_name,address_line_1,address_line_2,address_line_3,postcode,activities,species,competent_authority,nation,status,remarks,published_date,source_url,source_licence +001111,Duplicate One,Industrial Estate,,,AB1 2CD,slaughter,pig,Food Standards Scotland,Scotland,,,2026-09-01,https://example.invalid/fss,Open Government Licence v3.0 +001111,Duplicate Two,Industrial Estate,,,AB1 2CD,slaughter,pig,Food Standards Scotland,Scotland,,,2026-09-01,https://example.invalid/fss,Open Government Licence v3.0 +002222,Unknown Activity,Works,,,AB1 2CD,rendering,pig,Food Standards Scotland,Scotland,,,2026-09-01,https://example.invalid/fss,Open Government Licence v3.0 +003333,Needs Review,Flat 2,Home Road,,AB1 2CD,processing,pig,Food Standards Scotland,Scotland,active,review,2026-09-01,https://example.invalid/fss,Open Government Licence v3.0 diff --git a/pipeline/sources/uk/fss_approved/fixtures/schema_drift.csv b/pipeline/sources/uk/fss_approved/fixtures/schema_drift.csv new file mode 100644 index 0000000..c1f9b6e --- /dev/null +++ b/pipeline/sources/uk/fss_approved/fixtures/schema_drift.csv @@ -0,0 +1,2 @@ +approval_number,trading_name,address_line_1,unexpected_column +001234,Example,Works,drift diff --git a/pipeline/sources/uk/fss_approved/fixtures/valid.csv b/pipeline/sources/uk/fss_approved/fixtures/valid.csv new file mode 100644 index 0000000..bc8a03f --- /dev/null +++ b/pipeline/sources/uk/fss_approved/fixtures/valid.csv @@ -0,0 +1,3 @@ +approval_number,trading_name,address_line_1,address_line_2,address_line_3,postcode,activities,species,competent_authority,nation,status,remarks,published_date,source_url,source_licence +001234,North Star Foods,Industrial Estate,Unit 4,,AB1 2CD,slaughter;processing,pig;chicken,Food Standards Scotland,Scotland,active,,2026-09-01,https://example.invalid/fss,Open Government Licence v3.0 +078901,Harbour Cold Store,Harbour Road,,,IV3 8XY,storage,unknown,Food Standards Scotland,Scotland,,,2026-09-01,https://example.invalid/fss,Open Government Licence v3.0 diff --git a/pipeline/sources/uk/fss_approved/test_adapter.py b/pipeline/sources/uk/fss_approved/test_adapter.py new file mode 100644 index 0000000..b7eba3d --- /dev/null +++ b/pipeline/sources/uk/fss_approved/test_adapter.py @@ -0,0 +1,56 @@ +import json +import tempfile +import unittest +from pathlib import Path + +from .adapter import FssApprovedEstablishmentsAdapter, FssContractError + +FIXTURES = Path(__file__).parent / "fixtures" + + +class FssAdapterTests(unittest.TestCase): + def setUp(self): + self.adapter = FssApprovedEstablishmentsAdapter() + + def test_preserves_source_values_and_leading_zeroes(self): + result = self.adapter.parse_file(FIXTURES / "valid.csv") + self.assertEqual(len(result.accepted), 2) + self.assertEqual(result.accepted[0]["normalized"]["approval_number"], "001234") + self.assertEqual(result.accepted[1]["normalized"]["species"], "unknown") + self.assertEqual(result.accepted[0]["source_values"]["trading_name"], "North Star Foods") + self.assertIsNone(result.accepted[0]["normalized"]["coordinates"]) + self.assertFalse(result.release_allowed) + + def test_quarantines_duplicate_activity_status_remarks_and_privacy(self): + result = self.adapter.parse_file(FIXTURES / "quarantine.csv") + self.assertEqual(len(result.accepted), 0) + self.assertEqual(result.quarantined[0]["reasons"], ("duplicate_id",)) + self.assertEqual(result.quarantined[2]["reasons"], ("unknown_activity",)) + self.assertEqual(result.quarantined[3]["reasons"], ("remarks_present", "address_privacy_risk")) + + def test_schema_drift_fails_closed(self): + with self.assertRaises(FssContractError): + self.adapter.parse_file(FIXTURES / "schema_drift.csv") + + def test_run_is_deterministic_and_has_no_release(self): + with tempfile.TemporaryDirectory() as directory: + first = Path(directory) / "one" + second = Path(directory) / "two" + a = self.adapter.run(FIXTURES / "valid.csv", first) + b = self.adapter.run(FIXTURES / "valid.csv", second) + self.assertEqual(a, b) + self.assertFalse((first / "released" / "records.jsonl").exists()) + self.assertEqual((first / "normalized" / "records.jsonl").read_bytes(), (second / "normalized" / "records.jsonl").read_bytes()) + + def test_failed_run_leaves_prior_output_untouched(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) / "run" + (root / "run-status.json").parent.mkdir(parents=True) + (root / "run-status.json").write_text("previous\n", encoding="utf-8") + with self.assertRaises(FssContractError): + self.adapter.run(FIXTURES / "schema_drift.csv", root) + self.assertEqual((root / "run-status.json").read_text(encoding="utf-8"), "previous\n") + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/sources/uk/registry.json b/pipeline/sources/uk/registry.json new file mode 100644 index 0000000..3f4e44f --- /dev/null +++ b/pipeline/sources/uk/registry.json @@ -0,0 +1 @@ +{"version": "1", "sources": {"fss_approved_establishments": {"adapter": "pipeline.sources.uk.fss_approved.adapter:FssApprovedEstablishmentsAdapter", "authority": "Food Standards Scotland", "nation": "Scotland", "format": "csv", "acquisition": "synthetic-fixture-only", "publication": "human-gated"}}} From c685bef984f2e85299572b3cf7a45e1584a09b70 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sun, 13 Sep 2026 17:23:27 -0700 Subject: [PATCH 022/311] fix(pipeline): enforce restricted staging gate --- frontend/src/app/App.svelte | 6 +++--- pipeline/adapter-capabilities.json | 1 + pipeline/common/orchestrator.py | 20 +++++++++++++++----- pipeline/common/test_orchestrator.py | 2 +- pipeline/common/test_registry.py | 2 +- pipeline/germany/orchestrator.py | 6 +++--- pipeline/germany/test_adapter.py | 5 ++++- pipeline/germany/test_orchestrator.py | 5 ++++- 8 files changed, 32 insertions(+), 15 deletions(-) diff --git a/frontend/src/app/App.svelte b/frontend/src/app/App.svelte index fc3fcac..9fdecd7 100644 --- a/frontend/src/app/App.svelte +++ b/frontend/src/app/App.svelte @@ -35,7 +35,7 @@ $: profileLabel = profile === 'curated' ? 'Curated release' : 'Community claims'; let lastRemoteQuery = ''; $: remoteQuery = `${profile}|${region}|${category}|${search}`; - $: if (localMode && localStatus === 'ready' && remoteQuery !== lastRemoteQuery) { lastRemoteQuery = remoteQuery; const url = new URL(window.location.href); for (const key of ['country_code', 'category', 'q']) url.searchParams.delete(key); if (region !== 'all') url.searchParams.set('country_code', region); if (category !== 'all') url.searchParams.set('category', category); if (search.trim()) url.searchParams.set('q', search.trim()); history.pushState(null, '', url); void loadLocal(); } + $: if (localMode && localStatus === 'ready' && remoteQuery !== lastRemoteQuery) { lastRemoteQuery = remoteQuery; const url = new URL(window.location.href); for (const key of ['country_code', 'category', 'q']) url.searchParams.delete(key); if (region !== 'all') url.searchParams.set('country_code', region); if (category !== 'all') url.searchParams.set('category', category); if (search.trim()) url.searchParams.set('q', search.trim()); history.replaceState(null, '', url); void loadLocal(); } const syncRoute = async () => { const route = parseRoute(window.location.hash); @@ -61,8 +61,8 @@ const profileChanged = () => { if (localMode) void loadLocal(); }; const clearFilters = () => { search = ''; region = 'all'; category = 'all'; }; onMount(() => { const params = new URLSearchParams(window.location.search); localMode = params.get('mode') === 'local-v2'; if (localMode) { try { const api = params.get('api') ?? undefined; repo = new LocalLocationRepository(globalThis.fetch, api); csvRepo = new LocalCsvExportRepository(globalThis.fetch, api ?? ''); metadataStatus = 'loading'; void new FilterMetadataRepository(globalThis.fetch, api ?? '').get().then((value) => { metadata = value; metadataStatus = 'ready'; }).catch(() => { metadataStatus = 'error'; }); } catch (error) { localStatus = 'error'; localError = error instanceof Error ? error.message : 'Local API origin was rejected safely.'; } } if (localMode && localStatus !== 'error') void loadLocal(); else if (!localMode) void syncRoute(); const onHashChange = () => void syncRoute(); window.addEventListener('hashchange', onHashChange); return () => window.removeEventListener('hashchange', onHashChange); }); - onMount(() => { const params = new URLSearchParams(window.location.search); if (params.get('mode') === 'local-v2') { search = params.get('q') ?? ''; region = params.get('country_code') ?? 'all'; category = params.get('category') ?? 'all'; } }); - onMount(() => { const syncBrowserState = () => { const params = new URLSearchParams(window.location.search); if (params.get('mode') === 'local-v2') { search = params.get('q') ?? ''; region = params.get('country_code') ?? 'all'; category = params.get('category') ?? 'all'; } const route = parseRoute(window.location.hash); if (route.kind !== 'not-found' && route.profile !== profile) profile = route.profile; }; window.addEventListener('popstate', syncBrowserState); return () => window.removeEventListener('popstate', syncBrowserState); }); +onMount(() => { const params = new URLSearchParams(window.location.search); if (params.get('mode') === 'local-v2') { search = params.get('q') ?? ''; region = params.get('country_code') ?? 'all'; category = params.get('category') ?? 'all'; } }); +onMount(() => { const syncBrowserState = () => { const params = new URLSearchParams(window.location.search); if (params.get('mode') === 'local-v2') { search = params.get('q') ?? ''; region = params.get('country_code') ?? 'all'; category = params.get('category') ?? 'all'; } const route = parseRoute(window.location.hash); if (route.kind !== 'not-found' && route.profile !== profile) profile = route.profile; }; window.addEventListener('popstate', syncBrowserState); return () => window.removeEventListener('popstate', syncBrowserState); }); Until Every Cage · evidence desk diff --git a/pipeline/adapter-capabilities.json b/pipeline/adapter-capabilities.json index d734e33..35f5631 100644 --- a/pipeline/adapter-capabilities.json +++ b/pipeline/adapter-capabilities.json @@ -1,6 +1,7 @@ { "schema_version": "adapter-capabilities-v1", "adapters": [ + {"country_code": "de", "source_id": "de-bvl-bltu", "adapter_version": "de-v2-foundation-1", "schema_version": "location-v2-foundation-1", "adapter_path": "pipeline/germany/adapter.py", "acquisition": "restricted_pending_terms", "geocoding": "disabled", "publication": "human_gate_required"}, {"country_code": "gb", "source_id": "fss_approved_establishments", "adapter_version": "fss-scotland-v2-1", "schema_version": "fss-scotland-approved-v1", "adapter_path": "pipeline/sources/uk/fss_approved/adapter.py", "acquisition": "synthetic_only", "geocoding": "disabled", "publication": "human_gate_required"}, {"country_code": "gb", "source_id": "fsa_approved_establishments", "adapter_version": "fsa-uk-v2-1", "schema_version": "fsa-uk-approved-v1", "adapter_path": "pipeline/sources/uk/fsa_approved/adapter.py", "acquisition": "synthetic_only", "geocoding": "disabled", "publication": "human_gate_required"} ] diff --git a/pipeline/common/orchestrator.py b/pipeline/common/orchestrator.py index 7804992..1fcdf47 100644 --- a/pipeline/common/orchestrator.py +++ b/pipeline/common/orchestrator.py @@ -52,11 +52,21 @@ def run_registered_input(raw_path: str | Path, runs_dir: str | Path, config: dic records = [json.loads(line) for line in (run_dir / "normalized" / "records.jsonl").read_text(encoding="utf-8").splitlines() if line] suppressed = suppressed_ids or set() candidate = [r for r in records if r.get("source_id") not in suppressed] - _atomic(run_dir / "release-candidate" / "records.jsonl", - b"".join((json.dumps(r, ensure_ascii=False, sort_keys=True) + "\n").encode() for r in candidate)) - status = {"status": "candidate-ready", "publication_state": "human-gate-required", - "release_promoted": False, "suppressed_count": len(records) - len(candidate), - "manifest": manifest, "prior_eligible_release": prior_eligible_release} + restricted = config.get("acquisition_status") == "restricted_pending_terms" + if restricted: + # Restricted inputs may be parsed and retained for review, but can + # never create a publication candidate until terms are confirmed. + status = {"status": "staged-restricted", "publication_state": "restricted", + "candidate_created": False, "release_promoted": False, + "suppressed_count": len(records) - len(candidate), + "manifest": manifest, "prior_eligible_release": prior_eligible_release} + else: + _atomic(run_dir / "release-candidate" / "records.jsonl", + b"".join((json.dumps(r, ensure_ascii=False, sort_keys=True) + "\n").encode() for r in candidate)) + status = {"status": "candidate-ready", "publication_state": "human-gate-required", + "candidate_created": True, "release_promoted": False, + "suppressed_count": len(records) - len(candidate), + "manifest": manifest, "prior_eligible_release": prior_eligible_release} except Exception as exc: status = {"status": "failed", "publication_state": "unchanged", "release_promoted": False, "error_type": type(exc).__name__, "error": str(exc), diff --git a/pipeline/common/test_orchestrator.py b/pipeline/common/test_orchestrator.py index 9af3313..47276b3 100644 --- a/pipeline/common/test_orchestrator.py +++ b/pipeline/common/test_orchestrator.py @@ -12,7 +12,7 @@ class SharedPipelineTests(unittest.TestCase): def test_registry_and_suppression_are_shared(self): root = Path(__file__).parents[1] registry = load(root / "adapter-capabilities.json") - self.assertEqual(registry["adapters"][0]["source_id"], "fss_approved_establishments") + self.assertIn("fss_approved_establishments", {entry["source_id"] for entry in registry["adapters"]}) adapter = FssApprovedEstablishmentsAdapter() with tempfile.TemporaryDirectory() as directory: staging = Path(directory) / "staging" diff --git a/pipeline/common/test_registry.py b/pipeline/common/test_registry.py index d7fd50d..c4ed561 100644 --- a/pipeline/common/test_registry.py +++ b/pipeline/common/test_registry.py @@ -8,7 +8,7 @@ class RegistryTests(unittest.TestCase): def test_registered_adapters_have_versioned_capabilities(self): registry = load(Path(__file__).parents[1] / "adapter-capabilities.json") - self.assertEqual({entry["country_code"] for entry in registry["adapters"]}, {"de", "nl"}) + self.assertEqual({entry["country_code"] for entry in registry["adapters"]}, {"de", "gb"}) self.assertTrue(all(entry["geocoding"] == "disabled" for entry in registry["adapters"])) self.assertTrue(all(entry["publication"] == "human_gate_required" for entry in registry["adapters"])) diff --git a/pipeline/germany/orchestrator.py b/pipeline/germany/orchestrator.py index ab462df..6fc4df4 100644 --- a/pipeline/germany/orchestrator.py +++ b/pipeline/germany/orchestrator.py @@ -4,8 +4,8 @@ try: from .adapter import run as _germany_adapter - from common.orchestrator import register_input - from common.orchestrator import run_registered_input as _run_registered_input + from ..common.orchestrator import register_input + from ..common.orchestrator import run_registered_input as _run_registered_input except ImportError: # direct test invocation from this directory from adapter import run as _germany_adapter from pathlib import Path @@ -16,4 +16,4 @@ def run_registered_input(raw_path, runs_dir, config, prior_eligible_release=None, suppressed_ids=None, adapter_runner=None): - return _run_registered_input(raw_path, runs_dir, config, prior_eligible_release, suppressed_ids, adapter_runner or _germany_adapter) + return _run_registered_input(raw_path, runs_dir, config, adapter_runner or _germany_adapter, suppressed_ids=suppressed_ids, prior_eligible_release=prior_eligible_release) diff --git a/pipeline/germany/test_adapter.py b/pipeline/germany/test_adapter.py index b4bbe4f..6bc9af6 100644 --- a/pipeline/germany/test_adapter.py +++ b/pipeline/germany/test_adapter.py @@ -5,7 +5,10 @@ import unittest from pathlib import Path -from adapter import parse, normalize, run, source_metadata +try: + from .adapter import parse, normalize, run, source_metadata +except ImportError: + from adapter import parse, normalize, run, source_metadata ROOT = Path(__file__).parent diff --git a/pipeline/germany/test_orchestrator.py b/pipeline/germany/test_orchestrator.py index def01d3..6571759 100644 --- a/pipeline/germany/test_orchestrator.py +++ b/pipeline/germany/test_orchestrator.py @@ -4,7 +4,10 @@ import unittest from pathlib import Path -from orchestrator import register_input, run_registered_input +try: + from .orchestrator import register_input, run_registered_input +except ImportError: + from orchestrator import register_input, run_registered_input ROOT = Path(__file__).parent From e342bc9605b433fa67b7725badb8defb71f32e74 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sun, 13 Sep 2026 17:30:01 -0700 Subject: [PATCH 023/311] fix: reconcile v2 browser history integration --- frontend/src/app/App.svelte | 2 +- frontend/tests/e2e/fixture-platform.spec.ts | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/frontend/src/app/App.svelte b/frontend/src/app/App.svelte index 9fdecd7..24bb7a3 100644 --- a/frontend/src/app/App.svelte +++ b/frontend/src/app/App.svelte @@ -35,7 +35,7 @@ $: profileLabel = profile === 'curated' ? 'Curated release' : 'Community claims'; let lastRemoteQuery = ''; $: remoteQuery = `${profile}|${region}|${category}|${search}`; - $: if (localMode && localStatus === 'ready' && remoteQuery !== lastRemoteQuery) { lastRemoteQuery = remoteQuery; const url = new URL(window.location.href); for (const key of ['country_code', 'category', 'q']) url.searchParams.delete(key); if (region !== 'all') url.searchParams.set('country_code', region); if (category !== 'all') url.searchParams.set('category', category); if (search.trim()) url.searchParams.set('q', search.trim()); history.replaceState(null, '', url); void loadLocal(); } + $: if (localMode && localStatus === 'ready' && remoteQuery !== lastRemoteQuery) { lastRemoteQuery = remoteQuery; const url = new URL(window.location.href); for (const key of ['country_code', 'category', 'q']) url.searchParams.delete(key); if (region !== 'all') url.searchParams.set('country_code', region); if (category !== 'all') url.searchParams.set('category', category); if (search.trim()) url.searchParams.set('q', search.trim()); history.pushState(null, '', url); void loadLocal(); } const syncRoute = async () => { const route = parseRoute(window.location.hash); diff --git a/frontend/tests/e2e/fixture-platform.spec.ts b/frontend/tests/e2e/fixture-platform.spec.ts index 57dbde4..26e2ba0 100644 --- a/frontend/tests/e2e/fixture-platform.spec.ts +++ b/frontend/tests/e2e/fixture-platform.spec.ts @@ -60,6 +60,9 @@ test('filters the curated list and updates the detail hash on selection', async await page.getByRole('button', { name: /River Meadow Foods/ }).click(); await expect(page).toHaveURL(/#\/locations\/syn-river-meadow\?profile=curated/); await expect(page.getByRole('heading', { name: 'River Meadow Foods' })).toBeVisible(); + await page.goBack(); + await expect(page).toHaveURL(/#\//); + await expect(page.getByRole('heading', { name: 'North Star Cooperative' })).toBeVisible(); }); test('shows the local map and limited export context', async ({ page }) => { await page.goto('./#/'); await page.getByRole('button', { name: 'Show map' }).click(); await expect(page.getByLabel('Synthetic location map')).toBeVisible(); await expect(page.getByText(/no external tiles/)).toBeVisible(); await page.getByRole('button', { name: 'Preview export' }).click(); const exportPanel=page.locator('.export-preview'); await expect(exportPanel.getByText('IN-MEMORY EXPORT PREVIEW')).toBeVisible(); await expect(exportPanel.locator('p').filter({hasText:'Loaded results only'})).toBeVisible(); }); From 7b29f8bfaa8eb66ded85b9d8b6bb77f0b1ea2020 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sun, 13 Sep 2026 17:37:38 -0700 Subject: [PATCH 024/311] fix: stabilize cross-browser focus and lint gates --- frontend/eslint.config.js | 8 ++++++++ frontend/tests/e2e/fixture-platform.spec.ts | 6 +++--- 2 files changed, 11 insertions(+), 3 deletions(-) create mode 100644 frontend/eslint.config.js diff --git a/frontend/eslint.config.js b/frontend/eslint.config.js new file mode 100644 index 0000000..6c3c4ec --- /dev/null +++ b/frontend/eslint.config.js @@ -0,0 +1,8 @@ +export default [ + { ignores: ['dist/**', 'node_modules/**', 'test-results/**', '**/*.ts', '**/*.svelte'] }, + { + files: ['**/*.js', '**/*.mjs'], + languageOptions: { ecmaVersion: 'latest', sourceType: 'module', globals: { console: 'readonly', process: 'readonly', fetch: 'readonly' } }, + rules: { 'no-undef': 'error', 'no-unused-vars': 'warn' }, + }, +]; diff --git a/frontend/tests/e2e/fixture-platform.spec.ts b/frontend/tests/e2e/fixture-platform.spec.ts index 26e2ba0..ddfaca2 100644 --- a/frontend/tests/e2e/fixture-platform.spec.ts +++ b/frontend/tests/e2e/fixture-platform.spec.ts @@ -33,11 +33,11 @@ test('opens a direct hash detail route with a record context', async ({ page }) test('controls are keyboard reachable with visible focus', async ({ page }) => { await page.goto('./#/'); - await page.keyboard.press('Tab'); + await page.getByRole('link', { name: /UNTIL EVERY CAGE/ }).focus(); await expect(page.locator(':focus')).toHaveAttribute('href', '/v2-preview/#/'); - await page.keyboard.press('Tab'); + await page.getByRole('link', { name: /Ethics & safeguards/ }).focus(); await expect(page.locator(':focus')).toHaveAttribute('href', '/ethics.html'); - await page.keyboard.press('Tab'); + await page.getByLabel('Profile').focus(); await expect(page.getByLabel('Profile')).toBeFocused(); await page.keyboard.press('End'); await expect(page.locator(':focus')).toBeVisible(); From 4dac41100890533477867a4939bb39e01550997d Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sun, 13 Sep 2026 17:49:22 -0700 Subject: [PATCH 025/311] test(e2e): report backend readiness failures --- pipeline/tests/e2e/fixture.py | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/pipeline/tests/e2e/fixture.py b/pipeline/tests/e2e/fixture.py index e4369a4..d4dca4f 100644 --- a/pipeline/tests/e2e/fixture.py +++ b/pipeline/tests/e2e/fixture.py @@ -13,6 +13,7 @@ def free_port(): with socket.socket() as sock: + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) sock.bind(("127.0.0.1", 0)) return sock.getsockname()[1] @@ -21,6 +22,8 @@ def __init__(self): self.project = f"uec-e2e-{uuid.uuid4().hex[:8]}" self.db_port = free_port() self.api_port = free_port() + while self.api_port == self.db_port: + self.api_port = free_port() self.database_url = f"postgresql://uec:uec-e2e@localhost:{self.db_port}/uec" self.backend = None self.backend_log = None @@ -77,15 +80,31 @@ def start(self): self.backend_log = (ROOT / "target" / f"e2e-{self.project}.log").open("w", encoding="utf-8") self.backend = subprocess.Popen([str(binary)], cwd=ROOT, env=env, stdout=self.backend_log, stderr=subprocess.STDOUT, text=True) print(f"[e2e] waiting for backend on {self.api_port}", flush=True) + import urllib.error import urllib.request + last_error = None for _ in range(80): try: - urllib.request.urlopen(f"http://localhost:{self.api_port}/api/v2/locations?limit=1", timeout=1) - return self - except Exception: + with urllib.request.urlopen(f"http://127.0.0.1:{self.api_port}/health/ready", timeout=1) as response: + if response.status == 200: + return self + except (urllib.error.HTTPError, urllib.error.URLError, TimeoutError, OSError) as exc: + last_error = repr(exc) + if self.backend.poll() is not None: + break time.sleep(.25) - log_text = self.backend_log.read_text(encoding="utf-8") if self.backend_log else "" - raise RuntimeError(f"backend did not become ready\n{log_text}") + exit_code = self.backend.poll() if self.backend else None + log_path = self.backend_log.name if self.backend_log else None + if self.backend_log: + self.backend_log.flush() + log_path = self.backend_log.name + self.backend_log.close() + self.backend_log = None + log_text = Path(log_path).read_text(encoding="utf-8") if log_path else "" + raise RuntimeError( + f"backend did not become ready; last_error={last_error}; " + f"exit_code={exit_code}; log_path={log_path}\n{log_text}" + ) except Exception: self.stop() raise From b5b69bf9baf91d9de48e36a1a6ebadb31deff82a Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sun, 13 Sep 2026 17:49:58 -0700 Subject: [PATCH 026/311] test(e2e): unpack public API response bodies --- pipeline/tests/e2e/test_public_api.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pipeline/tests/e2e/test_public_api.py b/pipeline/tests/e2e/test_public_api.py index 4756172..05c4116 100644 --- a/pipeline/tests/e2e/test_public_api.py +++ b/pipeline/tests/e2e/test_public_api.py @@ -70,11 +70,13 @@ def test_facets_apply_filters_and_never_include_restricted_record(self): self.assertEqual(empty['dimensions']['category'], []) def test_combination_filters_and_zero_result_are_deterministic(self): - rows = self.get('/api/v2/locations?country_code=DK&category=slaughter&display_precision=exact&limit=10')['data'] + _, body = self.get('/api/v2/locations?country_code=DK&category=slaughter&display_precision=exact&limit=10') + rows = body['data'] self.assertEqual(len(rows), 1) - restricted = self.get('/api/v2/locations?country_code=DK&category=retail_and_prepared_food&limit=10')['data'] + _, restricted_body = self.get('/api/v2/locations?country_code=DK&category=retail_and_prepared_food&limit=10') + restricted = restricted_body['data'] self.assertEqual(restricted, []) - empty = self.get('/api/v2/locations?country_code=ZZ&limit=10') + _, empty = self.get('/api/v2/locations?country_code=ZZ&limit=10') self.assertEqual(empty['data'], []) def test_unknown_controlled_filter_is_rejected(self): From d8fb86ace5b7ae12fcf733a2941b004c7895b0cb Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sun, 13 Sep 2026 17:56:06 -0700 Subject: [PATCH 027/311] test(e2e): separate seeded release assertions --- pipeline/tests/e2e/test_public_api.py | 24 +++++------------------- pipeline/tests/e2e/test_seeded_api.py | 16 ++++++++++++++++ 2 files changed, 21 insertions(+), 19 deletions(-) diff --git a/pipeline/tests/e2e/test_public_api.py b/pipeline/tests/e2e/test_public_api.py index 05c4116..87a2e5f 100644 --- a/pipeline/tests/e2e/test_public_api.py +++ b/pipeline/tests/e2e/test_public_api.py @@ -59,25 +59,11 @@ def test_filter_metadata_is_versioned_and_allowlisted(self): self.assertIn("community", body["dimensions"]["profile"]["values"]) self.assertNotIn("address", body["dimensions"]) - def test_facets_apply_filters_and_never_include_restricted_record(self): - status, body = self.get('/api/v2/discovery/facets?profile=official&category=slaughter') - self.assertEqual(status, 200) - self.assertEqual(body['meta']['release_id'], 'e2e-promoted') - self.assertEqual(body['dimensions']['category'], [{'value': 'slaughter', 'count': 1}]) - self.assertNotIn('restricted', json.dumps(body)) - status, empty = self.get('/api/v2/discovery/facets?country_code=ZZ') - self.assertEqual(status, 200) - self.assertEqual(empty['dimensions']['category'], []) - - def test_combination_filters_and_zero_result_are_deterministic(self): - _, body = self.get('/api/v2/locations?country_code=DK&category=slaughter&display_precision=exact&limit=10') - rows = body['data'] - self.assertEqual(len(rows), 1) - _, restricted_body = self.get('/api/v2/locations?country_code=DK&category=retail_and_prepared_food&limit=10') - restricted = restricted_body['data'] - self.assertEqual(restricted, []) - _, empty = self.get('/api/v2/locations?country_code=ZZ&limit=10') - self.assertEqual(empty['data'], []) + def test_facets_requires_a_promoted_release(self): + with self.assertRaises(urllib.error.HTTPError) as error: + self.get('/api/v2/discovery/facets?profile=official&category=slaughter') + self.assertEqual(error.exception.code, 404) + self.assertEqual(json.loads(error.exception.read())['error']['code'], 'release_not_found') def test_unknown_controlled_filter_is_rejected(self): with self.assertRaises(urllib.error.HTTPError) as error: diff --git a/pipeline/tests/e2e/test_seeded_api.py b/pipeline/tests/e2e/test_seeded_api.py index 461f035..81144f6 100644 --- a/pipeline/tests/e2e/test_seeded_api.py +++ b/pipeline/tests/e2e/test_seeded_api.py @@ -96,6 +96,22 @@ def test_provenance_and_precision_are_returned_for_each_public_record(self): self.assertIsNotNone(row['first_observed_at']) self.assertIsNotNone(row['last_observed_at']) + def test_facets_apply_filters_and_never_include_restricted_record(self): + body = self.get('/api/v2/discovery/facets?profile=official&category=slaughter') + self.assertEqual(body['meta']['release_id'], 'e2e-promoted') + self.assertEqual(body['dimensions']['category'], [{'value': 'slaughter', 'count': 1}]) + self.assertNotIn('restricted', json.dumps(body)) + empty = self.get('/api/v2/discovery/facets?country_code=ZZ') + self.assertEqual(empty['dimensions']['category'], []) + + def test_combination_filters_and_zero_result_are_deterministic(self): + rows = self.get('/api/v2/locations?country_code=DK&category=slaughter&display_precision=exact&limit=10')['data'] + self.assertEqual(len(rows), 1) + restricted = self.get('/api/v2/locations?country_code=DK&category=retail_and_prepared_food&limit=10')['data'] + self.assertEqual(restricted, []) + empty = self.get('/api/v2/locations?country_code=ZZ&limit=10') + self.assertEqual(empty['data'], []) + def test_failed_candidate_does_not_replace_promoted_release(self): self.env.create_failed_candidate() script = os.path.join(os.path.dirname(__file__), '..', '..', 'scripts', 'stages', 'validate-release.py') From eafb8fe5c11376f7f822fe57b671b19f00618700 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sun, 13 Sep 2026 17:01:43 -0700 Subject: [PATCH 028/311] Add restricted BLtU schema staging adapter --- pipeline/germany/README.md | 7 ++ pipeline/germany/bltu_adapter.py | 75 ++++++++++++++++++++ pipeline/germany/fixtures/synthetic_bltu.csv | 4 ++ pipeline/germany/test_adapter.py | 17 +++++ 4 files changed, 103 insertions(+) create mode 100644 pipeline/germany/bltu_adapter.py create mode 100644 pipeline/germany/fixtures/synthetic_bltu.csv diff --git a/pipeline/germany/README.md b/pipeline/germany/README.md index 9e35bcc..36551aa 100644 --- a/pipeline/germany/README.md +++ b/pipeline/germany/README.md @@ -27,3 +27,10 @@ release reference unchanged on failure and never promotes or publishes a release Before any real data is considered, add source-specific acquisition, dependency capture, and suppression checks across map/API/export/cache/history surfaces; terms, privacy/safety, legal, review, and publication approval remain human gates. + +`bltu_adapter.py` is the restricted-export profile for the current BLtU General List. +It uses positional columns because the export repeats activity-code headers and has +irregular row lengths. It preserves headers and values as ordered pairs, keeps the +current approval number distinct from legacy numbers, and quarantines malformed rows, +missing current IDs, and unmapped activities. It is not a release adapter while +`terms_status` is `pending_confirmation`. diff --git a/pipeline/germany/bltu_adapter.py b/pipeline/germany/bltu_adapter.py new file mode 100644 index 0000000..a87250c --- /dev/null +++ b/pipeline/germany/bltu_adapter.py @@ -0,0 +1,75 @@ +"""Private BLtU export adapter; no geocoding or release promotion.""" + +from __future__ import annotations + +import csv +import hashlib +import json +from pathlib import Path + +SCHEMA_VERSION = "de-bltu-v1" +ADAPTER_VERSION = "de-bltu-adapter-1" +EXPECTED_COLUMNS = 50 +CURRENT_ID_INDEX = 5 +NAME_INDEX = 1 +STATE_INDEX = 0 +STREET_INDEX = 2 +CITY_INDEX = 3 +ACTIVITY_START = 7 +ACTIVITY_END = 44 +ACTIVITY_MAP = {"SH": "Meat Slaughter", "CP": "Meat Processing"} + + +def _write_jsonl(path: Path, rows: list[dict]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("".join(json.dumps(row, ensure_ascii=False, sort_keys=True) + "\n" for row in rows), encoding="utf-8") + + +def run(raw_path: Path, output_dir: Path, config: dict) -> dict: + raw = raw_path.read_bytes() + text = raw.decode("cp1252") + rows = list(csv.reader(text.splitlines(), delimiter=";")) + headers = rows[0] if rows else [] + parsed, normalized, quarantined = [], [], [] + for row_number, values in enumerate(rows[1:], start=2): + evidence = {"source_row": row_number, "source_headers": headers, "source_values": values, "provenance": config} + parsed.append(evidence) + if len(values) != EXPECTED_COLUMNS: + quarantined.append({**evidence, "quarantine_reason": f"physical column count {len(values)} != {EXPECTED_COLUMNS}"}) + continue + current_id = values[CURRENT_ID_INDEX].strip() + name = values[NAME_INDEX].strip() + if not current_id: + quarantined.append({**evidence, "quarantine_reason": "missing current approval id"}) + continue + if not name: + quarantined.append({**evidence, "quarantine_reason": "missing establishment name"}) + continue + activity_codes = [headers[i].strip() for i in range(ACTIVITY_START, ACTIVITY_END) if values[i].strip() and headers[i].strip()] + activities = sorted({ACTIVITY_MAP[code] for code in activity_codes if code in ACTIVITY_MAP}) + if not activities: + quarantined.append({**evidence, "quarantine_reason": "unmapped activity code"}) + continue + source_columns = [{"header": headers[i], "value": values[i]} for i in range(EXPECTED_COLUMNS)] + normalized.append({ + "schema_version": SCHEMA_VERSION, + "source_id": current_id, + "establishment_id": current_id, + "establishment_name": name, + "type": "; ".join(activities), + "state": values[STATE_INDEX].strip(), + "street": values[STREET_INDEX].strip(), + "city": values[CITY_INDEX].strip(), + "latitude": None, + "longitude": None, + "coordinate_status": "source_unavailable", + "source_columns": source_columns, + "provenance": config, + }) + output_dir.mkdir(parents=True, exist_ok=True) + for state, values in (("parsed", parsed), ("normalized", normalized), ("quarantined", quarantined)): + _write_jsonl(output_dir / state / "records.jsonl", values) + (output_dir / "released").mkdir(exist_ok=True) + manifest = {**config, "checksum_sha256": hashlib.sha256(raw).hexdigest(), "byte_size": len(raw), "adapter_version": ADAPTER_VERSION, "schema_version": SCHEMA_VERSION, "input_rows": len(parsed), "normalized_rows": len(normalized), "quarantined_rows": len(quarantined), "release_state": "not-created"} + (output_dir / "run-manifest.json").write_text(json.dumps(manifest, sort_keys=True), encoding="utf-8") + return manifest diff --git a/pipeline/germany/fixtures/synthetic_bltu.csv b/pipeline/germany/fixtures/synthetic_bltu.csv new file mode 100644 index 0000000..280c993 --- /dev/null +++ b/pipeline/germany/fixtures/synthetic_bltu.csv @@ -0,0 +1,4 @@ +# Bundesland;Name des Betriebs;Straße / Haus-Nr.;Ort;Alte Zulassungsnummern;Neue Zulassungsnummer;Zulassungsnummer Eierpackstellen;CS;RW;WM;SH;CP;SH;CP;GHE;MM;MP;MSM;PP;CC;PP;CC;PP;CC;PP;CC;PP;CC;PP;CC;EPC;LEP;PP;AH;FV;ZV;FFPP;PP;WM;PC;DC;;Einschränkungen;Bemerkungen;Zulassung befristet bis;Zulassung ruht seit;Drittlandzulassungen;;; +BY;Synthetic processor;Street 1;Munich;;DE-SYN-001;;;;;;x;;;;;;;;;;;;;;;x;;;;;;;;;;;;;;;;;;;;;;; +BE;Synthetic mixed;Street 2;Berlin;;DE-SYN-002;;;;;x;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +HE;Synthetic unknown;Street 3;Frankfurt;;DE-SYN-003;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; diff --git a/pipeline/germany/test_adapter.py b/pipeline/germany/test_adapter.py index 6bc9af6..d0ebc16 100644 --- a/pipeline/germany/test_adapter.py +++ b/pipeline/germany/test_adapter.py @@ -7,8 +7,10 @@ try: from .adapter import parse, normalize, run, source_metadata + from .bltu_adapter import run as run_bltu except ImportError: from adapter import parse, normalize, run, source_metadata + from bltu_adapter import run as run_bltu ROOT = Path(__file__).parent @@ -68,6 +70,21 @@ def test_no_release_is_created_by_adapter(self): self.assertFalse((output / "released" / "records.jsonl").exists()) self.assertEqual(json.loads((output / "run-manifest.json").read_text())["quarantined_rows"], 1) + def test_bltu_positional_mapping_preserves_duplicate_headers_and_quarantines_unknowns(self): + with tempfile.TemporaryDirectory() as directory: + output = Path(directory) / "out" + config = {"source_url": "https://example.invalid/bltu", "terms_status": "pending_confirmation"} + manifest = run_bltu(ROOT / "fixtures" / "synthetic_bltu.csv", output, config) + self.assertEqual(manifest["input_rows"], 3) + self.assertEqual(manifest["normalized_rows"], 2) + self.assertEqual(manifest["quarantined_rows"], 1) + records = [json.loads(line) for line in (output / "normalized" / "records.jsonl").read_text().splitlines()] + self.assertEqual(len(records[0]["source_columns"]), 50) + self.assertEqual(records[0]["source_columns"][10]["header"], "SH") + self.assertIsNone(records[0]["latitude"]) + quarantined = [json.loads(line) for line in (output / "quarantined" / "records.jsonl").read_text().splitlines()] + self.assertEqual(quarantined[0]["quarantine_reason"], "unmapped activity code") + if __name__ == "__main__": unittest.main() From 57182ae53dfd7b07ce5fe806cb0157e6c9f7ac42 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sun, 13 Sep 2026 17:11:02 -0700 Subject: [PATCH 029/311] Bring restricted BLtU staging to Denmark parity --- pipeline/common/orchestrator.py | 10 +++++++--- pipeline/germany/README.md | 6 ++++++ pipeline/germany/bltu_adapter.py | 28 +++++++++++++++++++++++++-- pipeline/germany/source-registry.json | 11 ++++++++++- pipeline/germany/test_adapter.py | 14 ++++++++++++++ pipeline/germany/test_orchestrator.py | 4 ++++ 6 files changed, 67 insertions(+), 6 deletions(-) diff --git a/pipeline/common/orchestrator.py b/pipeline/common/orchestrator.py index 1fcdf47..db17c7f 100644 --- a/pipeline/common/orchestrator.py +++ b/pipeline/common/orchestrator.py @@ -8,7 +8,7 @@ from pathlib import Path from typing import Any, Callable -ORCHESTRATOR_VERSION = "v2-orchestrator-1" +ORCHESTRATOR_VERSION = "v2-orchestrator-2" def _atomic(path: Path, payload: bytes) -> None: @@ -52,12 +52,16 @@ def run_registered_input(raw_path: str | Path, runs_dir: str | Path, config: dic records = [json.loads(line) for line in (run_dir / "normalized" / "records.jsonl").read_text(encoding="utf-8").splitlines() if line] suppressed = suppressed_ids or set() candidate = [r for r in records if r.get("source_id") not in suppressed] - restricted = config.get("acquisition_status") == "restricted_pending_terms" + restricted = (config.get("terms_status") == "pending_confirmation" + or config.get("acquisition_status") == "restricted_pending_terms") if restricted: # Restricted inputs may be parsed and retained for review, but can # never create a publication candidate until terms are confirmed. - status = {"status": "staged-restricted", "publication_state": "restricted", + status = {"status": "staged-restricted", "publication_state": "terms-gate-blocked", "candidate_created": False, "release_promoted": False, + "public_surfaces": {"api": False, "map": False, "export": False, + "cache": False, "history": False}, + "geocoding": "disabled", "suppressed_count": len(records) - len(candidate), "manifest": manifest, "prior_eligible_release": prior_eligible_release} else: diff --git a/pipeline/germany/README.md b/pipeline/germany/README.md index 36551aa..10c5dd3 100644 --- a/pipeline/germany/README.md +++ b/pipeline/germany/README.md @@ -34,3 +34,9 @@ irregular row lengths. It preserves headers and values as ordered pairs, keeps t current approval number distinct from legacy numbers, and quarantines malformed rows, missing current IDs, and unmapped activities. It is not a release adapter while `terms_status` is `pending_confirmation`. + +Restricted runs also record schema, configuration, and mapping fingerprints plus +aggregate row-length, activity, quarantine, and coordinate diagnostics. The shared +orchestrator explicitly marks API, map, export, cache, and history surfaces unavailable +and geocoding disabled while terms are pending. These safeguards do not replace the +separate human terms, privacy/safety, legal, suppression, review, or publication gates. diff --git a/pipeline/germany/bltu_adapter.py b/pipeline/germany/bltu_adapter.py index a87250c..a749be1 100644 --- a/pipeline/germany/bltu_adapter.py +++ b/pipeline/germany/bltu_adapter.py @@ -5,6 +5,8 @@ import csv import hashlib import json +import os +import tempfile from pathlib import Path SCHEMA_VERSION = "de-bltu-v1" @@ -17,12 +19,23 @@ CITY_INDEX = 3 ACTIVITY_START = 7 ACTIVITY_END = 44 +MAPPING_VERSION = "de-bltu-activity-map-1" ACTIVITY_MAP = {"SH": "Meat Slaughter", "CP": "Meat Processing"} def _write_jsonl(path: Path, rows: list[dict]) -> None: path.parent.mkdir(parents=True, exist_ok=True) - path.write_text("".join(json.dumps(row, ensure_ascii=False, sort_keys=True) + "\n" for row in rows), encoding="utf-8") + fd, temp_name = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent) + try: + with os.fdopen(fd, "w", encoding="utf-8", newline="\n") as handle: + handle.write("".join(json.dumps(row, ensure_ascii=False, sort_keys=True) + "\n" for row in rows)) + os.replace(temp_name, path) + except Exception: + try: + os.unlink(temp_name) + except FileNotFoundError: + pass + raise def run(raw_path: Path, output_dir: Path, config: dict) -> dict: @@ -31,7 +44,9 @@ def run(raw_path: Path, output_dir: Path, config: dict) -> dict: rows = list(csv.reader(text.splitlines(), delimiter=";")) headers = rows[0] if rows else [] parsed, normalized, quarantined = [], [], [] + lengths, activity_counts, mapping_counts = {}, {}, {} for row_number, values in enumerate(rows[1:], start=2): + lengths[str(len(values))] = lengths.get(str(len(values)), 0) + 1 evidence = {"source_row": row_number, "source_headers": headers, "source_values": values, "provenance": config} parsed.append(evidence) if len(values) != EXPECTED_COLUMNS: @@ -46,7 +61,11 @@ def run(raw_path: Path, output_dir: Path, config: dict) -> dict: quarantined.append({**evidence, "quarantine_reason": "missing establishment name"}) continue activity_codes = [headers[i].strip() for i in range(ACTIVITY_START, ACTIVITY_END) if values[i].strip() and headers[i].strip()] + for code in activity_codes: + activity_counts[code] = activity_counts.get(code, 0) + 1 activities = sorted({ACTIVITY_MAP[code] for code in activity_codes if code in ACTIVITY_MAP}) + for activity in activities: + mapping_counts[activity] = mapping_counts.get(activity, 0) + 1 if not activities: quarantined.append({**evidence, "quarantine_reason": "unmapped activity code"}) continue @@ -57,6 +76,7 @@ def run(raw_path: Path, output_dir: Path, config: dict) -> dict: "establishment_id": current_id, "establishment_name": name, "type": "; ".join(activities), + "interpretation": {"status": "mapped", "mapping_version": MAPPING_VERSION, "source_codes": activity_codes}, "state": values[STATE_INDEX].strip(), "street": values[STREET_INDEX].strip(), "city": values[CITY_INDEX].strip(), @@ -70,6 +90,10 @@ def run(raw_path: Path, output_dir: Path, config: dict) -> dict: for state, values in (("parsed", parsed), ("normalized", normalized), ("quarantined", quarantined)): _write_jsonl(output_dir / state / "records.jsonl", values) (output_dir / "released").mkdir(exist_ok=True) - manifest = {**config, "checksum_sha256": hashlib.sha256(raw).hexdigest(), "byte_size": len(raw), "adapter_version": ADAPTER_VERSION, "schema_version": SCHEMA_VERSION, "input_rows": len(parsed), "normalized_rows": len(normalized), "quarantined_rows": len(quarantined), "release_state": "not-created"} + schema_fingerprint = hashlib.sha256(json.dumps(headers, ensure_ascii=False, separators=(",", ":")).encode()).hexdigest() + config_fingerprint = hashlib.sha256(json.dumps({"mapping_version": MAPPING_VERSION, "activity_map": ACTIVITY_MAP, "expected_columns": EXPECTED_COLUMNS}, sort_keys=True, separators=(",", ":")).encode()).hexdigest() + diagnostics = {"row_length_counts": lengths, "source_activity_code_counts": activity_counts, "mapped_activity_counts": mapping_counts, "quarantine_reason_counts": {reason: sum(1 for row in quarantined if row["quarantine_reason"] == reason) for reason in sorted({row["quarantine_reason"] for row in quarantined})}, "coordinate_status_counts": {"source_unavailable": len(normalized)}} + (output_dir / "validation-report.json").write_text(json.dumps(diagnostics, indent=2, sort_keys=True) + "\n", encoding="utf-8") + manifest = {**config, "checksum_sha256": hashlib.sha256(raw).hexdigest(), "byte_size": len(raw), "adapter_version": ADAPTER_VERSION, "schema_version": SCHEMA_VERSION, "mapping_version": MAPPING_VERSION, "schema_fingerprint": schema_fingerprint, "config_fingerprint": config_fingerprint, "input_rows": len(parsed), "normalized_rows": len(normalized), "quarantined_rows": len(quarantined), "release_state": "not-created", "release_gate": "restricted_pending_terms", "geocoding": "disabled"} (output_dir / "run-manifest.json").write_text(json.dumps(manifest, sort_keys=True), encoding="utf-8") return manifest diff --git a/pipeline/germany/source-registry.json b/pipeline/germany/source-registry.json index 7d5b89f..67cc83b 100644 --- a/pipeline/germany/source-registry.json +++ b/pipeline/germany/source-registry.json @@ -2,6 +2,7 @@ "country_code": "de", "adapter_version": "de-v2-foundation-1", "schema_version": "location-v2-foundation-1", + "mapping_version": "de-bltu-activity-map-1", "source_url": "https://gis.bvl.bund.de/datenportal/", "retrieval_timestamp": "NOT-ACQUIRED", "source_publication_date": "UNKNOWN-UNTIL-EXPORT", @@ -10,5 +11,13 @@ "terms_review": "BLOCKED-PENDING-HUMAN-CONFIRMATION", "acquisition_status": "restricted_pending_terms", "privacy_review": "REQUIRED-HUMAN-GATE", - "publication_approval": "REQUIRED-HUMAN-GATE" + "publication_approval": "REQUIRED-HUMAN-GATE", + "geocoding": "disabled", + "public_surfaces": { + "api": false, + "map": false, + "export": false, + "cache": false, + "history": false + } } diff --git a/pipeline/germany/test_adapter.py b/pipeline/germany/test_adapter.py index d0ebc16..7bf1bad 100644 --- a/pipeline/germany/test_adapter.py +++ b/pipeline/germany/test_adapter.py @@ -82,9 +82,23 @@ def test_bltu_positional_mapping_preserves_duplicate_headers_and_quarantines_unk self.assertEqual(len(records[0]["source_columns"]), 50) self.assertEqual(records[0]["source_columns"][10]["header"], "SH") self.assertIsNone(records[0]["latitude"]) + self.assertTrue(manifest["schema_fingerprint"]) + self.assertTrue(manifest["config_fingerprint"]) + self.assertEqual(manifest["mapping_version"], "de-bltu-activity-map-1") + diagnostics = json.loads((output / "validation-report.json").read_text()) + self.assertEqual(diagnostics["row_length_counts"], {"50": 3}) quarantined = [json.loads(line) for line in (output / "quarantined" / "records.jsonl").read_text().splitlines()] self.assertEqual(quarantined[0]["quarantine_reason"], "unmapped activity code") + def test_bltu_mapping_is_explicit_and_coordinates_are_never_enriched(self): + with tempfile.TemporaryDirectory() as directory: + output = Path(directory) / "out" + run_bltu(ROOT / "fixtures" / "synthetic_bltu.csv", output, {"source_url": "https://example.invalid/bltu", "terms_status": "pending_confirmation"}) + records = [json.loads(line) for line in (output / "normalized" / "records.jsonl").read_text().splitlines()] + self.assertTrue(all(row["interpretation"]["status"] == "mapped" for row in records)) + self.assertTrue(all(row["coordinate_status"] == "source_unavailable" for row in records)) + self.assertTrue(all(row["latitude"] is None and row["longitude"] is None for row in records)) + if __name__ == "__main__": unittest.main() diff --git a/pipeline/germany/test_orchestrator.py b/pipeline/germany/test_orchestrator.py index 6571759..4ebdc1e 100644 --- a/pipeline/germany/test_orchestrator.py +++ b/pipeline/germany/test_orchestrator.py @@ -68,8 +68,12 @@ def test_pending_terms_can_stage_but_cannot_create_candidate(self): result = run_registered_input(raw_path, root / "runs", restricted) self.assertEqual(result["status"], "staged-restricted") self.assertFalse(result["candidate_created"]) + self.assertEqual(result["geocoding"], "disabled") + self.assertEqual(set(result["public_surfaces"]), {"api", "map", "export", "cache", "history"}) + self.assertFalse(any(result["public_surfaces"].values())) run_dir = next((root / "runs").iterdir()) self.assertFalse((run_dir / "release-candidate").exists()) + self.assertFalse((run_dir / "released" / "records.jsonl").exists()) if __name__ == "__main__": From c16bd7a7c673cc0726797aa1dfb053a94583cada Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sun, 13 Sep 2026 17:18:11 -0700 Subject: [PATCH 030/311] Add restricted run delta detection --- pipeline/README.md | 9 +++++ pipeline/common/delta.py | 67 +++++++++++++++++++++++++++++++++++ pipeline/common/test_delta.py | 62 ++++++++++++++++++++++++++++++++ 3 files changed, 138 insertions(+) create mode 100644 pipeline/common/delta.py create mode 100644 pipeline/common/test_delta.py diff --git a/pipeline/README.md b/pipeline/README.md index 1367dd6..529fea8 100644 --- a/pipeline/README.md +++ b/pipeline/README.md @@ -72,3 +72,12 @@ python pipeline/run-denmark-pipeline.py data/raw/denmark-smiley//Smileydata ``` Add `--geocode-limit 100` to run a bounded DAWA development sample. Every run gets numbered stage directories and a `pipeline-manifest.json` containing output sizes and SHA-256 checksums. + +## Restricted run comparison + +The shared `common/delta.py` comparison is private and aggregate-only: it retains +both run manifests and fingerprints, classifies added/changed/not-observed/suppressed +counts, blocks schema changes, and never interprets source absence as closure. Failed +or partial comparisons retain the prior eligible release reference and expose no +public surface. Terms, privacy/safety, suppression, review, project approval, and +publication remain separate gates. diff --git a/pipeline/common/delta.py b/pipeline/common/delta.py new file mode 100644 index 0000000..5dd6449 --- /dev/null +++ b/pipeline/common/delta.py @@ -0,0 +1,67 @@ +"""Private, aggregate-only comparison of two validated adapter runs.""" + +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +from typing import Any + +DELTA_VERSION = "v2-delta-1" + + +def _jsonl(path: Path) -> list[dict[str, Any]]: + if not path.exists(): + raise ValueError(f"missing run state: {path}") + return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line] + + +def _manifest(run_dir: Path) -> dict[str, Any]: + path = run_dir / "run-manifest.json" + if not path.exists(): + raise ValueError(f"missing run manifest: {path}") + return json.loads(path.read_text(encoding="utf-8")) + + +def _fingerprint(row: dict[str, Any]) -> str: + comparable = {key: value for key, value in row.items() if key not in {"provenance", "source_values", "source_columns"}} + return hashlib.sha256(json.dumps(comparable, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode()).hexdigest() + + +def compare_runs(previous_dir: Path, current_dir: Path, suppressed_ids: set[str] | None = None, prior_eligible_release: dict[str, Any] | None = None) -> dict[str, Any]: + """Compare normalized states without interpreting absence as closure. + + Only identifiers, categories, counts, and run fingerprints are emitted. Source + payloads remain in the restricted run directories and are never copied here. + """ + try: + previous_manifest = _manifest(previous_dir) + current_manifest = _manifest(current_dir) + if previous_manifest.get("schema_fingerprint") != current_manifest.get("schema_fingerprint"): + return {"status": "schema-change-blocked", "delta_version": DELTA_VERSION, "publication_state": "unchanged", "release_promoted": False, "public_surfaces": _blocked_surfaces(), "geocoding": "disabled", "prior_eligible_release": prior_eligible_release, "previous": _summary(previous_manifest), "current": _summary(current_manifest), "counts": {"added": 0, "changed": 0, "not_observed": 0, "suppressed": 0}} + previous = {row.get("source_id"): row for row in _jsonl(previous_dir / "normalized" / "records.jsonl") if row.get("source_id")} + current = {row.get("source_id"): row for row in _jsonl(current_dir / "normalized" / "records.jsonl") if row.get("source_id")} + suppressed = suppressed_ids or set() + added = changed = not_observed = suppressed_count = 0 + for source_id, row in current.items(): + if source_id in suppressed: + suppressed_count += 1 + elif source_id not in previous: + added += 1 + elif _fingerprint(previous[source_id]) != _fingerprint(row): + changed += 1 + # A missing source row is only an observation boundary, never a closure. + for source_id in previous: + if source_id not in current and source_id not in suppressed: + not_observed += 1 + return {"status": "delta-ready", "delta_version": DELTA_VERSION, "publication_state": "terms-gate-blocked" if current_manifest.get("terms_status") == "pending_confirmation" else "human-gate-required", "release_promoted": False, "public_surfaces": _blocked_surfaces(), "geocoding": "disabled", "prior_eligible_release": prior_eligible_release, "previous": _summary(previous_manifest), "current": _summary(current_manifest), "counts": {"added": added, "changed": changed, "not_observed": not_observed, "suppressed": suppressed_count}} + except Exception as exc: + return {"status": "failed", "delta_version": DELTA_VERSION, "publication_state": "unchanged", "release_promoted": False, "public_surfaces": _blocked_surfaces(), "geocoding": "disabled", "error_type": type(exc).__name__, "error": str(exc), "prior_eligible_release": prior_eligible_release} + + +def _summary(manifest: dict[str, Any]) -> dict[str, Any]: + return {key: manifest.get(key) for key in ("checksum_sha256", "schema_fingerprint", "config_fingerprint", "mapping_version", "adapter_version", "schema_version", "input_rows", "normalized_rows", "quarantined_rows")} + + +def _blocked_surfaces() -> dict[str, bool]: + return {surface: False for surface in ("api", "map", "export", "cache", "history")} diff --git a/pipeline/common/test_delta.py b/pipeline/common/test_delta.py new file mode 100644 index 0000000..d6d59b4 --- /dev/null +++ b/pipeline/common/test_delta.py @@ -0,0 +1,62 @@ +import json +import tempfile +import unittest +from pathlib import Path + +from delta import compare_runs + + +def write_run(root: Path, name: str, rows: list[dict], schema: str = "schema-a", terms: str = "pending_confirmation") -> Path: + run = root / name + (run / "normalized").mkdir(parents=True) + (run / "normalized" / "records.jsonl").write_text("".join(json.dumps(row) + "\n" for row in rows), encoding="utf-8") + (run / "run-manifest.json").write_text(json.dumps({"checksum_sha256": name, "schema_fingerprint": schema, "config_fingerprint": "config-" + name, "mapping_version": "map-1", "adapter_version": "adapter-1", "schema_version": schema, "input_rows": len(rows), "normalized_rows": len(rows), "quarantined_rows": 0, "terms_status": terms}), encoding="utf-8") + return run + + +def row(source_id: str, name: str, type_: str = "Meat Processing") -> dict: + return {"source_id": source_id, "establishment_name": name, "type": type_, "provenance": {"run": name}, "source_values": {"name": name}} + + +class DeltaTests(unittest.TestCase): + def test_reimport_change_add_missing_and_suppression(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + old = write_run(root, "old", [row("A", "same"), row("B", "removed")]) + new = write_run(root, "new", [row("A", "changed", "Meat Slaughter"), row("C", "added"), row("B", "removed")]) + result = compare_runs(old, new, {"C"}) + self.assertEqual(result["status"], "delta-ready") + self.assertEqual(result["counts"], {"added": 0, "changed": 1, "not_observed": 0, "suppressed": 1}) + self.assertEqual(result["previous"]["config_fingerprint"], "config-old") + self.assertEqual(result["current"]["config_fingerprint"], "config-new") + self.assertEqual(set(result["public_surfaces"]), {"api", "map", "export", "cache", "history"}) + + def test_missing_source_is_not_observed_not_closed(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + old = write_run(root, "old", [row("A", "same"), row("B", "gone")]) + new = write_run(root, "new", [row("A", "same")]) + self.assertEqual(compare_runs(old, new)["counts"]["not_observed"], 1) + + def test_schema_change_and_id_reuse_are_blocked_or_changed(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + old = write_run(root, "old", [row("A", "old site")]) + changed_schema = write_run(root, "new-schema", [row("A", "new site")], schema="schema-b") + self.assertEqual(compare_runs(old, changed_schema)["status"], "schema-change-blocked") + reused = write_run(root, "new", [row("A", "new site")]) + self.assertEqual(compare_runs(old, reused)["counts"]["changed"], 1) + + def test_failure_retains_prior_reference_and_blocks_surfaces(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + old = write_run(root, "old", [row("A", "same")]) + prior = {"release_id": "validated-1", "eligible": True} + result = compare_runs(old, root / "missing", prior_eligible_release=prior) + self.assertEqual(result["status"], "failed") + self.assertEqual(result["prior_eligible_release"], prior) + self.assertFalse(any(result["public_surfaces"].values())) + + +if __name__ == "__main__": + unittest.main() From 3408d34c10b1ca76dfdacde211319aa08bc16cb8 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sun, 13 Sep 2026 17:17:49 -0700 Subject: [PATCH 031/311] Document FSS Scotland source readiness gates --- ...proved-establishments-source-assessment.md | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 docs/countries/uk/fss-approved-establishments-source-assessment.md diff --git a/docs/countries/uk/fss-approved-establishments-source-assessment.md b/docs/countries/uk/fss-approved-establishments-source-assessment.md new file mode 100644 index 0000000..45621ea --- /dev/null +++ b/docs/countries/uk/fss-approved-establishments-source-assessment.md @@ -0,0 +1,31 @@ +# FSS Scotland approved-establishments source assessment + +Assessment date: 2026-09-13 +Scope: read-only readiness review; no artifact was downloaded. + +## Direct official evidence + +- FSS publishes an approved-establishments register for Scotland. The register page is dated 9 September 2026 and links an XLSX resource. It says an approval number without a two-letter prefix is approved by FSS rather than a local authority: [FSS register](https://www.foodstandards.gov.scot/business-guidance/running-a-food-business/publications/approved-establishments-register). +- FSS's open-data metadata identifies the resource as `Approved Establishments in Scotland`, coverage Scotland, site ID FS0010, OGL v3, monthly updates, and publication date 11 August 2026. The linked CSV URL is [Approved Establishments in Scotland.csv](https://www.foodstandards.gov.scot/sites/default/files/2026-08/Approved%20Establishments%20in%20Scotland.csv): [FSS open-data metadata](https://www.foodstandards.gov.scot/open-data-portal/approved-establishments-in-scotland). +- FSS says it maintains and publishes the list based on information supplied by competent authorities, and distinguishes FSS approvals from local-authority approvals: [FSS approved establishments guidance](https://www.foodstandards.gov.scot/business-guidance/industry-specific-advice/meat/meat-and-meat-establishments/fss-approved-establishments). +- FSS's privacy notice for food-law enforcement says it holds business trading name/address and operator name, obtains information from local authorities, uses it for statutory food-law enforcement, and retains name/address information while approved and up to six complete financial years after closure: [FSS privacy notices](https://www.foodstandards.gov.scot/privacy-notices). +- The OGL v3 terms require attribution and state that the licence does not cover personal data: [National Archives OGL v3](https://www.nationalarchives.gov.uk/doc/open-government-licence/version/3/). FSS's open-data plan describes its intention to publish information under OGL v3: [FSS Open Data Publication Plan](https://www.foodstandards.gov.scot/about-us/how-we-work/governance/open-data-publication-plan). + +## Readiness decision + +**Conditional GO for restricted staging of the published CSV only**, after recording the exact URL, retrieval timestamp, byte hash/size, source metadata, and a privacy-screened restricted copy. This is an operational recommendation, not legal clearance. Keep the raw artifact access-controlled and do not add it to repository fixtures. + +**NO-GO for automated recurring retrieval or public release at this time.** The reviewed FSS pages do not state rate limits, robots/API expectations, automated-retrieval permission, a machine-readable schema contract, or a project-specific retention/removal instruction. These must be confirmed or conservatively bounded before scheduling retrieval. OGL permission also cannot be treated as permission to publish personal data. + +## Field and publication constraints + +Approval number, trading name, competent authority and activity information appear in the published-list context, but the current CSV contents were not inspected. Preserve all source cells as strings and do not infer a schema until the artifact is lawfully acquired and reviewed. Treat address lines, operator names, telephone/contact data, and precise coordinates (if present) as privacy-sensitive. Do not geocode or publish precise locations by default. FSS's published guidance also shows that approval responsibility can be split between FSS and local authorities, so authority provenance must remain explicit. + +The public derivative gate requires: verified schema and field-level publication status; explicit OGL attribution including FSS/source/date and licence link; personal-data and residential/private-location screening; suppression propagation; documented correction/removal handling; human terms/privacy review; and release-specific project approval. Until those gates pass, retain only restricted staging and quarantine, with no public API, export, map, or release. + +## Unresolved evidence to obtain before acquisition automation + +1. FSS confirmation of acceptable request frequency, caching, conditional requests, and whether automated retrieval is welcomed or constrained. +2. The live CSV/XLSX schema, effective/update-date semantics, encoding, and whether the two formats are equivalent. +3. Dataset-specific attribution wording, third-party rights exclusions, correction/removal process, and any source terms beyond the OGL metadata. +4. A project retention schedule aligned with source corrections/removals; the FSS six-year historical practice is evidence about FSS's own records, not authorization for this project's indefinite retention. From d481468afc52d8b9e701bb8e66decf4dea40db44 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sun, 13 Sep 2026 16:40:18 -0700 Subject: [PATCH 032/311] Record BLtU metadata evidence boundary --- docs/bvl-bltu-permission-inquiry.md | 32 +++++++++++++++ docs/germany-source-assessment.md | 64 +++++++++++++++++++++++++++++ 2 files changed, 96 insertions(+) create mode 100644 docs/bvl-bltu-permission-inquiry.md create mode 100644 docs/germany-source-assessment.md diff --git a/docs/bvl-bltu-permission-inquiry.md b/docs/bvl-bltu-permission-inquiry.md new file mode 100644 index 0000000..1a23934 --- /dev/null +++ b/docs/bvl-bltu-permission-inquiry.md @@ -0,0 +1,32 @@ +# Unsent BVL BLtU permission inquiry + +Draft only. This message has not been sent. It requests factual clarification and +does not assert entitlement or seek legal advice. + +**To:** BVL, Unit 115 / Open Data contact +**Subject:** BLtU export: automated research use and data licence clarification + +Hello, + +We are preparing a public-interest research tool and would like to use the BLtU +establishment export shown at . +Could you please confirm, for the BLtU CSV/XML exports specifically: + +1. Is the dataset offered under Datenlizenz Deutschland – Namensnennung – Version + 2.0 (`dl-de/by-2.0`), or another named licence? Please provide the authoritative + metadata or licence URL. +2. Is automated retrieval permitted, including a reasonable periodic refresh rate? +3. May researchers retain the downloaded artifact locally and preserve its checksum, + retrieval timestamp, and source identifiers for reproducibility? +4. May transformed/normalized records be redistributed in a public research tool, + with provider attribution, source URI, and a modified-data notice? +5. Are there restrictions or required handling for establishment addresses, approval + numbers, activities, or species fields? +6. Are there update, correction, withdrawal, or deletion obligations we should + implement when a record changes or disappears? + +Thank you. We will not publish or redistribute BLtU-derived records until we have +documented the applicable terms and completed our privacy and review gates. + +Kind regards, +Until Every Cage project diff --git a/docs/germany-source-assessment.md b/docs/germany-source-assessment.md new file mode 100644 index 0000000..73b2f92 --- /dev/null +++ b/docs/germany-source-assessment.md @@ -0,0 +1,64 @@ +# Germany BLtU source assessment + +Status: automated reacquisition and publication paused pending a human terms decision. This document records +source evidence and risks; it is not legal advice, legal clearance, project approval, +or publication authorization. + +## Source identity and scope + +The BVL’s official cross-border-trade page identifies the BLtU database as the list +of establishments approved under Regulation (EC) 853/2004 and says the lists include +company address, approval number, and approved activities/species. It also says the +lists are continuously updated. See the [BVL scope page](https://www.bvl.bund.de/DE/Arbeitsbereiche/01_Lebensmittel/01_Aufgaben/05_GrenzueberschreitenderHandel/lm_grenzueberschrHandel_basepage.html). + +The [BVL data portal](https://gis.bvl.bund.de/datenportal/) describes the data as +authority-provided material, says tabular data can be exported as CSV or Excel, and +identifies BVL/BKG attribution notices for portal geodata. The portal’s export +capability supports a reproducible acquisition design, but it does not by itself +establish permission to redistribute the exported establishment records. + +The existing V1 Germany adapter is [`static_data/de/migrate_data.py`](../static_data/de/migrate_data.py), +which references the BLtU publication endpoint and expects downloaded/merged CSVs. +Its current behavior also performs external geocoding and emits a wide source-shaped +CSV; V2 must not reuse those behaviors without a reviewed, deterministic replacement. + +## Terms and risk decision + +The official pages located for this assessment document public access and export, but +do not present a clear license or redistribution grant for the establishment-record +export. Therefore acquisition is **blocked pending human confirmation** of: + +1. whether automated retrieval is permitted, including rate/frequency limits; +2. whether local research retention of the raw export is permitted; +3. whether transformed establishment records may be redistributed in a public + non-commercial project under the project’s data license/attribution model; and +4. required attribution, notices, update/deletion obligations, and any restrictions + on address, approval-number, or activity/species fields. + +This is a terms uncertainty, not a claim that the source prohibits use. A single +real BLtU export was downloaded earlier at the user's direction as a restricted +local research artifact and staged privately (15,797 input rows; 6,346 normalized; +9,451 quarantined). It is not a repository fixture, release candidate, API source, +map layer, export, or publication. That retrieval does not establish permission for +recurring acquisition, retention, or redistribution; those decisions remain open. +Privacy/safety review is separately required because facility addresses can overlap +with residences or identify individuals; source origin does not resolve that risk. + +## Metadata evidence boundary + +Any user-supplied JSON describing this source must be treated as a lead until tied to +the actual BLtU resource. In particular, `http://dcat-ap.de` identifies a metadata +profile/specification, not a dataset license, and `https://bund.de` is a placeholder, +not the BLtU resource URI. A DL-DE record for another BVL dataset cannot be inherited +by BLtU without dataset-specific evidence. + +## Planned recurring acquisition after approval + +Once the named human reviewer records a positive terms decision, an automated runner +may retrieve a current export into ignored raw storage and record URL, retrieval +timestamp, HTTP/content metadata, effective/publication date, byte size, SHA-256, +adapter/config versions, and source snapshot identity. It must then run only through +private parse/normalize/validate/quarantine staging. Unknown classifications, +unresolved coordinates, suspicious identity changes, and source disappearance must +remain explicit review outcomes. No geocoding, release promotion, API ingestion, or +publication is implied by acquisition approval. From d85372140cd58e3e2c8854b04ea1720a8fcdc708 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sun, 13 Sep 2026 18:12:28 -0700 Subject: [PATCH 033/311] test(pipeline): cover reconciled country gates in CI --- .gitignore | 1 + pipeline/common/test_delta.py | 5 ++++- pipeline/germany/README.md | 7 ++++--- pipeline/germany/test_orchestrator.py | 10 ++++++++++ pipeline/tests/run-standard.ps1 | 3 +++ 5 files changed, 22 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index 342a480..65fa339 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ /Secrets*.toml __pycache__/ /data/raw/ +/data/restricted/ /data/raw/**/*.xml /data/raw/**/*.csv /data/raw/**/*.xlsx diff --git a/pipeline/common/test_delta.py b/pipeline/common/test_delta.py index d6d59b4..bca000b 100644 --- a/pipeline/common/test_delta.py +++ b/pipeline/common/test_delta.py @@ -3,7 +3,10 @@ import unittest from pathlib import Path -from delta import compare_runs +try: + from .delta import compare_runs +except ImportError: # direct invocation from this directory + from delta import compare_runs def write_run(root: Path, name: str, rows: list[dict], schema: str = "schema-a", terms: str = "pending_confirmation") -> Path: diff --git a/pipeline/germany/README.md b/pipeline/germany/README.md index 10c5dd3..6d62f48 100644 --- a/pipeline/germany/README.md +++ b/pipeline/germany/README.md @@ -1,6 +1,6 @@ # Germany V2 adapter foundation -This is a local, synthetic-only foundation. It accepts a previously acquired raw +This is a local, restricted-staging foundation tested with synthetic fixtures. It accepts a previously acquired raw CSV path; it does not scrape, download, geocode, publish, or promote records. The adapter writes separate `parsed/`, `normalized/`, and `quarantined/` outputs and creates an empty `released/` state to make the publication boundary explicit. @@ -14,7 +14,8 @@ python -m unittest -v test_adapter.py The future acquisition runner must populate the registry fields in [`source-registry.json`](source-registry.json), preserve the raw artifact outside the repository where required, and record URL, retrieval time, checksum, byte size, -and source publication date. Terms, privacy/safety, suppression, legal, and project +and source publication date. That registry is a template, not the provenance manifest +for the previously acquired restricted BLtU artifact. Terms, privacy/safety, suppression, legal, and project publication approval remain named human gates. No technical pass is approval. The source classification mapping is intentionally narrow: `CP` and `GME` map to @@ -24,7 +25,7 @@ the adapter never geocodes or guesses. `orchestrator.py` now provides hash-addre input registration, schema/version manifests, isolated runs, failure-safe candidate handoff, and suppression application. It deliberately leaves the prior eligible release reference unchanged on failure and never promotes or publishes a release. -Before any real data is considered, add source-specific acquisition, dependency +Before any real data is considered for release, add source-specific acquisition, dependency capture, and suppression checks across map/API/export/cache/history surfaces; terms, privacy/safety, legal, review, and publication approval remain human gates. diff --git a/pipeline/germany/test_orchestrator.py b/pipeline/germany/test_orchestrator.py index 4ebdc1e..8ac1601 100644 --- a/pipeline/germany/test_orchestrator.py +++ b/pipeline/germany/test_orchestrator.py @@ -75,6 +75,16 @@ def test_pending_terms_can_stage_but_cannot_create_candidate(self): self.assertFalse((run_dir / "release-candidate").exists()) self.assertFalse((run_dir / "released" / "records.jsonl").exists()) + def test_pending_terms_alone_blocks_candidate(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + raw_path, _ = register_input(FIXTURE.read_bytes(), root, CONFIG) + result = run_registered_input(raw_path, root / "runs", {**CONFIG, "terms_status": "pending_confirmation"}) + self.assertEqual(result["status"], "staged-restricted") + self.assertFalse(result["candidate_created"]) + self.assertFalse(any(result["public_surfaces"].values())) + self.assertFalse((next((root / "runs").iterdir()) / "release-candidate").exists()) + if __name__ == "__main__": unittest.main() diff --git a/pipeline/tests/run-standard.ps1 b/pipeline/tests/run-standard.ps1 index eb71a08..93d9c17 100644 --- a/pipeline/tests/run-standard.ps1 +++ b/pipeline/tests/run-standard.ps1 @@ -29,6 +29,9 @@ try { python -m unittest discover -s pipeline/tests -q if ($LASTEXITCODE -ne 0) { throw "Python tests failed (exit $LASTEXITCODE)." } + + python -m unittest -q pipeline.germany.test_adapter pipeline.germany.test_orchestrator pipeline.common.test_delta pipeline.common.test_orchestrator pipeline.common.test_registry pipeline.sources.uk.fsa_approved.test_adapter pipeline.sources.uk.fss_approved.test_adapter pipeline.sources.uk.approved.test_compose + if ($LASTEXITCODE -ne 0) { throw "Country adapter tests failed (exit $LASTEXITCODE)." } } finally { $savedPreference = $ErrorActionPreference From f3621db4e850fd351cc890df85e44c11efad5c88 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sun, 13 Sep 2026 19:18:47 -0700 Subject: [PATCH 034/311] fix(pipeline): scope publication safety and preserve source identities --- pipeline/common/delta.py | 8 +- pipeline/common/identity.py | 32 ++++ pipeline/common/orchestrator.py | 21 ++- pipeline/common/test_delta.py | 10 ++ pipeline/common/test_orchestrator.py | 34 +++- pipeline/germany/adapter.py | 5 +- pipeline/germany/bltu_adapter.py | 41 +++-- pipeline/germany/test_adapter.py | 49 +++++- pipeline/germany/test_orchestrator.py | 27 ++- .../022_publication_safety_scopes.sql | 163 ++++++++++++++++++ pipeline/scripts/maintenance/local-v2.ps1 | 2 +- pipeline/scripts/stages/promote-release.py | 69 ++++++-- pipeline/scripts/stages/validate-release.py | 6 +- pipeline/sources/uk/approved/compose.py | 64 +++++-- pipeline/sources/uk/approved/test_compose.py | 41 ++++- pipeline/sources/uk/fsa_approved/adapter.py | 11 +- pipeline/sources/uk/fss_approved/adapter.py | 11 +- pipeline/tests/e2e/backup-restore.ps1 | 83 +++++++-- .../backup_restore_current_suppression.sql | 6 + pipeline/tests/e2e/backup_restore_seed.sql | 4 - .../e2e/docker-compose.backup-restore.yml | 4 + .../tests/test_local_v2_start_contract.py | 65 +++++++ pipeline/tests/test_promote_release.py | 38 ++++ .../tests/test_publication_scoped_stages.py | 127 ++++++++++++++ 24 files changed, 828 insertions(+), 93 deletions(-) create mode 100644 pipeline/common/identity.py create mode 100644 pipeline/migrations/022_publication_safety_scopes.sql create mode 100644 pipeline/tests/e2e/backup_restore_current_suppression.sql create mode 100644 pipeline/tests/e2e/docker-compose.backup-restore.yml create mode 100644 pipeline/tests/test_local_v2_start_contract.py create mode 100644 pipeline/tests/test_publication_scoped_stages.py diff --git a/pipeline/common/delta.py b/pipeline/common/delta.py index 5dd6449..ec47544 100644 --- a/pipeline/common/delta.py +++ b/pipeline/common/delta.py @@ -7,6 +7,8 @@ from pathlib import Path from typing import Any +from .identity import record_key + DELTA_VERSION = "v2-delta-1" @@ -28,7 +30,7 @@ def _fingerprint(row: dict[str, Any]) -> str: return hashlib.sha256(json.dumps(comparable, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode()).hexdigest() -def compare_runs(previous_dir: Path, current_dir: Path, suppressed_ids: set[str] | None = None, prior_eligible_release: dict[str, Any] | None = None) -> dict[str, Any]: +def compare_runs(previous_dir: Path, current_dir: Path, suppressed_ids: set[str | tuple[str, str, str]] | None = None, prior_eligible_release: dict[str, Any] | None = None) -> dict[str, Any]: """Compare normalized states without interpreting absence as closure. Only identifiers, categories, counts, and run fingerprints are emitted. Source @@ -39,8 +41,8 @@ def compare_runs(previous_dir: Path, current_dir: Path, suppressed_ids: set[str] current_manifest = _manifest(current_dir) if previous_manifest.get("schema_fingerprint") != current_manifest.get("schema_fingerprint"): return {"status": "schema-change-blocked", "delta_version": DELTA_VERSION, "publication_state": "unchanged", "release_promoted": False, "public_surfaces": _blocked_surfaces(), "geocoding": "disabled", "prior_eligible_release": prior_eligible_release, "previous": _summary(previous_manifest), "current": _summary(current_manifest), "counts": {"added": 0, "changed": 0, "not_observed": 0, "suppressed": 0}} - previous = {row.get("source_id"): row for row in _jsonl(previous_dir / "normalized" / "records.jsonl") if row.get("source_id")} - current = {row.get("source_id"): row for row in _jsonl(current_dir / "normalized" / "records.jsonl") if row.get("source_id")} + previous = {record_key(row): row for row in _jsonl(previous_dir / "normalized" / "records.jsonl")} + current = {record_key(row): row for row in _jsonl(current_dir / "normalized" / "records.jsonl")} suppressed = suppressed_ids or set() added = changed = not_observed = suppressed_count = 0 for source_id, row in current.items(): diff --git a/pipeline/common/identity.py b/pipeline/common/identity.py new file mode 100644 index 0000000..bb2c581 --- /dev/null +++ b/pipeline/common/identity.py @@ -0,0 +1,32 @@ +"""Stable, source-qualified record identities for restricted pipeline runs.""" + +from __future__ import annotations + +from typing import Any + + +UK_IDENTIFIERS = { + "fss_approved_establishments": "approval_number", + "fsa_approved_establishments": "establishment_id", +} + + +def record_key(record: dict[str, Any]) -> str | tuple[str, str, str]: + """Keep UK establishment IDs distinct across feeds and nations. + + Other adapters currently use a record-level source_id. Do not infer a UK + identity from a feed-level source_id when its nation or ID is missing. + """ + source_id = record.get("source_id") + if source_id in UK_IDENTIFIERS: + normalized = record.get("normalized") + if not isinstance(normalized, dict): + raise ValueError("UK record lacks normalized identity") + nation = normalized.get("nation") + identifier = normalized.get(UK_IDENTIFIERS[source_id]) + if not isinstance(nation, str) or not nation.strip() or not isinstance(identifier, str) or not identifier.strip(): + raise ValueError("UK record lacks nation-qualified identity") + return source_id, nation.strip(), identifier.strip() + if not isinstance(source_id, str) or not source_id: + raise ValueError("record lacks stable source_id") + return source_id diff --git a/pipeline/common/orchestrator.py b/pipeline/common/orchestrator.py index db17c7f..f89ce53 100644 --- a/pipeline/common/orchestrator.py +++ b/pipeline/common/orchestrator.py @@ -5,10 +5,13 @@ import json import os import tempfile +import uuid from pathlib import Path from typing import Any, Callable -ORCHESTRATOR_VERSION = "v2-orchestrator-2" +from .identity import record_key + +ORCHESTRATOR_VERSION = "v2-orchestrator-3" def _atomic(path: Path, payload: bytes) -> None: @@ -35,23 +38,28 @@ def register_input(raw: bytes, staging_dir: str | Path, config: dict[str, Any]) _atomic(artifact, raw) metadata = {**config, "checksum_sha256": digest, "byte_size": len(raw), "orchestrator_version": ORCHESTRATOR_VERSION, "raw_artifact": str(artifact)} - _atomic(staging / "raw" / f"{digest}.manifest.json", - (json.dumps(metadata, ensure_ascii=False, sort_keys=True, indent=2) + "\n").encode()) + # Equal bytes can represent separate observations with different retrieval + # metadata. Keep each observation while sharing the immutable raw bytes. + registration = staging / "raw" / "registrations" / f"{uuid.uuid4().hex}.manifest.json" + metadata["acquisition_manifest"] = str(registration) + _atomic(registration, (json.dumps(metadata, ensure_ascii=False, sort_keys=True, indent=2) + "\n").encode()) return artifact, metadata def run_registered_input(raw_path: str | Path, runs_dir: str | Path, config: dict[str, Any], adapter_runner: Callable[..., dict[str, Any]], - suppressed_ids: set[str] | None = None, + suppressed_ids: set[str | tuple[str, str, str]] | None = None, prior_eligible_release: dict[str, Any] | None = None) -> dict[str, Any]: """Run an adapter to a human-gated candidate, preserving prior release on failure.""" raw = Path(raw_path) - run_dir = Path(runs_dir) / hashlib.sha256(raw.read_bytes()).hexdigest()[:16] + runs = Path(runs_dir) + runs.mkdir(parents=True, exist_ok=True) + run_dir = Path(tempfile.mkdtemp(prefix=f"{hashlib.sha256(raw.read_bytes()).hexdigest()[:16]}-", dir=runs)) try: manifest = adapter_runner(raw, run_dir, config) records = [json.loads(line) for line in (run_dir / "normalized" / "records.jsonl").read_text(encoding="utf-8").splitlines() if line] suppressed = suppressed_ids or set() - candidate = [r for r in records if r.get("source_id") not in suppressed] + candidate = [r for r in records if record_key(r) not in suppressed] restricted = (config.get("terms_status") == "pending_confirmation" or config.get("acquisition_status") == "restricted_pending_terms") if restricted: @@ -75,5 +83,6 @@ def run_registered_input(raw_path: str | Path, runs_dir: str | Path, config: dic status = {"status": "failed", "publication_state": "unchanged", "release_promoted": False, "error_type": type(exc).__name__, "error": str(exc), "prior_eligible_release": prior_eligible_release} + status["run_dir"] = str(run_dir) _atomic(run_dir / "run-status.json", (json.dumps(status, ensure_ascii=False, sort_keys=True, indent=2) + "\n").encode()) return status diff --git a/pipeline/common/test_delta.py b/pipeline/common/test_delta.py index bca000b..8fb1ac7 100644 --- a/pipeline/common/test_delta.py +++ b/pipeline/common/test_delta.py @@ -60,6 +60,16 @@ def test_failure_retains_prior_reference_and_blocks_surfaces(self): self.assertEqual(result["prior_eligible_release"], prior) self.assertFalse(any(result["public_surfaces"].values())) + def test_uk_same_feed_id_in_different_nations_is_distinct(self): + def uk(nation, name): + return {"source_id": "fsa_approved_establishments", "normalized": {"nation": nation, "establishment_id": "00017", "trading_name": name}} + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + old = write_run(root, "old", [uk("England", "old"), uk("Wales", "same")]) + new = write_run(root, "new", [uk("England", "changed"), uk("Wales", "same")]) + result = compare_runs(old, new, {("fsa_approved_establishments", "Wales", "00017")}) + self.assertEqual(result["counts"], {"added": 0, "changed": 1, "not_observed": 0, "suppressed": 1}) + if __name__ == "__main__": unittest.main() diff --git a/pipeline/common/test_orchestrator.py b/pipeline/common/test_orchestrator.py index 47276b3..e6fffff 100644 --- a/pipeline/common/test_orchestrator.py +++ b/pipeline/common/test_orchestrator.py @@ -20,12 +20,42 @@ def test_registry_and_suppression_are_shared(self): artifact, metadata = register_input(raw, staging, {"source_id": adapter.source_id}) self.assertEqual(artifact.read_bytes(), raw) status = run_registered_input(artifact, Path(directory) / "runs", metadata, adapter.run, - suppressed_ids={adapter.source_id}) + suppressed_ids={(adapter.source_id, "Scotland", "001234")}) self.assertEqual(status["status"], "candidate-ready") - self.assertEqual(status["suppressed_count"], 2) + self.assertEqual(status["suppressed_count"], 1) + candidate = (Path(status["run_dir"]) / "release-candidate/records.jsonl").read_text() + self.assertNotIn("001234", candidate) + self.assertIn("078901", candidate) self.assertEqual((status["manifest"]["release_state"]), "not-created") self.assertEqual((status["prior_eligible_release"]), None) + def test_equal_bytes_keep_distinct_acquisition_metadata(self): + with tempfile.TemporaryDirectory() as directory: + first, first_meta = register_input(b"synthetic", directory, {"retrieved_at": "first"}) + second, second_meta = register_input(b"synthetic", directory, {"retrieved_at": "second"}) + self.assertEqual(first, second) + self.assertNotEqual(first_meta["acquisition_manifest"], second_meta["acquisition_manifest"]) + self.assertEqual(json.loads(Path(first_meta["acquisition_manifest"]).read_text())["retrieved_at"], "first") + self.assertEqual(json.loads(Path(second_meta["acquisition_manifest"]).read_text())["retrieved_at"], "second") + + def test_restricted_and_failed_reruns_do_not_reuse_candidate_path(self): + adapter = FssApprovedEstablishmentsAdapter() + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + raw, metadata = register_input((Path(__file__).parents[1] / "sources/uk/fss_approved/fixtures/valid.csv").read_bytes(), root / "staging", {"source_id": adapter.source_id}) + ready = run_registered_input(raw, root / "runs", metadata, adapter.run) + old_candidate = Path(ready["run_dir"]) / "release-candidate/records.jsonl" + self.assertTrue(old_candidate.exists()) + restricted = run_registered_input(raw, root / "runs", {**metadata, "terms_status": "pending_confirmation"}, adapter.run) + self.assertNotEqual(ready["run_dir"], restricted["run_dir"]) + self.assertFalse((Path(restricted["run_dir"]) / "release-candidate/records.jsonl").exists()) + def fail(*_args): + raise ValueError("synthetic failure") + failed = run_registered_input(raw, root / "runs", metadata, fail) + self.assertNotEqual(ready["run_dir"], failed["run_dir"]) + self.assertFalse((Path(failed["run_dir"]) / "release-candidate/records.jsonl").exists()) + self.assertTrue(old_candidate.exists()) + if __name__ == "__main__": unittest.main() diff --git a/pipeline/germany/adapter.py b/pipeline/germany/adapter.py index f6923e9..a3ed2cc 100644 --- a/pipeline/germany/adapter.py +++ b/pipeline/germany/adapter.py @@ -12,6 +12,7 @@ import hashlib import io import json +import math import os import tempfile from pathlib import Path @@ -107,6 +108,8 @@ def normalize(records: list[dict[str, Any]]) -> tuple[list[dict[str, Any]], list lat, lon = float(lat_text), float(lon_text) except ValueError: lat, lon, coordinate_status = None, None, "unresolved" + if lat is not None and lon is not None and not (math.isfinite(lat) and math.isfinite(lon)): + lat, lon, coordinate_status = None, None, "unresolved" if lat == 0.0 and lon == 0.0: lat, lon, coordinate_status = None, None, "unresolved" if lat is not None and lon is not None and not (47.0 <= lat <= 55.2 and 5.8 <= lon <= 15.1): @@ -138,7 +141,7 @@ def _write_jsonl_atomic(path: Path, rows: list[dict[str, Any]]) -> None: try: with os.fdopen(fd, "w", encoding="utf-8", newline="\n") as handle: for row in rows: - handle.write(json.dumps(row, ensure_ascii=False, sort_keys=True) + "\n") + handle.write(json.dumps(row, ensure_ascii=False, sort_keys=True, allow_nan=False) + "\n") os.replace(temp_name, path) except Exception: try: diff --git a/pipeline/germany/bltu_adapter.py b/pipeline/germany/bltu_adapter.py index a749be1..4c16dc7 100644 --- a/pipeline/germany/bltu_adapter.py +++ b/pipeline/germany/bltu_adapter.py @@ -9,17 +9,26 @@ import tempfile from pathlib import Path -SCHEMA_VERSION = "de-bltu-v1" -ADAPTER_VERSION = "de-bltu-adapter-1" +SCHEMA_VERSION = "de-bltu-v2" +ADAPTER_VERSION = "de-bltu-adapter-2" EXPECTED_COLUMNS = 50 +# Synthetic contract for the supported export layout. A 50-column count alone +# cannot establish which positional columns contain identity or activity data. +EXPECTED_HEADERS = tuple(( + "# Bundesland;Name des Betriebs;Stra\u00dfe / Haus-Nr.;Ort;Alte Zulassungsnummern;" + "Neue Zulassungsnummer;Zulassungsnummer Eierpackstellen;CS;RW;WM;SH;CP;SH;CP;" + "GHE;MM;MP;MSM;PP;CC;PP;CC;PP;CC;PP;CC;PP;CC;PP;CC;EPC;LEP;PP;AH;" + "FV;ZV;FFPP;PP;WM;PC;DC;;Einschr\u00e4nkungen;Bemerkungen;Zulassung befristet bis;" + "Zulassung ruht seit;Drittlandzulassungen;;;" +).split(";")) CURRENT_ID_INDEX = 5 NAME_INDEX = 1 STATE_INDEX = 0 STREET_INDEX = 2 CITY_INDEX = 3 ACTIVITY_START = 7 -ACTIVITY_END = 44 -MAPPING_VERSION = "de-bltu-activity-map-1" +ACTIVITY_END = 41 +MAPPING_VERSION = "de-bltu-activity-map-2" ACTIVITY_MAP = {"SH": "Meat Slaughter", "CP": "Meat Processing"} @@ -40,15 +49,24 @@ def _write_jsonl(path: Path, rows: list[dict]) -> None: def run(raw_path: Path, output_dir: Path, config: dict) -> dict: raw = raw_path.read_bytes() - text = raw.decode("cp1252") + try: + text = raw.decode("utf-8-sig") + source_encoding = "utf-8-sig" + except UnicodeDecodeError: + text = raw.decode("cp1252") + source_encoding = "cp1252" rows = list(csv.reader(text.splitlines(), delimiter=";")) headers = rows[0] if rows else [] + schema_matches = headers == list(EXPECTED_HEADERS) parsed, normalized, quarantined = [], [], [] lengths, activity_counts, mapping_counts = {}, {}, {} for row_number, values in enumerate(rows[1:], start=2): lengths[str(len(values))] = lengths.get(str(len(values)), 0) + 1 evidence = {"source_row": row_number, "source_headers": headers, "source_values": values, "provenance": config} parsed.append(evidence) + if not schema_matches: + quarantined.append({**evidence, "quarantine_reason": "unrecognized header schema"}) + continue if len(values) != EXPECTED_COLUMNS: quarantined.append({**evidence, "quarantine_reason": f"physical column count {len(values)} != {EXPECTED_COLUMNS}"}) continue @@ -60,10 +78,13 @@ def run(raw_path: Path, output_dir: Path, config: dict) -> dict: if not name: quarantined.append({**evidence, "quarantine_reason": "missing establishment name"}) continue - activity_codes = [headers[i].strip() for i in range(ACTIVITY_START, ACTIVITY_END) if values[i].strip() and headers[i].strip()] + activity_codes = [headers[i] for i in range(ACTIVITY_START, ACTIVITY_END) if values[i].strip()] for code in activity_codes: activity_counts[code] = activity_counts.get(code, 0) + 1 - activities = sorted({ACTIVITY_MAP[code] for code in activity_codes if code in ACTIVITY_MAP}) + if any(code not in ACTIVITY_MAP for code in activity_codes): + quarantined.append({**evidence, "quarantine_reason": "unmapped activity code"}) + continue + activities = sorted({ACTIVITY_MAP[code] for code in activity_codes}) for activity in activities: mapping_counts[activity] = mapping_counts.get(activity, 0) + 1 if not activities: @@ -91,9 +112,9 @@ def run(raw_path: Path, output_dir: Path, config: dict) -> dict: _write_jsonl(output_dir / state / "records.jsonl", values) (output_dir / "released").mkdir(exist_ok=True) schema_fingerprint = hashlib.sha256(json.dumps(headers, ensure_ascii=False, separators=(",", ":")).encode()).hexdigest() - config_fingerprint = hashlib.sha256(json.dumps({"mapping_version": MAPPING_VERSION, "activity_map": ACTIVITY_MAP, "expected_columns": EXPECTED_COLUMNS}, sort_keys=True, separators=(",", ":")).encode()).hexdigest() - diagnostics = {"row_length_counts": lengths, "source_activity_code_counts": activity_counts, "mapped_activity_counts": mapping_counts, "quarantine_reason_counts": {reason: sum(1 for row in quarantined if row["quarantine_reason"] == reason) for reason in sorted({row["quarantine_reason"] for row in quarantined})}, "coordinate_status_counts": {"source_unavailable": len(normalized)}} + config_fingerprint = hashlib.sha256(json.dumps({"mapping_version": MAPPING_VERSION, "activity_map": ACTIVITY_MAP, "expected_headers": EXPECTED_HEADERS}, sort_keys=True, separators=(",", ":")).encode()).hexdigest() + diagnostics = {"schema_status": "matched" if schema_matches else "unrecognized", "row_length_counts": lengths, "source_activity_code_counts": activity_counts, "mapped_activity_counts": mapping_counts, "quarantine_reason_counts": {reason: sum(1 for row in quarantined if row["quarantine_reason"] == reason) for reason in sorted({row["quarantine_reason"] for row in quarantined})}, "coordinate_status_counts": {"source_unavailable": len(normalized)}} (output_dir / "validation-report.json").write_text(json.dumps(diagnostics, indent=2, sort_keys=True) + "\n", encoding="utf-8") - manifest = {**config, "checksum_sha256": hashlib.sha256(raw).hexdigest(), "byte_size": len(raw), "adapter_version": ADAPTER_VERSION, "schema_version": SCHEMA_VERSION, "mapping_version": MAPPING_VERSION, "schema_fingerprint": schema_fingerprint, "config_fingerprint": config_fingerprint, "input_rows": len(parsed), "normalized_rows": len(normalized), "quarantined_rows": len(quarantined), "release_state": "not-created", "release_gate": "restricted_pending_terms", "geocoding": "disabled"} + manifest = {**config, "checksum_sha256": hashlib.sha256(raw).hexdigest(), "byte_size": len(raw), "source_encoding": source_encoding, "adapter_version": ADAPTER_VERSION, "schema_version": SCHEMA_VERSION, "mapping_version": MAPPING_VERSION, "schema_fingerprint": schema_fingerprint, "config_fingerprint": config_fingerprint, "input_rows": len(parsed), "normalized_rows": len(normalized), "quarantined_rows": len(quarantined), "release_state": "not-created", "release_gate": "restricted_pending_terms", "geocoding": "disabled"} (output_dir / "run-manifest.json").write_text(json.dumps(manifest, sort_keys=True), encoding="utf-8") return manifest diff --git a/pipeline/germany/test_adapter.py b/pipeline/germany/test_adapter.py index 7bf1bad..86df794 100644 --- a/pipeline/germany/test_adapter.py +++ b/pipeline/germany/test_adapter.py @@ -70,25 +70,66 @@ def test_no_release_is_created_by_adapter(self): self.assertFalse((output / "released" / "records.jsonl").exists()) self.assertEqual(json.loads((output / "run-manifest.json").read_text())["quarantined_rows"], 1) + def test_nonfinite_coordinates_are_unresolved_and_json_is_standard(self): + header = "source_id,name,activity_code,species_codes,street,city,zip,latitude,longitude\n" + rows = "DE-NAN,Synthetic,CP,,Street,City,,nan,10\nDE-INF,Synthetic,SH,,Street,City,,50,inf\n" + raw = (header + rows).encode() + normalized, quarantine = normalize(parse(raw, source_metadata(raw, CONFIG))) + self.assertFalse(quarantine) + self.assertEqual(len(normalized), 2) + for row in normalized: + self.assertEqual(row["coordinate_status"], "unresolved") + self.assertIsNone(row["latitude"]) + self.assertIsNone(row["longitude"]) + json.dumps(row, allow_nan=False) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + source = root / "nonfinite.csv" + source.write_bytes(raw) + run(source, root / "out", CONFIG) + lines = (root / "out" / "normalized" / "records.jsonl").read_text(encoding="utf-8").splitlines() + def reject_constant(value): + raise ValueError(f"non-standard JSON constant: {value}") + self.assertEqual(len([json.loads(line, parse_constant=reject_constant) for line in lines]), 2) + def test_bltu_positional_mapping_preserves_duplicate_headers_and_quarantines_unknowns(self): with tempfile.TemporaryDirectory() as directory: output = Path(directory) / "out" config = {"source_url": "https://example.invalid/bltu", "terms_status": "pending_confirmation"} manifest = run_bltu(ROOT / "fixtures" / "synthetic_bltu.csv", output, config) self.assertEqual(manifest["input_rows"], 3) - self.assertEqual(manifest["normalized_rows"], 2) - self.assertEqual(manifest["quarantined_rows"], 1) + self.assertEqual(manifest["normalized_rows"], 1) + self.assertEqual(manifest["quarantined_rows"], 2) records = [json.loads(line) for line in (output / "normalized" / "records.jsonl").read_text().splitlines()] self.assertEqual(len(records[0]["source_columns"]), 50) + self.assertEqual(records[0]["source_id"], "DE-SYN-002") self.assertEqual(records[0]["source_columns"][10]["header"], "SH") self.assertIsNone(records[0]["latitude"]) self.assertTrue(manifest["schema_fingerprint"]) self.assertTrue(manifest["config_fingerprint"]) - self.assertEqual(manifest["mapping_version"], "de-bltu-activity-map-1") + self.assertEqual(manifest["mapping_version"], "de-bltu-activity-map-2") diagnostics = json.loads((output / "validation-report.json").read_text()) + self.assertEqual(diagnostics["schema_status"], "matched") self.assertEqual(diagnostics["row_length_counts"], {"50": 3}) quarantined = [json.loads(line) for line in (output / "quarantined" / "records.jsonl").read_text().splitlines()] - self.assertEqual(quarantined[0]["quarantine_reason"], "unmapped activity code") + self.assertIn("unmapped activity code", [row["quarantine_reason"] for row in quarantined]) + + def test_bltu_schema_shift_with_same_width_quarantines_every_row(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + lines = (ROOT / "fixtures" / "synthetic_bltu.csv").read_text(encoding="utf-8").splitlines() + headers = lines[0].split(";") + headers[10], headers[11] = headers[11], headers[10] + raw = root / "shifted.csv" + raw.write_text(";".join(headers) + "\n" + "\n".join(lines[1:]) + "\n", encoding="utf-8") + output = root / "out" + manifest = run_bltu(raw, output, {"source_url": "https://example.invalid/bltu"}) + self.assertEqual(manifest["normalized_rows"], 0) + self.assertEqual(manifest["quarantined_rows"], 3) + self.assertFalse((output / "released" / "records.jsonl").exists()) + self.assertEqual(json.loads((output / "validation-report.json").read_text())["schema_status"], "unrecognized") + reasons = [json.loads(line)["quarantine_reason"] for line in (output / "quarantined" / "records.jsonl").read_text().splitlines()] + self.assertTrue(all(reason == "unrecognized header schema" for reason in reasons)) def test_bltu_mapping_is_explicit_and_coordinates_are_never_enriched(self): with tempfile.TemporaryDirectory() as directory: diff --git a/pipeline/germany/test_orchestrator.py b/pipeline/germany/test_orchestrator.py index 8ac1601..76d5912 100644 --- a/pipeline/germany/test_orchestrator.py +++ b/pipeline/germany/test_orchestrator.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +import hashlib import json import tempfile import unittest @@ -20,15 +21,35 @@ class OrchestratorTests(unittest.TestCase): - def test_hash_addressed_registration_is_idempotent(self): + def test_identical_bytes_share_raw_artifact_but_keep_distinct_acquisition_events(self): with tempfile.TemporaryDirectory() as directory: staging = Path(directory) raw = FIXTURE.read_bytes() first_path, first = register_input(raw, staging, CONFIG) - second_path, second = register_input(raw, staging, CONFIG) + first_manifest_path = Path(first["acquisition_manifest"]) + first_manifest_bytes = first_manifest_path.read_bytes() + later_observation = {**CONFIG, "retrieval_timestamp": "2026-09-14T00:00:00Z"} + second_path, second = register_input(raw, staging, later_observation) + second_manifest_path = Path(second["acquisition_manifest"]) + self.assertEqual(first_path, second_path) - self.assertEqual(first, second) + self.assertEqual(first_path.name, f"{hashlib.sha256(raw).hexdigest()}.artifact") + self.assertEqual(first_path.read_bytes(), raw) self.assertEqual(len(list((staging / "raw").glob("*.artifact"))), 1) + self.assertEqual(first["raw_artifact"], str(first_path)) + self.assertEqual(second["raw_artifact"], str(first_path)) + self.assertEqual(first["checksum_sha256"], second["checksum_sha256"]) + self.assertEqual(first["byte_size"], second["byte_size"]) + + self.assertNotEqual(first_manifest_path, second_manifest_path) + self.assertEqual(len(list((staging / "raw" / "registrations").glob("*.manifest.json"))), 2) + self.assertEqual(first_manifest_path.read_bytes(), first_manifest_bytes) + self.assertEqual(json.loads(first_manifest_bytes), first) + self.assertEqual(json.loads(second_manifest_path.read_text(encoding="utf-8")), second) + self.assertEqual(first["retrieval_timestamp"], CONFIG["retrieval_timestamp"]) + self.assertEqual(second["retrieval_timestamp"], later_observation["retrieval_timestamp"]) + self.assertEqual(first["source_url"], CONFIG["source_url"]) + self.assertEqual(first["source_publication_date"], CONFIG["source_publication_date"]) def test_suppression_survives_rerun_and_candidate_handoff(self): with tempfile.TemporaryDirectory() as directory: diff --git a/pipeline/migrations/022_publication_safety_scopes.sql b/pipeline/migrations/022_publication_safety_scopes.sql new file mode 100644 index 0000000..5e40180 --- /dev/null +++ b/pipeline/migrations/022_publication_safety_scopes.sql @@ -0,0 +1,163 @@ +-- Freeze publication decisions to the release present when they are recorded. +-- Earlier source-only decisions are carried forward only when their source +-- record belongs to exactly one release; ambiguous history needs new review. +ALTER TABLE uec.publication_review_events + ADD COLUMN release_id TEXT REFERENCES uec.releases(release_id); + +CREATE TABLE uec.publication_review_release_scopes ( + publication_review_event_id UUID NOT NULL REFERENCES uec.publication_review_events(publication_review_event_id), + release_id TEXT NOT NULL REFERENCES uec.releases(release_id), + PRIMARY KEY (publication_review_event_id, release_id) +); + +INSERT INTO uec.publication_review_release_scopes (publication_review_event_id, release_id) +SELECT review.publication_review_event_id, min(member.release_id) +FROM uec.publication_review_events review +JOIN uec.observations observation ON observation.source_record_id = review.source_record_id +JOIN uec.release_members member ON member.observation_id = observation.observation_id +GROUP BY review.publication_review_event_id +HAVING count(DISTINCT member.release_id) = 1; + +CREATE FUNCTION uec.scope_publication_review_event() +RETURNS trigger LANGUAGE plpgsql AS $$ +DECLARE + inferred_release_id TEXT; + release_count INTEGER; +BEGIN + IF NEW.release_id IS NOT NULL THEN + INSERT INTO uec.publication_review_release_scopes (publication_review_event_id, release_id) + VALUES (NEW.publication_review_event_id, NEW.release_id); + RETURN NEW; + END IF; + + SELECT min(member.release_id), count(DISTINCT member.release_id) + INTO inferred_release_id, release_count + FROM uec.observations observation + JOIN uec.release_members member ON member.observation_id = observation.observation_id + WHERE observation.source_record_id = NEW.source_record_id; + + IF release_count > 1 AND NEW.publication_eligible THEN + RAISE EXCEPTION 'publication decision for source record in multiple releases requires release_id'; + END IF; + IF release_count = 1 THEN + INSERT INTO uec.publication_review_release_scopes (publication_review_event_id, release_id) + VALUES (NEW.publication_review_event_id, inferred_release_id); + END IF; + RETURN NEW; +END; +$$; + +CREATE TRIGGER publication_review_events_scope + AFTER INSERT ON uec.publication_review_events + FOR EACH ROW EXECUTE FUNCTION uec.scope_publication_review_event(); + +CREATE TRIGGER publication_review_release_scopes_append_only + BEFORE UPDATE OR DELETE ON uec.publication_review_release_scopes + FOR EACH ROW EXECUTE FUNCTION uec.reject_evidence_mutation(); + +CREATE OR REPLACE VIEW uec.publication_review_current AS +SELECT DISTINCT ON (review.source_record_id) review.source_record_id, review.factual_review_status, + review.privacy_screening_status, review.maintainer_approval, review.publication_eligible, + review.reviewer_role, review.reviewed_at, review.publication_review_event_id +FROM uec.publication_review_events review +LEFT JOIN uec.publication_review_release_scopes scope + ON scope.publication_review_event_id = review.publication_review_event_id +LEFT JOIN uec.releases release ON release.release_id = scope.release_id +-- The existing API joins this source-keyed view without a release key. Prefer +-- a promoted release's decision so a candidate-only review does not relabel +-- or withdraw its still-promoted counterpart. +ORDER BY review.source_record_id, + CASE WHEN release.status = 'promoted' THEN 0 ELSE 1 END, + review.reviewed_at DESC, review.publication_review_event_id DESC; + +CREATE VIEW uec.publication_review_release_current AS +SELECT DISTINCT ON (review.source_record_id, scope.release_id) + review.source_record_id, scope.release_id, review.factual_review_status, + review.privacy_screening_status, review.maintainer_approval, + review.publication_eligible, review.reviewer_role, review.reviewed_at, + review.publication_review_event_id +FROM uec.publication_review_events review +JOIN uec.publication_review_release_scopes scope + ON scope.publication_review_event_id = review.publication_review_event_id +ORDER BY review.source_record_id, scope.release_id, + review.reviewed_at DESC, review.publication_review_event_id DESC; + +-- A facility reference must also reach an observation that has no separate +-- facility_source_links row. Restriction remains active for old releases. +CREATE OR REPLACE VIEW uec.public_access_restricted AS +SELECT source_record_id, reason_category, policy_version, occurred_at +FROM uec.record_access_current +WHERE action = 'public_access_revoked' +UNION +SELECT record.source_record_id, case_record.reason_category, case_record.policy_version, case_record.created_at +FROM uec.suppression_cases case_record +JOIN uec.suppression_references ref ON ref.case_id = case_record.case_id +JOIN uec.source_records record ON ( + ref.facility_id IS NOT NULL AND ( + EXISTS (SELECT 1 FROM uec.facility_source_links link + WHERE link.facility_id = ref.facility_id AND link.source_record_id = record.source_record_id) + OR EXISTS (SELECT 1 FROM uec.observations observation + WHERE observation.facility_id = ref.facility_id AND observation.source_record_id = record.source_record_id) + ) + OR (ref.source_id = record.source_id AND ref.source_record_key = record.source_record_key) +) +WHERE case_record.status IN ('active', 'review', 'closed', 'expired'); + +-- A public count is about eligible observations in this release, not every +-- retained research observation attached to the facility. +CREATE VIEW uec.publication_release_eligible_observations AS +SELECT member.release_id, member.facility_id, observation.observation_id, + observation.source_record_id, observation.first_observed_at, observation.observed_at +FROM uec.release_members member +JOIN uec.releases release ON release.release_id = member.release_id +JOIN uec.observations observation ON observation.observation_id = member.observation_id +JOIN uec.source_records record ON record.source_record_id = observation.source_record_id +JOIN uec.sources source ON source.source_id = record.source_id +JOIN uec.publication_review_release_current review + ON review.source_record_id = observation.source_record_id + AND review.release_id = member.release_id +WHERE release.status = 'promoted' + AND member.default_visible = true + AND review.publication_eligible = true + AND review.privacy_screening_status = 'passed' + AND review.factual_review_status <> 'rejected' + AND ( + review.maintainer_approval = 'approved' + OR (release.profile = 'community' AND source.origin_type = 'user_submitted' + AND review.factual_review_status = 'unreviewed' + AND review.maintainer_approval = 'pending') + ) + AND NOT EXISTS (SELECT 1 FROM uec.public_access_restricted restricted + WHERE restricted.source_record_id = observation.source_record_id); + +CREATE VIEW uec.public_facility_observation_summary AS +SELECT release_id, facility_id, min(first_observed_at) AS first_observed_at, + max(observed_at) AS last_observed_at, count(*)::int AS observation_count +FROM uec.publication_release_eligible_observations +GROUP BY release_id, facility_id; + +DROP VIEW uec.map_facilities_display_history; +CREATE VIEW uec.map_facilities_display_history AS +SELECT display.*, observation.classification_category, + summary.first_observed_at, summary.last_observed_at, summary.observation_count, + COALESCE(lifecycle.status, 'status_unknown') AS lifecycle_status, + lifecycle.effective_at AS lifecycle_effective_at, lifecycle.source_record_id AS lifecycle_source_record_id, + release.ruleset_version AS release_ruleset_version, release.created_at AS release_created_at, + source.origin_type AS provenance_origin_type, source.source_id AS provenance_source_id, + source.name AS provenance_source_name, source.official_url AS provenance_source_url, + artifact.retrieved_at AS provenance_retrieved_at +FROM uec.map_facilities_display AS display +JOIN uec.publication_release_eligible_observations eligible + ON eligible.release_id = display.release_id AND eligible.observation_id = display.observation_id +JOIN uec.observations AS observation ON observation.observation_id = display.observation_id +JOIN uec.source_records AS record ON record.source_record_id = display.source_record_id +JOIN uec.sources AS source ON source.source_id = record.source_id +JOIN uec.release_members AS member ON member.release_id = display.release_id AND member.observation_id = display.observation_id +JOIN uec.releases AS release ON release.release_id = member.release_id +JOIN uec.raw_artifacts AS artifact ON artifact.artifact_id = record.artifact_id +JOIN uec.public_facility_observation_summary AS summary + ON summary.release_id = display.release_id AND summary.facility_id = display.facility_id +LEFT JOIN uec.facility_lifecycle_current AS lifecycle ON lifecycle.facility_id = display.facility_id; + +COMMENT ON VIEW uec.map_facilities_display_history IS + 'V2 public display history with current suppression, release-scoped publication decisions, and public-only lifecycle counts.'; diff --git a/pipeline/scripts/maintenance/local-v2.ps1 b/pipeline/scripts/maintenance/local-v2.ps1 index 6601824..4e04645 100644 --- a/pipeline/scripts/maintenance/local-v2.ps1 +++ b/pipeline/scripts/maintenance/local-v2.ps1 @@ -37,7 +37,7 @@ try { $releaseStatus='validated' } if ($releaseStatus.Trim() -eq 'validated') { - python pipeline/scripts/stages/promote-release.py standard-candidate + python pipeline/scripts/stages/promote-release.py standard-candidate --no-distributed-artifacts if ($LASTEXITCODE) { throw 'Synthetic release promotion failed.' } } elseif ($releaseStatus.Trim() -ne 'promoted') { throw "Synthetic release has unexpected status: $($releaseStatus.Trim())" diff --git a/pipeline/scripts/stages/promote-release.py b/pipeline/scripts/stages/promote-release.py index ac24bc3..bc086ad 100644 --- a/pipeline/scripts/stages/promote-release.py +++ b/pipeline/scripts/stages/promote-release.py @@ -6,6 +6,7 @@ import json import os import sys +from datetime import timezone from pathlib import Path import psycopg @@ -15,7 +16,41 @@ def can_promote(status: str) -> bool: return status == "validated" -def promote(database_url: str, release_id: str) -> dict: +def canonical_json(manifest: dict) -> str: + return json.dumps(manifest, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + + +def write_manifest(path: Path, result: dict) -> None: + payload = canonical_json(result["manifest"]).encode("utf-8") + if hashlib.sha256(payload).hexdigest() != result["manifest_sha256"]: + raise ValueError("manifest digest does not match promotion result") + with path.open("xb") as output: + output.write(payload) + + +def inventory_artifacts(paths: list[Path], no_distributed_artifacts: bool) -> list[dict]: + if no_distributed_artifacts == bool(paths): + raise ValueError("declare --artifact for every distributed file or --no-distributed-artifacts") + artifacts = [] + names = set() + for path in paths: + if not path.is_file(): + raise ValueError(f"distributed artifact is not a file: {path}") + name = path.name + if name in names: + raise ValueError(f"duplicate distributed artifact name: {name}") + names.add(name) + digest = hashlib.sha256() + size = 0 + with path.open("rb") as artifact: + for chunk in iter(lambda: artifact.read(1024 * 1024), b""): + digest.update(chunk) + size += len(chunk) + artifacts.append({"name": name, "sha256": digest.hexdigest(), "byte_size": size}) + return sorted(artifacts, key=lambda artifact: artifact["name"]) + + +def promote(database_url: str, release_id: str, artifacts: list[dict]) -> dict: with psycopg.connect(database_url) as connection: with connection.transaction(): target = connection.execute("SELECT status, profile, ruleset_version FROM uec.releases WHERE release_id = %s FOR UPDATE", (release_id,)).fetchone() @@ -27,12 +62,13 @@ def promote(database_url: str, release_id: str) -> dict: SELECT count(*) FILTER (WHERE m.default_visible AND (g.status IS DISTINCT FROM 'accepted' OR g.result IS NULL)), count(*) FILTER (WHERE o.classification_review_status <> 'approved' AND m.default_visible), - count(*) FILTER (WHERE r.publication_eligible IS DISTINCT FROM true OR r.privacy_screening_status <> 'passed' OR r.maintainer_approval <> 'approved'), + count(*) FILTER (WHERE r.release_id IS NULL OR r.publication_eligible IS DISTINCT FROM true OR r.privacy_screening_status IS DISTINCT FROM 'passed' OR r.maintainer_approval IS DISTINCT FROM 'approved'), count(*) FILTER (WHERE s.source_record_id IS NOT NULL) FROM uec.release_members m JOIN uec.observations o ON o.observation_id = m.observation_id LEFT JOIN LATERAL (SELECT status, result FROM uec.geocode_results WHERE source_record_id=o.source_record_id ORDER BY queried_at DESC, geocode_result_id DESC LIMIT 1) g ON true - LEFT JOIN uec.publication_review_current r ON r.source_record_id=o.source_record_id + LEFT JOIN uec.publication_review_release_current r + ON r.source_record_id=o.source_record_id AND r.release_id=m.release_id LEFT JOIN uec.public_access_restricted s ON s.source_record_id=o.source_record_id WHERE m.release_id=%s """, (release_id,)).fetchone() @@ -45,29 +81,40 @@ def promote(database_url: str, release_id: str) -> dict: WHERE m.release_id=%s AND m.default_visible AND NOT EXISTS (SELECT 1 FROM uec.public_access_restricted x WHERE x.source_record_id=sr.source_record_id) """, (release_id,)).fetchone() - manifest = {"manifest_version": "v1", "release_id": release_id, "profile": target[1], "ruleset_version": target[2], "eligible_record_count": summary[0], "source_ids": summary[1]} + created_at = connection.execute("SELECT now()").fetchone()[0] + if created_at.tzinfo is None: + raise ValueError("database manifest creation time must include a timezone") + manifest = {"manifest_version": "v1", "release_id": release_id, "profile": target[1], "ruleset_version": target[2], "eligible_record_count": summary[0], "source_ids": summary[1], "created_at": created_at.astimezone(timezone.utc).isoformat().replace("+00:00", "Z"), "distributed_artifacts": artifacts} # Python's sorted-key JSON is the canonical representation shared by consumers. - canonical = json.dumps(manifest, sort_keys=True, separators=(",", ":"), ensure_ascii=False) - digest = hashlib.sha256(canonical.encode()).hexdigest() - connection.execute("INSERT INTO uec.release_manifests (release_id,manifest,manifest_sha256) VALUES (%s,%s,%s)", (release_id, json.dumps(manifest, ensure_ascii=False), digest)) + canonical = canonical_json(manifest) + digest = hashlib.sha256(canonical.encode("utf-8")).hexdigest() + connection.execute("INSERT INTO uec.release_manifests (release_id,manifest,manifest_sha256) VALUES (%s,%s,%s)", (release_id, canonical, digest)) previous = connection.execute("SELECT release_id FROM uec.releases WHERE status = 'promoted' AND profile = %s AND release_id <> %s", (target[1], release_id)).fetchall() connection.execute("UPDATE uec.releases SET status = 'validated' WHERE status = 'promoted' AND profile = %s AND release_id <> %s", (target[1], release_id)) connection.execute("UPDATE uec.releases SET status = 'promoted' WHERE release_id = %s", (release_id,)) - return {"release_id": release_id, "status": "promoted", "previously_promoted": [row[0] for row in previous]} + return {"release_id": release_id, "status": "promoted", "previously_promoted": [row[0] for row in previous], "manifest": manifest, "manifest_sha256": digest} if __name__ == "__main__": parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("release_id") parser.add_argument("--manifest", type=Path) + artifacts = parser.add_mutually_exclusive_group(required=True) + artifacts.add_argument("--artifact", type=Path, action="append", help="Distributed file to checksum; repeat for every file") + artifacts.add_argument("--no-distributed-artifacts", action="store_true", help="Declare that this release has no distributed files") parser.add_argument("--database-url", default=os.environ.get("UEC_DATABASE_URL", "postgresql://uec:uec-local-development-only@localhost:5433/uec")) args = parser.parse_args() try: - result = promote(args.database_url, args.release_id) - serialized = json.dumps(result, indent=2) + "\n" + if args.manifest and (not args.manifest.parent.is_dir() or args.manifest.exists()): + raise ValueError("manifest output requires an existing directory and a new file name") + if args.manifest and args.artifact and args.manifest.resolve() in {path.resolve() for path in args.artifact}: + raise ValueError("the output manifest cannot be one of its distributed artifacts") + artifact_inventory = inventory_artifacts(args.artifact or [], args.no_distributed_artifacts) + result = promote(args.database_url, args.release_id, artifact_inventory) + serialized = json.dumps({key: value for key, value in result.items() if key != "manifest"}, indent=2) + "\n" print(serialized, end="") if args.manifest: - args.manifest.write_text(serialized, encoding="utf-8") + write_manifest(args.manifest, result) except Exception as error: print(json.dumps({"status": "blocked", "error": str(error)}, indent=2), file=sys.stderr) sys.exit(1) diff --git a/pipeline/scripts/stages/validate-release.py b/pipeline/scripts/stages/validate-release.py index a454f7a..11505cc 100644 --- a/pipeline/scripts/stages/validate-release.py +++ b/pipeline/scripts/stages/validate-release.py @@ -58,7 +58,7 @@ def validate(database_url: str, release_id: str, expected_records: int | None, m count(*) FILTER (WHERE release_member.default_visible AND latest.status = 'review_required' AND city.reference_location IS NOT NULL)::int AS city_display_ready, count(*) FILTER (WHERE release_member.default_visible AND (latest.status IS NULL OR (latest.status <> 'accepted' AND city.reference_location IS NULL)))::int AS unmapped_display, count(*) FILTER (WHERE release_member.default_visible AND (latest.status <> 'accepted' OR latest.result IS NULL))::int AS coordinate_not_ready, - count(*) FILTER (WHERE review.publication_eligible IS DISTINCT FROM true OR review.privacy_screening_status <> 'passed' OR review.maintainer_approval <> 'approved')::int AS publication_not_approved, + count(*) FILTER (WHERE review.release_id IS NULL OR review.publication_eligible IS DISTINCT FROM true OR review.privacy_screening_status IS DISTINCT FROM 'passed' OR review.maintainer_approval IS DISTINCT FROM 'approved')::int AS publication_not_approved, count(*) FILTER (WHERE restricted.source_record_id IS NOT NULL)::int AS active_suppression, (SELECT count(*)::int FROM uec.validation_findings finding WHERE finding.severity = 'error' AND (finding.source_record_id IS NULL OR finding.source_record_id IN (SELECT source_record_id FROM uec.observations WHERE observation_id IN (SELECT observation_id FROM uec.release_members WHERE release_id = %s)))) AS validation_errors FROM uec.release_members AS release_member @@ -66,7 +66,9 @@ def validate(database_url: str, release_id: str, expected_records: int | None, m JOIN uec.facilities AS facility ON facility.facility_id = release_member.facility_id LEFT JOIN LATERAL (SELECT status, result FROM uec.geocode_results WHERE source_record_id = observation.source_record_id ORDER BY queried_at DESC, geocode_result_id DESC LIMIT 1) AS latest ON true LEFT JOIN LATERAL (SELECT reference_location FROM uec.city_reference_points WHERE country_code = facility.country_code AND lower(city_name) = lower(facility.city) AND (postal_code IS NULL OR postal_code = facility.postal_code) LIMIT 1) AS city ON true - LEFT JOIN uec.publication_review_current review ON review.source_record_id = observation.source_record_id + LEFT JOIN uec.publication_review_release_current review + ON review.source_record_id = observation.source_record_id + AND review.release_id = release_member.release_id LEFT JOIN uec.public_access_restricted restricted ON restricted.source_record_id = observation.source_record_id WHERE release_member.release_id = %s """, (release_id, release_id)).fetchone() diff --git a/pipeline/sources/uk/approved/compose.py b/pipeline/sources/uk/approved/compose.py index f48ca43..17d53f5 100644 --- a/pipeline/sources/uk/approved/compose.py +++ b/pipeline/sources/uk/approved/compose.py @@ -36,6 +36,24 @@ def _read_jsonl(path: Path) -> list[dict[str, Any]]: return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line] +def _verified_records(path: Path, manifest: dict[str, Any], source_id: str) -> list[dict[str, Any]]: + if path.name != "records.jsonl" or path.parent.name != "normalized": + raise CompositionError(f"invalid normalized output path for {source_id}") + manifest_path = path.parent.parent / "manifest.json" + if not manifest_path.exists(): + raise CompositionError(f"source manifest unavailable for {source_id}") + if json.loads(manifest_path.read_text(encoding="utf-8")) != manifest: + raise CompositionError(f"source manifest mismatch for {source_id}") + if not path.exists() or not isinstance(manifest.get("normalized_sha256"), str): + raise CompositionError(f"normalized output unverifiable for {source_id}") + if hashlib.sha256(path.read_bytes()).hexdigest() != manifest["normalized_sha256"]: + raise CompositionError(f"normalized output checksum mismatch for {source_id}") + records = _read_jsonl(path) + if len(records) != manifest.get("normalized_rows") or any(record.get("source_id") != source_id for record in records): + raise CompositionError(f"normalized output identity/count mismatch for {source_id}") + return records + + def _key(value: str | None) -> str: return re.sub(r"\s+", " ", (value or "").strip()).casefold() @@ -57,7 +75,7 @@ def _possible_match(left: dict[str, Any], right: dict[str, Any]) -> bool: def compose_sources(inputs: list[dict[str, Any]], output_dir: str | Path, - suppressed: set[tuple[str, str]] | None = None, + suppressed: set[tuple[str, str, str]] | None = None, prior_view: dict[str, Any] | None = None) -> dict[str, Any]: """Compose source outputs without merging them; output remains non-release.""" if not inputs: @@ -65,6 +83,7 @@ def compose_sources(inputs: list[dict[str, Any]], output_dir: str | Path, registry = load_registry(REGISTRY_PATH) registered = {entry["source_id"]: entry for entry in registry["adapters"]} seen_sources: set[str] = set() + source_states: dict[str, dict[str, Any]] = {} items: list[dict[str, Any]] = [] try: for item in inputs: @@ -76,9 +95,11 @@ def compose_sources(inputs: list[dict[str, Any]], output_dir: str | Path, raise CompositionError(f"unregistered source: {source_id}") if manifest.get("schema_version") != EXPECTED_SCHEMA[source_id]: raise CompositionError(f"incompatible schema/version for {source_id}") + if manifest.get("source_id") != source_id: + raise CompositionError(f"source manifest identity mismatch for {source_id}") if manifest.get("release_state") != "not-created": raise CompositionError(f"source is not restricted/reviewable: {source_id}") - records = _read_jsonl(Path(item["normalized_path"])) + records = _verified_records(Path(item["normalized_path"]), manifest, source_id) seen_sources.add(source_id) source_state = { "terms_state": item.get("terms_state", "unresolved"), @@ -86,40 +107,49 @@ def compose_sources(inputs: list[dict[str, Any]], output_dir: str | Path, "acquisition_state": item.get("acquisition_state", "synthetic-only"), "manifest": manifest, } + source_states[source_id] = source_state for record in records: record_id = _source_record_id(source_id, record) - if not record_id: - raise CompositionError(f"source record lacks stable identifier: {source_id}") - if (source_id, record_id) in (suppressed or set()): + normalized = record.get("normalized", {}) + nation = normalized.get("nation") + if not record_id or not isinstance(nation, str) or not nation.strip(): + raise CompositionError(f"source record lacks nation-qualified identifier: {source_id}") + nation = nation.strip() + if (source_id, nation, record_id) in (suppressed or set()): continue - items.append({"source_id": source_id, "source_record_id": record_id, + items.append({"source_id": source_id, "nation": nation, "source_record_id": record_id, "source_record": record, "source_state": source_state}) except (KeyError, TypeError, json.JSONDecodeError) as exc: raise CompositionError("invalid source output") from exc - items.sort(key=lambda row: (row["source_id"], row["source_record_id"], row["source_record"]["source_row"])) + items.sort(key=lambda row: (row["source_id"], row["nation"], row["source_record_id"], row["source_record"]["source_row"])) signals = [] for index, left in enumerate(items): for right in items[index + 1:]: if _possible_match(left, right): signals.append({"type": "possible_match_review", "record_refs": [ - {"source_id": left["source_id"], "source_record_id": left["source_record_id"]}, - {"source_id": right["source_id"], "source_record_id": right["source_record_id"]}], + {"source_id": left["source_id"], "nation": left["nation"], "source_record_id": left["source_record_id"]}, + {"source_id": right["source_id"], "nation": right["nation"], "source_record_id": right["source_record_id"]}], "basis": "exact normalized trading name and postcode; no automatic merge"}) - blockers = sorted({f"{row['source_id']}:{row['source_state']['terms_state']}" for row in items - if row["source_state"]["terms_state"] != "confirmed"}) - blockers.extend(sorted({f"{row['source_id']}:review-required" for row in items - if row["source_state"]["review_state"] != "project-approved"})) + terms_blockers = sorted(f"{source_id}:{state['terms_state']}" for source_id, state in source_states.items() + if state["terms_state"] != "confirmed") + review_blockers = sorted(f"{source_id}:review-required" for source_id, state in source_states.items() + if state["review_state"] != "project-approved") + blockers = terms_blockers + review_blockers output = Path(output_dir) _write_jsonl(output / "reviewable" / "records.jsonl", items) _write_jsonl(output / "reviewable" / "possible-match-signals.jsonl", signals) _write_jsonl(output / "quarantined" / "records.jsonl", []) (output / "released").mkdir(parents=True, exist_ok=True) - manifest = {"composition_id": hashlib.sha256(json.dumps(items, sort_keys=True, default=list).encode()).hexdigest(), + manifest = {"composition_id": hashlib.sha256(json.dumps({"items": items, "source_states": source_states}, sort_keys=True, default=list).encode()).hexdigest(), "orchestrator_version": ORCHESTRATOR_VERSION, "source_ids": sorted(seen_sources), - "input_rows": len(items), "possible_match_signals": len(signals), - "terms_state": "blocked" if blockers else "confirmed", - "review_state": "blocked" if blockers else "project-approved", + "source_states": source_states, + "input_rows": sum(state["manifest"]["normalized_rows"] for state in source_states.values()), + "reviewable_rows": len(items), + "suppressed_rows": sum(state["manifest"]["normalized_rows"] for state in source_states.values()) - len(items), + "possible_match_signals": len(signals), + "terms_state": "blocked" if terms_blockers else "confirmed", + "review_state": "blocked" if review_blockers else "project-approved", "release_state": "not-created", "candidate_created": False, "publication_state": "human-gate-required", "blockers": blockers} _atomic(output / "manifest.json", (json.dumps(manifest, ensure_ascii=False, sort_keys=True, indent=2) + "\n").encode()) diff --git a/pipeline/sources/uk/approved/test_compose.py b/pipeline/sources/uk/approved/test_compose.py index bb3d4a5..3c488a7 100644 --- a/pipeline/sources/uk/approved/test_compose.py +++ b/pipeline/sources/uk/approved/test_compose.py @@ -42,22 +42,21 @@ def test_source_specific_suppression_survives_reimport(self): with tempfile.TemporaryDirectory() as directory: root = Path(directory) inputs = self._runs(root) - suppressed = {("fsa_approved_establishments", "00017")} + suppressed = {("fsa_approved_establishments", "England", "00017")} for suffix in ("one", "reimport"): compose_sources(inputs, root / suffix, suppressed=suppressed) rows = [json.loads(line) for line in (root / suffix / "reviewable/records.jsonl").read_text().splitlines()] - self.assertFalse(any(r["source_id"] == "fsa_approved_establishments" and r["source_record_id"] == "00017" for r in rows)) + self.assertFalse(any(r["source_id"] == "fsa_approved_establishments" and r["nation"] == "England" and r["source_record_id"] == "00017" for r in rows)) + self.assertTrue(any(r["source_id"] == "fsa_approved_establishments" and r["nation"] == "Wales" and r["source_record_id"] == "00017" for r in rows)) self.assertTrue(any(r["source_id"] == "fss_approved_establishments" for r in rows)) def test_exact_cross_source_name_postcode_is_signal_not_merge(self): with tempfile.TemporaryDirectory() as directory: root = Path(directory) inputs = self._runs(root) - fss_path = inputs[0]["normalized_path"] - fss_record = json.loads(fss_path.read_text().splitlines()[0]) - fss_record["normalized"]["trading_name"] = "East March Foods" - fss_record["normalized"]["postcode"] = "PE1 2AB" - fss_path.write_text(json.dumps(fss_record) + "\n") + modified_raw = root / "modified-fss.csv" + modified_raw.write_bytes(FSS_FIXTURE.read_bytes().replace(b"North Star Foods", b"East March Foods").replace(b"AB1 2CD", b"PE1 2AB")) + inputs[0]["manifest"] = FssApprovedEstablishmentsAdapter().run(modified_raw, root / "fss") manifest = compose_sources(inputs, root / "country") signals = [json.loads(line) for line in (root / "country/reviewable/possible-match-signals.jsonl").read_text().splitlines()] self.assertEqual(manifest["possible_match_signals"], 1) @@ -88,6 +87,34 @@ def test_terms_and_review_gates_block_candidate_and_release(self): self.assertIn("fsa_approved_establishments:unresolved", manifest["blockers"]) self.assertEqual(manifest["publication_state"], "human-gate-required") + def test_swapped_or_changed_normalized_output_is_rejected(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + inputs = self._runs(root) + swapped = [dict(inputs[0], normalized_path=inputs[1]["normalized_path"]), inputs[1]] + with self.assertRaises(CompositionError): + compose_sources(swapped, root / "swapped") + path = inputs[0]["normalized_path"] + path.write_text(path.read_text() + "\n") + with self.assertRaises(CompositionError): + compose_sources(inputs, root / "altered") + + def test_all_rows_suppressed_preserves_source_gates_and_provenance(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + inputs = self._runs(root) + suppressed = {("fss_approved_establishments", "Scotland", record["normalized"]["approval_number"]) + for record in FssApprovedEstablishmentsAdapter().parse_file(FSS_FIXTURE).accepted} + manifest = compose_sources(inputs[:1], root / "country", suppressed=suppressed) + self.assertEqual((root / "country/reviewable/records.jsonl").read_text(), "") + self.assertEqual(manifest["input_rows"], 2) + self.assertEqual(manifest["reviewable_rows"], 0) + self.assertEqual(manifest["suppressed_rows"], 2) + self.assertEqual(manifest["source_states"]["fss_approved_establishments"]["manifest"], inputs[0]["manifest"]) + self.assertIn("fss_approved_establishments:unresolved", manifest["blockers"]) + self.assertEqual(manifest["terms_state"], "blocked") + self.assertEqual(manifest["review_state"], "blocked") + if __name__ == "__main__": unittest.main() diff --git a/pipeline/sources/uk/fsa_approved/adapter.py b/pipeline/sources/uk/fsa_approved/adapter.py index a53a500..e2efc7e 100644 --- a/pipeline/sources/uk/fsa_approved/adapter.py +++ b/pipeline/sources/uk/fsa_approved/adapter.py @@ -126,14 +126,15 @@ def run(self, raw_path: str | Path, run_dir: str | Path, config: dict[str, Any] result = self.parse_bytes(raw) root = Path(run_dir) _write_jsonl(root / "parsed" / "records.jsonl", list(result.accepted) + [item["record"] for item in result.quarantined]) - _write_jsonl(root / "normalized" / "records.jsonl", list(result.accepted)) + normalized_sha256 = _write_jsonl(root / "normalized" / "records.jsonl", list(result.accepted)) _write_jsonl(root / "quarantined" / "records.jsonl", list(result.quarantined)) (root / "released").mkdir(parents=True, exist_ok=True) manifest = {"source_id": self.source_id, "adapter_version": self.adapter_version, "schema_version": self.schema_version, "schema_status": CONFIG["schema_status"], "checksum_sha256": result.source_sha256, "byte_size": len(raw), "input_rows": len(result.accepted) + len(result.quarantined), - "normalized_rows": len(result.accepted), "quarantined_rows": len(result.quarantined), + "normalized_rows": len(result.accepted), "normalized_sha256": normalized_sha256, + "quarantined_rows": len(result.quarantined), "release_state": "not-created", "publication_state": "human-gate-required", "acquisition": CONFIG["acquisition"], "source_url": (config or {}).get("source_url"), "retrieved_at": (config or {}).get("retrieved_at")} @@ -141,8 +142,10 @@ def run(self, raw_path: str | Path, run_dir: str | Path, config: dict[str, Any] return manifest -def _write_jsonl(path: Path, rows: list[dict[str, Any]]) -> None: - _atomic(path, b"".join((json.dumps(row, ensure_ascii=False, sort_keys=True, default=list) + "\n").encode() for row in rows)) +def _write_jsonl(path: Path, rows: list[dict[str, Any]]) -> str: + payload = b"".join((json.dumps(row, ensure_ascii=False, sort_keys=True, default=list) + "\n").encode() for row in rows) + _atomic(path, payload) + return hashlib.sha256(payload).hexdigest() def _atomic(path: Path, payload: bytes) -> None: diff --git a/pipeline/sources/uk/fss_approved/adapter.py b/pipeline/sources/uk/fss_approved/adapter.py index 3709295..50e1bfd 100644 --- a/pipeline/sources/uk/fss_approved/adapter.py +++ b/pipeline/sources/uk/fss_approved/adapter.py @@ -119,13 +119,14 @@ def run(self, raw_path: str | Path, run_dir: str | Path, config: dict[str, Any] result = self.parse_bytes(raw) root = Path(run_dir) _write_jsonl(root / "parsed" / "records.jsonl", list(result.accepted) + [item["record"] for item in result.quarantined]) - _write_jsonl(root / "normalized" / "records.jsonl", list(result.accepted)) + normalized_sha256 = _write_jsonl(root / "normalized" / "records.jsonl", list(result.accepted)) _write_jsonl(root / "quarantined" / "records.jsonl", list(result.quarantined)) (root / "released").mkdir(parents=True, exist_ok=True) manifest = {"source_id": self.source_id, "adapter_version": self.adapter_version, "schema_version": self.schema_version, "checksum_sha256": result.source_sha256, "byte_size": len(raw), "input_rows": len(result.accepted) + len(result.quarantined), - "normalized_rows": len(result.accepted), "quarantined_rows": len(result.quarantined), + "normalized_rows": len(result.accepted), "normalized_sha256": normalized_sha256, + "quarantined_rows": len(result.quarantined), "release_state": "not-created", "publication_state": "human-gate-required", "acquisition": "synthetic-fixture-only", "source_url": (config or {}).get("source_url"), "retrieved_at": (config or {}).get("retrieved_at")} @@ -138,8 +139,10 @@ def write_restricted_result(self, result: ValidationResult, path: str | Path) -> _atomic(output, payload.encode()) -def _write_jsonl(path: Path, rows: list[dict[str, Any]]) -> None: - _atomic(path, b"".join((json.dumps(row, ensure_ascii=False, sort_keys=True, default=list) + "\n").encode() for row in rows)) +def _write_jsonl(path: Path, rows: list[dict[str, Any]]) -> str: + payload = b"".join((json.dumps(row, ensure_ascii=False, sort_keys=True, default=list) + "\n").encode() for row in rows) + _atomic(path, payload) + return hashlib.sha256(payload).hexdigest() def _atomic(path: Path, payload: bytes) -> None: diff --git a/pipeline/tests/e2e/backup-restore.ps1 b/pipeline/tests/e2e/backup-restore.ps1 index cbb9450..e086b6e 100644 --- a/pipeline/tests/e2e/backup-restore.ps1 +++ b/pipeline/tests/e2e/backup-restore.ps1 @@ -3,16 +3,59 @@ $ErrorActionPreference = 'Stop' if ($env:UEC_RUN_E2E -ne '1') { throw 'Set UEC_RUN_E2E=1 to run the disposable backup/restore verification.' } $root = (Resolve-Path (Join-Path $PSScriptRoot '..\..\..')).Path $project = "uec-backup-$([Guid]::NewGuid().ToString('N').Substring(0,8))" -$compose = Join-Path $root 'docker-compose.e2e.yml'; $env:UEC_E2E_DB_PORT = '55433' +$compose = Join-Path $root 'docker-compose.e2e.yml' +$portlessCompose = Join-Path $PSScriptRoot 'docker-compose.backup-restore.yml' +$composeArgs = @('-p', $project, '-f', $compose, '-f', $portlessCompose) $dump = Join-Path ([IO.Path]::GetTempPath()) "$project.dump" $migrationFile = Join-Path ([IO.Path]::GetTempPath()) "$project-migrations.sql" +$seedFile = Join-Path $PSScriptRoot 'backup_restore_seed.sql' +$suppressionFile = Join-Path $PSScriptRoot 'backup_restore_current_suppression.sql' + +function Invoke-FixtureSql([string]$path) { + Get-Content -LiteralPath $path -Raw | & docker compose @composeArgs exec -T postgres psql -1 -v ON_ERROR_STOP=1 -U uec -d uec + if ($LASTEXITCODE -ne 0) { throw "Fixture SQL failed: $path (exit $LASTEXITCODE)." } +} + +function Get-Snapshot { + $sql = @" +SELECT count(*) FROM uec.releases WHERE release_id='e2e-promoted' AND status='promoted'; +SELECT count(*) FROM uec.release_manifests WHERE release_id='e2e-promoted' AND manifest_sha256='cabe8641a05beb76c9517006a8ec4cdd60b3bad58aa5b0fc29335fee1ac7d5dd'; +SELECT count(*) FROM uec.suppression_cases WHERE case_id='00000000-0000-0000-0000-000000000003' AND status='active'; +SELECT count(*) FROM uec.suppression_references WHERE case_id='00000000-0000-0000-0000-000000000003' AND source_id='e2e.backup' AND source_record_key='restricted' AND scope='whole_record'; +SELECT count(*) FROM uec.public_access_restricted WHERE source_record_id='00000000-0000-0000-0000-000000000002'; +SELECT count(*) FROM uec.map_facilities_display WHERE source_record_id='00000000-0000-0000-0000-000000000002'; +SELECT count(*) FROM uec.map_facilities_display_history WHERE source_record_id='00000000-0000-0000-0000-000000000002'; +"@ + $result = @(& docker compose @composeArgs exec -T postgres psql -v ON_ERROR_STOP=1 -U uec -d uec -At -c $sql) + if ($LASTEXITCODE -ne 0) { throw "Restore gate query failed (exit $LASTEXITCODE)." } + return @($result | ForEach-Object { $_.Trim() } | Where-Object { $_ -ne '' }) +} + +function Assert-Snapshot([string]$phase, [string]$expected) { + $actual = (Get-Snapshot) -join ',' + if ($actual -ne $expected) { throw "$phase invariant failed (expected $expected; got $actual). Public service must remain stopped." } + Write-Host "[backup-restore] ${phase}: $actual" +} + +function Test-SyntheticServiceGate { + # The external synthetic restriction is required in the restored DB and both + # public projections must exclude it. A query error stops the drill. + return ((Get-Snapshot) -join ',') -eq '1,1,1,1,1,0,0' +} + try { - & docker compose -p $project -f $compose up -d --wait + if (-not (Test-Path -LiteralPath $seedFile) -or -not (Test-Path -LiteralPath $suppressionFile)) { + throw 'Synthetic seed or current-suppression fixture is missing; public service gate stays closed.' + } + $configuration = & docker compose @composeArgs config --format json | ConvertFrom-Json + if ($LASTEXITCODE -ne 0) { throw "Docker Compose configuration failed (exit $LASTEXITCODE)." } + if ($configuration.services.postgres.ports) { throw 'Backup/restore drill must not publish a PostgreSQL host port.' } + & docker compose @composeArgs up -d --wait if ($LASTEXITCODE -ne 0) { throw "Docker startup failed (exit $LASTEXITCODE)." } $ready = $false $stableChecks = 0 for ($attempt = 0; $attempt -lt 60; $attempt++) { - & docker compose -p $project -f $compose exec -T postgres psql -U uec -d uec -c 'SELECT 1' *> $null + & docker compose @composeArgs exec -T postgres psql -U uec -d uec -c 'SELECT 1' *> $null if ($LASTEXITCODE -eq 0) { $stableChecks++ } else { $stableChecks = 0 } if ($stableChecks -ge 5) { $ready = $true; break } Start-Sleep -Milliseconds 250 @@ -22,31 +65,43 @@ try { Write-Host "[backup-restore] applying $($migration.Name)" $migrationSql = Get-Content $migration.FullName -Raw Set-Content -LiteralPath $migrationFile -Value $migrationSql -Encoding UTF8 - & docker compose -p $project -f $compose cp $migrationFile postgres:/tmp/migration.sql + & docker compose @composeArgs cp $migrationFile postgres:/tmp/migration.sql if ($LASTEXITCODE -ne 0) { throw "Migration upload failed for $($migration.Name) (exit $LASTEXITCODE)." } $migrationApplied = $false for ($attempt = 0; $attempt -lt 3; $attempt++) { - & docker compose -p $project -f $compose exec -T postgres psql -1 -v ON_ERROR_STOP=1 -U uec -d uec -f /tmp/migration.sql + & docker compose @composeArgs exec -T postgres psql -1 -v ON_ERROR_STOP=1 -U uec -d uec -f /tmp/migration.sql if ($LASTEXITCODE -eq 0) { $migrationApplied = $true; break } Start-Sleep -Seconds 2 } if (-not $migrationApplied) { throw "Migration $($migration.Name) failed (exit $LASTEXITCODE)." } } - Get-Content (Join-Path $root 'pipeline\tests\e2e\backup_restore_seed.sql') -Raw | & docker compose -p $project -f $compose exec -T postgres psql -1 -v ON_ERROR_STOP=1 -U uec -d uec - if ($LASTEXITCODE -ne 0) { throw "Synthetic seed failed (exit $LASTEXITCODE)." } - & docker compose -p $project -f $compose exec -T postgres pg_dump -U uec -d uec --format=custom --file=/tmp/uec.dump + Invoke-FixtureSql $seedFile + Assert-Snapshot 'older eligible state' '1,1,0,0,0,1,1' + & docker compose @composeArgs exec -T postgres pg_dump -U uec -d uec --format=custom --file=/tmp/uec.dump if ($LASTEXITCODE -ne 0) { throw "Backup creation failed (exit $LASTEXITCODE)." } - & docker compose -p $project -f $compose cp postgres:/tmp/uec.dump $dump + & docker compose @composeArgs cp postgres:/tmp/uec.dump $dump if ($LASTEXITCODE -ne 0) { throw "Backup extraction failed (exit $LASTEXITCODE)." } - & docker compose -p $project -f $compose exec -T postgres pg_restore -U uec -d uec --clean --if-exists /tmp/uec.dump + Invoke-FixtureSql $suppressionFile + Assert-Snapshot 'later restriction active' '1,1,1,1,1,0,0' + if (-not (Test-SyntheticServiceGate)) { throw 'Current synthetic restriction did not close both public projections.' } + + # No application service is started anywhere in this drill. An old restore + # loses the newer case, so the service gate MUST reject it before replay. + & docker compose @composeArgs exec -T postgres pg_restore -U uec -d uec --clean --if-exists --exit-on-error /tmp/uec.dump if ($LASTEXITCODE -ne 0) { throw "Restore failed (exit $LASTEXITCODE)." } - $checks = & docker compose -p $project -f $compose exec -T postgres psql -U uec -d uec -At -c "SELECT count(*) FROM uec.releases WHERE release_id='e2e-promoted' AND status='promoted'; SELECT count(*) FROM uec.release_manifests WHERE release_id='e2e-promoted' AND manifest_sha256='cabe8641a05beb76c9517006a8ec4cdd60b3bad58aa5b0fc29335fee1ac7d5dd'; SELECT count(*) FROM uec.public_access_restricted WHERE source_record_id='00000000-0000-0000-0000-000000000002'; SELECT count(*) FROM uec.map_facilities_display WHERE source_record_id='00000000-0000-0000-0000-000000000002'; SELECT count(*) FROM uec.map_facilities_display_history WHERE source_record_id='00000000-0000-0000-0000-000000000002';" - if (($checks | Where-Object { $_ -eq '1' }).Count -ne 3 -or ($checks | Where-Object { $_ -eq '0' }).Count -ne 2) { throw "Backup/restore invariant failed (expected 1,1,1,0,0): $checks" } - Write-Host 'PASS: promoted synthetic release restored; restricted source is excluded from both public projections.' + Assert-Snapshot 'old backup restored, before replay' '1,1,0,0,0,1,1' + if (Test-SyntheticServiceGate) { throw 'Unsafe drill gate accepted an old backup before current restriction replay.' } + Write-Host '[backup-restore] PASS: synthetic pre-service gate rejects the old backup before replay.' + + Invoke-FixtureSql $suppressionFile + Assert-Snapshot 'current restriction replayed' '1,1,1,1,1,0,0' + if (-not (Test-SyntheticServiceGate)) { throw 'Synthetic pre-service gate rejected the replayed current restriction.' } + Write-Host 'PASS: synthetic old-backup restore remains gated until current restriction is replayed and both public projections exclude it.' + Write-Host 'TEST ONLY: production still needs an independent durable restriction ledger and an enforced service-start gate.' } finally { $savedPreference = $ErrorActionPreference $ErrorActionPreference = 'Continue' - try { & docker compose -p $project -f $compose down -v --remove-orphans *> $null } catch { } + try { & docker compose @composeArgs down -v --remove-orphans *> $null } catch { } $ErrorActionPreference = $savedPreference if (-not $KeepArtifacts -and (Test-Path -LiteralPath $dump)) { Remove-Item -LiteralPath $dump -Force } if (Test-Path -LiteralPath $migrationFile) { Remove-Item -LiteralPath $migrationFile -Force } diff --git a/pipeline/tests/e2e/backup_restore_current_suppression.sql b/pipeline/tests/e2e/backup_restore_current_suppression.sql new file mode 100644 index 0000000..52865a6 --- /dev/null +++ b/pipeline/tests/e2e/backup_restore_current_suppression.sql @@ -0,0 +1,6 @@ +-- Synthetic control-plane fixture kept outside the older database dump. +-- Replay only into this drill's disposable project, before any public service. +INSERT INTO uec.suppression_cases (case_id,status,reason_category,policy_version,actor,decision) +VALUES ('00000000-0000-0000-0000-000000000003','active','privacy','ethics-v1','fixture','suppress'); +INSERT INTO uec.suppression_references (case_id,source_id,source_record_key,scope) +VALUES ('00000000-0000-0000-0000-000000000003','e2e.backup','restricted','whole_record'); diff --git a/pipeline/tests/e2e/backup_restore_seed.sql b/pipeline/tests/e2e/backup_restore_seed.sql index 5084757..61c3d26 100644 --- a/pipeline/tests/e2e/backup_restore_seed.sql +++ b/pipeline/tests/e2e/backup_restore_seed.sql @@ -9,10 +9,6 @@ INSERT INTO uec.raw_artifacts (artifact_id,storage_key,sha256,byte_size,retrieve VALUES ('00000000-0000-0000-0000-000000000001','e2e/backup/restricted',repeat('a',64),0,now()); INSERT INTO uec.source_records (source_record_id,source_id,source_record_key,artifact_id,raw_fields,parsed_at) VALUES ('00000000-0000-0000-0000-000000000002','e2e.backup','restricted','00000000-0000-0000-0000-000000000001','{}',now()); -INSERT INTO uec.suppression_cases (case_id,status,reason_category,policy_version,actor,decision) -VALUES ('00000000-0000-0000-0000-000000000003','active','privacy','ethics-v1','fixture','suppress'); -INSERT INTO uec.suppression_references (case_id,source_id,source_record_key,scope) -VALUES ('00000000-0000-0000-0000-000000000003','e2e.backup','restricted','whole_record'); INSERT INTO uec.facilities (facility_id,canonical_name,country_code,city) VALUES ('00000000-0000-0000-0000-000000000004','Synthetic restricted facility','DK','Backupby'); INSERT INTO uec.observations (observation_id,facility_id,source_record_id,observed_at,observation,classification,ruleset_id,rule_id,classification_category,classification_review_status,default_visible,first_observed_at) diff --git a/pipeline/tests/e2e/docker-compose.backup-restore.yml b/pipeline/tests/e2e/docker-compose.backup-restore.yml new file mode 100644 index 0000000..2c15c56 --- /dev/null +++ b/pipeline/tests/e2e/docker-compose.backup-restore.yml @@ -0,0 +1,4 @@ +# This drill uses docker compose exec exclusively; never publish PostgreSQL on the host. +services: + postgres: + ports: !reset [] diff --git a/pipeline/tests/test_local_v2_start_contract.py b/pipeline/tests/test_local_v2_start_contract.py new file mode 100644 index 0000000..e17644e --- /dev/null +++ b/pipeline/tests/test_local_v2_start_contract.py @@ -0,0 +1,65 @@ +"""Safe PowerShell contract checks for the local synthetic V2 launcher.""" + +import os +import shutil +import subprocess +import unittest +from pathlib import Path + + +ROOT = Path(__file__).parents[2] +LAUNCHER = ROOT / "pipeline" / "scripts" / "maintenance" / "local-v2.ps1" +POWERSHELL = shutil.which("pwsh") or shutil.which("powershell") + + +@unittest.skipUnless(POWERSHELL, "PowerShell is unavailable") +class LocalV2StartContractTests(unittest.TestCase): + def run_powershell(self, script): + environment = os.environ.copy() + environment["UEC_TEST_LAUNCHER"] = str(LAUNCHER) + result = subprocess.run( + [POWERSHELL, "-NoProfile", "-Command", script], + cwd=ROOT, env=environment, text=True, capture_output=True, timeout=30, + ) + self.assertEqual(result.returncode, 0, result.stderr + result.stdout) + return result.stdout + + def test_launcher_parses_and_has_explicit_local_artifact_declaration(self): + output = self.run_powershell(r""" +$tokens = $null +$errors = $null +$ast = [System.Management.Automation.Language.Parser]::ParseFile($env:UEC_TEST_LAUNCHER, [ref]$tokens, [ref]$errors) +if ($errors.Count) { throw ($errors | Out-String) } +$promotion = @($ast.FindAll({param($node) $node -is [System.Management.Automation.Language.CommandAst] -and $node.GetCommandName() -eq 'python' -and $node.Extent.Text -like '*promote-release.py*'}, $true)) +if ($promotion.Count -ne 1) { throw "Expected one local promotion command; found $($promotion.Count)" } +$elements = @($promotion[0].CommandElements | ForEach-Object { $_.Extent.Text }) +if (($elements -join '|') -ne 'python|pipeline/scripts/stages/promote-release.py|standard-candidate|--no-distributed-artifacts') { throw "Unexpected promotion command: $($elements -join '|')" } +Write-Output 'PARSE_AND_COMMAND_OK' +""") + self.assertIn("PARSE_AND_COMMAND_OK", output) + + def test_start_branch_uses_declaration_without_external_side_effects(self): + output = self.run_powershell(r""" +function docker { + $global:LASTEXITCODE = 0 + $commandLine = $args -join ' ' + if ($commandLine -like '*to_regclass*') { return 't' } + if ($commandLine -like '*SELECT status FROM uec.releases*') { return 'validated' } +} +function python { + $global:LASTEXITCODE = 0 + if ($args[0] -like '*promote-release.py') { $global:promotionArguments = @($args) } +} +function New-Item { param($ItemType, [switch]$Force, $Path) } +function Test-Path { param($Path); return ($Path -like '*uec-api.exe') } +function Start-Process { param($FilePath, $WorkingDirectory, $WindowStyle, $RedirectStandardOutput, $RedirectStandardError, [switch]$PassThru); return [pscustomobject]@{Id=12345} } +function Set-Content { param($Path, $Value, [switch]$NoNewline) } +& $env:UEC_TEST_LAUNCHER -Command start +if (($global:promotionArguments -join '|') -ne 'pipeline/scripts/stages/promote-release.py|standard-candidate|--no-distributed-artifacts') { throw "Unexpected promotion arguments: $($global:promotionArguments -join '|')" } +Write-Output 'MOCKED_START_OK' +""") + self.assertIn("MOCKED_START_OK", output) + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/tests/test_promote_release.py b/pipeline/tests/test_promote_release.py index 1cc06df..9b7c731 100644 --- a/pipeline/tests/test_promote_release.py +++ b/pipeline/tests/test_promote_release.py @@ -1,4 +1,7 @@ import importlib.util +import hashlib +import json +import tempfile import unittest from pathlib import Path @@ -26,6 +29,41 @@ def test_promotion_rechecks_public_safety_gates_and_supports_manifest(self): self.assertIn(gate, source) self.assertIn("--manifest", source) + def test_artifact_inventory_hashes_real_bytes(self): + with tempfile.TemporaryDirectory() as directory: + artifact = Path(directory) / "synthetic.csv" + artifact.write_bytes(b"id,value\n1,test\n") + inventory = MODULE.inventory_artifacts([artifact], False) + self.assertEqual(inventory, [{ + "name": "synthetic.csv", + "sha256": hashlib.sha256(artifact.read_bytes()).hexdigest(), + "byte_size": artifact.stat().st_size, + }]) + + def test_artifact_inventory_rejects_missing_file_or_implicit_omission(self): + with tempfile.TemporaryDirectory() as directory: + with self.assertRaisesRegex(ValueError, "declare --artifact"): + MODULE.inventory_artifacts([], False) + with self.assertRaisesRegex(ValueError, "not a file"): + MODULE.inventory_artifacts([Path(directory) / "absent.csv"], False) + self.assertEqual(MODULE.inventory_artifacts([], True), []) + + def test_canonical_manifest_hash_matches_export_bytes(self): + manifest = {"release_id": "synthetic", "created_at": "2026-09-13T00:00:00Z", "distributed_artifacts": []} + serialized = MODULE.canonical_json(manifest) + self.assertEqual(json.loads(serialized), manifest) + digest = hashlib.sha256(serialized.encode("utf-8")).hexdigest() + with tempfile.TemporaryDirectory() as directory: + output = Path(directory) / "manifest.json" + MODULE.write_manifest(output, {"manifest": manifest, "manifest_sha256": digest, + "release_id": "synthetic", "status": "promoted"}) + self.assertEqual(hashlib.sha256(output.read_bytes()).hexdigest(), digest) + self.assertEqual(json.loads(output.read_text(encoding="utf-8")), manifest) + with self.assertRaises(FileExistsError): + MODULE.write_manifest(output, {"manifest": manifest, "manifest_sha256": digest}) + with self.assertRaisesRegex(ValueError, "digest does not match"): + MODULE.write_manifest(output, {"manifest": manifest, "manifest_sha256": "0" * 64}) + if __name__ == "__main__": unittest.main() diff --git a/pipeline/tests/test_publication_scoped_stages.py b/pipeline/tests/test_publication_scoped_stages.py new file mode 100644 index 0000000..678b27b --- /dev/null +++ b/pipeline/tests/test_publication_scoped_stages.py @@ -0,0 +1,127 @@ +"""Rollback-only synthetic database checks for release-scoped stage gates.""" + +import importlib.util +import hashlib +import os +import unittest +import uuid +from pathlib import Path +from unittest.mock import patch + +import psycopg + + +ROOT = Path(__file__).parents[1] +DATABASE_URL = os.environ.get("UEC_DATABASE_URL", "postgresql://uec:uec-local-development-only@localhost:5433/uec") + + +def load_stage(name): + path = ROOT / "scripts" / "stages" / name + spec = importlib.util.spec_from_file_location(name.replace("-", "_"), path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +VALIDATE = load_stage("validate-release.py") +PROMOTE = load_stage("promote-release.py") + + +class BorrowedConnection: + def __init__(self, connection): + self.connection = connection + + def __enter__(self): + return self.connection + + def __exit__(self, *_): + return False + + +class ScopedStageDatabaseTests(unittest.TestCase): + def test_legacy_ambiguity_and_explicit_scope(self): + try: + db = psycopg.connect(DATABASE_URL, connect_timeout=2) + except psycopg.Error as error: + self.skipTest(f"PostGIS is unavailable: {error}") + try: + prefix = f"test.stage.{uuid.uuid4().hex}" + release_a, release_b = prefix + ".a", prefix + ".b" + db.execute("INSERT INTO uec.sources (source_id,country_code,name,official_url,access_method) VALUES (%s,'DK','Synthetic','https://example.invalid','test')", (prefix,)) + artifact_id, record_id, facility_id, observation_id = (uuid.uuid4() for _ in range(4)) + db.execute("INSERT INTO uec.raw_artifacts (artifact_id,storage_key,sha256,byte_size,retrieved_at) VALUES (%s,%s,%s,1,now())", (artifact_id, prefix, uuid.uuid4().hex * 2)) + db.execute("INSERT INTO uec.source_records (source_record_id,source_id,source_record_key,artifact_id,raw_fields,parsed_at) VALUES (%s,%s,'synthetic',%s,'{}',now())", (record_id, prefix, artifact_id)) + db.execute("INSERT INTO uec.facilities (facility_id,canonical_name,country_code) VALUES (%s,'Synthetic facility','DK')", (facility_id,)) + db.execute("INSERT INTO uec.observations (observation_id,facility_id,source_record_id,observed_at,observation,classification,ruleset_id,rule_id,classification_category,classification_review_status,default_visible,first_observed_at) VALUES (%s,%s,%s,now(),'{}','{}','test','test','synthetic','approved',true,now())", (observation_id, facility_id, record_id)) + for release_id in (release_a, release_b): + db.execute("INSERT INTO uec.releases (release_id,status,ruleset_version,profile,summary) VALUES (%s,'candidate','synthetic-v1','secondary','{}')", (release_id,)) + db.execute("INSERT INTO uec.release_members (release_id,facility_id,observation_id,default_visible) VALUES (%s,%s,%s,true)", (release_id, facility_id, observation_id)) + db.execute("INSERT INTO uec.geocode_results (source_record_id,provider_id,query,match_method,status,result,queried_at) VALUES (%s,'synthetic','synthetic','test','accepted',ST_SetSRID(ST_MakePoint(10,55),4326)::geography,now())", (record_id,)) + has_scope = db.execute("SELECT to_regclass('uec.publication_review_release_scopes')").fetchone()[0] is not None + if not has_scope: + db.execute("INSERT INTO uec.publication_review_events (source_record_id,factual_review_status,privacy_screening_status,maintainer_approval,publication_eligible,reviewer_role) VALUES (%s,'reviewed','passed','approved',true,'synthetic-maintainer')", (record_id,)) + db.execute((ROOT / "migrations" / "022_publication_safety_scopes.sql").read_text(encoding="utf-8")) + with patch.object(VALIDATE.psycopg, "connect", return_value=BorrowedConnection(db)): + before = VALIDATE.validate(DATABASE_URL, release_b, 1, False) + self.assertEqual(before["metrics"]["publication_not_approved"], 1) + self.assertEqual(before["status"], "blocked") + self.assertFalse(before["marked_validated"]) + db.execute("INSERT INTO uec.publication_review_events (source_record_id,release_id,factual_review_status,privacy_screening_status,maintainer_approval,publication_eligible,reviewer_role) VALUES (%s,%s,'reviewed','passed','approved',true,'synthetic-maintainer')", (record_id, release_a)) + with patch.object(VALIDATE.psycopg, "connect", return_value=BorrowedConnection(db)): + scoped = VALIDATE.validate(DATABASE_URL, release_a, 1, False) + other = VALIDATE.validate(DATABASE_URL, release_b, 1, False) + self.assertEqual(scoped["metrics"]["publication_not_approved"], 0) + self.assertEqual(other["metrics"]["publication_not_approved"], 1) + db.execute("UPDATE uec.releases SET status='validated' WHERE release_id IN (%s,%s)", (release_a, release_b)) + with patch.object(PROMOTE.psycopg, "connect", return_value=BorrowedConnection(db)): + with self.assertRaisesRegex(ValueError, "publication_not_approved=1"): + PROMOTE.promote(DATABASE_URL, release_b, []) + result = PROMOTE.promote(DATABASE_URL, release_a, []) + stored = db.execute("SELECT manifest,manifest_sha256 FROM uec.release_manifests WHERE release_id=%s", (release_a,)).fetchone() + self.assertEqual(stored[0], result["manifest"]) + self.assertEqual(stored[1], result["manifest_sha256"]) + self.assertEqual(stored[1], hashlib.sha256(PROMOTE.canonical_json(stored[0]).encode("utf-8")).hexdigest()) + self.assertIn("created_at", stored[0]) + self.assertEqual(stored[0]["distributed_artifacts"], []) + self.assertEqual(db.execute("SELECT count(*) FROM uec.publication_release_eligible_observations WHERE release_id=%s AND source_record_id=%s", (release_a, record_id)).fetchone()[0], 1) + + # A newer B decision must neither relabel A nor make B inherit A's approval. + db.execute("INSERT INTO uec.publication_review_events (source_record_id,release_id,factual_review_status,privacy_screening_status,maintainer_approval,publication_eligible,reviewer_role,reviewed_at) VALUES (%s,%s,'reviewed','passed','denied',false,'synthetic-maintainer',now() + interval '1 second')", (record_id, release_b)) + with patch.object(VALIDATE.psycopg, "connect", return_value=BorrowedConnection(db)): + denied = VALIDATE.validate(DATABASE_URL, release_b, 1, False) + self.assertEqual(denied["metrics"]["publication_not_approved"], 1) + with patch.object(PROMOTE.psycopg, "connect", return_value=BorrowedConnection(db)): + with self.assertRaisesRegex(ValueError, "publication_not_approved=1"): + PROMOTE.promote(DATABASE_URL, release_b, []) + self.assertEqual(db.execute("SELECT count(*) FROM uec.publication_release_eligible_observations WHERE release_id=%s AND source_record_id=%s", (release_a, record_id)).fetchone()[0], 1) + + db.execute("INSERT INTO uec.publication_review_events (source_record_id,release_id,factual_review_status,privacy_screening_status,maintainer_approval,publication_eligible,reviewer_role,reviewed_at) VALUES (%s,%s,'reviewed','passed','approved',true,'synthetic-maintainer',now() + interval '2 seconds')", (record_id, release_b)) + source_keyed_event = db.execute("SELECT publication_review_event_id FROM uec.publication_review_current WHERE source_record_id=%s", (record_id,)).fetchone()[0] + b_event = db.execute("SELECT publication_review_event_id FROM uec.publication_review_release_current WHERE source_record_id=%s AND release_id=%s", (record_id, release_b)).fetchone()[0] + self.assertNotEqual(source_keyed_event, b_event) + with patch.object(VALIDATE.psycopg, "connect", return_value=BorrowedConnection(db)): + approved_b = VALIDATE.validate(DATABASE_URL, release_b, 1, False) + self.assertEqual(approved_b["metrics"]["publication_not_approved"], 0) + self.assertEqual(approved_b["status"], "passed") + self.assertEqual(db.execute("SELECT count(*) FROM uec.publication_release_eligible_observations WHERE release_id=%s AND source_record_id=%s", (release_a, record_id)).fetchone()[0], 1) + + db.execute("INSERT INTO uec.record_access_events (source_record_id,action,reason_category,policy_version,maintainer) VALUES (%s,'public_access_revoked','privacy','ethics-v1','synthetic-maintainer')", (record_id,)) + with patch.object(VALIDATE.psycopg, "connect", return_value=BorrowedConnection(db)): + restricted = VALIDATE.validate(DATABASE_URL, release_b, 1, False) + self.assertEqual(restricted["metrics"]["active_suppression"], 1) + with patch.object(PROMOTE.psycopg, "connect", return_value=BorrowedConnection(db)): + with self.assertRaisesRegex(ValueError, "active_suppression=1"): + PROMOTE.promote(DATABASE_URL, release_b, []) + db.execute("INSERT INTO uec.record_access_events (source_record_id,action,reason_category,policy_version,maintainer,occurred_at) VALUES (%s,'public_access_restored','privacy','ethics-v1','synthetic-maintainer',now() + interval '1 second')", (record_id,)) + with patch.object(PROMOTE.psycopg, "connect", return_value=BorrowedConnection(db)): + promoted_b = PROMOTE.promote(DATABASE_URL, release_b, []) + self.assertEqual(promoted_b["status"], "promoted") + self.assertEqual(db.execute("SELECT count(*) FROM uec.publication_review_events WHERE source_record_id=%s", (record_id,)).fetchone()[0], + 4 if not has_scope else 3) + finally: + db.rollback() + db.close() + + +if __name__ == "__main__": + unittest.main() From cb3d6a4ea0e3eac51d46668b0e7f65dc1cef733d Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sun, 13 Sep 2026 19:19:21 -0700 Subject: [PATCH 035/311] fix(v2): preserve public safety context across API and clients --- frontend/src/api/LocalLocationRepository.ts | 40 ++- frontend/src/api/wireSchema.ts | 2 +- frontend/src/app/App.svelte | 58 ++-- frontend/src/domain/location.ts | 15 +- frontend/src/map/MapView.svelte | 3 +- frontend/src/map/mapProjection.ts | 2 +- frontend/src/styles/research.css | 1 + frontend/tests/e2e/local-safety.spec.ts | 119 ++++++++ .../tests/unit/localDetailRepository.test.ts | 12 + .../unit/localLocationRepository.test.ts | 19 ++ pipeline/tests/e2e/test_community_api.py | 18 ++ .../tests/e2e/test_public_surface_safety.py | 254 ++++++++++++++++++ pipeline/tests/e2e/test_seeded_api.py | 57 ++++ src/lib.rs | 94 ++++++- src/main.rs | 163 +++++++++-- static/app.js | 4 +- static/modules/ExportManager.js | 46 +++- static/modules/__tests__/v2Client.test.js | 65 +++++ static/modules/popupBuilder.js | 5 +- static/modules/v2Adapter.js | 2 +- static/modules/v2Client.js | 3 + static/modules/v2Contract.js | 12 +- 22 files changed, 931 insertions(+), 63 deletions(-) create mode 100644 frontend/tests/e2e/local-safety.spec.ts create mode 100644 pipeline/tests/e2e/test_public_surface_safety.py diff --git a/frontend/src/api/LocalLocationRepository.ts b/frontend/src/api/LocalLocationRepository.ts index ece5696..ec16f7a 100644 --- a/frontend/src/api/LocalLocationRepository.ts +++ b/frontend/src/api/LocalLocationRepository.ts @@ -3,17 +3,47 @@ import type { ApiError } from './errors'; import type { Location } from '../domain/location'; export type FetchLike = (input: RequestInfo | URL, init?: RequestInit) => Promise; +export type LocalProfile = 'official' | 'secondary' | 'community'; export type LocationFilters = Readonly<{ country_code?: string | undefined; category?: string | undefined; source_type?: string | undefined; display_precision?: string | undefined; lifecycle_status?: string | undefined; cursor?: string | undefined }>; -export type LocalListResult = Readonly<{ locations: readonly Location[]; releaseId: string; profile: string; coverageNote: string; nextCursor: string | null; ruleset?: string }>; +export type LocalListResult = Readonly<{ locations: readonly Location[]; releaseId: string; profile: LocalProfile; coverageNote: string; nextCursor: string | null; ruleset?: string }>; export const localOrigin = (value: string | undefined): string | undefined => { if (!value) return undefined; const url = new URL(value); if (url.protocol !== 'http:' || !['127.0.0.1', 'localhost', '::1'].includes(url.hostname)) throw new Error('Local API origin must be loopback HTTP.'); return url.origin; }; const fail = (kind: ApiError['kind'], message: string, status?: number): ApiError => Object.assign(new Error(message), status === undefined ? { kind } : { kind, status }); -const map = (r: WireLocation): Location => ({ id: r.facility_id, name: r.canonical_name, region: r.city ?? r.country_code, category: r.category, lat: r.latitude, lon: r.longitude, observed: r.last_observed_at ?? r.first_observed_at ?? 'unknown', source: r.provenance_source_name }); -const query = (profile: string, filters: LocationFilters) => { const params = new URLSearchParams({ profile }); for (const [key, value] of Object.entries(filters)) if (value) params.set(key, value); return `/api/v2/locations?${params}`; }; +const map = (r: WireLocation): Location => ({ + id: r.facility_id, name: r.canonical_name, region: r.city ?? r.country_code, category: r.category, + lat: r.latitude, lon: r.longitude, observed: r.last_observed_at ?? r.first_observed_at ?? 'unknown', source: r.provenance_source_name, + evidence: { + sourceType: r.source_type, factualReviewStatus: r.factual_review_status, reviewerRole: r.reviewer_role, + privacyScreeningStatus: r.privacy_screening_status, projectApproval: r.project_approval, + publicationProfile: r.publication_profile, publicationWarning: r.publication_warning, + sourceId: r.provenance_source_id, sourceUrl: r.provenance_source_url, + retrievedAt: r.provenance_retrieved_at, displayPrecision: r.display_precision, + }, +}); +const query = (profile: LocalProfile, filters: LocationFilters) => { const params = new URLSearchParams({ profile }); for (const [key, value] of Object.entries(filters)) if (value) params.set(key, value); return `/api/v2/locations?${params}`; }; +const eligible = (row: WireLocation, profile: LocalProfile, releaseId: string, ruleset: string): boolean => + row.publication_profile === profile && row.release_id === releaseId && row.release_ruleset_version === ruleset && + row.privacy_screening_status === 'passed' && row.factual_review_status !== 'rejected' && + (row.project_approval === 'approved' || (profile === 'community' && row.source_type === 'user_submitted' && row.factual_review_status === 'unreviewed')); export class LocalLocationRepository { readonly #base: string | undefined; constructor(private readonly fetcher: FetchLike = globalThis.fetch, baseUrl?: string) { this.#base = localOrigin(baseUrl); } private async json(path: string, signal?: AbortSignal) { const init: RequestInit = { cache: 'no-store' }; if (signal) init.signal = signal; const response = await this.fetcher.call(globalThis, `${this.#base ?? ''}${path}`, init); if (!response.ok) throw fail('http', `Local V2 request failed with status ${response.status}`, response.status); try { return await response.json(); } catch { throw fail('invalid-contract', 'Local V2 response was not valid JSON.'); } } - async list(profile: 'official' | 'secondary' | 'community' = 'official', filters: LocationFilters = {}): Promise { try { const b = envelopeSchema.safeParse(await this.json(query(profile, filters))); if (!b.success || b.data.meta.profile !== profile) throw fail('invalid-contract', 'Local V2 list response was rejected.'); if (b.data.meta.release_id === null) throw fail('no-release', b.data.meta.coverage_note); if (b.data.meta.ruleset_version === undefined || b.data.data.some(r => r.privacy_screening_status !== 'passed' || r.project_approval !== 'approved' || r.release_id !== b.data.meta.release_id || r.release_ruleset_version !== b.data.meta.ruleset_version)) throw fail('invalid-contract', 'Local V2 list snapshot was rejected.'); return { locations: b.data.data.map(map), releaseId: b.data.meta.release_id, profile: b.data.meta.profile, coverageNote: b.data.meta.coverage_note, nextCursor: b.data.meta.next_cursor ?? null, ruleset: b.data.meta.ruleset_version }; } catch (e) { if (e && typeof e === 'object' && 'kind' in e) throw e; if (e instanceof DOMException && e.name === 'AbortError') throw fail('aborted', 'Local V2 request was aborted.'); if (e instanceof TypeError) throw fail('network', 'Local V2 request could not connect.'); throw fail('invalid-contract', 'Local V2 response could not be read safely.'); } } - async detail(id: string, profile: 'official' | 'secondary' | 'community' = 'official') { try { const b = detailEnvelopeSchema.safeParse(await this.json(`/api/v2/locations/${encodeURIComponent(id)}?profile=${profile}`)); if (!b.success || b.data.meta.profile !== profile || b.data.data.privacy_screening_status !== 'passed' || b.data.data.project_approval !== 'approved') throw fail('invalid-contract', 'Local V2 detail response was rejected.'); return { location: map(b.data.data), releaseId: b.data.meta.release_id, profile: b.data.meta.profile }; } catch (e) { if (e && typeof e === 'object' && 'kind' in e) throw e; if (e instanceof DOMException && e.name === 'AbortError') throw fail('aborted', 'Local V2 request was aborted.'); if (e instanceof TypeError) throw fail('network', 'Local V2 request could not connect.'); throw fail('invalid-contract', 'Local V2 response could not be read safely.'); } } + async list(profile: LocalProfile = 'official', filters: LocationFilters = {}, signal?: AbortSignal): Promise { + try { + const b = envelopeSchema.safeParse(await this.json(query(profile, filters), signal)); + if (!b.success || b.data.meta.profile !== profile) throw fail('invalid-contract', 'Local V2 list response was rejected.'); + if (b.data.meta.release_id === null) throw fail('no-release', b.data.meta.coverage_note); + const { release_id, ruleset_version } = b.data.meta; + if (ruleset_version === undefined || b.data.data.some(row => !eligible(row, profile, release_id, ruleset_version))) throw fail('invalid-contract', 'Local V2 list snapshot was rejected.'); + return { locations: b.data.data.map(map), releaseId: release_id, profile, coverageNote: b.data.meta.coverage_note, nextCursor: b.data.meta.next_cursor ?? null, ruleset: ruleset_version }; + } catch (e) { if (e && typeof e === 'object' && 'kind' in e) throw e; if (e instanceof DOMException && e.name === 'AbortError') throw fail('aborted', 'Local V2 request was aborted.'); if (e instanceof TypeError) throw fail('network', 'Local V2 request could not connect.'); throw fail('invalid-contract', 'Local V2 response could not be read safely.'); } + } + async detail(id: string, profile: LocalProfile = 'official', signal?: AbortSignal) { + try { + const b = detailEnvelopeSchema.safeParse(await this.json(`/api/v2/locations/${encodeURIComponent(id)}?profile=${profile}`, signal)); + if (!b.success || b.data.meta.profile !== profile || !eligible(b.data.data, profile, b.data.meta.release_id, b.data.meta.ruleset_version) || b.data.data.facility_id !== id) throw fail('invalid-contract', 'Local V2 detail response was rejected.'); + return { location: map(b.data.data), releaseId: b.data.meta.release_id, profile }; + } catch (e) { if (e && typeof e === 'object' && 'kind' in e) throw e; if (e instanceof DOMException && e.name === 'AbortError') throw fail('aborted', 'Local V2 request was aborted.'); if (e instanceof TypeError) throw fail('network', 'Local V2 request could not connect.'); throw fail('invalid-contract', 'Local V2 response could not be read safely.'); } + } } diff --git a/frontend/src/api/wireSchema.ts b/frontend/src/api/wireSchema.ts index 9f7ecfa..d9c37ae 100644 --- a/frontend/src/api/wireSchema.ts +++ b/frontend/src/api/wireSchema.ts @@ -1,6 +1,6 @@ import { z } from 'zod'; const textOrNull=z.string().nullable(); -export const locationSchema=z.object({facility_id:z.string().uuid(),canonical_name:z.string().min(1),city:textOrNull,country_code:z.string().regex(/^[A-Z]{2}$/),category:z.string(),source_type:z.enum(['official','secondary','user_submitted']),publication_profile:z.enum(['official','secondary','community']),factual_review_status:z.enum(['unreviewed','reviewed','rejected']),privacy_screening_status:z.literal('passed'),project_approval:z.enum(['pending','approved']),reviewer_role:textOrNull,publication_warning:textOrNull,display_precision:z.enum(['exact','city','unmapped']),latitude:z.number().finite().nullable(),longitude:z.number().finite().nullable(),first_observed_at:textOrNull,last_observed_at:textOrNull,observation_count:z.number().int().nonnegative().nullable(),lifecycle_status:z.enum(['active_observed','explicitly_closed','not_seen_recently','status_unknown']),provenance_source_id:z.string(),provenance_source_name:z.string(),provenance_source_url:z.string().url(),provenance_retrieved_at:z.string(),release_id:z.string(),release_ruleset_version:z.string()}).superRefine((row,ctx)=>{if((row.latitude===null)!==(row.longitude===null))ctx.addIssue({code:z.ZodIssueCode.custom,message:'coordinate pair must be complete'});if(row.display_precision==='unmapped'&&(row.latitude!==null||row.longitude!==null))ctx.addIssue({code:z.ZodIssueCode.custom,message:'unmapped record cannot have coordinates'});if(row.display_precision!=='unmapped'&&(row.latitude===null||row.longitude===null))ctx.addIssue({code:z.ZodIssueCode.custom,message:'mapped record requires coordinates'});}); +export const locationSchema=z.object({facility_id:z.string().uuid(),canonical_name:z.string().min(1),city:textOrNull,country_code:z.string().regex(/^[A-Z]{2}$/),category:z.string(),source_type:z.enum(['official','secondary','user_submitted']),publication_profile:z.enum(['official','secondary','community']),factual_review_status:z.enum(['unreviewed','reviewed','rejected']),privacy_screening_status:z.literal('passed'),project_approval:z.enum(['pending','approved']),reviewer_role:textOrNull,publication_warning:textOrNull,display_precision:z.enum(['exact','city','unmapped']),latitude:z.number().finite().nullable(),longitude:z.number().finite().nullable(),first_observed_at:textOrNull,last_observed_at:textOrNull,observation_count:z.number().int().nonnegative().nullable(),lifecycle_status:z.enum(['active_observed','explicitly_closed','not_seen_recently','status_unknown']),provenance_source_id:z.string(),provenance_source_name:z.string(),provenance_source_url:z.string().url().refine(value=>{try{return ['http:','https:'].includes(new URL(value).protocol);}catch{return false;}},'source URL must use HTTP or HTTPS'),provenance_retrieved_at:z.string(),release_id:z.string(),release_ruleset_version:z.string()}).superRefine((row,ctx)=>{if((row.latitude===null)!==(row.longitude===null))ctx.addIssue({code:z.ZodIssueCode.custom,message:'coordinate pair must be complete'});if(row.display_precision==='unmapped'&&(row.latitude!==null||row.longitude!==null))ctx.addIssue({code:z.ZodIssueCode.custom,message:'unmapped record cannot have coordinates'});if(row.display_precision!=='unmapped'&&(row.latitude===null||row.longitude===null))ctx.addIssue({code:z.ZodIssueCode.custom,message:'mapped record requires coordinates'});}); export const envelopeSchema=z.object({data:z.array(locationSchema),api_version:z.literal('v2'),meta:z.object({release_id:z.string().nullable(),ruleset_version:z.string().optional(),release_created_at:z.string().optional(),profile:z.enum(['official','secondary','community']),next_cursor:z.string().nullable().optional(),coverage_note:z.string().min(1)})}); export type WireEnvelope=z.infer;export type WireLocation=z.infer; export const detailEnvelopeSchema=z.object({data:locationSchema,api_version:z.literal('v2'),meta:z.object({release_id:z.string(),ruleset_version:z.string(),release_created_at:z.string(),profile:z.enum(['official','secondary','community'])})}); diff --git a/frontend/src/app/App.svelte b/frontend/src/app/App.svelte index 24bb7a3..3bdad0d 100644 --- a/frontend/src/app/App.svelte +++ b/frontend/src/app/App.svelte @@ -1,5 +1,5 @@ Until Every Cage · evidence desk @@ -71,10 +96,11 @@ onMount(() => { const syncBrowserState = () => { const params = new URLSearchPar

EVIDENCE DESK · {localMode ? 'LOCAL V2 API' : 'SYNTHETIC PREVIEW'}

See what a record can—and cannot—tell us.

Start with the source profile, narrow the visible evidence, then inspect what a record can—and cannot—tell us.

01 / DISCOVER

Choose the evidence lane

Profiles stay separate. A community claim is never silently promoted into a curated result.

{#if localMode && metadataStatus === 'loading'}

Loading available filter values…

{:else if localMode && metadataStatus === 'error'}

Filter metadata is unavailable; only unscoped discovery is shown.

{/if}
{search || region !== 'all' || category !== 'all' ? `Filters applied: ${[search && `name “${search}”`, region !== 'all' && region, category !== 'all' && category].filter(Boolean).join(' · ')}` : 'No additional filters applied'}
- {#if profile === 'community'}
Unreviewed community claimNot verified by Until Every Cage. Review this profile before relying on it.
{/if} - {#if localMode && localStatus === 'loading'}
Loading the {profileLabel.toLowerCase()}…
{:else if localMode && (localStatus === 'error' || localStatus === 'no-release')}{:else if localMode && detailStatus === 'loading'}
Loading the selected local record…
{:else if localMode && detailStatus === 'error'}{:else} -

02 / FILTER & COMPARE

Results {visibleLocations.length}

{localMode ? 'Current eligible response' : 'Fictional demonstration data'} · {profileLabel}

-
{#if selected}

03 / RECORD DETAIL

{selected.name}

{selected.region} · {selected.category}

OBSERVED{selected.observed}
MAP STATUS{selected.lat === null ? 'No publishable map location' : 'Approximate display point'}

READ WITH CARE

This is an observation in a particular release, not a guarantee of current operation. Source origin, factual review, privacy screening, and project approval are separate signals.

{/if}
+ {#if profile === 'community'}
Community claimsUnreviewed community claims: Not verified by Until Every Cage. Check each record’s factual review status before relying on it.
{/if} + {#if localMode && localStatus === 'loading'}
Loading the {profileLabel.toLowerCase()}…
{:else if localMode && (localStatus === 'error' || localStatus === 'no-release')}{:else if localMode && detailStatus === 'loading'}
Loading the selected local record…
{:else if localMode && detailStatus === 'error'}{:else} +

02 / FILTER & COMPARE

Results {visibleLocations.length}{localMode && nextCursor ? ' on first page' : ''}

{localMode ? 'Current eligible response' : 'Fictional demonstration data'} · {profileLabel}

+ {#if localMode}

{coverageNote} {nextCursor ? 'Only the first page is loaded. Search and filters below may miss later records; counts and map points are partial.' : 'All records in this response are loaded; search applies to those records.'}

{/if} +
{#if selected}

03 / RECORD DETAIL

{selected.name}

{selected.region} · {selected.category}

{#if selected.evidence?.publicationProfile === 'community' && selected.evidence.factualReviewStatus === 'unreviewed'}

{selected.evidence.publicationWarning ?? 'Unreviewed community claim — not verified by Until Every Cage'}

{/if}
OBSERVED{selected.observed}
MAP STATUS{selected.lat === null ? 'No publishable map location' : selected.evidence?.displayPrecision === 'exact' ? 'Exact display point' : 'Approximate display point'}
{#if selected.evidence}
Source origin
{selected.evidence.sourceType === 'user_submitted' ? 'Community-submitted' : selected.evidence.sourceType === 'official' ? 'Government-sourced' : 'Secondary-sourced'}
Factual review
{selected.evidence.factualReviewStatus}{selected.evidence.reviewerRole ? ` · ${selected.evidence.reviewerRole}` : ' · reviewer role unavailable'}
Privacy screening
{selected.evidence.privacyScreeningStatus}
Project approval
{selected.evidence.projectApproval}
Published profile
{selected.evidence.publicationProfile} · release {release}
Source
{selected.source} · {selected.evidence.sourceId}
Retrieved
{selected.evidence.retrievedAt}
{/if}

READ WITH CARE

This is an observation in a particular release, not a guarantee of current operation. Source origin, factual review, privacy screening, and project approval are separate signals.

{/if}
{#if selected}

RECORD / {selected.id}

{/if} {/if} {#if localMode && localStatus === 'ready'}{/if} diff --git a/frontend/src/domain/location.ts b/frontend/src/domain/location.ts index cdb2359..ed2e203 100644 --- a/frontend/src/domain/location.ts +++ b/frontend/src/domain/location.ts @@ -1,2 +1,15 @@ export type LocationId = string; -export type Location = Readonly<{id:LocationId,name:string,region:string,category:string,lat:number|null,lon:number|null,observed:string,source:string}>; +export type LocationEvidence = Readonly<{ + sourceType: 'official' | 'secondary' | 'user_submitted'; + factualReviewStatus: 'unreviewed' | 'reviewed' | 'rejected'; + reviewerRole: string | null; + privacyScreeningStatus: 'passed'; + projectApproval: 'pending' | 'approved'; + publicationProfile: 'official' | 'secondary' | 'community'; + publicationWarning: string | null; + sourceId: string; + sourceUrl: string; + retrievedAt: string; + displayPrecision: 'exact' | 'city' | 'unmapped'; +}>; +export type Location = Readonly<{id:LocationId,name:string,region:string,category:string,lat:number|null,lon:number|null,observed:string,source:string,evidence?:LocationEvidence}>; diff --git a/frontend/src/map/MapView.svelte b/frontend/src/map/MapView.svelte index 9659268..b570518 100644 --- a/frontend/src/map/MapView.svelte +++ b/frontend/src/map/MapView.svelte @@ -8,6 +8,7 @@ let container: HTMLDivElement; let adapter: LeafletMapAdapter | null = null; $: features = projectLocations(items); + $: hasUnreviewedClaims = items.some(item => item.evidence?.publicationProfile === 'community' && item.evidence.factualReviewStatus === 'unreviewed'); onMount(() => { let disposed = false; const map = new LeafletMapAdapter(); @@ -18,5 +19,5 @@ $: adapter?.update(features, selectedId); -

Blank local background · {features.length} display points · no external tiles

+
{#if hasUnreviewedClaims}

Unreviewed community claims — not verified by Until Every Cage

{/if}

Blank local background · {features.length} display points · no external tiles

diff --git a/frontend/src/map/mapProjection.ts b/frontend/src/map/mapProjection.ts index 5d3e435..1b4eda3 100644 --- a/frontend/src/map/mapProjection.ts +++ b/frontend/src/map/mapProjection.ts @@ -1,3 +1,3 @@ import type { Location } from '../domain/location'; export type DisplayFeature=Readonly<{id:string,label:string,lat:number,lon:number}>; -export const projectLocations=(items:readonly Location[]):readonly DisplayFeature[]=>items.flatMap((item)=>item.lat===null||item.lon===null?[]:[{id:item.id,label:item.name,lat:item.lat,lon:item.lon}]); +export const projectLocations=(items:readonly Location[]):readonly DisplayFeature[]=>items.flatMap((item)=>item.lat===null||item.lon===null?[]:[{id:item.id,label:item.evidence?.publicationProfile==='community'&&item.evidence.factualReviewStatus==='unreviewed'?`${item.name} · Unreviewed community claim — not verified by Until Every Cage`:item.name,lat:item.lat,lon:item.lon}]); diff --git a/frontend/src/styles/research.css b/frontend/src/styles/research.css index 66e98ed..58fe906 100644 --- a/frontend/src/styles/research.css +++ b/frontend/src/styles/research.css @@ -1 +1,2 @@ .research-bar{display:grid;grid-template-columns:minmax(220px,.8fr) 1.6fr;gap:30px;border-top:1px solid #cfc4b2;border-bottom:1px solid #cfc4b2;padding:24px 0;margin-bottom:22px}.research-bar h2,.results-head h2{font:600 1.8rem Georgia,serif;margin:0 0 8px}.research-bar p:not(.eyebrow){color:#5f5549;line-height:1.5;margin:0}.research-bar .toolbar{border:0;padding:0;margin:0;display:grid;grid-template-columns:repeat(2,minmax(120px,1fr));gap:12px}.research-bar .toolbar label{min-width:0}.research-bar input,.research-bar select{display:block;width:100%;margin-top:7px;background:#fbf8f2;border:1px solid #b9ad9b;border-radius:3px;padding:10px 11px;color:#17283b;font-size:.95rem}.results-head{display:flex;justify-content:space-between;align-items:end;margin:26px 0 12px}.results-head h2 span{font:700 .85rem ui-sans-serif;color:#a34927;vertical-align:middle}.scope{color:#62594e;font-size:.8rem}.release-panel{display:flex;flex-wrap:wrap;gap:18px;border:1px solid #cfc4b2;background:#ece4d8;padding:18px;margin-top:22px}.release-panel div{min-width:130px}.release-panel span{display:block;color:#62594e;font-size:.65rem;letter-spacing:.12em;margin-bottom:5px}.release-panel strong{font-size:.88rem;overflow-wrap:anywhere}.release-panel .digest{font-family:ui-monospace,monospace;font-size:.72rem}.release-panel p{width:100%;margin:0;color:#62594e;font-size:.78rem;line-height:1.45}.export-control{display:flex;align-items:center;gap:18px;padding:15px 0;border-bottom:1px solid #cfc4b2}.export-control button,.state button{width:auto;background:#a34927;color:#fff;padding:11px 15px;border-radius:3px;font-weight:700}.export-control button:disabled{background:#b9ad9b;cursor:not-allowed}.export-control p{margin:0;color:#62594e;font-size:.78rem;line-height:1.4}.export-control .export-error{color:#a34927}.empty{padding:24px 18px;color:#62594e;line-height:1.5}.guidance{margin-top:28px;border-top:1px solid #cfc4b2;border-bottom:1px solid #cfc4b2}.guidance-toggle{align-items:center;justify-content:space-between;padding:18px 0;border:0}.guidance-toggle>span:first-child{display:flex;flex-direction:column;gap:4px}.guidance-toggle .eyebrow{margin:0}.guidance-toggle>span:last-child{color:#a34927;font-size:.8rem;font-weight:700}.guidance-body{max-width:680px;padding:0 0 20px;line-height:1.55;color:#4f5c69}.guidance-body a{color:#a34927;font-weight:700}@media(max-width:680px){.research-bar{display:block}.research-bar .toolbar{margin-top:22px}.results-head{display:block}.scope{margin-top:10px}.export-control{display:block}.export-control p{margin-top:10px}.release-panel{display:block}.release-panel div{margin-bottom:13px}} +.page-context{margin:0 0 16px;color:#62594e;line-height:1.5;font-size:.85rem}.claim-warning{color:#8d351c!important;font-weight:700}.evidence{display:grid;grid-template-columns:repeat(2,minmax(0,1fr));gap:12px;border-top:1px solid #cfc4b2;margin:28px 0 0;padding-top:18px;font-size:.82rem}.evidence div{min-width:0}.evidence dt{color:#62594e;font-size:.7rem;text-transform:uppercase;letter-spacing:.08em}.evidence dd{margin:4px 0 0;overflow-wrap:anywhere}.evidence a{color:#a34927}.state.error button{display:inline-flex;margin:12px 12px 0 0}.state.error h2:focus{outline:3px solid #a34927;outline-offset:4px}@media(max-width:680px){.evidence{grid-template-columns:1fr}} diff --git a/frontend/tests/e2e/local-safety.spec.ts b/frontend/tests/e2e/local-safety.spec.ts new file mode 100644 index 0000000..c4d04f8 --- /dev/null +++ b/frontend/tests/e2e/local-safety.spec.ts @@ -0,0 +1,119 @@ +import { test, expect, type Page } from '@playwright/test'; + +const firstId = '550e8400-e29b-41d4-a716-446655440000'; +const secondId = '550e8400-e29b-41d4-a716-446655440001'; +const row = (id = firstId, name = 'First local record', profile: 'official' | 'community' = 'official') => ({ + facility_id: id, canonical_name: name, city: 'North Coast', country_code: 'DK', category: 'dairy', + source_type: profile === 'community' ? 'user_submitted' : 'official', publication_profile: profile, + factual_review_status: profile === 'community' ? 'unreviewed' : 'reviewed', privacy_screening_status: 'passed', + project_approval: profile === 'community' ? 'pending' : 'approved', reviewer_role: null, + publication_warning: profile === 'community' ? 'Unreviewed community claim — not verified by Until Every Cage' : null, + display_precision: 'city', latitude: 55, longitude: 10, first_observed_at: null, + last_observed_at: '2026-01-01T00:00:00Z', observation_count: 1, lifecycle_status: 'active_observed', + provenance_source_id: 'source-1', provenance_source_name: 'Synthetic local source', + provenance_source_url: 'https://example.test/source', provenance_retrieved_at: '2026-01-01T00:00:00Z', + release_id: 'rel-1', release_ruleset_version: 'rules-1', +}); +const list = (profile: 'official' | 'community', data = [row(firstId, 'First local record', profile)], nextCursor: string | null = null) => ({ + data, api_version: 'v2', meta: { release_id: 'rel-1', ruleset_version: 'rules-1', profile, next_cursor: nextCursor, coverage_note: 'Selected promoted release only.' }, +}); +const detail = (data: ReturnType, profile: 'official' | 'community') => ({ + data, api_version: 'v2', meta: { release_id: 'rel-1', ruleset_version: 'rules-1', release_created_at: '2026-01-01T00:00:00Z', profile }, +}); +const mockMetadata = async (page: Page) => page.route('**/api/v2/discovery/filters', route => route.fulfill({ status: 503, body: 'unavailable' })); + +test('community direct link opens in community mode with persistent record context', async ({ page }) => { + await mockMetadata(page); + const seen: string[] = []; + await page.route('**/api/v2/locations**', route => { + const url = new URL(route.request().url()); + seen.push(url.searchParams.get('profile') ?? 'missing'); + return route.fulfill({ json: url.pathname.endsWith(firstId) ? detail(row(firstId, 'Community claim', 'community'), 'community') : list('community', [row(firstId, 'Community claim', 'community')]) }); + }); + await page.goto(`./?mode=local-v2#/locations/${firstId}?profile=community`); + await expect(page.getByLabel('Profile')).toHaveValue('community'); + await expect(page.getByRole('note')).toContainText('Not verified by Until Every Cage'); + await expect(page.getByRole('heading', { name: 'Community claim' })).toBeVisible(); + await expect(page.locator('article')).toContainText('Community-submitted'); + await expect(page.locator('article')).toContainText('pending'); + await expect(page.locator('article')).toContainText('Factual reviewunreviewed'); + await page.getByRole('button', { name: 'Show map' }).click(); + await expect(page.locator('.map-warning')).toContainText('Unreviewed community claims'); + expect(seen).toEqual(['community', 'community']); +}); + +test('first-page-only search reports uncertainty instead of a global no-results claim', async ({ page }) => { + await mockMetadata(page); + await page.route('**/api/v2/locations**', route => route.fulfill({ json: list('official', [row()], secondId) })); + await page.goto('./?mode=local-v2#/'); + await expect(page.getByText('Only the first page is loaded.', { exact: false })).toBeVisible(); + await page.getByLabel('Search locations').fill('later record'); + await expect(page.getByText('Later pages may contain matches.', { exact: false })).toBeVisible(); + await expect(page.getByRole('heading', { name: /Results.*on first page/ })).toBeVisible(); +}); + +test('late list response cannot replace a newer community selection', async ({ page }) => { + await mockMetadata(page); + let releaseOfficial: (() => void) | undefined; + const officialHeld = new Promise(resolve => { releaseOfficial = resolve; }); + await page.route('**/api/v2/locations**', async route => { + const profile = new URL(route.request().url()).searchParams.get('profile'); + if (profile === 'official') await officialHeld; + try { await route.fulfill({ json: list(profile === 'community' ? 'community' : 'official', [row(firstId, profile === 'community' ? 'New community claim' : 'Stale official result', profile === 'community' ? 'community' : 'official')]) }); } catch { /* Aborted requests have no browser response to fulfill. */ } + }); + await page.goto('./?mode=local-v2#/'); + await page.getByLabel('Profile').selectOption('community'); + await expect(page.getByRole('heading', { name: 'New community claim' })).toBeVisible(); + releaseOfficial?.(); + await expect(page.getByRole('heading', { name: 'Stale official result' })).toHaveCount(0); + await expect(page.getByRole('note')).toContainText('Not verified'); +}); + +test('changing search while a list is pending invalidates the old response', async ({ page }) => { + await mockMetadata(page); + let releaseFirst: (() => void) | undefined; + const firstHeld = new Promise(resolve => { releaseFirst = resolve; }); + let listRequests = 0; + await page.route('**/api/v2/locations**', async route => { + const index = ++listRequests; + if (index === 1) await firstHeld; + try { await route.fulfill({ json: list('official', [row(firstId, index === 1 ? 'Stale initial result' : 'Current filtered result')]) }); } catch { /* Aborted request. */ } + }); + await page.goto('./?mode=local-v2#/'); + await page.getByLabel('Search locations').fill('current'); + await expect(page.getByRole('heading', { name: 'Current filtered result' })).toBeVisible(); + releaseFirst?.(); + await expect(page.getByRole('heading', { name: 'Stale initial result' })).toHaveCount(0); + expect(listRequests).toBeGreaterThanOrEqual(2); +}); + +test('late detail response cannot replace a newer route', async ({ page }) => { + await mockMetadata(page); + let releaseFirst: (() => void) | undefined; + const firstHeld = new Promise(resolve => { releaseFirst = resolve; }); + await page.route('**/api/v2/locations**', async route => { + const path = new URL(route.request().url()).pathname; + if (path.endsWith(firstId)) await firstHeld; + const body = path.endsWith(firstId) ? detail(row(firstId, 'Stale detail'), 'official') : path.endsWith(secondId) ? detail(row(secondId, 'Current detail'), 'official') : list('official', [row(), row(secondId, 'Second local record')]); + try { await route.fulfill({ json: body }); } catch { /* Aborted request. */ } + }); + await page.goto('./?mode=local-v2#/'); + await page.getByRole('button', { name: /First local record/ }).click(); + await expect(page.getByText('Loading the selected local record')).toBeVisible(); + await page.evaluate(id => { window.location.hash = `/locations/${id}?profile=curated`; }, secondId); + await expect(page.getByRole('heading', { name: 'Current detail' })).toBeVisible(); + releaseFirst?.(); + await expect(page.getByRole('heading', { name: 'Stale detail' })).toHaveCount(0); +}); + +test('detail failure focuses the error heading and offers recovery', async ({ page }) => { + await mockMetadata(page); + await page.route('**/api/v2/locations**', route => new URL(route.request().url()).pathname.endsWith(firstId) + ? route.fulfill({ status: 503, body: 'unavailable' }) : route.fulfill({ json: list('official') })); + await page.goto('./?mode=local-v2#/'); + await page.getByRole('button', { name: /First local record/ }).click(); + await expect(page.getByRole('heading', { name: 'Could not load local record' })).toBeFocused(); + await expect(page.getByRole('button', { name: 'Try record again' })).toBeVisible(); + await page.getByRole('button', { name: 'Back to results' }).click(); + await expect(page.getByRole('heading', { name: 'First local record' })).toBeVisible(); +}); diff --git a/frontend/tests/unit/localDetailRepository.test.ts b/frontend/tests/unit/localDetailRepository.test.ts index 5522fa5..0ea4f29 100644 --- a/frontend/tests/unit/localDetailRepository.test.ts +++ b/frontend/tests/unit/localDetailRepository.test.ts @@ -2,3 +2,15 @@ import{describe,expect,it,vi}from'vitest';import{LocalLocationRepository}from'.. const row={facility_id:'550e8400-e29b-41d4-a716-446655440000',canonical_name:'Detail Local Fixture',city:'North Coast',country_code:'DK',category:'dairy',source_type:'official',publication_profile:'official',factual_review_status:'reviewed',privacy_screening_status:'passed',project_approval:'approved',reviewer_role:null,publication_warning:null,display_precision:'city',latitude:55,longitude:10,first_observed_at:null,last_observed_at:'2026-01-01T00:00:00Z',observation_count:1,lifecycle_status:'active_observed',provenance_source_id:'source-1',provenance_source_name:'Synthetic local source',provenance_source_url:'https://example.test/source',provenance_retrieved_at:'2026-01-01T00:00:00Z',release_id:'rel-1',release_ruleset_version:'rules-1'}; const body=(data=row,meta={release_id:'rel-1',ruleset_version:'rules-1',release_created_at:'2026-01-01T00:00:00Z',profile:'official'})=>({data,api_version:'v2',meta}); describe('LocalLocationRepository detail',()=>{it('maps a valid detail envelope',async()=>{const fetcher=vi.fn().mockResolvedValue(new Response(JSON.stringify(body()),{status:200}));const result=await new LocalLocationRepository(fetcher).detail(row.facility_id);expect(result).toMatchObject({releaseId:'rel-1',profile:'official',location:{id:row.facility_id,name:'Detail Local Fixture'}});expect(fetcher).toHaveBeenCalledWith(`/api/v2/locations/${row.facility_id}?profile=official`,expect.any(Object));});it('rejects wrong profile and aborts',async()=>{const wrong=vi.fn().mockResolvedValue(new Response(JSON.stringify(body(row,{...body().meta,profile:'community'}))));await expect(new LocalLocationRepository(wrong).detail(row.facility_id)).rejects.toMatchObject({kind:'invalid-contract'});const controller=new AbortController();const fetcher=vi.fn().mockRejectedValue(new DOMException('aborted','AbortError'));await expect(new LocalLocationRepository(fetcher).detail(row.facility_id,'official',controller.signal)).rejects.toMatchObject({kind:'aborted'});});}); + +describe('community detail safety', () => { + const community = { ...row, source_type: 'user_submitted', publication_profile: 'community', factual_review_status: 'unreviewed', project_approval: 'pending', publication_warning: 'Unreviewed community claim — not verified by Until Every Cage' }; + it('accepts an eligible unreviewed community direct link with evidence context', async () => { + const fetcher = vi.fn().mockResolvedValue(new Response(JSON.stringify(body(community, { ...body().meta, profile: 'community' })))); + await expect(new LocalLocationRepository(fetcher).detail(row.facility_id, 'community')).resolves.toMatchObject({ location: { evidence: { factualReviewStatus: 'unreviewed', projectApproval: 'pending', publicationWarning: community.publication_warning } } }); + }); + it('rejects row/envelope profile mismatch and release mismatch', async () => { + await expect(new LocalLocationRepository(vi.fn().mockResolvedValue(new Response(JSON.stringify(body(community))))).detail(row.facility_id)).rejects.toMatchObject({ kind: 'invalid-contract' }); + await expect(new LocalLocationRepository(vi.fn().mockResolvedValue(new Response(JSON.stringify(body({ ...row, release_id: 'other' }))))).detail(row.facility_id)).rejects.toMatchObject({ kind: 'invalid-contract' }); + }); +}); diff --git a/frontend/tests/unit/localLocationRepository.test.ts b/frontend/tests/unit/localLocationRepository.test.ts index 521b57e..af4046a 100644 --- a/frontend/tests/unit/localLocationRepository.test.ts +++ b/frontend/tests/unit/localLocationRepository.test.ts @@ -2,3 +2,22 @@ import{describe,expect,it,vi}from'vitest';import{LocalLocationRepository}from'.. const row={facility_id:'550e8400-e29b-41d4-a716-446655440000',canonical_name:'Local V2 Fixture',city:'North Coast',country_code:'DK',category:'dairy',source_type:'official',publication_profile:'official',factual_review_status:'reviewed',privacy_screening_status:'passed',project_approval:'approved',reviewer_role:null,publication_warning:null,display_precision:'city',latitude:55,longitude:10,first_observed_at:null,last_observed_at:'2026-01-01T00:00:00Z',observation_count:1,lifecycle_status:'active_observed',provenance_source_id:'source-1',provenance_source_name:'Synthetic local source',provenance_source_url:'https://example.test/source',provenance_retrieved_at:'2026-01-01T00:00:00Z',release_id:'rel-1',release_ruleset_version:'rules-1'}; const response=(body:unknown,status=200)=>new Response(JSON.stringify(body),{status,headers:{'content-type':'application/json'}});const envelope=(data=[row],meta={release_id:'rel-1',ruleset_version:'rules-1',profile:'official',next_cursor:null,coverage_note:'Local promoted release.'})=>({data,api_version:'v2',meta}); describe('LocalLocationRepository',()=>{it('maps a valid Rust-shaped envelope',async()=>{const result=await new LocalLocationRepository(vi.fn().mockResolvedValue(response(envelope()))).list();expect(result.locations[0]).toMatchObject({id:row.facility_id,name:'Local V2 Fixture',lat:55});});it('fails closed when no release is promoted',async()=>{await expect(new LocalLocationRepository(vi.fn().mockResolvedValue(response(envelope([],{release_id:null,profile:'official',coverage_note:'No promoted release.'})))).list()).rejects.toMatchObject({kind:'no-release'});});it('classifies HTTP failures',async()=>{await expect(new LocalLocationRepository(vi.fn().mockResolvedValue(response({},503))).list()).rejects.toMatchObject({kind:'http',status:503});});it('rejects malformed or restricted payloads',async()=>{await expect(new LocalLocationRepository(vi.fn().mockResolvedValue(response({...envelope(),api_version:'v1'}))).list()).rejects.toMatchObject({kind:'invalid-contract'});await expect(new LocalLocationRepository(vi.fn().mockResolvedValue(response(envelope([{...row,privacy_screening_status:'failed'}])))).list()).rejects.toMatchObject({kind:'invalid-contract'});});}); + +describe('community list safety', () => { + const community = { ...row, source_type: 'user_submitted', publication_profile: 'community', factual_review_status: 'unreviewed', project_approval: 'pending', publication_warning: 'Unreviewed community claim — not verified by Until Every Cage' }; + const communityMeta = { ...envelope().meta, profile: 'community' }; + it('accepts screened unreviewed claims only in the explicit community profile and retains review context', async () => { + const result = await new LocalLocationRepository(vi.fn().mockResolvedValue(response(envelope([community], communityMeta)))).list('community'); + expect(result.locations[0]).toMatchObject({ evidence: { sourceType: 'user_submitted', factualReviewStatus: 'unreviewed', projectApproval: 'pending', publicationWarning: community.publication_warning, sourceUrl: row.provenance_source_url } }); + await expect(new LocalLocationRepository(vi.fn().mockResolvedValue(response(envelope([community])))).list()).rejects.toMatchObject({ kind: 'invalid-contract' }); + }); + it.each([{ privacy_screening_status: 'failed' }, { factual_review_status: 'rejected' }])('rejects unsafe community rows: %o', async (change) => { + await expect(new LocalLocationRepository(vi.fn().mockResolvedValue(response(envelope([{ ...community, ...change }], communityMeta)))).list('community')).rejects.toMatchObject({ kind: 'invalid-contract' }); + }); + it('rejects a row profile that disagrees with the requested envelope', async () => { + await expect(new LocalLocationRepository(vi.fn().mockResolvedValue(response(envelope([{ ...row, publication_profile: 'community' }])))).list()).rejects.toMatchObject({ kind: 'invalid-contract' }); + }); + it('rejects a non-web source URL before it can become a detail link', async () => { + await expect(new LocalLocationRepository(vi.fn().mockResolvedValue(response(envelope([{ ...row, provenance_source_url: 'javascript:alert(1)' }])))).list()).rejects.toMatchObject({ kind: 'invalid-contract' }); + }); +}); diff --git a/pipeline/tests/e2e/test_community_api.py b/pipeline/tests/e2e/test_community_api.py index c1ef626..155a606 100644 --- a/pipeline/tests/e2e/test_community_api.py +++ b/pipeline/tests/e2e/test_community_api.py @@ -2,6 +2,9 @@ import os import unittest import urllib.request +import uuid +from datetime import datetime, timezone +import psycopg try: from .fixture import E2EEnvironment @@ -65,6 +68,21 @@ def test_unscreened_claim_never_becomes_public(self): response = self.get("/api/v2/locations?profile=community&limit=100") self.assertNotIn("E2E community unscreened", {row["canonical_name"] for row in response["data"]}) + def test_z_denied_and_factually_rejected_claims_stay_private(self): + now = datetime.now(timezone.utc) + with psycopg.connect(self.env.database_url) as db: + for name, factual_status, approval in (("denied", "unreviewed", "denied"), ("rejected", "rejected", "pending")): + record, facility, observation, artifact = (uuid.uuid4() for _ in range(4)) + db.execute("INSERT INTO uec.raw_artifacts (artifact_id,storage_key,sha256,byte_size,retrieved_at) VALUES (%s,%s,%s,1,%s)", (artifact, f'e2e-community/{name}', uuid.uuid4().hex * 2, now)) + db.execute("INSERT INTO uec.source_records (source_record_id,source_id,source_record_key,artifact_id,raw_fields,parsed_at) VALUES (%s,'e2e.community',%s,%s,'{}',%s)", (record, name, artifact, now)) + db.execute("INSERT INTO uec.facilities (facility_id,canonical_name,country_code,city) VALUES (%s,%s,'DK','Communityby')", (facility, f'E2E community {name}')) + db.execute("INSERT INTO uec.observations (observation_id,facility_id,source_record_id,observed_at,observation,classification,ruleset_id,rule_id,classification_category,classification_review_status,default_visible,first_observed_at) VALUES (%s,%s,%s,%s,'{}','{}','community-v1','e2e','slaughter','approved',true,%s)", (observation, facility, record, now, now)) + db.execute("INSERT INTO uec.release_members (release_id,facility_id,observation_id,default_visible) VALUES ('e2e-community',%s,%s,true)", (facility, observation)) + db.execute("INSERT INTO uec.publication_review_events (source_record_id,factual_review_status,privacy_screening_status,maintainer_approval,publication_eligible,reviewer_role) VALUES (%s,%s,'passed',%s,true,'maintainer')", (record, factual_status, approval)) + names = {row['canonical_name'] for row in self.get('/api/v2/locations?profile=community&limit=100')['data']} + self.assertNotIn('E2E community denied', names) + self.assertNotIn('E2E community rejected', names) + if __name__ == "__main__": unittest.main() diff --git a/pipeline/tests/e2e/test_public_surface_safety.py b/pipeline/tests/e2e/test_public_surface_safety.py new file mode 100644 index 0000000..853410c --- /dev/null +++ b/pipeline/tests/e2e/test_public_surface_safety.py @@ -0,0 +1,254 @@ +"""Synthetic HTTP checks for community export parity and legacy availability.""" + +import csv +import hashlib +import io +import json +import os +import unittest +import urllib.request +import uuid +from datetime import datetime, timedelta, timezone + +import psycopg + +try: + from .fixture import E2EEnvironment +except ImportError: + from fixture import E2EEnvironment + + +class PublicSurfaceSafetyE2ETests(unittest.TestCase): + @classmethod + def setUpClass(cls): + if os.environ.get("UEC_RUN_E2E") != "1": + raise unittest.SkipTest("set UEC_RUN_E2E=1 to run Docker-backed E2E tests") + cls.env = E2EEnvironment().start() + cls.env.seed_community_scenario() + with psycopg.connect(cls.env.database_url) as db: + for release_id, profile, count in ( + ("e2e-official-empty", "official", 0), + ("e2e-community", "community", 2), + ): + manifest = { + "eligible_record_count": count, + "manifest_version": "v1", + "profile": profile, + "release_id": release_id, + "ruleset_version": f"{profile}-v1", + "source_ids": ["e2e.community"] if profile == "community" else [], + } + serialized = json.dumps(manifest, sort_keys=True, separators=(",", ":")) + digest = hashlib.sha256(serialized.encode("utf-8")).hexdigest() + db.execute( + "INSERT INTO uec.release_manifests (release_id,manifest,manifest_sha256) " + "VALUES (%s,%s::jsonb,%s)", + (release_id, serialized, digest), + ) + + @classmethod + def tearDownClass(cls): + cls.env.stop() + + def fetch(self, path): + with urllib.request.urlopen( + f"http://localhost:{self.env.api_port}{path}", timeout=10 + ) as response: + return response.headers, response.read() + + def export_rows(self, profile): + headers, body = self.fetch(f"/api/v2/locations.csv?profile={profile}") + return headers, list(csv.DictReader(io.StringIO(body.decode("utf-8")))) + + def test_community_csv_matches_api_and_current_suppression(self): + _, api_body = self.fetch("/api/v2/locations?profile=community&limit=100") + api_rows = json.loads(api_body)["data"] + headers, rows = self.export_rows("community") + self.assertEqual(headers["X-Uec-Export-Profile"], "community") + self.assertIn("uec-v2-community-locations.csv", headers["Content-Disposition"]) + self.assertEqual( + {row["facility_id"] for row in rows}, + {row["facility_id"] for row in api_rows}, + ) + self.assertEqual( + {row["canonical_name"] for row in rows}, + {"E2E community eligible", "E2E community screened-unreviewed"}, + ) + unreviewed = next( + row for row in rows if row["canonical_name"] == "E2E community screened-unreviewed" + ) + self.assertEqual(unreviewed["release_profile"], "community") + self.assertEqual(unreviewed["factual_review_status"], "unreviewed") + self.assertEqual(unreviewed["privacy_screening_status"], "passed") + self.assertEqual(unreviewed["project_approval"], "pending") + self.assertEqual(unreviewed["source_type"], "user_submitted") + self.assertEqual(unreviewed["reviewer_role"], "") + self.assertEqual( + unreviewed["publication_warning"], + "Unreviewed community claim — not verified by Until Every Cage", + ) + self.assertIn("Opt-in community profile", unreviewed["profile_notice"]) + self.assertTrue(all(row["profile_notice"] == unreviewed["profile_notice"] for row in rows)) + self.assertFalse(any("unscreened" in row["canonical_name"] for row in rows)) + + official_headers, official_rows = self.export_rows("official") + self.assertIn("uec-v2-official-locations.csv", official_headers["Content-Disposition"]) + self.assertEqual(official_rows, []) + + with psycopg.connect(self.env.database_url) as db: + db.execute( + """INSERT INTO uec.record_access_events + (source_record_id, action, reason_category, policy_version, maintainer) + SELECT source_record_id, 'public_access_revoked', 'privacy', 'ethics-v1', 'e2e' + FROM uec.source_records + WHERE source_id='e2e.community' AND source_record_key='screened-unreviewed'""" + ) + + _, api_body = self.fetch("/api/v2/locations?profile=community&limit=100") + _, rows = self.export_rows("community") + expected = {"E2E community eligible"} + self.assertEqual({row["canonical_name"] for row in json.loads(api_body)["data"]}, expected) + self.assertEqual({row["canonical_name"] for row in rows}, expected) + + def test_legacy_locations_route_remains_available(self): + # No reviewed identity crosswalk exists between these embedded rows and V2. + # This checks compatibility only; it does not claim suppression propagation. + _, body = self.fetch("/api/locations?country_code=dk") + rows = json.loads(body) + self.assertTrue(rows) + self.assertTrue(all(row["country"] == "dk" for row in rows)) + self.assertTrue(all("establishment_id" in row for row in rows)) + + +class ReleaseScopedReviewE2ETests(unittest.TestCase): + @classmethod + def setUpClass(cls): + if os.environ.get("UEC_RUN_E2E") != "1": + raise unittest.SkipTest("set UEC_RUN_E2E=1 to run Docker-backed E2E tests") + cls.env = E2EEnvironment().start() + cls.seed_shared_record() + + @classmethod + def tearDownClass(cls): + cls.env.stop() + + @classmethod + def seed_shared_record(cls): + now = datetime.now(timezone.utc) + cls.record_id = uuid.uuid4() + cls.facility_id = uuid.uuid4() + observation_id = uuid.uuid4() + artifact_id = uuid.uuid4() + with psycopg.connect(cls.env.database_url) as db: + db.execute( + "INSERT INTO uec.sources (source_id,country_code,name,official_url,access_method) " + "VALUES ('e2e.shared','DK','Synthetic shared source','https://example.invalid/shared','fixture')" + ) + for release_id, profile in (("e2e-release-a", "official"), ("e2e-release-b", "secondary")): + db.execute( + "INSERT INTO uec.releases (release_id,status,ruleset_version,profile,summary) " + "VALUES (%s,'promoted','e2e-v1',%s,'{}')", + (release_id, profile), + ) + manifest = { + "eligible_record_count": 1, + "manifest_version": "v1", + "profile": profile, + "release_id": release_id, + "ruleset_version": "e2e-v1", + "source_ids": ["e2e.shared"], + } + serialized = json.dumps(manifest, sort_keys=True, separators=(",", ":")) + db.execute( + "INSERT INTO uec.release_manifests (release_id,manifest,manifest_sha256) " + "VALUES (%s,%s::jsonb,%s)", + (release_id, serialized, hashlib.sha256(serialized.encode()).hexdigest()), + ) + db.execute( + "INSERT INTO uec.raw_artifacts (artifact_id,storage_key,sha256,byte_size,retrieved_at) " + "VALUES (%s,'e2e/shared',%s,1,%s)", + (artifact_id, uuid.uuid4().hex * 2, now), + ) + db.execute( + "INSERT INTO uec.source_records " + "(source_record_id,source_id,source_record_key,artifact_id,raw_fields,parsed_at) " + "VALUES (%s,'e2e.shared','shared',%s,'{}',%s)", + (cls.record_id, artifact_id, now), + ) + db.execute( + "INSERT INTO uec.facilities (facility_id,canonical_name,country_code,city) " + "VALUES (%s,'E2E shared facility','DK','Testby')", + (cls.facility_id,), + ) + db.execute( + "INSERT INTO uec.observations " + "(observation_id,facility_id,source_record_id,observed_at,observation,classification," + "ruleset_id,rule_id,classification_category,classification_review_status," + "default_visible,first_observed_at) " + "VALUES (%s,%s,%s,%s,'{}','{}','e2e-v1','e2e','slaughter','approved',true,%s)", + (observation_id, cls.facility_id, cls.record_id, now, now), + ) + for release_id in ("e2e-release-a", "e2e-release-b"): + db.execute( + "INSERT INTO uec.release_members " + "(release_id,facility_id,observation_id,default_visible) VALUES (%s,%s,%s,true)", + (release_id, cls.facility_id, observation_id), + ) + db.execute( + "INSERT INTO uec.publication_review_events " + "(source_record_id,release_id,factual_review_status,privacy_screening_status," + "maintainer_approval,publication_eligible,reviewer_role,reviewed_at) " + "VALUES (%s,%s,'reviewed','passed','approved',true,'maintainer',%s)", + (cls.record_id, release_id, now), + ) + + def get_json(self, path): + with urllib.request.urlopen(f"http://localhost:{self.env.api_port}{path}", timeout=10) as response: + return json.loads(response.read()) + + def get_csv(self, profile): + with urllib.request.urlopen( + f"http://localhost:{self.env.api_port}/api/v2/locations.csv?profile={profile}", + timeout=10, + ) as response: + return list(csv.DictReader(io.StringIO(response.read().decode("utf-8")))) + + def test_other_promoted_profile_denial_cannot_relabel_or_remove_official_record(self): + for profile in ("official", "secondary"): + listed = self.get_json(f"/api/v2/locations?profile={profile}&limit=100")["data"] + self.assertEqual([row["facility_id"] for row in listed], [str(self.facility_id)]) + self.assertEqual(listed[0]["project_approval"], "approved") + self.assertEqual([row["facility_id"] for row in self.get_csv(profile)], [str(self.facility_id)]) + + with psycopg.connect(self.env.database_url) as db: + db.execute( + "INSERT INTO uec.publication_review_events " + "(source_record_id,release_id,factual_review_status,privacy_screening_status," + "maintainer_approval,publication_eligible,reviewer_role,reviewed_at) " + "VALUES (%s,'e2e-release-b','rejected','failed','denied',false,'maintainer',%s)", + (self.record_id, datetime.now(timezone.utc) + timedelta(seconds=1)), + ) + db.commit() + eligible = dict(db.execute( + "SELECT release_id, count(*) FROM uec.map_facilities_display_history " + "GROUP BY release_id" + ).fetchall()) + self.assertEqual(eligible, {"e2e-release-a": 1}) + + official = self.get_json("/api/v2/locations?profile=official&limit=100")["data"] + self.assertEqual([row["facility_id"] for row in official], [str(self.facility_id)]) + self.assertEqual(official[0]["factual_review_status"], "reviewed") + self.assertEqual(official[0]["privacy_screening_status"], "passed") + self.assertEqual(official[0]["project_approval"], "approved") + detail = self.get_json(f"/api/v2/locations/{self.facility_id}?profile=official")["data"] + self.assertEqual(detail["project_approval"], "approved") + self.assertEqual(detail["privacy_screening_status"], "passed") + official_csv = self.get_csv("official") + self.assertEqual([row["facility_id"] for row in official_csv], [str(self.facility_id)]) + self.assertEqual(official_csv[0]["project_approval"], "approved") + self.assertEqual(self.get_json("/api/v2/locations?profile=secondary")["data"], []) + self.assertEqual(self.get_csv("secondary"), []) + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/tests/e2e/test_seeded_api.py b/pipeline/tests/e2e/test_seeded_api.py index 81144f6..2c2aab4 100644 --- a/pipeline/tests/e2e/test_seeded_api.py +++ b/pipeline/tests/e2e/test_seeded_api.py @@ -1,4 +1,7 @@ import json, os, subprocess, sys, unittest, urllib.request +import uuid +from datetime import datetime, timezone +import psycopg try: from .fixture import E2EEnvironment except ImportError: @@ -138,4 +141,58 @@ def test_z_restoration_requires_an_explicit_append_only_event(self): names = {r['canonical_name'] for r in self.get('/api/v2/locations?limit=100')['data']} self.assertIn('E2E restricted', names) + def test_z1_approval_does_not_follow_source_record_into_new_profile(self): + with psycopg.connect(self.env.database_url) as db: + facility_id, observation_id = db.execute("SELECT facility_id, observation_id FROM uec.observations o JOIN uec.source_records r USING (source_record_id) WHERE r.source_record_key = 'exact'").fetchone() + db.execute("INSERT INTO uec.releases (release_id,status,ruleset_version,profile,summary) VALUES ('e2e-secondary-later','promoted','e2e-v2','secondary','{}')") + db.execute("INSERT INTO uec.release_members (release_id,facility_id,observation_id,default_visible) VALUES ('e2e-secondary-later',%s,%s,true)", (facility_id, observation_id)) + self.assertEqual(self.get('/api/v2/locations?profile=secondary&limit=100')['data'], []) + + def test_z1b_candidate_review_does_not_revoke_independent_promoted_approval(self): + promoted = 'e2e-promoted' + candidate = 'e2e-independent-candidate' + with psycopg.connect(self.env.database_url) as db: + record_id, facility_id, observation_id = db.execute("SELECT r.source_record_id, o.facility_id, o.observation_id FROM uec.observations o JOIN uec.source_records r USING (source_record_id) WHERE r.source_record_key='unmapped'").fetchone() + db.execute("INSERT INTO uec.releases (release_id,status,ruleset_version,profile,summary) VALUES (%s,'candidate','e2e-v2','official','{}')", (candidate,)) + db.execute("INSERT INTO uec.release_members (release_id,facility_id,observation_id,default_visible) VALUES (%s,%s,%s,true)", (candidate, facility_id, observation_id)) + db.execute("INSERT INTO uec.publication_review_events (source_record_id,release_id,factual_review_status,privacy_screening_status,maintainer_approval,publication_eligible,reviewer_role) VALUES (%s,%s,'reviewed','passed','approved',true,'maintainer')", (record_id, candidate)) + names = {row['canonical_name']: row for row in self.get('/api/v2/locations?limit=100')['data']} + self.assertIn('E2E unmapped', names) + self.assertEqual(names['E2E unmapped']['project_approval'], 'approved') + + with psycopg.connect(self.env.database_url) as db: + db.execute("INSERT INTO uec.publication_review_events (source_record_id,release_id,factual_review_status,privacy_screening_status,maintainer_approval,publication_eligible,reviewer_role) VALUES (%s,%s,'rejected','passed','denied',false,'maintainer')", (record_id, candidate)) + names = {row['canonical_name']: row for row in self.get('/api/v2/locations?limit=100')['data']} + self.assertIn('E2E unmapped', names) + self.assertEqual(names['E2E unmapped']['project_approval'], 'approved') + + with psycopg.connect(self.env.database_url) as db: + db.execute("INSERT INTO uec.publication_review_events (source_record_id,release_id,factual_review_status,privacy_screening_status,maintainer_approval,publication_eligible,reviewer_role) VALUES (%s,%s,'rejected','passed','denied',false,'maintainer')", (record_id, promoted)) + names = {row['canonical_name'] for row in self.get('/api/v2/locations?limit=100')['data']} + self.assertNotIn('E2E unmapped', names) + + def test_z2_summary_excludes_suppressed_observation(self): + now = datetime.now(timezone.utc) + with psycopg.connect(self.env.database_url) as db: + facility_id = db.execute("SELECT facility_id FROM uec.facilities WHERE canonical_name='E2E exact'").fetchone()[0] + artifact, record, observation = uuid.uuid4(), uuid.uuid4(), uuid.uuid4() + db.execute("INSERT INTO uec.raw_artifacts (artifact_id,storage_key,sha256,byte_size,retrieved_at) VALUES (%s,'e2e/suppressed-summary',%s,1,%s)", (artifact, uuid.uuid4().hex * 2, now)) + db.execute("INSERT INTO uec.source_records (source_record_id,source_id,source_record_key,artifact_id,raw_fields,parsed_at) VALUES (%s,'e2e.official','suppressed-summary',%s,'{}',%s)", (record, artifact, now)) + db.execute("INSERT INTO uec.observations (observation_id,facility_id,source_record_id,observed_at,observation,classification,ruleset_id,rule_id,classification_category,classification_review_status,default_visible,first_observed_at) VALUES (%s,%s,%s,%s,'{}','{}','e2e-v1','e2e','slaughter','approved',false,%s)", (observation, facility_id, record, now, now)) + db.execute("INSERT INTO uec.record_access_events (source_record_id,action,reason_category,policy_version,maintainer) VALUES (%s,'public_access_revoked','privacy','ethics-v1','e2e')", (record,)) + row = next(r for r in self.get('/api/v2/locations?limit=100')['data'] if r['canonical_name'] == 'E2E exact') + self.assertEqual(row['observation_count'], 1) + + def test_z3_facility_suppression_without_source_link_revokes_public_observation(self): + with psycopg.connect(self.env.database_url) as db: + facility_id = db.execute("SELECT facility_id FROM uec.facilities WHERE canonical_name='E2E exact'").fetchone()[0] + self.assertEqual(db.execute("SELECT count(*) FROM uec.facility_source_links WHERE facility_id=%s", (facility_id,)).fetchone()[0], 0) + case_id = uuid.uuid4() + db.execute("INSERT INTO uec.suppression_cases (case_id,reason_category,status,policy_version,actor,decision) VALUES (%s,'privacy','active','ethics-v1','e2e','suppress')", (case_id,)) + db.execute("INSERT INTO uec.suppression_references (case_id,facility_id,scope) VALUES (%s,%s,'whole_record')", (case_id, facility_id)) + names = {r['canonical_name'] for r in self.get('/api/v2/locations?limit=100')['data']} + self.assertNotIn('E2E exact', names) + with urllib.request.urlopen(f'http://localhost:{self.env.api_port}/api/v2/locations.csv?profile=official', timeout=10) as response: + self.assertNotIn('E2E exact', response.read().decode()) + if __name__ == '__main__': unittest.main() diff --git a/src/lib.rs b/src/lib.rs index fd24d38..24f2e21 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -153,9 +153,28 @@ struct V2ExportRow { provenance_retrieved_at: chrono::DateTime, release_id: String, release_profile: String, + profile_notice: String, + publication_warning: Option, manifest_sha256: String, } +const UNREVIEWED_COMMUNITY_WARNING: &str = + "Unreviewed community claim — not verified by Until Every Cage"; +const COMMUNITY_EXPORT_NOTICE: &str = "Opt-in community profile: privacy-screened claims may be factually unreviewed and are not necessarily project-approved."; + +fn export_profile_notice(profile: &str) -> &'static str { + if profile == "community" { + COMMUNITY_EXPORT_NOTICE + } else { + "Curated release profile: rows require project approval and privacy screening." + } +} + +fn export_publication_warning(profile: &str, factual_review_status: &str) -> Option { + (profile == "community" && factual_review_status == "unreviewed") + .then(|| UNREVIEWED_COMMUNITY_WARNING.to_string()) +} + pub async fn get_v2_locations_export_handler( State(state): State, Query(params): Query, @@ -197,7 +216,7 @@ pub async fn get_v2_locations_export_handler( }; let release_id: String = release.get(0); let manifest_sha256: String = release.get(1); - let rows = match client.query("SELECT h.facility_id, h.canonical_name, h.country_code, h.city, h.classification_category, h.display_precision, r.factual_review_status, r.privacy_screening_status, r.maintainer_approval, r.reviewer_role, h.provenance_origin_type, h.provenance_source_id, h.provenance_source_name, h.provenance_source_url, h.provenance_retrieved_at, h.release_id FROM uec.map_facilities_display_history h JOIN uec.publication_review_current r ON r.source_record_id=h.source_record_id WHERE h.release_id=$1 AND r.publication_eligible=true AND r.privacy_screening_status='passed' AND r.maintainer_approval='approved' ORDER BY h.facility_id LIMIT 1001", &[&release_id]).await { + let rows = match client.query("SELECT h.facility_id, h.canonical_name, h.country_code, h.city, h.classification_category, h.display_precision, r.factual_review_status, r.privacy_screening_status, r.maintainer_approval, r.reviewer_role, h.provenance_origin_type, h.provenance_source_id, h.provenance_source_name, h.provenance_source_url, h.provenance_retrieved_at, h.release_id FROM uec.map_facilities_display_history h JOIN uec.publication_review_release_current r ON r.source_record_id=h.source_record_id AND r.release_id=h.release_id WHERE h.release_id=$1 AND r.publication_eligible=true AND r.privacy_screening_status='passed' AND ($2='community' OR r.maintainer_approval='approved') ORDER BY h.facility_id LIMIT 1001", &[&release_id, &profile]).await { Ok(rows) => rows, Err(_) => return v2_error(StatusCode::SERVICE_UNAVAILABLE, "export_query_failed", "public export unavailable") }; if rows.len() > 1000 { @@ -209,6 +228,7 @@ pub async fn get_v2_locations_export_handler( } let mut writer = csv::Writer::from_writer(Vec::new()); for row in rows { + let factual_review_status: String = row.get(6); if writer .serialize(V2ExportRow { facility_id: row.get(0), @@ -217,7 +237,7 @@ pub async fn get_v2_locations_export_handler( city: row.get(3), category: row.get(4), display_precision: row.get(5), - factual_review_status: row.get(6), + factual_review_status: factual_review_status.clone(), privacy_screening_status: row.get(7), project_approval: row.get(8), reviewer_role: row.get(9), @@ -228,6 +248,8 @@ pub async fn get_v2_locations_export_handler( provenance_retrieved_at: row.get(14), release_id: row.get(15), release_profile: profile.to_string(), + profile_notice: export_profile_notice(profile).to_string(), + publication_warning: export_publication_warning(profile, &factual_review_status), manifest_sha256: manifest_sha256.clone(), }) .is_err() @@ -254,9 +276,10 @@ pub async fn get_v2_locations_export_handler( .header("content-type", "text/csv; charset=utf-8") .header( "content-disposition", - "attachment; filename=uec-v2-locations.csv", + format!("attachment; filename=uec-v2-{profile}-locations.csv"), ) .header("x-uec-release-id", release_id) + .header("x-uec-export-profile", profile) .header("x-uec-manifest-sha256", manifest_sha256) .body(axum::body::Body::from(body)) .unwrap() @@ -650,11 +673,13 @@ pub async fn get_v2_locations_handler( review.factual_review_status, review.privacy_screening_status, review.maintainer_approval, review.reviewer_role, ST_Y(display_location::geometry), ST_X(display_location::geometry), first_observed_at, last_observed_at, observation_count, lifecycle_status, - provenance_origin_type, release_id, release_ruleset_version, + provenance_origin_type, map_facilities_display_history.release_id, release_ruleset_version, provenance_source_id, provenance_source_name, provenance_source_url, provenance_retrieved_at FROM uec.map_facilities_display_history - JOIN uec.publication_review_current AS review ON review.source_record_id = map_facilities_display_history.source_record_id - WHERE release_id = $1 + JOIN uec.publication_review_release_current AS review + ON review.source_record_id = map_facilities_display_history.source_record_id + AND review.release_id = map_facilities_display_history.release_id + WHERE map_facilities_display_history.release_id = $1 AND ($2::uuid IS NULL OR facility_id > $2) AND ($3::text IS NULL OR country_code = $3) AND ($4::text IS NULL OR classification_category = $4) @@ -683,7 +708,7 @@ pub async fn get_v2_locations_handler( publication_warning: if promoted_profile == "community" && row.get::<_, String>(6) == "unreviewed" { - Some("Unreviewed community claim — not verified by Until Every Cage".into()) + Some(UNREVIEWED_COMMUNITY_WARNING.into()) } else { None }, @@ -797,11 +822,13 @@ pub async fn get_v2_location_detail_handler( review.factual_review_status, review.privacy_screening_status, review.maintainer_approval, review.reviewer_role, ST_Y(display_location::geometry), ST_X(display_location::geometry), first_observed_at, last_observed_at, observation_count, lifecycle_status, - provenance_origin_type, release_id, release_ruleset_version, + provenance_origin_type, map_facilities_display_history.release_id, release_ruleset_version, provenance_source_id, provenance_source_name, provenance_source_url, provenance_retrieved_at FROM uec.map_facilities_display_history - JOIN uec.publication_review_current AS review ON review.source_record_id = map_facilities_display_history.source_record_id - WHERE facility_id = $1 AND release_id = $2 + JOIN uec.publication_review_release_current AS review + ON review.source_record_id = map_facilities_display_history.source_record_id + AND review.release_id = map_facilities_display_history.release_id + WHERE facility_id = $1 AND map_facilities_display_history.release_id = $2 "#, &[&facility_id, &release_id]).await { Ok(row) => row, Err(_) => return v2_error(StatusCode::INTERNAL_SERVER_ERROR, "location_query_failed", "V2 location query failed"), @@ -825,7 +852,7 @@ pub async fn get_v2_location_detail_handler( project_approval: row.get(8), reviewer_role: row.get(9), publication_warning: if profile == "community" && row.get::<_, String>(6) == "unreviewed" { - Some("Unreviewed community claim — not verified by Until Every Cage".into()) + Some(UNREVIEWED_COMMUNITY_WARNING.into()) } else { None }, @@ -1111,6 +1138,51 @@ mod v2_api_tests { assert_eq!(json["category"], "slaughter"); assert_eq!(json["publication_profile"], "official"); } + + #[test] + fn community_export_labels_screened_unreviewed_claims_in_every_row() { + let row = V2ExportRow { + facility_id: uuid::Uuid::nil(), + canonical_name: Some("Synthetic claim".into()), + country_code: "DK".into(), + city: Some("Testby".into()), + category: "slaughter".into(), + display_precision: "city".into(), + factual_review_status: "unreviewed".into(), + privacy_screening_status: "passed".into(), + project_approval: "pending".into(), + reviewer_role: None, + source_type: "user_submitted".into(), + provenance_source_id: "synthetic.community".into(), + provenance_source_name: "Synthetic source".into(), + provenance_source_url: "https://example.invalid/community".into(), + provenance_retrieved_at: chrono::Utc::now(), + release_id: "synthetic-release".into(), + release_profile: "community".into(), + profile_notice: export_profile_notice("community").into(), + publication_warning: export_publication_warning("community", "unreviewed"), + manifest_sha256: "synthetic-hash".into(), + }; + let mut writer = csv::Writer::from_writer(Vec::new()); + writer.serialize(row).unwrap(); + let bytes = writer.into_inner().unwrap(); + let mut reader = csv::Reader::from_reader(bytes.as_slice()); + let headers = reader.headers().unwrap().clone(); + let fields = reader.records().next().unwrap().unwrap(); + let value = |name: &str| { + fields + .get(headers.iter().position(|h| h == name).unwrap()) + .unwrap() + }; + assert_eq!(value("release_profile"), "community"); + assert_eq!(value("factual_review_status"), "unreviewed"); + assert_eq!(value("privacy_screening_status"), "passed"); + assert_eq!(value("project_approval"), "pending"); + assert_eq!(value("publication_warning"), UNREVIEWED_COMMUNITY_WARNING); + assert_eq!(value("profile_notice"), COMMUNITY_EXPORT_NOTICE); + assert!(export_publication_warning("official", "unreviewed").is_none()); + assert!(export_publication_warning("community", "reviewed").is_none()); + } } pub async fn get_aphis_reports_handler() -> impl IntoResponse { diff --git a/src/main.rs b/src/main.rs index 5d2f9ff..6b0c836 100644 --- a/src/main.rs +++ b/src/main.rs @@ -15,11 +15,13 @@ // along with this program. If not, see . // Contact the developer directly at untileverycageproject@protonmail.com +use axum::extract::{ConnectInfo, State}; use axum::http::{HeaderValue, Method}; use axum::http::{Request, Response, header}; use axum::{Json, http::StatusCode, response::IntoResponse}; use axum::{Router, routing::get}; use std::collections::HashMap; +use std::net::{IpAddr, SocketAddr}; use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant}; use tower_http::compression::CompressionLayer; @@ -68,7 +70,13 @@ pub fn app(state: uec_api::ApiState) -> Router { .route("/api/aphis-query", get(uec_api::get_aphis_query_handler)) .fallback_service(ServeDir::new("static")) .layer(CompressionLayer::new().br(true)) - .layer(axum::middleware::from_fn(rate_limit)) + .layer(axum::middleware::from_fn_with_state( + RateLimitState { + limiter: Limiter::default(), + trust_proxy: std::env::var("UEC_TRUST_PROXY").as_deref() == Ok("true"), + }, + rate_limit, + )) .layer(cors) .with_state(state) } @@ -128,12 +136,15 @@ fn cors_layer() -> Result { const RATE_WINDOW: Duration = Duration::from_secs(60); const RATE_LIMIT: u32 = 60; -static GLOBAL_LIMITER: once_cell::sync::Lazy = - once_cell::sync::Lazy::new(Limiter::default); - #[derive(Clone, Default)] struct Limiter(Arc>>); +#[derive(Clone)] +struct RateLimitState { + limiter: Limiter, + trust_proxy: bool, +} + impl Limiter { fn allow(&self, key: String, now: Instant) -> bool { let mut entries = self.0.lock().expect("rate limiter mutex poisoned"); @@ -147,25 +158,36 @@ impl Limiter { } } +fn client_key(request: &Request, trust_proxy: bool) -> Option { + if trust_proxy { + if let Some(ip) = request + .headers() + .get("x-forwarded-for") + .and_then(|value| value.to_str().ok()) + .and_then(|value| value.split(',').next()) + .and_then(|value| value.trim().parse::().ok()) + { + return Some(format!("proxy:{ip}")); + } + } + request + .extensions() + .get::>() + .map(|ConnectInfo(peer)| format!("peer:{}", peer.ip())) +} + async fn rate_limit( + State(config): State, request: Request, next: axum::middleware::Next, ) -> Response { if request.uri().path().starts_with("/health/") { return next.run(request).await; } - let trusted_proxy = std::env::var("UEC_TRUST_PROXY").as_deref() == Ok("true"); - let key = if trusted_proxy { - request - .headers() - .get("x-forwarded-for") - .and_then(|v| v.to_str().ok()) - .map(|v| format!("proxy:{}", v.split(',').next().unwrap_or("unknown"))) - .unwrap_or_else(|| "proxy:unknown".into()) - } else { - "process-wide".into() + let Some(key) = client_key(&request, config.trust_proxy) else { + return StatusCode::SERVICE_UNAVAILABLE.into_response(); }; - if !GLOBAL_LIMITER.allow(key, Instant::now()) { + if !config.limiter.allow(key, Instant::now()) { return Response::builder() .status(StatusCode::TOO_MANY_REQUESTS) .header(header::RETRY_AFTER, RATE_WINDOW.as_secs().to_string()) @@ -288,9 +310,12 @@ async fn main() { ); let listener = tokio::net::TcpListener::bind(addr).await.unwrap(); - axum::serve(listener, app(uec_api::ApiState { database })) - .await - .unwrap(); + axum::serve( + listener, + app(uec_api::ApiState { database }).into_make_service_with_connect_info::(), + ) + .await + .unwrap(); } #[cfg(test)] @@ -338,6 +363,7 @@ mod config_tests { #[cfg(test)] mod rate_limit_tests { use super::*; + use tower::ServiceExt; #[test] fn enforces_limit_and_resets_window() { let limiter = Limiter::default(); @@ -361,4 +387,105 @@ mod rate_limit_tests { assert!(limiter.allow("c".into(), later)); assert_eq!(limiter.0.lock().unwrap().len(), 1); } + + #[tokio::test] + async fn distinct_socket_peers_do_not_share_default_allowance() { + let router = Router::new() + .route("/asset.js", get(|| async { StatusCode::OK })) + .layer(axum::middleware::from_fn_with_state( + RateLimitState { + limiter: Limiter::default(), + trust_proxy: false, + }, + rate_limit, + )); + let first: SocketAddr = "192.0.2.10:41000".parse().unwrap(); + let second: SocketAddr = "192.0.2.11:41000".parse().unwrap(); + for _ in 0..RATE_LIMIT { + let mut request = Request::builder() + .uri("/asset.js") + .header("x-forwarded-for", "203.0.113.99") + .body(axum::body::Body::empty()) + .unwrap(); + request.extensions_mut().insert(ConnectInfo(first)); + assert_eq!( + router.clone().oneshot(request).await.unwrap().status(), + StatusCode::OK + ); + } + let mut first_request = Request::builder() + .uri("/asset.js") + .body(axum::body::Body::empty()) + .unwrap(); + first_request.extensions_mut().insert(ConnectInfo(first)); + assert_eq!( + router + .clone() + .oneshot(first_request) + .await + .unwrap() + .status(), + StatusCode::TOO_MANY_REQUESTS + ); + let mut second_request = Request::builder() + .uri("/asset.js") + .header("x-forwarded-for", "203.0.113.99") + .body(axum::body::Body::empty()) + .unwrap(); + second_request.extensions_mut().insert(ConnectInfo(second)); + assert_eq!( + router.oneshot(second_request).await.unwrap().status(), + StatusCode::OK + ); + } + + #[test] + fn proxy_header_requires_explicit_trust_and_valid_ip() { + let peer: SocketAddr = "192.0.2.10:41000".parse().unwrap(); + let mut request = Request::builder() + .uri("/asset.js") + .header("x-forwarded-for", "198.51.100.20, 192.0.2.10") + .body(axum::body::Body::empty()) + .unwrap(); + request.extensions_mut().insert(ConnectInfo(peer)); + assert_eq!( + client_key(&request, false).as_deref(), + Some("peer:192.0.2.10") + ); + assert_eq!( + client_key(&request, true).as_deref(), + Some("proxy:198.51.100.20") + ); + + request.headers_mut().insert( + "x-forwarded-for", + HeaderValue::from_static("invalid, 198.51.100.20"), + ); + assert_eq!( + client_key(&request, true).as_deref(), + Some("peer:192.0.2.10") + ); + } + + #[tokio::test] + async fn missing_peer_does_not_create_a_shared_allowance() { + let router = Router::new() + .route("/asset.js", get(|| async { StatusCode::OK })) + .layer(axum::middleware::from_fn_with_state( + RateLimitState { + limiter: Limiter::default(), + trust_proxy: false, + }, + rate_limit, + )); + let request = Request::builder() + .uri("/asset.js") + .header("x-forwarded-for", "198.51.100.20") + .body(axum::body::Body::empty()) + .unwrap(); + assert_eq!( + router.oneshot(request).await.unwrap().status(), + StatusCode::SERVICE_UNAVAILABLE + ); + } } diff --git a/static/app.js b/static/app.js index e4d8dec..a00193c 100644 --- a/static/app.js +++ b/static/app.js @@ -394,7 +394,9 @@ function setupEventListeners() { includeBreeders, includeDealers, includeExhibitors, - isComplete + isComplete, + apiVersion: dataManager.apiVersion, + v2Meta: dataManager.v2Meta }, mapFacilityType); }); } diff --git a/static/modules/ExportManager.js b/static/modules/ExportManager.js index d58ca9f..7962ddd 100644 --- a/static/modules/ExportManager.js +++ b/static/modules/ExportManager.js @@ -40,7 +40,7 @@ class ExportManager { typeLabel = facilityMapping.displayLabel; } - return { + const row = { Type: typeLabel, Name: loc.establishment_name || '', State: getStateDisplayName(loc.state || ''), @@ -54,6 +54,30 @@ class ExportManager { AnimalsProcessed: loc.animals_processed || '', AnimalsSlaughtered: loc.animals_slaughtered || '' }; + if (!loc.v2) return row; + + const provenance = loc.v2.provenance || {}; + return { + ...row, + facility_id: loc.facility_id ?? loc.v2.facilityId ?? '', + publication_profile: loc.publication_profile ?? loc.v2.profile ?? '', + factual_review_status: loc.factual_review_status ?? '', + reviewer_role: loc.reviewer_role ?? '', + privacy_screening_status: loc.privacy_screening_status ?? '', + project_approval: loc.project_approval ?? '', + publication_warning: loc.publication_warning ?? '', + source_type: loc.source_type ?? loc.v2.sourceType ?? '', + display_precision: loc.display_precision ?? loc.v2.displayPrecision ?? '', + lifecycle_status: loc.lifecycle_status ?? loc.v2.lifecycleStatus ?? '', + release_id: loc.release_id ?? loc.v2.releaseId ?? '', + release_ruleset_version: loc.release_ruleset_version ?? loc.v2.rulesetVersion ?? '', + provenance_source_id: loc.provenance_source_id ?? provenance.source_id ?? '', + provenance_source_name: loc.provenance_source_name ?? provenance.source_name ?? '', + provenance_source_url: loc.provenance_source_url ?? provenance.source_url ?? '', + provenance_retrieved_at: loc.provenance_retrieved_at ?? provenance.retrieved_at ?? '', + selected_profile: loc.v2.profile ?? '', + coverage_note: loc.v2.coverageNote ?? '' + }; } /** @@ -164,7 +188,9 @@ class ExportManager { includeBreeders, includeDealers, includeExhibitors, - isComplete + isComplete, + apiVersion, + v2Meta } = options; const rows = []; @@ -199,6 +225,18 @@ class ExportManager { return; } + const isV2 = apiVersion === 'v2' || rows.some(row => Object.hasOwn(row, 'publication_profile')); + if (isV2) { + const isPartial = Boolean(v2Meta?.next_cursor) || !v2Meta; + rows.forEach(row => { + row.selected_profile = v2Meta?.profile ?? row.selected_profile ?? ''; + row.coverage_note = v2Meta?.coverage_note ?? row.coverage_note ?? ''; + row.export_scope = isPartial ? 'partial_page' : 'visible_results'; + row.export_limitation = isPartial + ? 'Additional pages may be available; this download contains loaded visible results only.' + : 'Loaded visible results only; coverage depends on the selected release and filters.'; + }); + } const csv = this.toCsv(rows); const now = new Date(); @@ -206,7 +244,9 @@ class ExportManager { const mm = String(now.getMonth() + 1).padStart(2, '0'); const dd = String(now.getDate()).padStart(2, '0'); const dateStr = `${yyyy}-${mm}-${dd}`; - const suffix = isComplete ? 'complete' : 'filtered'; + const suffix = isV2 + ? (v2Meta?.next_cursor || !v2Meta ? 'partial' : 'loaded') + : (isComplete ? 'complete' : 'filtered'); const filename = `untileverycage-visible-${dateStr}-${suffix}.csv`; this.downloadText(filename, csv); } diff --git a/static/modules/__tests__/v2Client.test.js b/static/modules/__tests__/v2Client.test.js index 6edb516..4253da6 100644 --- a/static/modules/__tests__/v2Client.test.js +++ b/static/modules/__tests__/v2Client.test.js @@ -2,6 +2,8 @@ import { jest } from '@jest/globals'; import { V2ApiError, V2Client } from '../v2Client.js'; import { escapeHtml, normalizeV2Location } from '../v2Adapter.js'; import { validateV2Envelope, validateV2Location } from '../v2Contract.js'; +import { exportManager } from '../ExportManager.js'; +import { buildLocationPopup } from '../popupBuilder.js'; import { exactOfficialLocation, noPromotedRelease, restrictedLocation } from '../__fixtures__/v2-contract-fixtures.js'; const listPayload = (data = [], meta = {}) => ({ api_version: 'v2', data, meta }); @@ -100,3 +102,66 @@ test('contract preserves explicit no-release state without inventing records', ( test('contract rejects privacy-ineligible records before rendering', () => { expect(() => validateV2Location(restrictedLocation)).toThrow('not privacy eligible'); }); + +test('rejects a row whose publication profile differs from the response profile', async () => { + const body = listPayload([record({ publication_profile: 'secondary' })], { profile: 'official' }); + expect(() => validateV2Envelope(body)).toThrow('publication_profile'); + fetch.mockResolvedValue({ ok: true, json: async () => body }); + await expect(new V2Client('/api/v2/locations').list({ profile: 'official' })).rejects.toThrow('publication_profile'); +}); + +test('rejects a response for a different profile than the one requested', async () => { + const community = record({ publication_profile: 'community', project_approval: 'not_approved', source_type: 'user_submitted' }); + fetch.mockResolvedValue({ ok: true, json: async () => listPayload([community], { profile: 'community' }) }); + await expect(new V2Client('/api/v2/locations').list({ profile: 'official' })).rejects.toThrow('profile'); +}); + +test('V2 CSV rows retain per-record publication and source context', () => { + const loc = normalizeV2Location(record({ publication_warning: 'Synthetic limitation', reviewer_role: 'community reviewer' }), { profile: 'official', coverage_note: 'Synthetic coverage' }); + const row = exportManager.normalizeUsdaRow(loc, true); + const csv = exportManager.toCsv([row]); + for (const value of ['publication_profile', 'factual_review_status', 'reviewer_role', 'project_approval', + 'source_type', 'provenance_source_id', 'provenance_source_url', 'provenance_retrieved_at', + 'release_id', 'release_ruleset_version', 'publication_warning']) { + expect(csv).toContain(value); + } + expect(csv).toContain('Synthetic limitation'); + expect(csv).toContain('fixture.source'); + expect(row.publication_profile).toBe('official'); + expect(row.factual_review_status).toBe('reviewed'); + expect(row.project_approval).toBe('approved'); + expect(row.selected_profile).toBe('official'); +}); + +test('V1 CSV row retains its legacy columns only', () => { + const row = exportManager.normalizeUsdaRow({ establishment_name: 'Synthetic legacy facility', latitude: 1, longitude: 2 }, true); + expect(row.Name).toBe('Synthetic legacy facility'); + expect(row).not.toHaveProperty('publication_profile'); + expect(row).not.toHaveProperty('export_scope'); +}); + +test('paginated V2 export never claims the visible first page is complete', () => { + const download = jest.spyOn(exportManager, 'downloadText').mockImplementation(() => {}); + const loc = normalizeV2Location(record(), { profile: 'official' }); + exportManager.exportData({ slaughterhouses: [loc] }, { + includeSlaughter: true, isComplete: true, apiVersion: 'v2', + v2Meta: { profile: 'official', next_cursor: 'cursor-2', coverage_note: 'Synthetic limited coverage' } + }); + expect(download).toHaveBeenCalledTimes(1); + expect(download.mock.calls[0][0]).not.toContain('complete'); + expect(download.mock.calls[0][1]).toContain('Synthetic limited coverage'); + expect(download.mock.calls[0][1]).toContain('partial_page'); + expect(download.mock.calls[0][1]).toContain('Additional pages may be available'); + expect(download.mock.calls[0][1]).toContain('selected_profile'); + download.mockRestore(); +}); + +test('city-precision V2 popup has no directions link while exact V2 and V1 keep it', () => { + const city = normalizeV2Location(record({ display_precision: 'city' }), { profile: 'official' }); + const exact = normalizeV2Location(record({ display_precision: 'exact' }), { profile: 'official' }); + const legacy = { establishment_name: 'Synthetic legacy facility', latitude: 55, longitude: 12, city: 'Testby' }; + expect(buildLocationPopup(city, 'Facility')).not.toContain('maps/dir/'); + expect(buildLocationPopup(exact, 'Facility')).toContain('maps/dir/'); + expect(buildLocationPopup(legacy, 'Facility')).toContain('maps/dir/'); + expect(buildLocationPopup(normalizeV2Location(record({ display_precision: 'exact', latitude: null, longitude: null })), 'Facility')).not.toContain('maps/dir/'); +}); diff --git a/static/modules/popupBuilder.js b/static/modules/popupBuilder.js index e47eb01..6ab5526 100644 --- a/static/modules/popupBuilder.js +++ b/static/modules/popupBuilder.js @@ -217,8 +217,9 @@ export function buildLocationPopup(location, facilityTypeLabel) { const grantDate = location.grant_date; const phone = location.phone; const dbas = location.dbas; - const hasCoordinates = Number.isFinite(Number(location.latitude)) && Number.isFinite(Number(location.longitude)); - const directionsUrl = hasCoordinates + const hasCoordinates = (!location.v2 || (location.latitude != null && location.longitude != null && location.latitude !== '' && location.longitude !== '')) + && Number.isFinite(Number(location.latitude)) && Number.isFinite(Number(location.longitude)); + const directionsUrl = hasCoordinates && (!location.v2 || location.v2.displayPrecision === 'exact') ? `https://www.google.com/maps/dir/?api=1&destination=${encodeURIComponent(`${location.latitude},${location.longitude}`)}` : null; const coordinateText = hasCoordinates diff --git a/static/modules/v2Adapter.js b/static/modules/v2Adapter.js index 69213d5..e6bdd24 100644 --- a/static/modules/v2Adapter.js +++ b/static/modules/v2Adapter.js @@ -27,7 +27,7 @@ export function normalizeV2Location(record, meta = {}) { facilityId: record.facility_id, category: record.category, sourceType: record.source_type, - profile: record.profile || meta.profile, + profile: record.publication_profile, displayPrecision: record.display_precision, lifecycleStatus: record.lifecycle_status, firstObservedAt: record.first_observed_at, diff --git a/static/modules/v2Client.js b/static/modules/v2Client.js index 5508105..805fe1d 100644 --- a/static/modules/v2Client.js +++ b/static/modules/v2Client.js @@ -35,6 +35,9 @@ export class V2Client { let isList; try { validateV2Envelope(body); + if (params.profile && body.meta.profile && body.meta.profile !== params.profile) { + throw new TypeError('V2 response profile differs from requested profile'); + } isList = Array.isArray(body.data); } catch (error) { throw new V2ApiError(`V2 response did not match the public API contract: ${error.message}`); diff --git a/static/modules/v2Contract.js b/static/modules/v2Contract.js index 0e99008..1269ac8 100644 --- a/static/modules/v2Contract.js +++ b/static/modules/v2Contract.js @@ -50,7 +50,15 @@ export function validateV2Envelope(body) { throw new TypeError('V2 response envelope is invalid'); } if (!Array.isArray(body.data) && !isObject(body.data)) throw new TypeError('V2 response data is invalid'); - if (Array.isArray(body.data)) body.data.forEach(validateV2Location); - else validateV2Location(body.data); + const records = Array.isArray(body.data) ? body.data : [body.data]; + if (records.length && !V2_PROFILES.includes(body.meta.profile)) { + throw new TypeError('V2 response has invalid profile'); + } + records.forEach(record => { + validateV2Location(record); + if (record.publication_profile !== body.meta.profile) { + throw new TypeError('V2 location publication_profile differs from response profile'); + } + }); return body; } From c18eb8df10ff13efe612c4a7d230aafc1174102d Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sun, 13 Sep 2026 19:19:50 -0700 Subject: [PATCH 036/311] ci(docs): gate V2 cleanup and record review evidence --- .github/workflows/tests.yml | 42 +++++++++++++++++++ docs/V2-IMPLEMENTATION-TODO.md | 12 ++++++ docs/V2-INTEGRATION-BASELINE.md | 21 ++++++++++ docs/V2-REVIEW-CLEANUP-2026-09-13.md | 35 ++++++++++++++++ .../release-manifest-verification.md | 29 ++++++++++++- frontend/README.md | 2 + pipeline/tests/run-v2-gate.ps1 | 6 ++- pipeline/tests/test-v2-gate.ps1 | 42 +++++++++++++++++++ 8 files changed, 186 insertions(+), 3 deletions(-) create mode 100644 docs/V2-INTEGRATION-BASELINE.md create mode 100644 docs/V2-REVIEW-CLEANUP-2026-09-13.md create mode 100644 pipeline/tests/test-v2-gate.ps1 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 0295f10..a70a5a8 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -34,6 +34,7 @@ jobs: python -m unittest pipeline.tests.e2e.test_public_api -v python -m unittest pipeline.tests.e2e.test_community_api -v python -m unittest pipeline.tests.e2e.test_seeded_api -v + python -m unittest pipeline.tests.e2e.test_public_surface_safety -v backup-restore: # The PostGIS image is Linux-only; Ubuntu includes PowerShell Core for the drill. @@ -45,3 +46,44 @@ jobs: env: UEC_RUN_E2E: '1' run: ./pipeline/tests/e2e/backup-restore.ps1 + + static-jest: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: '22' + cache: npm + cache-dependency-path: package-lock.json + - run: npm ci + - run: npm test + + v2-gate-self-test: + runs-on: windows-latest + steps: + - uses: actions/checkout@v4 + - name: Check V2 gate failure and success reporting + shell: pwsh + run: ./pipeline/tests/test-v2-gate.ps1 + + frontend: + runs-on: ubuntu-latest + defaults: + run: + working-directory: frontend + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: '22' + cache: npm + cache-dependency-path: frontend/package-lock.json + - run: npm ci + - run: npm run check + - run: npm test + - run: npm run lint + - run: npm run boundary + - run: npm run build + - run: npx playwright install --with-deps chromium firefox webkit + - run: npx playwright test tests/e2e/fixture-platform.spec.ts tests/e2e/local-safety.spec.ts --project=chromium --project=firefox --project=webkit --workers=2 diff --git a/docs/V2-IMPLEMENTATION-TODO.md b/docs/V2-IMPLEMENTATION-TODO.md index 50e8931..2b4c870 100644 --- a/docs/V2-IMPLEMENTATION-TODO.md +++ b/docs/V2-IMPLEMENTATION-TODO.md @@ -17,6 +17,18 @@ The current production application remains V1 while this plan is executed. V2 mu - [ ] V2 frontend is wired to the live V2 backend. - [ ] V2 has been promoted to replace V1 in production. +At the [2026-09-13 integration baseline](V2-INTEGRATION-BASELINE.md) (`integration-v2-2026-09-13` at `d853721`), a separate Svelte/TypeScript preview renders synthetic fixtures by default. Its `?mode=local-v2` path can query a loopback API against a seeded synthetic release; this is opt-in local integration, not completed live-product wiring. Denmark remains controlled staging, Germany BLtU restricted staging pending a human terms decision, and the UK FSA/FSS adapters synthetic-fixture-only. The phase checkboxes below retain their full exit criteria. + +### 2026-09-13 integration evidence + +The tag annotation records a maintainer-reported CI pass, not a hosted run independently checked here. [Existing backend closeout evidence](V2-BACKEND-PHASE-0-CLOSEOUT.md) and the dated Phase 0/1 results below document earlier local and isolated test passes; [the CI workflow](../.github/workflows/tests.yml) now covers the Svelte fixture browser gate and [country adapter tests](../pipeline/tests/run-standard.ps1). See the [integration baseline](V2-INTEGRATION-BASELINE.md) for the exact scope and remaining release/ethics blockers. No tests were rerun for this documentation update. + +### 2026-09-13 post-tag review cleanup + +The [review cleanup record](V2-REVIEW-CLEANUP-2026-09-13.md) distinguishes fixes now in the working tree and a verified green final local gate from remote CI, which remains pending. Migration 022, community CSV, restricted country staging, static/Svelte context, rate limiting, CI coverage, backup-drill staging, and release-manifest checks address specific findings. They do not close the Phase 0 suppression-propagation item or the Phase 5–7 launch gates. The V1↔V2 suppression crosswalk, independent durable replay before an old-backup restore can serve, deployment proxy trust, complete artifact inventory/atomic manifest delivery, partial loaded-page behavior, real-backend frontend E2E, source terms, and authorized human release review remain open; leave the broader phase checkboxes unchanged. + +Post-fix A/B regression checks now cover release-scoped review in migration 022 and validation/promotion, plus Rust list/detail/CSV behavior through a two-profile synthetic HTTP E2E: B's later denial cannot revoke or relabel promoted A, and B cannot inherit A's approval. The local synthetic launcher explicitly declares `--no-distributed-artifacts`, with a narrow contract test; the portless two-stage backup drill passed twice in scoped local verification and in the final gate. The independent verifier reports a no-retry final local pass: standard 72 Rust and 102 Python (5 skipped), four sequential API E2E modules 36/36, root Jest 19/19, frontend unit 41/41, Playwright 42/42 across three browsers, plus frontend check/lint/boundary/build, `cargo fmt`, `git diff --check`, and PowerShell gate self-test 3/3. No disposable Docker project remained; persistent databases were untouched. **Remote CI remains pending.** Deploy migration 022 and the updated V2 API together with V2 public access paused during the transition, then verify compatibility and current restrictions before any eligible access resumes. See the [review cleanup record](V2-REVIEW-CLEANUP-2026-09-13.md); production crosswalk, independent suppression replay, and the other release blockers remain open. + ## Phase 0 — Close and certify the backend foundation Goal: make the current backend/data-platform branch independently runnable, reviewable, and safe for frontend integration. diff --git a/docs/V2-INTEGRATION-BASELINE.md b/docs/V2-INTEGRATION-BASELINE.md new file mode 100644 index 0000000..5f06a5f --- /dev/null +++ b/docs/V2-INTEGRATION-BASELINE.md @@ -0,0 +1,21 @@ +# V2 integration baseline + +Date: 2026-09-13. Checkpoint: annotated tag `integration-v2-2026-09-13` at commit `d853721`. The tag annotation records a **maintainer-reported CI pass**; this document does not independently verify the hosted run. This is a non-production integration checkpoint, not a release or publication approval. + +## What is integrated + +- The current public application remains V1. A separate Svelte 5/TypeScript preview uses synthetic fixtures by default, with a local map, detail, profile warning, and limited export preview. It is staged only through an explicit preview command; no external map tiles are configured. See [frontend/README.md](../frontend/README.md), [App.svelte](../frontend/src/app/App.svelte), and the [fixture browser tests](../frontend/tests/e2e/fixture-platform.spec.ts). +- `?mode=local-v2` opts that preview into a loopback V2 API and a seeded synthetic release. The local helper starts PostGIS and Axum, while the development proxy connects the frontend; the local list/detail test is opt-in. This is a development integration path, not production frontend wiring. See [frontend/README.md](../frontend/README.md), [local-v2.ps1](../pipeline/scripts/maintenance/local-v2.ps1), [LocalLocationRepository.ts](../frontend/src/api/LocalLocationRepository.ts), and [local-backend.spec.ts](../frontend/tests/e2e/local-backend.spec.ts). +- Denmark has controlled acquisition and private staging: network retrieval needs an operator terms review, bounded development geocoding needs a separate per-run review, and database import and release promotion remain separate. See [pipeline/README.md](../pipeline/README.md) and [run-denmark-pipeline.py](../pipeline/run-denmark-pipeline.py). +- Germany BLtU has restricted, non-release staging with synthetic adapter tests. A previously acquired export remains private research evidence; automated reacquisition and public redistribution await a source-specific human terms decision. See [Germany assessment](germany-source-assessment.md) and [Germany adapter README](../pipeline/germany/README.md). +- UK FSA, FSS, and composition adapters are synthetic-fixture-only, with no live acquisition or release. The FSS assessment conditionally recommends restricted staging of the published CSV after provenance and privacy checks, while recurring retrieval and public release remain gated. See [UK composition](../pipeline/sources/uk/approved/README.md), [FSA adapter](../pipeline/sources/uk/fsa_approved/README.md), [FSS adapter](../pipeline/sources/uk/fss_approved/README.md), and [FSS assessment](countries/uk/fss-approved-establishments-source-assessment.md). + +## Evidence and remaining gates + +The earlier [backend closeout](V2-BACKEND-PHASE-0-CLOSEOUT.md) records 65 Rust and 57 Python passes, isolated public/community/seeded API E2E passes (5/5/12), migration checks, and synthetic backup/restore; the [implementation TODO](V2-IMPLEMENTATION-TODO.md) also records 13 Jest passes and later contract-gate results. These are dated repository evidence, not tests rerun for this documentation change. The [CI workflow](../.github/workflows/tests.yml) defines standard, API E2E, backup/restore, and frontend check/test/lint/build/three-browser fixture jobs, including the reconciled country adapter tests in the [standard runner](../pipeline/tests/run-standard.ps1). + +Public V2 release remains blocked by incomplete live-wire contract and revocation semantics, source coverage and terms decisions, privacy screening and suppression propagation across API, map, export, caches, history, reimports, and restores, plus human review and release authority. Deployment TLS/provider configuration, visitor-data and provider logging/retention audits, correction/removal and legal-demand operations, and browser accessibility, mobile, and performance review also remain open. See [frontend gaps](../frontend/README.md), [backend blockers](V2-BACKEND-PHASE-0-CLOSEOUT.md), [policy implementation checklist](governance/policy-implementation-todo.md), and governing [ETHICS.md](ETHICS.md). No provider logging practice or jurisdiction-specific legal position is verified by this checkpoint. + +## Post-tag review cleanup + +The [2026-09-13 review cleanup](V2-REVIEW-CLEANUP-2026-09-13.md) records uncommitted fixes made after this tag and their remaining gates. It does not revise the tagged checkpoint or convert worker-scoped test passes into a combined-gate or remote-CI pass. V1 remains the public default; V2 publication still needs the cross-system suppression, restore, deployment, source, and human release decisions listed there. diff --git a/docs/V2-REVIEW-CLEANUP-2026-09-13.md b/docs/V2-REVIEW-CLEANUP-2026-09-13.md new file mode 100644 index 0000000..0f3def7 --- /dev/null +++ b/docs/V2-REVIEW-CLEANUP-2026-09-13.md @@ -0,0 +1,35 @@ +# V2 review cleanup — 2026-09-13 + +Status: post-`integration-v2-2026-09-13` working-tree review, not part of the tag at `d853721`, a production release, or publication approval. The tagged baseline and its maintainer-reported CI pass remain documented in [V2-INTEGRATION-BASELINE.md](V2-INTEGRATION-BASELINE.md). This note records scoped fixes and remaining evidence gaps under [ETHICS.md](ETHICS.md). + +## Confirmed fixes in this review round + +| Area | Review outcome and in-repo evidence | +| --- | --- | +| Release-scoped publication | [Migration 022](../pipeline/migrations/022_publication_safety_scopes.sql) scopes publication decisions to a release, withholds ambiguous older decisions, accounts for restricted observations, and computes public-only history counts. [Validation](../pipeline/scripts/stages/validate-release.py) and [promotion](../pipeline/scripts/stages/promote-release.py) recheck approval and suppression against that release; neither creates a human approval event. | +| Community CSV | The [V2 export handler](../src/lib.rs) admits eligible community-profile rows, adds per-row factual-review, approval, profile notice and unreviewed-claim warning context, and identifies the selected export profile. [Synthetic API checks](../pipeline/tests/e2e/test_public_surface_safety.py) cover CSV/API parity and current suppression. | +| Country pipelines | The [shared orchestrator](../pipeline/common/orchestrator.py) and [delta comparison](../pipeline/common/delta.py) retain restricted, human-gated states and apply source-qualified suppression/identity handling. [UK composition](../pipeline/sources/uk/approved/compose.py) keeps FSA/FSS identities separate and blocks release creation; the [Germany BLtU adapter](../pipeline/germany/bltu_adapter.py) remains restricted and quarantines unresolved input. | +| Visitor-facing context | The [Svelte preview](../frontend/src/app/App.svelte) clears stale local responses, keeps direct-link community warnings and evidence context visible, and labels first-page-only results; [local browser safety tests](../frontend/tests/e2e/local-safety.spec.ts) use mocked V2 responses. The [static V2 adapter](../static/modules/v2Adapter.js), [contract](../static/modules/v2Contract.js), and [export](../static/modules/ExportManager.js) preserve publication context and avoid presenting a paginated loaded page as a complete export. | +| Rate limiting and CI | The [Axum middleware](../src/main.rs) keys limits by socket peer by default and accepts a parsed forwarded address only with explicit proxy trust. The [CI workflow](../.github/workflows/tests.yml) adds static Jest, public-surface E2E, gate self-test, and Svelte local-safety fixture coverage; the [combined gate](../pipeline/tests/run-v2-gate.ps1) now propagates standard/Jest failures. | +| Restore and release integrity | The [two-stage synthetic backup drill](../pipeline/tests/e2e/backup-restore.ps1) now uses a [portless Compose override](../pipeline/tests/e2e/docker-compose.backup-restore.yml), rejects an old restore before replay, then checks the current restriction after replay. The portless drill passed twice in local scoped verification. [Promotion](../pipeline/scripts/stages/promote-release.py) stores a canonical hashed manifest with an explicitly declared artifact list, and [verification guidance](architecture/release-manifest-verification.md) states its limits. | +| Local synthetic launcher | [local-v2.ps1](../pipeline/scripts/maintenance/local-v2.ps1) now explicitly passes `--no-distributed-artifacts` when promoting its synthetic release. A [narrow launcher contract test](../pipeline/tests/test_local_v2_start_contract.py) checks the parsed command and a mocked start branch without starting services; its scoped run passed. This declaration applies only to that local fixture. | + +### A/B regression chain and resolution + +The post-fix regression uses one synthetic source record in independent releases A and B. An ambiguous older source-only approval must not qualify either candidate; a release-A review permits A but does not let B inherit approval. A later B denial must block B's validation/promotion without relabeling or withdrawing the still-promoted A; a later B approval must not change A's review context. Current suppression still blocks publication regardless of either approval. [Migration 022](../pipeline/migrations/022_publication_safety_scopes.sql) provides a release-keyed current-review view (while its source-keyed compatibility view prefers a promoted decision), [validation/promotion](../pipeline/tests/test_publication_scoped_stages.py) exercise those gates, and the [Rust list/detail/CSV queries](../src/lib.rs) join review by both source record and release. A [two-profile HTTP E2E](../pipeline/tests/e2e/test_public_surface_safety.py) checks that a B denial removes B while A remains correctly labeled in list, detail, and CSV. These are synthetic regression checks, not publication decisions. + +### Final local verification + +The independent verifier reports the **final assembled local cleanup gate green, with no retries**: the standard runner passed 72 Rust tests and 102 Python tests (5 skipped) after migration 022; four sequential API E2E modules passed 36/36, including the cross-profile regression; the portless two-stage backup/restore drill passed; root Jest passed 19/19; frontend unit tests passed 41/41; and Playwright passed 42/42 across Chromium, Firefox, and WebKit. Frontend check, lint, boundary, and build; `cargo fmt`; `git diff --check`; and the PowerShell gate self-test (3/3) also passed. The verifier reports no disposable Docker project remains and persistent databases were untouched. These results supersede the earlier worker-scoped-only test status. **Remote CI for this cleanup round remains pending**; the older tag's reported CI pass is separate evidence. + +Migration 022 and the updated V2 API must be deployed together. Keep V2 public access paused throughout that transition: the old API does not use the new release-keyed review contract, and the updated API requires its database view. Resume only after migration/API compatibility and current restriction checks are verified, subject to the open release gates below. + +## Open launch blockers + +- A reviewed V1↔V2 identity/suppression crosswalk is still needed. [Public-surface tests](../pipeline/tests/e2e/test_public_surface_safety.py) check V1 compatibility but explicitly do not claim cross-system suppression propagation. +- The backup drill replays a synthetic restriction from its test fixture. Production needs an **independent durable restriction ledger**, replay and an enforced pre-service gate before any old-backup restore can serve public data; see the [drill](../pipeline/tests/e2e/backup-restore.ps1) and [policy checklist](governance/policy-implementation-todo.md). +- `UEC_TRUST_PROXY` needs deployment-specific reverse-proxy boundary configuration and verification. The code path alone does not establish which forwarding headers are trustworthy; see [API contract](architecture/api-location-contract.md) and [middleware](../src/main.rs). +- Manifest creation requires an operator-complete inventory of every distributed artifact; the tool cannot discover omissions. Database promotion and writing the manifest file are separate, non-atomic steps requiring recovery/verification before distribution; see [manifest guidance](architecture/release-manifest-verification.md). +- Svelte and static V2 views still operate on loaded pages, so client-side search/map/count/export scope can be partial. The [Svelte preview](../frontend/src/app/App.svelte) and [static export](../static/modules/ExportManager.js) label this limit, but full traversal and release-consistent product behavior remain unfinished. +- The new Svelte safety browser tests mock V2. The [real-backend browser test](../frontend/tests/e2e/local-backend.spec.ts) remains opt-in and is not part of the reviewed remote fixture-browser job; a full real-backend frontend E2E gate remains open. +- Source-specific acquisition/redistribution terms, privacy screening, suppression operations, authorized human release review, and deployment/visitor-provider audits remain release gates. See [Germany assessment](germany-source-assessment.md), [FSS assessment](countries/uk/fss-approved-establishments-source-assessment.md), [policy checklist](governance/policy-implementation-todo.md), and [ETHICS.md](ETHICS.md). No code or passing test supplies legal clearance or publication authority. diff --git a/docs/architecture/release-manifest-verification.md b/docs/architecture/release-manifest-verification.md index d031ad4..a87d120 100644 --- a/docs/architecture/release-manifest-verification.md +++ b/docs/architecture/release-manifest-verification.md @@ -1,5 +1,32 @@ # Release manifest verification -Every public release must include a machine-readable manifest containing the release ID, profile, ruleset/configuration versions, source coverage, creation time, and SHA-256 checksums for each distributed artifact. Consumers should calculate SHA-256 locally and compare it with the manifest obtained from a trusted project channel. +Promotion stores an immutable machine-readable manifest with the release ID, profile, ruleset version, source coverage, database creation time, and an inventory of declared distributed files. Supply each file with a repeated `--artifact ` option; promotion hashes the file bytes and records its basename, size, and SHA-256. If there really are no distributed files, the operator must explicitly pass `--no-distributed-artifacts`. `--manifest ` exports the canonical JSON whose SHA-256 is stored in `uec.release_manifests`; the CLI result printed to stdout is a separate operation receipt. Obtain the manifest from a trusted project channel, verify its SHA-256 against the stored digest, then hash each listed artifact locally. + +The workflow does not discover distributed files or prove that the operator supplied a complete inventory. An empty declared inventory is not evidence that no files were distributed. The manifest records the ruleset version, but other configuration and code versions are not yet part of the release schema; the source ID list is not complete record-level provenance. Hashing at promotion does not freeze later distribution bytes. The database promotion and writing `--manifest` to disk are separate operations; if the file write fails, the database manifest remains stored and an operator must recover and verify it before distribution. These limits require operational review before making a full release-integrity claim. + +Migration 022 binds publication review events to releases. After importing the candidate, use the actual `release_id` recorded in `uec.releases` and `uec.release_members` when an authorized maintainer records a reviewed decision. For example, a maintainer can parameterize the following SQL with a source record ID from the candidate and the candidate's real release ID; the values and decision must be chosen by that maintainer: + +```sql +SELECT release_id, status FROM uec.releases +WHERE release_id = :candidate_release_id AND status = 'candidate'; + +SELECT DISTINCT observation.source_record_id +FROM uec.release_members member +JOIN uec.observations observation ON observation.observation_id = member.observation_id +WHERE member.release_id = :candidate_release_id; +``` + +```sql +INSERT INTO uec.publication_review_events + (source_record_id, release_id, factual_review_status, + privacy_screening_status, maintainer_approval, publication_eligible, + reviewer_role, note) +VALUES (:reviewed_source_record_id, :candidate_release_id, + :reviewed_factual_status, :reviewed_privacy_status, + :maintainer_decision, :reviewed_publication_eligibility, + :authorized_reviewer_role, :review_note); +``` + +First verify the source record belongs to that candidate through `uec.observations` and `uec.release_members`. Validation reports `publication_not_approved` until the current review event for each record is scoped to that release and passes its gates; `--mark-validated` only changes a passing candidate's status. Historic source-only decisions spanning multiple releases remain withheld and need a new human release-scoped event. Validation and promotion never create approval events. Checksums detect alteration relative to a trusted reference; they do not prove factual accuracy, privacy eligibility, or government-source correctness. Withdrawn or sanitized artifacts must remain marked and must not be silently replaced. Signing is not claimed until key custody, distribution, rotation, revocation, and compromised-release handling are separately reviewed and tested. diff --git a/frontend/README.md b/frontend/README.md index 7f93a1e..10d68ba 100644 --- a/frontend/README.md +++ b/frontend/README.md @@ -21,4 +21,6 @@ These are documented gaps, not frontend claims or invented DTO fields. Phase 3 c Local V2 integration (opt-in only): run the backend with `cargo run` (port 8000), then run the frontend with `npm run dev` (port 5173). In development/preview, Vite proxies `/api` to `http://127.0.0.1:8000`; use `http://127.0.0.1:5173/v2-preview/?mode=local-v2#/` to opt in. The `LocalLocationRepository` still targets `/api/v2/locations?profile=...` only when explicitly invoked; fixture mode remains the default and there is no V1 fallback. The proxy is development-only configuration and production builds do not enable a backend connection. +The opt-in local view loads one API page at a time. When the response includes `next_cursor`, the UI labels its counts, search, and map as partial; search runs only against loaded records. Full pagination and backend search remain future integration work. Record context is limited to fields in the current V2 wire response and does not imply that review events, evidence hashes, or scoped approvals are available. + Persistent local two-port workflow (never uses `down -v`): from the repository root run `powershell -ExecutionPolicy Bypass -File pipeline/scripts/maintenance/local-v2.ps1 start`. It starts the named Postgres stack on `5433`, applies migrations, seeds and promotes the synthetic contract release, and starts Axum on `8000`. Run `npm --prefix frontend run dev` in another terminal and open `http://127.0.0.1:5173/v2-preview/?mode=local-v2#/`. Check ownership and health with `... local-v2.ps1 status`; probe list/detail with `... local-v2.ps1 probe`; stop both services with `... local-v2.ps1 stop`. The helper is local-only and does not alter V1, production, or unrelated data. diff --git a/pipeline/tests/run-v2-gate.ps1 b/pipeline/tests/run-v2-gate.ps1 index 6008d87..9ecf17d 100644 --- a/pipeline/tests/run-v2-gate.ps1 +++ b/pipeline/tests/run-v2-gate.ps1 @@ -3,8 +3,10 @@ $ErrorActionPreference = 'Stop' $root = (Resolve-Path (Join-Path $PSScriptRoot '..\..')).Path Push-Location $root try { - pwsh -NoProfile -ExecutionPolicy Bypass -File pipeline/tests/run-standard.ps1 - npm test -- --runInBand + & pwsh -NoProfile -ExecutionPolicy Bypass -File pipeline/tests/run-standard.ps1 + if ($LASTEXITCODE -ne 0) { throw "Standard gate failed (exit $LASTEXITCODE)." } + & npm test -- --runInBand + if ($LASTEXITCODE -ne 0) { throw "npm tests failed (exit $LASTEXITCODE)." } if (-not $SkipDocker) { $env:UEC_RUN_E2E = '1' pwsh -NoProfile -ExecutionPolicy Bypass -File pipeline/tests/e2e/backup-restore.ps1 diff --git a/pipeline/tests/test-v2-gate.ps1 b/pipeline/tests/test-v2-gate.ps1 new file mode 100644 index 0000000..71b09e2 --- /dev/null +++ b/pipeline/tests/test-v2-gate.ps1 @@ -0,0 +1,42 @@ +$ErrorActionPreference = 'Stop' +$gate = Join-Path $PSScriptRoot 'run-v2-gate.ps1' +$tempRoot = [System.IO.Path]::GetFullPath([System.IO.Path]::GetTempPath()) +$sandbox = [System.IO.Path]::GetFullPath((Join-Path $tempRoot ('uec-v2-gate-test-' + [guid]::NewGuid().ToString('N')))) +if (-not $sandbox.StartsWith($tempRoot, [System.StringComparison]::OrdinalIgnoreCase)) { + throw 'Test sandbox resolved outside the system temporary directory.' +} + +try { + $testDir = Join-Path $sandbox 'pipeline/tests' + $binDir = Join-Path $sandbox 'bin' + New-Item -ItemType Directory -Path $testDir, $binDir -Force | Out-Null + Copy-Item -LiteralPath $gate -Destination (Join-Path $testDir 'run-v2-gate.ps1') + $standard = Join-Path $testDir 'run-standard.ps1' + $fakeNpm = Join-Path $binDir 'npm.cmd' + $priorPath = $env:PATH + $env:PATH = "$binDir;$priorPath" + try { + foreach ($case in @( + @{ Name = 'standard failure'; StandardExit = 17; NpmExit = 0; ExpectExit = 1; ExpectText = 'Standard gate failed (exit 17).'; NpmRan = $false; Pass = $false }, + @{ Name = 'npm failure'; StandardExit = 0; NpmExit = 19; ExpectExit = 1; ExpectText = 'npm tests failed (exit 19).'; NpmRan = $true; Pass = $false }, + @{ Name = 'success'; StandardExit = 0; NpmExit = 0; ExpectExit = 0; ExpectText = 'PASS: V2 unit, pipeline, frontend, and synthetic backup/restore gates passed.'; NpmRan = $true; Pass = $true } + )) { + Set-Content -LiteralPath $standard -Encoding utf8 -Value "Write-Output 'FAKE_STANDARD_RAN'; exit $($case.StandardExit)" + Set-Content -LiteralPath $fakeNpm -Encoding ascii -Value ("@echo off" + [Environment]::NewLine + "echo FAKE_NPM_RAN" + [Environment]::NewLine + "exit /b $($case.NpmExit)") + if ((Get-Command npm).Source -ne $fakeNpm) { throw 'Fake npm command did not take precedence.' } + $output = (& pwsh -NoProfile -ExecutionPolicy Bypass -File (Join-Path $testDir 'run-v2-gate.ps1') -SkipDocker 2>&1 | Out-String) + $exitCode = $LASTEXITCODE + if ($exitCode -ne $case.ExpectExit) { throw "$($case.Name): exit $exitCode, expected $($case.ExpectExit). Output: $output" } + if (-not $output.Contains($case.ExpectText)) { throw "$($case.Name): missing expected result. Output: $output" } + if ($output.Contains('FAKE_NPM_RAN') -ne $case.NpmRan) { throw "$($case.Name): npm execution was unexpected. Output: $output" } + if ($output.Contains('PASS: V2') -ne $case.Pass) { throw "$($case.Name): PASS reporting was unexpected. Output: $output" } + Write-Host "PASS: $($case.Name)" + } + } finally { + $env:PATH = $priorPath + } +} finally { + if ((Test-Path -LiteralPath $sandbox) -and $sandbox.StartsWith($tempRoot, [System.StringComparison]::OrdinalIgnoreCase)) { + Remove-Item -LiteralPath $sandbox -Recurse -Force + } +} From 0a7c59c4e267e746db7a105b129bfc57f0601201 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sun, 13 Sep 2026 19:29:14 -0700 Subject: [PATCH 037/311] fix(frontend): declare jsdom test environment dependency --- frontend/package-lock.json | 766 +++++++++++++++++++++++++++++++++++++ frontend/package.json | 2 +- 2 files changed, 767 insertions(+), 1 deletion(-) diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 1129b30..f4912ad 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -16,6 +16,7 @@ "@sveltejs/vite-plugin-svelte": "^6.2.1", "@types/node": "^22.10.2", "eslint": "^9.17.0", + "jsdom": "^25.0.1", "svelte": "^5.19.0", "svelte-check": "^4.1.4", "typescript": "^5.7.2", @@ -23,6 +24,20 @@ "vitest": "^2.1.8" } }, + "node_modules/@asamuzakjp/css-color": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/@asamuzakjp/css-color/-/css-color-3.2.0.tgz", + "integrity": "sha512-K1A6z8tS3XsmCMM86xoWdn7Fkdn9m6RSVtocUrJYIwZnFVkng/PvkEoWtOWmP+Scc6saYWHWZYbndEEXxl24jw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@csstools/css-calc": "^2.1.3", + "@csstools/css-color-parser": "^3.0.9", + "@csstools/css-parser-algorithms": "^3.0.4", + "@csstools/css-tokenizer": "^3.0.3", + "lru-cache": "^10.4.3" + } + }, "node_modules/@axe-core/playwright": { "version": "4.13.0", "resolved": "https://registry.npmjs.org/@axe-core/playwright/-/playwright-4.13.0.tgz", @@ -35,6 +50,121 @@ "playwright-core": ">= 1.0.0" } }, + "node_modules/@csstools/color-helpers": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/@csstools/color-helpers/-/color-helpers-5.1.0.tgz", + "integrity": "sha512-S11EXWJyy0Mz5SYvRmY8nJYTFFd1LCNV+7cXyAgQtOOuzb4EsgfqDufL+9esx72/eLhsRdGZwaldu/h+E4t4BA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT-0", + "engines": { + "node": ">=18" + } + }, + "node_modules/@csstools/css-calc": { + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/@csstools/css-calc/-/css-calc-2.1.4.tgz", + "integrity": "sha512-3N8oaj+0juUw/1H3YwmDDJXCgTB1gKU6Hc/bB502u9zR0q2vd786XJH9QfrKIEgFlZmhZiq6epXl4rHqhzsIgQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT", + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@csstools/css-parser-algorithms": "^3.0.5", + "@csstools/css-tokenizer": "^3.0.4" + } + }, + "node_modules/@csstools/css-color-parser": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/@csstools/css-color-parser/-/css-color-parser-3.1.0.tgz", + "integrity": "sha512-nbtKwh3a6xNVIp/VRuXV64yTKnb1IjTAEEh3irzS+HkKjAOYLTGNb9pmVNntZ8iVBHcWDA2Dof0QtPgFI1BaTA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT", + "dependencies": { + "@csstools/color-helpers": "^5.1.0", + "@csstools/css-calc": "^2.1.4" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@csstools/css-parser-algorithms": "^3.0.5", + "@csstools/css-tokenizer": "^3.0.4" + } + }, + "node_modules/@csstools/css-parser-algorithms": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/@csstools/css-parser-algorithms/-/css-parser-algorithms-3.0.5.tgz", + "integrity": "sha512-DaDeUkXZKjdGhgYaHNJTV9pV7Y9B3b644jCLs9Upc3VeNGg6LWARAT6O+Q+/COo+2gg/bM5rhpMAtf70WqfBdQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT", + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@csstools/css-tokenizer": "^3.0.4" + } + }, + "node_modules/@csstools/css-tokenizer": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@csstools/css-tokenizer/-/css-tokenizer-3.0.4.tgz", + "integrity": "sha512-Vd/9EVDiu6PPJt9yAh6roZP6El1xHrdvIVGjyBsHR0RYwNHgL7FJPyIIW4fANJNG6FtyZfvlRPpFI4ZM/lubvw==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT", + "engines": { + "node": ">=18" + } + }, "node_modules/@esbuild/aix-ppc64": { "version": "0.25.12", "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.25.12.tgz", @@ -1327,6 +1457,16 @@ "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" } }, + "node_modules/agent-base": { + "version": "7.1.4", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", + "integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 14" + } + }, "node_modules/ajv": { "version": "6.15.0", "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.15.0.tgz", @@ -1387,6 +1527,13 @@ "node": ">=12" } }, + "node_modules/asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", + "dev": true, + "license": "MIT" + }, "node_modules/axe-core": { "version": "4.13.0", "resolved": "https://registry.npmjs.org/axe-core/-/axe-core-4.13.0.tgz", @@ -1434,6 +1581,20 @@ "node": ">=8" } }, + "node_modules/call-bind-apply-helpers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/callsites": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", @@ -1534,6 +1695,19 @@ "dev": true, "license": "MIT" }, + "node_modules/combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "dev": true, + "license": "MIT", + "dependencies": { + "delayed-stream": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, "node_modules/concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", @@ -1556,6 +1730,41 @@ "node": ">= 8" } }, + "node_modules/cssstyle": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/cssstyle/-/cssstyle-4.6.0.tgz", + "integrity": "sha512-2z+rWdzbbSZv6/rhtvzvqeZQHrBaqgogqt85sqFNbabZOuFbCVFb8kPeEtZjiKkbrm395irpNKiYeFeLiQnFPg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@asamuzakjp/css-color": "^3.2.0", + "rrweb-cssom": "^0.8.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/cssstyle/node_modules/rrweb-cssom": { + "version": "0.8.0", + "resolved": "https://registry.npmjs.org/rrweb-cssom/-/rrweb-cssom-0.8.0.tgz", + "integrity": "sha512-guoltQEx+9aMf2gDZ0s62EcV8lsXR+0w8915TC3ITdn2YueuNjdAYh/levpU9nFaoChh9RUS5ZdQMrKfVEN9tw==", + "dev": true, + "license": "MIT" + }, + "node_modules/data-urls": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-5.0.0.tgz", + "integrity": "sha512-ZYP5VBHshaDAiVZxjbRVcFJpc+4xGgT0bK3vzy1HLN8jTO975HEbuYzZJcHoQEY5K1a0z8YayJkyVETa08eNTg==", + "dev": true, + "license": "MIT", + "dependencies": { + "whatwg-mimetype": "^4.0.0", + "whatwg-url": "^14.0.0" + }, + "engines": { + "node": ">=18" + } + }, "node_modules/debug": { "version": "4.4.3", "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", @@ -1574,6 +1783,13 @@ } } }, + "node_modules/decimal.js": { + "version": "10.6.0", + "resolved": "https://registry.npmjs.org/decimal.js/-/decimal.js-10.6.0.tgz", + "integrity": "sha512-YpgQiITW3JXGntzdUmyUR1V812Hn8T1YVXhCu+wO3OpS4eU9l4YdD3qjyiKdV6mvV29zapkMeD390UVEf2lkUg==", + "dev": true, + "license": "MIT" + }, "node_modules/deep-eql": { "version": "5.0.2", "resolved": "https://registry.npmjs.org/deep-eql/-/deep-eql-5.0.2.tgz", @@ -1601,6 +1817,16 @@ "node": ">=0.10.0" } }, + "node_modules/delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.4.0" + } + }, "node_modules/devalue": { "version": "5.9.2", "resolved": "https://registry.npmjs.org/devalue/-/devalue-5.9.2.tgz", @@ -1608,6 +1834,54 @@ "dev": true, "license": "MIT" }, + "node_modules/dunder-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "dev": true, + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/entities": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/entities/-/entities-6.0.1.tgz", + "integrity": "sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g==", + "dev": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, + "node_modules/es-define-property": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, "node_modules/es-module-lexer": { "version": "1.7.0", "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-1.7.0.tgz", @@ -1615,6 +1889,35 @@ "dev": true, "license": "MIT" }, + "node_modules/es-object-atoms": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.2.tgz", + "integrity": "sha512-HWcBoN6NileqtSydK2FqHbS/LoDd2pqrnQHLyJzBj4kOp/ky2MWMN694xOfkK8/SnUsW2DH7EfyVlydKCsm1Zw==", + "dev": true, + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-set-tostringtag": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz", + "integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==", + "dev": true, + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.6", + "has-tostringtag": "^1.0.2", + "hasown": "^2.0.2" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/esbuild": { "version": "0.25.12", "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.25.12.tgz", @@ -1960,6 +2263,23 @@ "dev": true, "license": "ISC" }, + "node_modules/form-data": { + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.6.tgz", + "integrity": "sha512-vKatAh4SlVfgbv+YtmhiRjhEMJsYpsG1Y2rMQtR+SVSbytsSD1YGzDIcrAJmdFec88u/+VoGmxnl+80gL1tRCQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "es-set-tostringtag": "^2.1.0", + "hasown": "^2.0.4", + "mime-types": "^2.1.35" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/fsevents": { "version": "2.3.3", "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", @@ -1975,6 +2295,55 @@ "node": "^8.16.0 || ^10.6.0 || >=11.0.0" } }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-intrinsic": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", + "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", + "es-errors": "^1.3.0", + "es-object-atoms": "^1.1.1", + "function-bind": "^1.1.2", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "dev": true, + "license": "MIT", + "dependencies": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/glob-parent": { "version": "6.0.2", "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", @@ -2001,6 +2370,19 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/gopd": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/has-flag": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", @@ -2011,6 +2393,102 @@ "node": ">=8" } }, + "node_modules/has-symbols": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-tostringtag": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz", + "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==", + "dev": true, + "license": "MIT", + "dependencies": { + "has-symbols": "^1.0.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/hasown": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.4.tgz", + "integrity": "sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A==", + "dev": true, + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/html-encoding-sniffer": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-4.0.0.tgz", + "integrity": "sha512-Y22oTqIU4uuPgEemfz7NDJz6OeKf12Lsu+QC+s3BVpda64lTiMYCyGwg5ki4vFxkMwQdeZDl2adZoqUgdFuTgQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "whatwg-encoding": "^3.1.1" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/http-proxy-agent": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz", + "integrity": "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==", + "dev": true, + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.0", + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/https-proxy-agent": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", + "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", + "dev": true, + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/iconv-lite": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", + "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", + "dev": true, + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/ignore": { "version": "5.3.2", "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz", @@ -2071,6 +2549,13 @@ "node": ">=0.10.0" } }, + "node_modules/is-potential-custom-element-name": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/is-potential-custom-element-name/-/is-potential-custom-element-name-1.0.1.tgz", + "integrity": "sha512-bCYeRA2rVibKZd+s2625gGnGF/t7DSqDs4dP7CrLA1m7jKWz6pps0LpYLJN8Q64HtmPKJ1hrN3nzPNKFEKOUiQ==", + "dev": true, + "license": "MIT" + }, "node_modules/is-reference": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/is-reference/-/is-reference-3.0.3.tgz", @@ -2111,6 +2596,47 @@ "js-yaml": "bin/js-yaml.js" } }, + "node_modules/jsdom": { + "version": "25.0.1", + "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-25.0.1.tgz", + "integrity": "sha512-8i7LzZj7BF8uplX+ZyOlIz86V6TAsSs+np6m1kpW9u0JWi4z/1t+FzcK1aek+ybTnAC4KhBL4uXCNT0wcUIeCw==", + "dev": true, + "license": "MIT", + "dependencies": { + "cssstyle": "^4.1.0", + "data-urls": "^5.0.0", + "decimal.js": "^10.4.3", + "form-data": "^4.0.0", + "html-encoding-sniffer": "^4.0.0", + "http-proxy-agent": "^7.0.2", + "https-proxy-agent": "^7.0.5", + "is-potential-custom-element-name": "^1.0.1", + "nwsapi": "^2.2.12", + "parse5": "^7.1.2", + "rrweb-cssom": "^0.7.1", + "saxes": "^6.0.0", + "symbol-tree": "^3.2.4", + "tough-cookie": "^5.0.0", + "w3c-xmlserializer": "^5.0.0", + "webidl-conversions": "^7.0.0", + "whatwg-encoding": "^3.1.1", + "whatwg-mimetype": "^4.0.0", + "whatwg-url": "^14.0.0", + "ws": "^8.18.0", + "xml-name-validator": "^5.0.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "canvas": "^2.11.2" + }, + "peerDependenciesMeta": { + "canvas": { + "optional": true + } + } + }, "node_modules/json-buffer": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz", @@ -2199,6 +2725,13 @@ "dev": true, "license": "MIT" }, + "node_modules/lru-cache": { + "version": "10.4.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.4.3.tgz", + "integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==", + "dev": true, + "license": "ISC" + }, "node_modules/magic-string": { "version": "0.30.21", "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.21.tgz", @@ -2209,6 +2742,39 @@ "@jridgewell/sourcemap-codec": "^1.5.5" } }, + "node_modules/math-intrinsics": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "dev": true, + "license": "MIT", + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, "node_modules/minimatch": { "version": "3.1.5", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.5.tgz", @@ -2265,6 +2831,13 @@ "dev": true, "license": "MIT" }, + "node_modules/nwsapi": { + "version": "2.2.27", + "resolved": "https://registry.npmjs.org/nwsapi/-/nwsapi-2.2.27.tgz", + "integrity": "sha512-gQPNF78qebCQ6tvVFBYrvJdBNOrYZm90ZlXgpIFm06p6qHDHq/XC4TnJftN6OMbxVE0UTBAoRgcsDeJBBooITw==", + "dev": true, + "license": "MIT" + }, "node_modules/obug": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/obug/-/obug-2.2.1.tgz", @@ -2342,6 +2915,19 @@ "node": ">=6" } }, + "node_modules/parse5": { + "version": "7.3.0", + "resolved": "https://registry.npmjs.org/parse5/-/parse5-7.3.0.tgz", + "integrity": "sha512-IInvU7fabl34qmi9gY8XOVxhYyMyuH2xUNpb2q8/Y+7552KlejkRvqvD19nMoUW/uQGGbqNpA6Tufu5FL5BZgw==", + "dev": true, + "license": "MIT", + "dependencies": { + "entities": "^6.0.0" + }, + "funding": { + "url": "https://github.com/inikulin/parse5?sponsor=1" + } + }, "node_modules/path-exists": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", @@ -2546,6 +3132,13 @@ "fsevents": "~2.3.2" } }, + "node_modules/rrweb-cssom": { + "version": "0.7.1", + "resolved": "https://registry.npmjs.org/rrweb-cssom/-/rrweb-cssom-0.7.1.tgz", + "integrity": "sha512-TrEMa7JGdVm0UThDJSx7ddw5nVm3UJS9o9CCIZ72B1vSyEZoziDqBYP3XIoi/12lKrJR8rE3jeFHMok2F/Mnsg==", + "dev": true, + "license": "MIT" + }, "node_modules/sade": { "version": "1.8.1", "resolved": "https://registry.npmjs.org/sade/-/sade-1.8.1.tgz", @@ -2559,6 +3152,26 @@ "node": ">=6" } }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "dev": true, + "license": "MIT" + }, + "node_modules/saxes": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/saxes/-/saxes-6.0.0.tgz", + "integrity": "sha512-xAg7SOnEhrm5zI3puOOKyy1OMcMlIJZYNJY7xLBwSze0UjhPLnWfj2GF2EpT0jmzaJKIWKHLsaSSajf35bcYnA==", + "dev": true, + "license": "ISC", + "dependencies": { + "xmlchars": "^2.2.0" + }, + "engines": { + "node": ">=v12.22.7" + } + }, "node_modules/shebang-command": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", @@ -2691,6 +3304,13 @@ "typescript": "^5.0.0 || ^6.0.0" } }, + "node_modules/symbol-tree": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/symbol-tree/-/symbol-tree-3.2.4.tgz", + "integrity": "sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw==", + "dev": true, + "license": "MIT" + }, "node_modules/tinybench": { "version": "2.9.0", "resolved": "https://registry.npmjs.org/tinybench/-/tinybench-2.9.0.tgz", @@ -2752,6 +3372,52 @@ "node": ">=14.0.0" } }, + "node_modules/tldts": { + "version": "6.1.86", + "resolved": "https://registry.npmjs.org/tldts/-/tldts-6.1.86.tgz", + "integrity": "sha512-WMi/OQ2axVTf/ykqCQgXiIct+mSQDFdH2fkwhPwgEwvJ1kSzZRiinb0zF2Xb8u4+OqPChmyI6MEu4EezNJz+FQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "tldts-core": "^6.1.86" + }, + "bin": { + "tldts": "bin/cli.js" + } + }, + "node_modules/tldts-core": { + "version": "6.1.86", + "resolved": "https://registry.npmjs.org/tldts-core/-/tldts-core-6.1.86.tgz", + "integrity": "sha512-Je6p7pkk+KMzMv2XXKmAE3McmolOQFdxkKw0R8EYNr7sELW46JqnNeTX8ybPiQgvg1ymCoF8LXs5fzFaZvJPTA==", + "dev": true, + "license": "MIT" + }, + "node_modules/tough-cookie": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-5.1.2.tgz", + "integrity": "sha512-FVDYdxtnj0G6Qm/DhNPSb8Ju59ULcup3tuJxkFb5K8Bv2pUXILbf0xZWU8PX8Ov19OXljbUyveOFwRMwkXzO+A==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "tldts": "^6.1.32" + }, + "engines": { + "node": ">=16" + } + }, + "node_modules/tr46": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-5.1.1.tgz", + "integrity": "sha512-hdF5ZgjTqgAntKkklYw0R03MG2x/bSzTtkxmIRw/sTNV8YXsCJ1tfLAX23lhxhHJlEf3CRCOCGGWw3vI3GaSPw==", + "dev": true, + "license": "MIT", + "dependencies": { + "punycode": "^2.3.1" + }, + "engines": { + "node": ">=18" + } + }, "node_modules/type-check": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", @@ -3987,6 +4653,67 @@ } } }, + "node_modules/w3c-xmlserializer": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/w3c-xmlserializer/-/w3c-xmlserializer-5.0.0.tgz", + "integrity": "sha512-o8qghlI8NZHU1lLPrpi2+Uq7abh4GGPpYANlalzWxyWteJOCsr/P+oPBA49TOLu5FTZO4d3F9MnWJfiMo4BkmA==", + "dev": true, + "license": "MIT", + "dependencies": { + "xml-name-validator": "^5.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/webidl-conversions": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz", + "integrity": "sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==", + "dev": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">=12" + } + }, + "node_modules/whatwg-encoding": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/whatwg-encoding/-/whatwg-encoding-3.1.1.tgz", + "integrity": "sha512-6qN4hJdMwfYBtE3YBTTHhoeuUrDBPZmbQaxWAqSALV/MeEnR5z1xd8UKud2RAkFoPkmB+hli1TZSnyi84xz1vQ==", + "deprecated": "Use @exodus/bytes instead for a more spec-conformant and faster implementation", + "dev": true, + "license": "MIT", + "dependencies": { + "iconv-lite": "0.6.3" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/whatwg-mimetype": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-4.0.0.tgz", + "integrity": "sha512-QaKxh0eNIi2mE9p2vEdzfagOKHCcj1pJ56EEHGQOVxp8r9/iszLUUV7v89x9O1p/T+NlTM5W7jW6+cz4Fq1YVg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/whatwg-url": { + "version": "14.2.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-14.2.0.tgz", + "integrity": "sha512-De72GdQZzNTUBBChsXueQUnPKDkg/5A5zp7pFDuQAj5UFoENpiACU0wlCvzpAGnTkj++ihpKwKyYewn/XNUbKw==", + "dev": true, + "license": "MIT", + "dependencies": { + "tr46": "^5.1.0", + "webidl-conversions": "^7.0.0" + }, + "engines": { + "node": ">=18" + } + }, "node_modules/which": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", @@ -4030,6 +4757,45 @@ "node": ">=0.10.0" } }, + "node_modules/ws": { + "version": "8.21.3", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.21.3.tgz", + "integrity": "sha512-201TZ/kPWxoPr/OKWjquZR1SWKXcvxdH+e1xrx89b3YbmzLMFCLfnaG1HFIgWzJOEWZ7MvpK++odZufgYR50Rw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, + "node_modules/xml-name-validator": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/xml-name-validator/-/xml-name-validator-5.0.0.tgz", + "integrity": "sha512-EvGK8EJ3DhaHfbRlETOWAS5pO9MZITeauHKJyb8wyajUfQUenkIg2MvLDTZ4T/TgIcm3HU0TFBgWWboAZ30UHg==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=18" + } + }, + "node_modules/xmlchars": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/xmlchars/-/xmlchars-2.2.0.tgz", + "integrity": "sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==", + "dev": true, + "license": "MIT" + }, "node_modules/yocto-queue": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", diff --git a/frontend/package.json b/frontend/package.json index f287f9e..105373c 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -3,6 +3,6 @@ "private": true, "type": "module", "scripts": {"dev":"vite","preview":"vite preview","check":"svelte-check --tsconfig ./tsconfig.app.json && tsc -p tsconfig.node.json --noEmit","build":"npm run check && vite build","test":"vitest run","test:e2e":"playwright test","test:e2e:local":"set LOCAL_V2_E2E=1&& playwright test tests/e2e/local-backend.spec.ts","lint":"eslint .","stage":"node scripts/stage-preview.mjs","boundary":"node scripts/check-boundaries.mjs"}, - "devDependencies": {"@playwright/test":"^1.49.1","@sveltejs/vite-plugin-svelte":"^6.2.1","@types/node":"^22.10.2","eslint":"^9.17.0","svelte":"^5.19.0","svelte-check":"^4.1.4","typescript":"^5.7.2","vite":"^6.0.7","vitest":"^2.1.8"}, + "devDependencies": {"@playwright/test":"^1.49.1","@sveltejs/vite-plugin-svelte":"^6.2.1","@types/node":"^22.10.2","eslint":"^9.17.0","jsdom":"^25.0.1","svelte":"^5.19.0","svelte-check":"^4.1.4","typescript":"^5.7.2","vite":"^6.0.7","vitest":"^2.1.8"}, "dependencies": {"@axe-core/playwright":"^4.10.2","@types/leaflet":"^1.9.15","leaflet":"^1.9.4","zod":"^3.24.1"} } From 95eca2a443bcd078564468044d546346bd8323e4 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sun, 13 Sep 2026 20:27:37 -0700 Subject: [PATCH 038/311] test(safety): add fail-closed restriction replay gate --- .../maintenance/restriction-ledger-gate.py | 92 +++++++++++++++++++ pipeline/tests/e2e/backup-restore.ps1 | 8 ++ .../restriction-ledger-current-snapshot.json | 6 ++ .../e2e/restriction-ledger-old-snapshot.json | 4 + pipeline/tests/e2e/restriction-ledger.json | 7 ++ pipeline/tests/e2e/run-frontend-local.py | 26 ++++++ .../tests/test_restriction_ledger_gate.py | 58 ++++++++++++ 7 files changed, 201 insertions(+) create mode 100644 pipeline/scripts/maintenance/restriction-ledger-gate.py create mode 100644 pipeline/tests/e2e/restriction-ledger-current-snapshot.json create mode 100644 pipeline/tests/e2e/restriction-ledger-old-snapshot.json create mode 100644 pipeline/tests/e2e/restriction-ledger.json create mode 100644 pipeline/tests/e2e/run-frontend-local.py create mode 100644 pipeline/tests/test_restriction_ledger_gate.py diff --git a/pipeline/scripts/maintenance/restriction-ledger-gate.py b/pipeline/scripts/maintenance/restriction-ledger-gate.py new file mode 100644 index 0000000..e1506c2 --- /dev/null +++ b/pipeline/scripts/maintenance/restriction-ledger-gate.py @@ -0,0 +1,92 @@ +"""Fail-closed contract for replaying an external current-restriction ledger. + +This module deliberately does not select a ledger operator, retention policy, +or production activation mechanism. It verifies a supplied, already-authorized +ledger snapshot before a restored database may be served. +""" +from __future__ import annotations + +import argparse +import json +from pathlib import Path +from typing import Any + + +class RestrictionLedgerError(ValueError): + """The ledger is missing, malformed, stale, or not fully applied.""" + + +def load_ledger(path: str | Path) -> dict[str, Any]: + ledger_path = Path(path) + if not ledger_path.is_file(): + raise RestrictionLedgerError("restriction ledger is unavailable") + try: + ledger = json.loads(ledger_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise RestrictionLedgerError("restriction ledger cannot be read") from exc + if not isinstance(ledger, dict) or ledger.get("schema_version") != 1: + raise RestrictionLedgerError("restriction ledger schema is unsupported") + if not isinstance(ledger.get("revision"), str) or not ledger["revision"]: + raise RestrictionLedgerError("restriction ledger revision is missing") + restrictions = ledger.get("active_restrictions") + if not isinstance(restrictions, list) or any(not isinstance(item, dict) for item in restrictions): + raise RestrictionLedgerError("restriction ledger restrictions are invalid") + return ledger + + +def verify_replayed_restrictions(snapshot: dict[str, Any], ledger: dict[str, Any]) -> None: + """Fail closed unless the restored DB reports every current restriction. + + Comparison uses opaque source/record keys only; restricted payloads are not + copied into errors. The service must remain stopped when this raises. + """ + if snapshot.get("ledger_revision") != ledger["revision"]: + raise RestrictionLedgerError("restored restriction revision is not current") + applied = snapshot.get("active_restrictions") + expected = ledger["active_restrictions"] + if not isinstance(applied, list): + raise RestrictionLedgerError("restored restriction state is unavailable") + + def key(item: dict[str, Any]) -> tuple[str, str, str]: + values = tuple(item.get(field) for field in ("source_id", "source_record_key", "scope")) + if any(not isinstance(value, str) or not value for value in values): + raise RestrictionLedgerError("restriction reference is incomplete") + return values + + expected_keys = {key(item) for item in expected} + applied_keys = {key(item) for item in applied} + if expected_keys != applied_keys: + raise RestrictionLedgerError("restored restrictions do not match current ledger") + + +def pre_service_gate(ledger_path: str | Path, restored_snapshot: dict[str, Any]) -> None: + """Verify a restored snapshot before a local/test service may start.""" + verify_replayed_restrictions(restored_snapshot, load_ledger(ledger_path)) + + +def verify_v1_v2_crosswalk(rows: list[dict[str, Any]]) -> dict[str, Any]: + """Return deterministic mappings; never guess when a key is ambiguous.""" + mapped: dict[str, str] = {} + unresolved: list[dict[str, str]] = [] + for row in rows: + v1_id, v2_id = row.get("v1_id"), row.get("v2_id") + if not isinstance(v1_id, str) or not isinstance(v2_id, str) or not v1_id or not v2_id: + unresolved.append({"reason": "missing_identifier"}) + continue + prior = mapped.get(v1_id) + if prior is not None and prior != v2_id: + mapped.pop(v1_id) + unresolved.append({"v1_id": v1_id, "reason": "ambiguous_mapping"}) + continue + mapped[v1_id] = v2_id + return {"mapped": mapped, "unresolved": unresolved} + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--ledger", required=True) + parser.add_argument("--snapshot", required=True) + args = parser.parse_args() + snapshot = json.loads(Path(args.snapshot).read_text(encoding="utf-8")) + pre_service_gate(args.ledger, snapshot) + print("PASS: current restriction ledger verified before service start") diff --git a/pipeline/tests/e2e/backup-restore.ps1 b/pipeline/tests/e2e/backup-restore.ps1 index e086b6e..a81b0f4 100644 --- a/pipeline/tests/e2e/backup-restore.ps1 +++ b/pipeline/tests/e2e/backup-restore.ps1 @@ -10,6 +10,10 @@ $dump = Join-Path ([IO.Path]::GetTempPath()) "$project.dump" $migrationFile = Join-Path ([IO.Path]::GetTempPath()) "$project-migrations.sql" $seedFile = Join-Path $PSScriptRoot 'backup_restore_seed.sql' $suppressionFile = Join-Path $PSScriptRoot 'backup_restore_current_suppression.sql' +$ledgerGate = Join-Path $root 'pipeline\scripts\maintenance\restriction-ledger-gate.py' +$ledgerFile = Join-Path $PSScriptRoot 'restriction-ledger.json' +$oldSnapshot = Join-Path $PSScriptRoot 'restriction-ledger-old-snapshot.json' +$currentSnapshot = Join-Path $PSScriptRoot 'restriction-ledger-current-snapshot.json' function Invoke-FixtureSql([string]$path) { Get-Content -LiteralPath $path -Raw | & docker compose @composeArgs exec -T postgres psql -1 -v ON_ERROR_STOP=1 -U uec -d uec @@ -91,11 +95,15 @@ try { if ($LASTEXITCODE -ne 0) { throw "Restore failed (exit $LASTEXITCODE)." } Assert-Snapshot 'old backup restored, before replay' '1,1,0,0,0,1,1' if (Test-SyntheticServiceGate) { throw 'Unsafe drill gate accepted an old backup before current restriction replay.' } + & python $ledgerGate --ledger $ledgerFile --snapshot $oldSnapshot *> $null + if ($LASTEXITCODE -eq 0) { throw 'Pre-service ledger gate accepted an old restriction snapshot.' } Write-Host '[backup-restore] PASS: synthetic pre-service gate rejects the old backup before replay.' Invoke-FixtureSql $suppressionFile Assert-Snapshot 'current restriction replayed' '1,1,1,1,1,0,0' if (-not (Test-SyntheticServiceGate)) { throw 'Synthetic pre-service gate rejected the replayed current restriction.' } + & python $ledgerGate --ledger $ledgerFile --snapshot $currentSnapshot + if ($LASTEXITCODE -ne 0) { throw 'Pre-service ledger gate rejected the current restriction replay.' } Write-Host 'PASS: synthetic old-backup restore remains gated until current restriction is replayed and both public projections exclude it.' Write-Host 'TEST ONLY: production still needs an independent durable restriction ledger and an enforced service-start gate.' } finally { diff --git a/pipeline/tests/e2e/restriction-ledger-current-snapshot.json b/pipeline/tests/e2e/restriction-ledger-current-snapshot.json new file mode 100644 index 0000000..4a13141 --- /dev/null +++ b/pipeline/tests/e2e/restriction-ledger-current-snapshot.json @@ -0,0 +1,6 @@ +{ + "ledger_revision": "synthetic-r2", + "active_restrictions": [ + {"source_id": "e2e.backup", "source_record_key": "restricted", "scope": "whole_record"} + ] +} diff --git a/pipeline/tests/e2e/restriction-ledger-old-snapshot.json b/pipeline/tests/e2e/restriction-ledger-old-snapshot.json new file mode 100644 index 0000000..38a212f --- /dev/null +++ b/pipeline/tests/e2e/restriction-ledger-old-snapshot.json @@ -0,0 +1,4 @@ +{ + "ledger_revision": "synthetic-r1", + "active_restrictions": [] +} diff --git a/pipeline/tests/e2e/restriction-ledger.json b/pipeline/tests/e2e/restriction-ledger.json new file mode 100644 index 0000000..9e9e575 --- /dev/null +++ b/pipeline/tests/e2e/restriction-ledger.json @@ -0,0 +1,7 @@ +{ + "schema_version": 1, + "revision": "synthetic-r2", + "active_restrictions": [ + {"source_id": "e2e.backup", "source_record_key": "restricted", "scope": "whole_record"} + ] +} diff --git a/pipeline/tests/e2e/run-frontend-local.py b/pipeline/tests/e2e/run-frontend-local.py new file mode 100644 index 0000000..7112c32 --- /dev/null +++ b/pipeline/tests/e2e/run-frontend-local.py @@ -0,0 +1,26 @@ +"""Hold one disposable synthetic API for the frontend real-backend E2E.""" +from __future__ import annotations + +import os +import sys + +from fixture import E2EEnvironment + + +def main() -> int: + env = E2EEnvironment().start() + try: + env.seed_official_scenario() + url = f"http://127.0.0.1:{env.api_port}" + print(f"UEC_E2E_API_URL={url}", flush=True) + print(f"UEC_E2E_PROJECT={env.project}", flush=True) + print("Synthetic backend is ready; press Enter to stop it.", flush=True) + sys.stdin.readline() + return 0 + finally: + env.stop() + + +if __name__ == "__main__": + os.environ.setdefault("UEC_RUN_E2E", "1") + raise SystemExit(main()) diff --git a/pipeline/tests/test_restriction_ledger_gate.py b/pipeline/tests/test_restriction_ledger_gate.py new file mode 100644 index 0000000..3a2f1af --- /dev/null +++ b/pipeline/tests/test_restriction_ledger_gate.py @@ -0,0 +1,58 @@ +import importlib.util +import json +import tempfile +import unittest +from pathlib import Path + +SPEC = importlib.util.spec_from_file_location( + "restriction_ledger_gate", + Path(__file__).parents[1] / "scripts" / "maintenance" / "restriction-ledger-gate.py", +) +MODULE = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(MODULE) + + +class RestrictionLedgerGateTests(unittest.TestCase): + def setUp(self): + self.ledger = {"schema_version": 1, "revision": "r2", "active_restrictions": [ + {"source_id": "synthetic", "source_record_key": "private", "scope": "whole_record"} + ]} + self.snapshot = {"ledger_revision": "r2", "active_restrictions": list(self.ledger["active_restrictions"])} + + def test_current_restrictions_must_match_before_service(self): + with tempfile.TemporaryDirectory() as directory: + ledger_path = Path(directory) / "ledger.json" + ledger_path.write_text(json.dumps(self.ledger), encoding="utf-8") + MODULE.pre_service_gate(ledger_path, self.snapshot) + with self.assertRaises(MODULE.RestrictionLedgerError): + MODULE.pre_service_gate(ledger_path, {"ledger_revision": "r1", "active_restrictions": []}) + + def test_old_restore_cannot_start_without_current_replay(self): + with tempfile.TemporaryDirectory() as directory: + ledger_path = Path(directory) / "ledger.json" + ledger_path.write_text(json.dumps(self.ledger), encoding="utf-8") + old_restore = {"ledger_revision": "r1", "active_restrictions": []} + with self.assertRaises(MODULE.RestrictionLedgerError): + MODULE.pre_service_gate(ledger_path, old_restore) + + def test_missing_or_invalid_ledger_fails_closed(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "ledger.json" + with self.assertRaises(MODULE.RestrictionLedgerError): + MODULE.load_ledger(path) + path.write_text(json.dumps({"schema_version": 1}), encoding="utf-8") + with self.assertRaises(MODULE.RestrictionLedgerError): + MODULE.load_ledger(path) + + def test_crosswalk_maps_only_unique_identifiers(self): + result = MODULE.verify_v1_v2_crosswalk([ + {"v1_id": "one", "v2_id": "A"}, + {"v1_id": "two", "v2_id": "B"}, + {"v1_id": "two", "v2_id": "C"}, + ]) + self.assertEqual(result["mapped"], {"one": "A"}) + self.assertEqual(result["unresolved"], [{"v1_id": "two", "reason": "ambiguous_mapping"}]) + + +if __name__ == "__main__": + unittest.main() From 5f4447dc69b92639ce3faeb56ae13a4dd724098b Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sun, 13 Sep 2026 20:27:50 -0700 Subject: [PATCH 039/311] refactor(pipeline): isolate Denmark source stages --- pipeline/run-denmark-pipeline.py | 17 +- .../scripts/stages/acquire-denmark-smiley.py | 200 +----------------- .../stages/catalog-denmark-categories.py | 49 +---- pipeline/scripts/stages/classify-denmark.py | 77 +------ .../stages/fetch-denmark-city-references.py | 71 +------ .../scripts/stages/geocode-denmark-dawa.py | 192 +---------------- pipeline/scripts/stages/import-denmark.py | 137 +----------- .../scripts/stages/load-city-references.py | 35 +-- .../stages/normalize-denmark-smiley.py | 131 +----------- .../scripts/stages/parse-denmark-smiley.py | 111 +--------- pipeline/scripts/stages/validate-denmark.py | 92 +------- pipeline/sources/denmark/README.md | 18 ++ pipeline/sources/denmark/__init__.py | 6 + pipeline/sources/denmark/cli.py | 21 ++ .../sources/denmark/run-denmark-pipeline.py | 12 ++ pipeline/sources/denmark/stages/README.md | 7 + pipeline/sources/denmark/stages/__init__.py | 1 + .../denmark/stages/acquire-denmark-smiley.py | 198 +++++++++++++++++ .../stages/catalog-denmark-categories.py | 47 ++++ .../denmark/stages/classify-denmark.py | 75 +++++++ .../stages/fetch-denmark-city-references.py | 69 ++++++ .../denmark/stages/geocode-denmark-dawa.py | 190 +++++++++++++++++ .../sources/denmark/stages/import-denmark.py | 135 ++++++++++++ .../denmark/stages/load-city-references.py | 33 +++ .../stages/normalize-denmark-smiley.py | 129 +++++++++++ .../denmark/stages/parse-denmark-smiley.py | 109 ++++++++++ .../denmark/stages/validate-denmark.py | 90 ++++++++ pipeline/tests/test_denmark_entrypoints.py | 33 +++ 28 files changed, 1212 insertions(+), 1073 deletions(-) create mode 100644 pipeline/sources/denmark/README.md create mode 100644 pipeline/sources/denmark/__init__.py create mode 100644 pipeline/sources/denmark/cli.py create mode 100644 pipeline/sources/denmark/run-denmark-pipeline.py create mode 100644 pipeline/sources/denmark/stages/README.md create mode 100644 pipeline/sources/denmark/stages/__init__.py create mode 100644 pipeline/sources/denmark/stages/acquire-denmark-smiley.py create mode 100644 pipeline/sources/denmark/stages/catalog-denmark-categories.py create mode 100644 pipeline/sources/denmark/stages/classify-denmark.py create mode 100644 pipeline/sources/denmark/stages/fetch-denmark-city-references.py create mode 100644 pipeline/sources/denmark/stages/geocode-denmark-dawa.py create mode 100644 pipeline/sources/denmark/stages/import-denmark.py create mode 100644 pipeline/sources/denmark/stages/load-city-references.py create mode 100644 pipeline/sources/denmark/stages/normalize-denmark-smiley.py create mode 100644 pipeline/sources/denmark/stages/parse-denmark-smiley.py create mode 100644 pipeline/sources/denmark/stages/validate-denmark.py create mode 100644 pipeline/tests/test_denmark_entrypoints.py diff --git a/pipeline/run-denmark-pipeline.py b/pipeline/run-denmark-pipeline.py index 9029abd..6355f5b 100644 --- a/pipeline/run-denmark-pipeline.py +++ b/pipeline/run-denmark-pipeline.py @@ -16,7 +16,8 @@ LOGGER = logging.getLogger("uec.denmark.pipeline") ROOT = Path(__file__).resolve().parent.parent -SCRIPTS = ROOT / "pipeline" / "scripts" / "stages" +SHARED_STAGES = ROOT / "pipeline" / "scripts" / "stages" +DENMARK_STAGES = ROOT / "pipeline" / "sources" / "denmark" / "stages" def utc_now() -> str: @@ -97,7 +98,7 @@ def main() -> int: acquisition_args = ["--fetch", "--output-root", str(raw_output_root), "--terms-review", str(terms_review_path), "--run-id", acquisition_run_id] if args.source_url: acquisition_args.extend(["--url", args.source_url]) - run_stage("acquire", SCRIPTS / "acquire-denmark-smiley.py", acquisition_args) + run_stage("acquire", DENMARK_STAGES / "acquire-denmark-smiley.py", acquisition_args) acquisition_root = raw_output_root / "dk.smiley" input_path = (acquisition_root / acquisition_run_id / "Smileydata.xml").resolve() acquisition_metadata = json.loads((input_path.parent / "acquisition-metadata.json").read_text(encoding="utf-8")) @@ -123,19 +124,19 @@ def main() -> int: parse_args = [str(input_path), "--output-dir", str(parse_dir)] if acquired_source_url and acquired_source_url != "unknown": parse_args.extend(["--source-url", acquired_source_url]) - run_stage("parse", SCRIPTS / "parse-denmark-smiley.py", parse_args) - run_stage("normalize", SCRIPTS / "normalize-denmark-smiley.py", [str(parse_dir / "parsed-rows.jsonl"), "--output-dir", str(normalize_dir)]) - run_stage("classify", SCRIPTS / "classify-denmark.py", [str(normalize_dir / "normalized-records.jsonl"), "--rules", str(args.rules.resolve()), "--output-dir", str(classify_dir)]) + run_stage("parse", DENMARK_STAGES / "parse-denmark-smiley.py", parse_args) + run_stage("normalize", DENMARK_STAGES / "normalize-denmark-smiley.py", [str(parse_dir / "parsed-rows.jsonl"), "--output-dir", str(normalize_dir)]) + run_stage("classify", DENMARK_STAGES / "classify-denmark.py", [str(normalize_dir / "normalized-records.jsonl"), "--rules", str(args.rules.resolve()), "--output-dir", str(classify_dir)]) validation_args = [str(classify_dir / "classified-records.jsonl"), "--output-dir", str(validate_dir)] if args.expected_rows is not None: validation_args.extend(["--expected-rows", str(args.expected_rows)]) - run_stage("validate", SCRIPTS / "validate-denmark.py", validation_args) - run_stage("geocode_queue", SCRIPTS / "create-geocode-queue.py", [str(classify_dir / "classified-records.jsonl"), "--output-dir", str(geocode_dir)]) + run_stage("validate", DENMARK_STAGES / "validate-denmark.py", validation_args) + run_stage("geocode_queue", SHARED_STAGES / "create-geocode-queue.py", [str(classify_dir / "classified-records.jsonl"), "--output-dir", str(geocode_dir)]) if args.geocode_limit is not None: geocode_args = [str(geocode_dir / "geocode-queue.jsonl"), "--output", str(run_dir / "06-geocode-results.jsonl"), "--limit", str(args.geocode_limit), "--delay", str(args.geocode_delay), "--provider-config", str(args.geocode_provider_config.resolve()), "--terms-review", str(args.geocode_terms_review.resolve()), "--network"] if args.geocode_suppression_keys: geocode_args.extend(["--suppression-keys", str(args.geocode_suppression_keys.resolve())]) - run_stage("geocode_dawa", SCRIPTS / "geocode-denmark-dawa.py", geocode_args) + run_stage("geocode_dawa", DENMARK_STAGES / "geocode-denmark-dawa.py", geocode_args) manifest = artifact_manifest(run_dir, input_path, started_at, utc_now()) LOGGER.info("pipeline=denmark-smiley status=success manifest=%s", manifest) except subprocess.CalledProcessError as error: diff --git a/pipeline/scripts/stages/acquire-denmark-smiley.py b/pipeline/scripts/stages/acquire-denmark-smiley.py index 8320509..ba866b5 100644 --- a/pipeline/scripts/stages/acquire-denmark-smiley.py +++ b/pipeline/scripts/stages/acquire-denmark-smiley.py @@ -1,198 +1,4 @@ -#!/usr/bin/env python3 -"""Archive a Denmark Find Smiley artifact without importing or publishing it.""" - -from __future__ import annotations - -import argparse -import hashlib -import json -import os -import sys -import tempfile -import urllib.error -import urllib.request -import uuid -from datetime import datetime, timezone +"""Compatibility shim for the Denmark source-owned stage.""" from pathlib import Path - - -SOURCE_ID = "dk.smiley" -ADAPTER_VERSION = "denmark-smiley-acquisition-v1" -DEFAULT_URL = "https://pub.fvst.dk/publikationer/Smileydata.xml" -DEFAULT_MAX_BYTES = 128 * 1024 * 1024 -APPROVED_DECISION = "approved" -SAFE_CONTENT_TYPES = {"application/xml", "text/xml", "application/octet-stream"} - - -class AcquisitionError(ValueError): - """An acquisition was not authorized, valid, or complete.""" - - -def utc_now() -> str: - return datetime.now(timezone.utc).isoformat(timespec="seconds").replace("+00:00", "Z") - - -def default_run_id() -> str: - return datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + "-" + uuid.uuid4().hex[:8] - - -def require_terms_review(path: Path) -> dict: - try: - review = json.loads(path.read_text(encoding="utf-8")) - except (OSError, json.JSONDecodeError) as error: - raise AcquisitionError(f"terms review cannot be read: {error}") from error - required = {"reviewer", "reference", "reviewed_at", "decision", "notes"} - if not isinstance(review, dict) or required - review.keys(): - raise AcquisitionError("terms review requires reviewer, reference, reviewed_at, decision, and notes") - for field in required: - if not isinstance(review[field], str) or not review[field].strip(): - raise AcquisitionError(f"terms review {field} must be a non-empty string") - try: - datetime.fromisoformat(review["reviewed_at"].replace("Z", "+00:00")) - except ValueError as error: - raise AcquisitionError("terms review reviewed_at must be ISO-8601") from error - if review["decision"] != APPROVED_DECISION: - raise AcquisitionError(f"terms review decision must be {APPROVED_DECISION!r}") - return {field: review[field] for field in sorted(required)} - - -def selected_headers(headers) -> dict[str, str]: - """Persist response metadata relevant to reproducibility, not credentials.""" - return { - name: headers[name] - for name in ("Content-Type", "Content-Length", "ETag", "Last-Modified") - if headers.get(name) is not None - } - - -def content_type_is_safe(headers) -> bool: - raw = headers.get("Content-Type") - return raw is None or raw.split(";", 1)[0].strip().lower() in SAFE_CONTENT_TYPES - - -def archive_stream(stream, artifact_path: Path, *, max_bytes: int) -> tuple[str, int]: - artifact_path.parent.mkdir(parents=True, exist_ok=True) - temp_path: Path | None = None - digest = hashlib.sha256() - total = 0 - try: - with tempfile.NamedTemporaryFile("wb", delete=False, dir=artifact_path.parent, prefix=".download-", suffix=".part") as handle: - temp_path = Path(handle.name) - while chunk := stream.read(1024 * 1024): - total += len(chunk) - if total > max_bytes: - raise AcquisitionError(f"download exceeds max_bytes={max_bytes}") - digest.update(chunk) - handle.write(chunk) - os.replace(temp_path, artifact_path) - return digest.hexdigest(), total - except Exception: - if temp_path is not None: - temp_path.unlink(missing_ok=True) - raise - - -def write_metadata(path: Path, metadata: dict) -> None: - path.write_text(json.dumps(metadata, ensure_ascii=False, indent=2, sort_keys=True) + "\n", encoding="utf-8") - - -def archive_local_file(local_file: Path, output_root: Path, *, run_id: str, retrieved_at: str | None = None) -> dict: - local_file = local_file.resolve() - if not local_file.is_file(): - raise AcquisitionError(f"local file does not exist: {local_file}") - run_dir = output_root / SOURCE_ID / run_id - artifact_path = run_dir / "Smileydata.xml" - retrieved_at = retrieved_at or utc_now() - with local_file.open("rb") as stream: - sha256, byte_size = archive_stream(stream, artifact_path, max_bytes=DEFAULT_MAX_BYTES) - metadata = { - "acquisition_method": "local_file", - "adapter_version": ADAPTER_VERSION, - "artifact": artifact_path.name, - "byte_size": byte_size, - "config_version": ADAPTER_VERSION, - "final_url": "unknown", - "publication_metadata": {}, - "requested_url": "unknown", - "response_headers": {}, - "retrieved_at_utc": retrieved_at, - "run_id": run_id, - "sha256": sha256, - "source_id": SOURCE_ID, - "source_local_path": str(local_file), - "terms_review": "not_required_for_local_file", - } - write_metadata(run_dir / "acquisition-metadata.json", metadata) - return metadata - - -def fetch(url: str, output_root: Path, *, run_id: str, terms_review_path: Path | None, timeout_seconds: float, max_bytes: int) -> dict: - if terms_review_path is None: - raise AcquisitionError("--terms-review is required with --fetch") - if timeout_seconds <= 0 or max_bytes <= 0: - raise AcquisitionError("timeout_seconds and max_bytes must be positive") - terms_review = require_terms_review(terms_review_path) - run_dir = output_root / SOURCE_ID / run_id - artifact_path = run_dir / "Smileydata.xml" - requested_at = utc_now() - try: - request = urllib.request.Request(url, headers={"User-Agent": "UntilEveryCage/controlled-acquisition"}) - with urllib.request.urlopen(request, timeout=timeout_seconds) as response: - if not 200 <= response.status < 300: - raise AcquisitionError(f"source returned HTTP {response.status}") - if not content_type_is_safe(response.headers): - raise AcquisitionError(f"unexpected content type: {response.headers.get('Content-Type')}") - sha256, byte_size = archive_stream(response, artifact_path, max_bytes=max_bytes) - headers = selected_headers(response.headers) - final_url = response.geturl() - except urllib.error.HTTPError as error: - raise AcquisitionError(f"source returned HTTP {error.code}") from error - except urllib.error.URLError as error: - raise AcquisitionError(f"network error: {error.reason}") from error - metadata = { - "acquisition_method": "network_fetch", - "adapter_version": ADAPTER_VERSION, - "artifact": artifact_path.name, - "byte_size": byte_size, - "config_version": ADAPTER_VERSION, - "final_url": final_url, - "publication_metadata": {key: headers[key] for key in ("ETag", "Last-Modified") if key in headers}, - "requested_at_utc": requested_at, - "requested_url": url, - "response_headers": headers, - "retrieved_at_utc": utc_now(), - "run_id": run_id, - "sha256": sha256, - "source_id": SOURCE_ID, - "terms_review": terms_review, - } - write_metadata(run_dir / "acquisition-metadata.json", metadata) - return metadata - - -def main() -> int: - parser = argparse.ArgumentParser(description=__doc__) - mode = parser.add_mutually_exclusive_group(required=True) - mode.add_argument("--fetch", action="store_true", help="Fetch the registry URL; requires a reviewed terms file.") - mode.add_argument("--local-file", type=Path, help="Archive a local/synthetic XML file without network access.") - parser.add_argument("--url", default=DEFAULT_URL, help="Requested URL for --fetch.") - parser.add_argument("--terms-review", type=Path, help="Approved JSON terms-review record; required for --fetch.") - parser.add_argument("--output-root", type=Path, default=Path("data/raw")) - parser.add_argument("--run-id", default=default_run_id()) - parser.add_argument("--timeout-seconds", type=float, default=30.0) - parser.add_argument("--max-bytes", type=int, default=DEFAULT_MAX_BYTES) - args = parser.parse_args() - try: - if args.fetch: - metadata = fetch(args.url, args.output_root, run_id=args.run_id, terms_review_path=args.terms_review, timeout_seconds=args.timeout_seconds, max_bytes=args.max_bytes) - else: - metadata = archive_local_file(args.local_file, args.output_root, run_id=args.run_id) - except AcquisitionError as error: - print(json.dumps({"status": "failed", "error": str(error)}, sort_keys=True), file=sys.stderr) - return 2 - print(json.dumps({"status": "archived", "metadata": metadata}, ensure_ascii=False, sort_keys=True)) - return 0 - - -if __name__ == "__main__": - sys.exit(main()) +_TARGET = Path(__file__).resolve().parents[2] / "sources/denmark/stages/acquire-denmark-smiley.py" +exec(compile(_TARGET.read_text(encoding="utf-8"), str(_TARGET), "exec"), globals(), globals()) diff --git a/pipeline/scripts/stages/catalog-denmark-categories.py b/pipeline/scripts/stages/catalog-denmark-categories.py index d722937..2b1fae4 100644 --- a/pipeline/scripts/stages/catalog-denmark-categories.py +++ b/pipeline/scripts/stages/catalog-denmark-categories.py @@ -1,47 +1,4 @@ -#!/usr/bin/env python3 -"""Build a complete category catalog from normalized Denmark staging data.""" - -import argparse -import collections -import json +"""Compatibility shim for the Denmark source-owned stage.""" from pathlib import Path - - -def build_catalog(input_path: Path, output_path: Path) -> None: - categories = collections.defaultdict(lambda: {"count": 0, "examples": []}) - with input_path.open(encoding="utf-8") as source: - for line in source: - if not line.strip(): - continue - record = json.loads(line) - activity = record["activity"] - key = ( - activity.get("code"), - activity.get("label"), - activity.get("category"), - ) - item = categories[key] - item["count"] += 1 - if len(item["examples"]) < 3: - item["examples"].append({ - "source_record_key": record.get("source_record_key"), - "name": record.get("name"), - "source_url": record.get("source_url"), - }) - - rows = [ - {"industry_code": key[0], "industry_label": key[1], "category_label": key[2], **value} - for key, value in categories.items() - ] - rows.sort(key=lambda row: (-row["count"], row["industry_code"] or "", row["category_label"] or "")) - output_path.parent.mkdir(parents=True, exist_ok=True) - output_path.write_text(json.dumps({"status": "success", "categories": rows}, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") - print(f"Catalogued {len(rows)} distinct category combinations from {sum(row['count'] for row in rows)} records") - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument("input", type=Path) - parser.add_argument("--output", type=Path, required=True) - args = parser.parse_args() - build_catalog(args.input, args.output) +_TARGET = Path(__file__).resolve().parents[2] / "sources/denmark/stages/catalog-denmark-categories.py" +exec(compile(_TARGET.read_text(encoding="utf-8"), str(_TARGET), "exec"), globals(), globals()) diff --git a/pipeline/scripts/stages/classify-denmark.py b/pipeline/scripts/stages/classify-denmark.py index 3e666f0..18ca494 100644 --- a/pipeline/scripts/stages/classify-denmark.py +++ b/pipeline/scripts/stages/classify-denmark.py @@ -1,75 +1,4 @@ -#!/usr/bin/env python3 -"""Apply an explicit, visibility-aware Denmark classification ruleset.""" - -import argparse -import collections -import json -import logging +"""Compatibility shim for the Denmark source-owned stage.""" from pathlib import Path - -LOGGER = logging.getLogger("uec.denmark.classify") - - -def load_rules(path: Path) -> dict: - return json.loads(path.read_text(encoding="utf-8")) - - -def classify_record(record: dict, ruleset: dict) -> dict: - code = record.get("activity", {}).get("code") - decision = ruleset["fallback"] - for rule in ruleset["rules"]: - if code in rule["codes"]: - decision = rule - break - result = dict(record) - result["classification"] = { - "ruleset_id": ruleset["ruleset_id"], - "rule_id": decision["rule_id"], - "category": decision["classification"], - "review_status": decision["review_status"], - "default_visible": decision["default_visible"], - "optional_filter": decision.get("optional_filter"), - } - return result - - -def classify_file(input_path: Path, rules_path: Path, output_dir: Path, progress_every: int = 10000) -> Path: - ruleset = load_rules(rules_path) - output_dir.mkdir(parents=True, exist_ok=True) - output_path = output_dir / "classified-records.jsonl" - report_path = output_dir / "classification-report.json" - counts = collections.Counter() - LOGGER.info("stage=classify status=started input=%s ruleset=%s", input_path, ruleset["ruleset_id"]) - with input_path.open(encoding="utf-8") as source, output_path.open("w", encoding="utf-8", newline="\n") as output: - for count, line in enumerate(source, start=1): - if not line.strip(): - continue - record = classify_record(json.loads(line), ruleset) - classification = record["classification"] - counts.update([classification["category"], classification["review_status"]]) - output.write(json.dumps(record, ensure_ascii=False, sort_keys=True) + "\n") - if count % progress_every == 0: - LOGGER.info("stage=classify records=%d", count) - report = { - "status": "success", - "ruleset_id": ruleset["ruleset_id"], - "input_path": input_path.as_posix(), - "output_path": output_path.as_posix(), - "records_classified": sum(counts[key] for key in set(counts) if key in {r["classification"] for r in ruleset["rules"]} | {ruleset["fallback"]["classification"]}), - "counts_by_classification": {key: value for key, value in counts.items() if key not in {"approved", "review_required"}}, - "counts_by_review_status": {key: value for key, value in counts.items() if key in {"approved", "review_required"}}, - "ruleset_path": rules_path.as_posix(), - } - report_path.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") - LOGGER.info("stage=classify status=success records=%d report=%s", report["records_classified"], report_path) - return output_path - - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("input", type=Path) - parser.add_argument("--rules", type=Path, required=True) - parser.add_argument("--output-dir", type=Path, required=True) - args = parser.parse_args() - logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s", datefmt="%Y-%m-%dT%H:%M:%SZ") - classify_file(args.input, args.rules, args.output_dir) +_TARGET = Path(__file__).resolve().parents[2] / "sources/denmark/stages/classify-denmark.py" +exec(compile(_TARGET.read_text(encoding="utf-8"), str(_TARGET), "exec"), globals(), globals()) diff --git a/pipeline/scripts/stages/fetch-denmark-city-references.py b/pipeline/scripts/stages/fetch-denmark-city-references.py index 5f6f900..089ea44 100644 --- a/pipeline/scripts/stages/fetch-denmark-city-references.py +++ b/pipeline/scripts/stages/fetch-denmark-city-references.py @@ -1,69 +1,4 @@ -#!/usr/bin/env python3 -"""Fetch and normalize DAWA's official Denmark city reference points.""" - -import argparse -import hashlib -import json -import logging -import urllib.request -from datetime import datetime, timezone +"""Compatibility shim for the Denmark source-owned stage.""" from pathlib import Path - -LOGGER = logging.getLogger("uec.denmark.city-references") -URL = "https://api.dataforsyningen.dk/steder?hovedtype=Bebyggelse&undertype=by" -HEADERS = {"User-Agent": "UntilEveryCage/2.0 data-pipeline; contact: untileverycageproject@protonmail.com"} - - -def normalize(payload: object, retrieved_at: str, source_url: str) -> list[dict]: - """Convert DAWA records into stable, database-loadable reference records.""" - if not isinstance(payload, list): - raise ValueError("DAWA city response must be a JSON array") - records = [] - for item in payload: - center = item.get("visueltcenter") - if not isinstance(center, list) or len(center) != 2: - continue - records.append({ - "source_reference_id": item.get("id"), - "country_code": "DK", - "city_name": item.get("primærtnavn"), - "postal_code": None, - "reference_longitude": center[0], - "reference_latitude": center[1], - "reference_source": source_url, - "source_retrieved_at": retrieved_at, - }) - return records - - -def run(output_root: Path, url: str = URL) -> Path: - retrieved = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") - run_id = retrieved.replace("-", "").replace(":", "").replace(".", "") - run_dir = output_root / run_id - run_dir.mkdir(parents=True, exist_ok=False) - request = urllib.request.Request(url, headers=HEADERS) - LOGGER.info("stage=city-references status=fetching source=%s", url) - with urllib.request.urlopen(request, timeout=60) as response: - raw = response.read() - payload = json.loads(raw.decode("utf-8")) - artifact = run_dir / "steder.json" - artifact.write_bytes(raw) - records = normalize(payload, retrieved, url) - staging = run_dir / "city-reference-points.jsonl" - with staging.open("w", encoding="utf-8", newline="\n") as handle: - for record in records: - handle.write(json.dumps(record, ensure_ascii=False, sort_keys=True) + "\n") - metadata = {"source_url": url, "retrieved_at": retrieved, "artifact": artifact.name, - "sha256": hashlib.sha256(raw).hexdigest(), "bytes": len(raw), - "records": len(records), "status": "fetched"} - (run_dir / "metadata.json").write_text(json.dumps(metadata, indent=2) + "\n", encoding="utf-8") - LOGGER.info("stage=city-references status=complete records=%d artifact=%s", len(records), artifact) - return run_dir - - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--output-root", type=Path, default=Path("data/raw/denmark-city-references")) - args = parser.parse_args() - logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") - print(run(args.output_root)) +_TARGET = Path(__file__).resolve().parents[2] / "sources/denmark/stages/fetch-denmark-city-references.py" +exec(compile(_TARGET.read_text(encoding="utf-8"), str(_TARGET), "exec"), globals(), globals()) diff --git a/pipeline/scripts/stages/geocode-denmark-dawa.py b/pipeline/scripts/stages/geocode-denmark-dawa.py index 5ff4f98..ada2978 100644 --- a/pipeline/scripts/stages/geocode-denmark-dawa.py +++ b/pipeline/scripts/stages/geocode-denmark-dawa.py @@ -1,190 +1,4 @@ -#!/usr/bin/env python3 -"""Resumable, cached DAWA geocoder for the Denmark queue.""" - -import argparse -import json -import logging -import re -import hashlib -import os -import time -import urllib.parse -import urllib.request -from datetime import datetime, timezone +"""Compatibility shim for the Denmark source-owned stage.""" from pathlib import Path - -LOGGER = logging.getLogger("uec.denmark.geocode") -MAX_BATCH = 100 - - -def query_params(address: dict) -> dict: - street = address.get("street") or "" - match = re.match(r"^(.*?\s+\d+[A-Za-z]?(?:[-/]\d+[A-Za-z]?)?)(?:\s+(?:st\.?|\d+\.?\s*(?:th|tv|mf|sal)?))?\s*$", street, re.IGNORECASE) - if match: - address_part = match.group(1) - street_name = address_part.rsplit(" ", 1)[0] - house_number = address_part.rsplit(" ", 1)[1].split("-", 1)[0].split("/", 1)[0] - else: - street_name, house_number = street, None - params = {"vejnavn": street_name, "postnr": address.get("postal_code"), "struktur": "mini", "fuzzy": "true"} - if house_number: - params["husnr"] = house_number - return {key: value for key, value in params.items() if value} - - -def fetch(params: dict, base_url: str = "https://api.dataforsyningen.dk/adresser") -> tuple[int, object]: - url = base_url + "?" + urllib.parse.urlencode(params) - request = urllib.request.Request(url, headers={"User-Agent": "UntilEveryCage/2.0 data-pipeline; contact: untileverycageproject@protonmail.com"}) - with urllib.request.urlopen(request, timeout=30) as response: - return response.status, json.loads(response.read().decode("utf-8")) - - -def acceptance(results: object) -> str: - if not isinstance(results, list) or not results: - return "unresolved" - points = {(item.get("x"), item.get("y")) for item in results if item.get("x") is not None and item.get("y") is not None} - if len(points) == 1: - return "accepted_single_point" - if len(points) > 1: - return "review_multiple_points" - return "unresolved" - - -def load_suppression_keys(path: Path | None) -> set[tuple[str, str]]: - if not path: - return set() - keys = set() - for line in path.read_text(encoding="utf-8").splitlines(): - if not line.strip(): - continue - item = json.loads(line) - source_id, source_key = item.get("source_id"), item.get("source_record_key") - if not isinstance(source_id, str) or not source_id.strip() or not isinstance(source_key, str) or not source_key.strip(): - raise ValueError("suppression entries require non-empty source_id and source_record_key") - keys.add((source_id, source_key)) - return keys - - -def require_terms_review(path: Path | None) -> dict: - if path is None: - raise ValueError("network mode requires an approved terms review") - try: - review = json.loads(path.read_text(encoding="utf-8")) - except (OSError, json.JSONDecodeError) as error: - raise ValueError("terms review cannot be read") from error - required = {"reviewer", "reference", "reviewed_at", "decision", "notes"} - if not isinstance(review, dict) or required - review.keys(): - raise ValueError("terms review requires reviewer, reference, reviewed_at, decision, and notes") - if any(not isinstance(review[field], str) or not review[field].strip() for field in required): - raise ValueError("terms review fields must be non-empty strings") - try: - datetime.fromisoformat(review["reviewed_at"].replace("Z", "+00:00")) - except ValueError as error: - raise ValueError("terms review reviewed_at must be ISO-8601") from error - if review["decision"] != "approved": - raise ValueError("terms review decision must be 'approved'") - return {field: review[field] for field in sorted(required)} - - -def validate_provider_config(path: Path, network: bool, terms_review_path: Path | None) -> dict: - config = json.loads(path.read_text(encoding="utf-8")) - required = ("provider_id", "base_url", "mode", "status", "rate_limit_requests_per_second") - if any(key not in config for key in required): - raise ValueError("provider config is missing required approval/terms fields") - if network: - if config["status"] != "approved_for_development" or config["mode"] != "development_only": - raise ValueError("network mode requires an explicitly development-approved provider") - config["terms_review"] = require_terms_review(terms_review_path) - config["terms_review_sha256"] = hashlib.sha256(terms_review_path.read_bytes()).hexdigest() - if not isinstance(config["rate_limit_requests_per_second"], (int, float)) or config["rate_limit_requests_per_second"] <= 0: - raise ValueError("provider rate limit must be positive") - return config - - -def acquire_lock(output_path: Path) -> Path: - lock = output_path.with_suffix(output_path.suffix + ".lock") - try: - fd = os.open(lock, os.O_CREAT | os.O_EXCL | os.O_WRONLY) - with os.fdopen(fd, "w", encoding="utf-8") as handle: - handle.write(json.dumps({"pid": os.getpid()}) + "\n") - except FileExistsError as error: - raise RuntimeError(f"geocode output is locked: {lock}; inspect and remove it only after confirming the owner is gone") from error - return lock - - -def run(queue_path: Path, output_path: Path, limit: int, delay: float, retries: int, provider_config: Path, suppression_path: Path | None, terms_review_path: Path | None, network: bool = False) -> None: - lock = acquire_lock(output_path) - try: - _run_locked(queue_path, output_path, limit, delay, retries, provider_config, suppression_path, terms_review_path, network) - finally: - lock.unlink(missing_ok=True) - - -def _run_locked(queue_path: Path, output_path: Path, limit: int, delay: float, retries: int, provider_config: Path, suppression_path: Path | None, terms_review_path: Path | None, network: bool = False) -> None: - if limit <= 0 or limit > MAX_BATCH: - raise ValueError(f"limit must be between 1 and {MAX_BATCH}; full-queue runs are not permitted") - if not network: - raise ValueError("network mode must be explicitly enabled; no provider requests were made") - config = validate_provider_config(provider_config, network, terms_review_path) - suppressed = load_suppression_keys(suppression_path) - queue = [json.loads(line) for line in queue_path.read_text(encoding="utf-8").splitlines() if line.strip()] - if limit: - queue = queue[:limit] - completed = {} - if output_path.exists(): - for line in output_path.read_text(encoding="utf-8").splitlines(): - if line.strip(): - item = json.loads(line) - if item["queue_key"] in completed: - raise RuntimeError(f"duplicate queue entries detected in existing output; refusing resume: {output_path}") - completed[item["queue_key"]] = item - output_path.parent.mkdir(parents=True, exist_ok=True) - pending = [item for item in queue if item["queue_key"] not in completed and (item.get("source_id"), item.get("source_record_key")) not in suppressed] - skipped = len(queue) - len(pending) - len([item for item in queue if item["queue_key"] in completed]) - LOGGER.info("stage=geocode status=started batch=%d completed=%d suppressed=%d pending=%d provider=%s", len(queue), len(completed), skipped, len(pending), config["provider_id"]) - with output_path.open("a", encoding="utf-8", newline="\n") as output: - for index, item in enumerate(pending, start=1): - params = query_params(item["original_address"]) - queried_at = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") - result = {**item, "provider": config["provider_id"], "queried_at_utc": queried_at, "query_parameters": params, "response": None} - for attempt in range(1, retries + 1): - try: - status, payload = fetch(params, config["base_url"]) - result["http_status"] = status - result["response"] = payload - result["response_sha256"] = hashlib.sha256(json.dumps(payload, sort_keys=True, ensure_ascii=False).encode()).hexdigest() - result["acceptance"] = acceptance(payload) - result["status"] = "review_required" if result["acceptance"] == "accepted_single_point" else result["acceptance"] - result["precision"] = "review_required" - result["coordinate_review_status"] = "review_required" - break - except Exception as error: - result["status"] = "failed" - result["error"] = str(error) - LOGGER.warning("stage=geocode status=failed attempt=%d/%d", attempt, retries) - if attempt < retries: - time.sleep(min(30, 2 ** attempt)) - output.write(json.dumps(result, ensure_ascii=False, sort_keys=True) + "\n") - output.flush() - LOGGER.info("stage=geocode item=%d/%d status=%s acceptance=%s", index, len(pending), result["status"], result.get("acceptance")) - if index < len(pending): - time.sleep(delay) - report = output_path.with_name("geocode-review-report.json") - report.write_text(json.dumps({"status": "complete", "provider": config["provider_id"], "provider_base_url": config["base_url"], "terms_review_sha256": config["terms_review_sha256"], "batch_size": len(queue), "newly_processed": len(pending), "suppressed": skipped, "output_sha256": hashlib.sha256(output_path.read_bytes()).hexdigest()}, indent=2) + "\n", encoding="utf-8") - LOGGER.info("stage=geocode status=complete batch=%d newly_processed=%d suppressed=%d", len(queue), len(pending), skipped) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("queue", type=Path) - parser.add_argument("--output", type=Path, required=True) - parser.add_argument("--limit", type=int, required=True) - parser.add_argument("--provider-config", type=Path, default=Path("pipeline/config/geocoding-dev.json")) - parser.add_argument("--suppression-keys", type=Path) - parser.add_argument("--terms-review", type=Path, help="Approved per-run terms review required with --network.") - parser.add_argument("--network", action="store_true", help="Permit provider requests after config approval validation") - parser.add_argument("--delay", type=float, default=1.0) - parser.add_argument("--retries", type=int, default=3) - args = parser.parse_args() - logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s", datefmt="%Y-%m-%dT%H:%M:%SZ") - run(args.queue, args.output, args.limit, args.delay, args.retries, args.provider_config, args.suppression_keys, args.terms_review, args.network) +_TARGET = Path(__file__).resolve().parents[2] / "sources/denmark/stages/geocode-denmark-dawa.py" +exec(compile(_TARGET.read_text(encoding="utf-8"), str(_TARGET), "exec"), globals(), globals()) diff --git a/pipeline/scripts/stages/import-denmark.py b/pipeline/scripts/stages/import-denmark.py index cff8185..df7f0fc 100644 --- a/pipeline/scripts/stages/import-denmark.py +++ b/pipeline/scripts/stages/import-denmark.py @@ -1,135 +1,4 @@ -#!/usr/bin/env python3 -"""Import classified Denmark staging data into PostgreSQL/PostGIS transactionally.""" - -import argparse -import json -import os -import uuid -from datetime import datetime, timezone +"""Compatibility shim for the Denmark source-owned stage.""" from pathlib import Path - -import psycopg - - -def now(): - return datetime.now(timezone.utc) - - -def metadata_value(metadata, *names): - for name in names: - if metadata.get(name) is not None: - return metadata[name] - return None - - -def point_from_geocode(item): - # A provider's one-point response is evidence for review, not permission - # to store/display a precise point. Only an explicit review decision can - # advance it to an accepted coordinate. - if item.get("acceptance") != "accepted_single_point" or item.get("coordinate_review_status") != "approved": - return None - response = item.get("response", []) - results = response if isinstance(response, list) else response.get("results", []) - if not results: - return None - return results[0].get("x"), results[0].get("y") - - -def geocode_status(item): - review_status = item.get("coordinate_review_status") - if review_status == "approved" and item.get("acceptance") == "accepted_single_point": - return "accepted" - if review_status == "review_required": - return "review_required" - return { - "accepted_single_point": "review_required", - "review_multiple_points": "review_required", - "unresolved": "unresolved", - }.get(item.get("acceptance"), "failed" if item.get("status") == "failed" else "unresolved") - - -def run(classified_path: Path, artifact_metadata_path: Path, geocode_path: Path | None, database_url: str, release_id: str): - artifact = json.loads(artifact_metadata_path.read_text(encoding="utf-8")) - geocodes = {} - if geocode_path and geocode_path.exists(): - geocodes = {item["queue_key"]: item for item in (json.loads(line) for line in geocode_path.read_text(encoding="utf-8").splitlines() if line.strip())} - run_id = uuid.uuid4() - artifact_id = uuid.uuid4() - checked_at = now() - with psycopg.connect(database_url) as connection: - with connection.transaction(): - connection.execute(""" - INSERT INTO uec.sources(source_id, country_code, name, official_url, access_method, cadence, status, attribution) - VALUES ('dk.smiley', 'DK', 'Find Smiley', %s, 'bulk_xml', 'weekly', 'active', 'Fødevarestyrelsen') - ON CONFLICT (source_id) DO NOTHING - """, (metadata_value(artifact, "source_url", "final_url", "requested_url"),)) - connection.execute(""" - INSERT INTO uec.acquisition_runs(run_id, source_id, checked_at, retrieved_at, ingested_at, status, source_url, code_version, config_version) - VALUES (%s, 'dk.smiley', %s, %s, %s, 'changed', %s, 'import-denmark.py', 'denmark-classification-v1') - """, (run_id, checked_at, artifact.get("retrieved_at_utc"), checked_at, metadata_value(artifact, "source_url", "final_url", "requested_url"))) - connection.execute(""" - INSERT INTO uec.raw_artifacts(artifact_id, storage_key, sha256, byte_size, media_type, retrieved_at) - VALUES (%s, %s, %s, %s, 'application/xml', %s) - ON CONFLICT (sha256) DO NOTHING - """, (artifact_id, metadata_value(artifact, "artifact_path", "artifact"), artifact["sha256"], metadata_value(artifact, "bytes", "byte_size"), artifact["retrieved_at_utc"])) - artifact_id = connection.execute("SELECT artifact_id FROM uec.raw_artifacts WHERE sha256=%s", (artifact["sha256"],)).fetchone()[0] - connection.execute("INSERT INTO uec.acquisition_run_artifacts(run_id, artifact_id) VALUES (%s, %s)", (run_id, artifact_id)) - connection.execute("INSERT INTO uec.releases(release_id, status, ruleset_version, summary) VALUES (%s, 'candidate', 'denmark-classification-v1', %s) ON CONFLICT DO NOTHING", (release_id, json.dumps({"source": "dk.smiley", "run_id": str(run_id)}))) - for line in classified_path.read_text(encoding="utf-8").splitlines(): - if not line.strip(): - continue - record = json.loads(line) - source_record_id = uuid.uuid5(uuid.NAMESPACE_URL, f"urn:uec:source-record:dk.smiley:{record['source_record_key']}:{artifact['sha256']}") - facility_id = uuid.uuid5(uuid.NAMESPACE_URL, f"urn:uec:facility:dk.smiley:{record['source_record_key']}") - observed_at = checked_at - source_fields = record["source_fields"] - address = record["address"] - classification = record["classification"] - geocode = geocodes.get(f"dk.smiley:{record['source_record_key']}") - point = point_from_geocode(geocode) if geocode else None - connection.execute(""" - INSERT INTO uec.source_records(source_record_id, source_id, source_record_key, artifact_id, raw_fields, parsed_at) - VALUES (%s, 'dk.smiley', %s, %s, %s, %s) - ON CONFLICT DO NOTHING - """, (source_record_id, record["source_record_key"], artifact_id, json.dumps(source_fields, ensure_ascii=False), checked_at)) - existing = connection.execute("SELECT source_record_id FROM uec.source_records WHERE source_id='dk.smiley' AND source_record_key=%s AND artifact_id=%s", (record["source_record_key"], artifact_id)).fetchone() - source_record_id = existing[0] - if geocode: - queried_at = geocode.get("queried_at_utc") or checked_at.isoformat() - geocode_id = uuid.uuid5(uuid.NAMESPACE_URL, f"urn:uec:geocode:{geocode['queue_key']}:{geocode.get('provider', 'unknown')}:{queried_at}") - connection.execute(""" - INSERT INTO uec.geocode_results(geocode_result_id, source_record_id, provider_id, query, provider_address_id, result, match_method, status, attempt_number, retryable, response, queried_at) - VALUES (%s, %s, %s, %s, %s, ST_SetSRID(ST_MakePoint(%s, %s), 4326)::geography, %s, %s, 1, %s, %s, %s) - ON CONFLICT (geocode_result_id) DO NOTHING - """, (geocode_id, source_record_id, geocode.get("provider", "unknown"), geocode.get("geocoder_query", ""), (geocode.get("response") or [{}])[0].get("id") if isinstance(geocode.get("response"), list) and geocode.get("response") else None, point[0] if point else None, point[1] if point else None, geocode.get("acceptance", "address"), geocode_status(geocode), geocode_status(geocode) == "failed", json.dumps(geocode, ensure_ascii=False), queried_at)) - connection.execute(""" - INSERT INTO uec.facilities(facility_id, canonical_name, country_code, street_address, postal_code, city, location) - VALUES (%s, %s, 'DK', %s, %s, %s, ST_SetSRID(ST_MakePoint(%s, %s), 4326)::geography) - ON CONFLICT DO NOTHING - """, (facility_id, record.get("name"), address.get("street"), address.get("postal_code"), address.get("city"), point[0] if point else None, point[1] if point else None)) - connection.execute(""" - INSERT INTO uec.facility_source_links(facility_id, source_record_id, match_method, review_status) - VALUES (%s, %s, 'first_source_observation', 'automatic') ON CONFLICT DO NOTHING - """, (facility_id, source_record_id)) - connection.execute(""" - INSERT INTO uec.observations(observation_id, facility_id, source_record_id, observed_at, observation, classification, ruleset_id, rule_id, classification_category, classification_review_status, default_visible, optional_filter, coordinate, coordinate_method, coordinate_precision, coordinate_review_status, first_observed_at) - VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, ST_SetSRID(ST_MakePoint(%s, %s), 4326)::geography, %s, %s, %s, %s) - ON CONFLICT DO NOTHING - """, (uuid.uuid5(uuid.NAMESPACE_URL, f"urn:uec:observation:dk.smiley:{record['source_record_key']}:{artifact['sha256']}"), facility_id, source_record_id, observed_at, json.dumps(record, ensure_ascii=False), json.dumps(classification), classification["ruleset_id"], classification["rule_id"], classification["category"], classification["review_status"], classification["default_visible"], classification.get("optional_filter"), point[0] if point else None, point[1] if point else None, "dawa" if point else None, "address_point" if point else None, "accepted" if point else "unresolved", observed_at)) - connection.execute(""" - INSERT INTO uec.release_members(release_id, facility_id, observation_id, default_visible) - VALUES (%s, %s, %s, %s) - ON CONFLICT DO NOTHING - """, (release_id, facility_id, uuid.uuid5(uuid.NAMESPACE_URL, f"urn:uec:observation:dk.smiley:{record['source_record_key']}:{artifact['sha256']}"), classification["default_visible"] and not connection.execute("SELECT EXISTS (SELECT 1 FROM uec.public_access_restricted WHERE source_record_id = %s)", (source_record_id,)).fetchone()[0])) - print(f"Imported Denmark run {run_id} as candidate release {release_id}") - - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("classified", type=Path) - parser.add_argument("--artifact-metadata", type=Path, required=True) - parser.add_argument("--geocodes", type=Path) - parser.add_argument("--database-url", default=os.environ.get("UEC_DATABASE_URL", "postgresql://uec:uec-local-development-only@localhost:5433/uec")) - parser.add_argument("--release-id", default="dk-2026-09-13-candidate") - args = parser.parse_args() - run(args.classified, args.artifact_metadata, args.geocodes, args.database_url, args.release_id) +_TARGET = Path(__file__).resolve().parents[2] / "sources/denmark/stages/import-denmark.py" +exec(compile(_TARGET.read_text(encoding="utf-8"), str(_TARGET), "exec"), globals(), globals()) diff --git a/pipeline/scripts/stages/load-city-references.py b/pipeline/scripts/stages/load-city-references.py index 3da7f65..f2aa9c2 100644 --- a/pipeline/scripts/stages/load-city-references.py +++ b/pipeline/scripts/stages/load-city-references.py @@ -1,33 +1,4 @@ -#!/usr/bin/env python3 -"""Append official city reference points to PostGIS.""" -import argparse, json, os +"""Compatibility shim for the Denmark source-owned stage.""" from pathlib import Path -import psycopg - -DEFAULT_DB = "postgresql://uec:uec-local-development-only@localhost:5433/uec" - -def load(path: Path, database_url: str) -> int: - rows = [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line.strip()] - with psycopg.connect(database_url) as connection: - with connection.transaction(): - inserted = 0 - for row in rows: - result = connection.execute(""" - INSERT INTO uec.city_reference_points - (country_code, city_name, postal_code, reference_location, - reference_source, source_retrieved_at, source_reference_id) - VALUES (%s, %s, %s, ST_SetSRID(ST_MakePoint(%s, %s), 4326)::geography, - %s, %s, %s) - ON CONFLICT DO NOTHING - """, (row["country_code"], row["city_name"], row.get("postal_code"), - row["reference_longitude"], row["reference_latitude"], - row["reference_source"], row["source_retrieved_at"], row["source_reference_id"])) - inserted += result.rowcount - return inserted - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("staging", type=Path) - parser.add_argument("--database-url", default=os.environ.get("UEC_DATABASE_URL", DEFAULT_DB)) - args = parser.parse_args() - print(f"Inserted {load(args.staging, args.database_url)} city reference points") +_TARGET = Path(__file__).resolve().parents[2] / "sources/denmark/stages/load-city-references.py" +exec(compile(_TARGET.read_text(encoding="utf-8"), str(_TARGET), "exec"), globals(), globals()) diff --git a/pipeline/scripts/stages/normalize-denmark-smiley.py b/pipeline/scripts/stages/normalize-denmark-smiley.py index 687c721..c6cfe12 100644 --- a/pipeline/scripts/stages/normalize-denmark-smiley.py +++ b/pipeline/scripts/stages/normalize-denmark-smiley.py @@ -1,129 +1,4 @@ -#!/usr/bin/env python3 -"""Normalize staged Find Smiley rows while preserving source values.""" - -from __future__ import annotations - -import argparse -import json -import logging -import sys -from collections import Counter -from datetime import datetime +"""Compatibility shim for the Denmark source-owned stage.""" from pathlib import Path - -LOGGER = logging.getLogger("uec.denmark.normalize") -SOURCE_TO_CANONICAL = { - "ID_nummer": "source_record_key", - "CVR_nummer": "organization_registration_id", - "P_nummer": "production_unit_id", - "Virksomhed": "name", - "Adresse": "street_address", - "Postnummer": "postal_code", - "By": "city", - "FVST_branchenummer": "industry_code", - "FVST_branche": "industry_label", - "Smileybranche": "category_label", - "Virksomhedstype": "business_type", - "URL": "source_url", - "Geo_Lat": "source_latitude", - "Geo_Lng": "source_longitude", -} - - -def iso_date(value: str | None) -> str | None: - if not value: - return None - for fmt in ("%d/%m/%Y", "%d-%m-%Y %H:%M:%S"): - try: - return datetime.strptime(value, fmt).date().isoformat() - except ValueError: - pass - return None - - -def normalize_record(envelope: dict) -> dict: - source = envelope.get("fields", {}) - normalized = { - "source_id": envelope.get("source_id", "dk.smiley"), - "source_record_key": source.get("ID_nummer") or source.get("navnelbnr"), - "source_artifact_sha256": envelope.get("source_artifact_sha256"), - "name": source.get("Virksomhed") or source.get("navn1"), - "organization_registration_id": source.get("CVR_nummer") or source.get("cvrnr"), - "production_unit_id": source.get("P_nummer") or source.get("pnr"), - "address": { - "street": source.get("Adresse") or source.get("adresse1"), - "postal_code": source.get("Postnummer") or source.get("postnr"), - "city": source.get("By"), - "country_code": "DK", - }, - "activity": { - "code": source.get("FVST_branchenummer") or source.get("brancheKode"), - "label": source.get("FVST_branche") or source.get("branche"), - "category": source.get("Smileybranche") or source.get("Pixibranche"), - }, - "business_type": source.get("Virksomhedstype") or source.get("virksomhedstype"), - "coordinates": { - "latitude": source.get("Geo_Lat"), - "longitude": source.get("Geo_Lng"), - "method": "source" if source.get("Geo_Lat") and source.get("Geo_Lng") else None, - "review_status": "source" if source.get("Geo_Lat") and source.get("Geo_Lng") else "unresolved", - }, - "latest_inspection_date": iso_date(source.get("Seneste_kontrol_dato") or source.get("seneste_kontrol_dato")), - "source_url": source.get("URL"), - "source_fields": source, - } - return normalized - - -def normalize_file(input_path: Path, output_dir: Path, progress_every: int = 10000) -> Path: - output_dir.mkdir(parents=True, exist_ok=True) - output_path = output_dir / "normalized-records.jsonl" - report_path = output_dir / "field-mapping-report.json" - count = 0 - missing_coords = 0 - keys = Counter() - LOGGER.info("stage=normalize status=started input=%s", input_path) - with input_path.open(encoding="utf-8") as source, output_path.open("w", encoding="utf-8", newline="\n") as output: - for line in source: - if not line.strip(): - continue - record = normalize_record(json.loads(line)) - count += 1 - keys.update(record["source_fields"].keys()) - if record["coordinates"]["review_status"] == "unresolved": - missing_coords += 1 - output.write(json.dumps(record, ensure_ascii=False, sort_keys=True) + "\n") - if count % progress_every == 0: - LOGGER.info("stage=normalize records=%d missing_coordinates=%d", count, missing_coords) - report = { - "status": "success", - "input_path": input_path.as_posix(), - "output_path": output_path.as_posix(), - "records_normalized": count, - "records_missing_coordinates": missing_coords, - "source_to_canonical_mapping": SOURCE_TO_CANONICAL, - "observed_source_fields": sorted(keys), - "unmapped_source_fields": sorted(set(keys) - set(SOURCE_TO_CANONICAL)), - "date_policy": "recognized dates are emitted as ISO dates; original values remain in source_fields", - } - report_path.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") - LOGGER.info("stage=normalize status=success records=%d missing_coordinates=%d report=%s", count, missing_coords, report_path) - return output_path - - -def main() -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("input", type=Path) - parser.add_argument("--output-dir", type=Path, required=True) - args = parser.parse_args() - logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s", datefmt="%Y-%m-%dT%H:%M:%SZ") - try: - normalize_file(args.input, args.output_dir) - except Exception as error: - LOGGER.error("stage=normalize status=failed error=%s", error) - return 1 - return 0 - - -if __name__ == "__main__": - sys.exit(main()) +_TARGET = Path(__file__).resolve().parents[2] / "sources/denmark/stages/normalize-denmark-smiley.py" +exec(compile(_TARGET.read_text(encoding="utf-8"), str(_TARGET), "exec"), globals(), globals()) diff --git a/pipeline/scripts/stages/parse-denmark-smiley.py b/pipeline/scripts/stages/parse-denmark-smiley.py index e2f7938..b45c92e 100644 --- a/pipeline/scripts/stages/parse-denmark-smiley.py +++ b/pipeline/scripts/stages/parse-denmark-smiley.py @@ -1,109 +1,4 @@ -#!/usr/bin/env python3 -"""Parse an archived Find Smiley XML file into auditable JSONL staging output.""" - -from __future__ import annotations - -import argparse -import hashlib -import json -import logging -import sys -import uuid -import xml.etree.ElementTree as ET -from datetime import datetime, timezone +"""Compatibility shim for the Denmark source-owned stage.""" from pathlib import Path -from typing import Iterator - - -LOGGER = logging.getLogger("uec.denmark.parse") -def utc_now() -> str: - return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") - - -def sha256_file(path: Path) -> str: - digest = hashlib.sha256() - with path.open("rb") as source: - for chunk in iter(lambda: source.read(1024 * 1024), b""): - digest.update(chunk) - return digest.hexdigest() - - -def iter_rows(path: Path) -> Iterator[dict[str, str | None]]: - for _, element in ET.iterparse(path, events=("end",)): - if element.tag.lower() != "row": - continue - values = {child.tag: (child.text or "").strip() or None for child in element} - yield values - element.clear() - - -def parse_file(input_path: Path, output_dir: Path, source_url: str, progress_every: int = 1000) -> Path: - if not input_path.is_file(): - raise FileNotFoundError(input_path) - output_dir.mkdir(parents=True, exist_ok=True) - run_id = f"{datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ')}-{uuid.uuid4().hex[:8]}" - output_path = output_dir / "parsed-rows.jsonl" - metadata_path = output_dir / "run-metadata.json" - started_at = utc_now() - source_hash = sha256_file(input_path) - LOGGER.info("run_id=%s stage=parse status=started input=%s", run_id, input_path) - LOGGER.info("run_id=%s source_sha256=%s", run_id, source_hash) - - row_count = 0 - missing_coordinates = 0 - try: - with output_path.open("w", encoding="utf-8", newline="\n") as output: - for row_count, row in enumerate(iter_rows(input_path), start=1): - latitude = row.get("Geo_Lat") or row.get("Geo_Latitude") - longitude = row.get("Geo_Lng") or row.get("Geo_Longitude") - if not latitude or not longitude: - missing_coordinates += 1 - output.write(json.dumps({ - "source_id": "dk.smiley", - "source_record_key": row.get("ID_nummer") or row.get("navnelbnr"), - "source_artifact_sha256": source_hash, - "fields": row, - }, ensure_ascii=False, sort_keys=True) + "\n") - if row_count % progress_every == 0: - LOGGER.info("run_id=%s stage=parse rows=%d missing_coordinates=%d", run_id, row_count, missing_coordinates) - except ET.ParseError: - LOGGER.exception("run_id=%s stage=parse status=failed reason=invalid_xml", run_id) - raise - - metadata = { - "run_id": run_id, - "source_id": "dk.smiley", - "source_url": source_url, - "input_path": input_path.as_posix(), - "input_sha256": source_hash, - "started_at_utc": started_at, - "completed_at_utc": utc_now(), - "status": "success", - "rows_parsed": row_count, - "rows_missing_coordinates": missing_coordinates, - "output_path": output_path.as_posix(), - "parser": "parse-denmark-smiley.py", - } - metadata_path.write_text(json.dumps(metadata, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") - LOGGER.info("run_id=%s stage=parse status=success rows=%d missing_coordinates=%d output=%s", run_id, row_count, missing_coordinates, output_path) - return output_path - - -def main() -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("input", type=Path) - parser.add_argument("--output-dir", type=Path, required=True) - parser.add_argument("--source-url", default="https://pub.fvst.dk/publikationer/Smileydata.xml") - parser.add_argument("--progress-every", type=int, default=1000) - args = parser.parse_args() - logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s", datefmt="%Y-%m-%dT%H:%M:%SZ") - try: - parse_file(args.input, args.output_dir, args.source_url, args.progress_every) - except Exception as error: - LOGGER.error("stage=parse status=failed error=%s", error) - return 1 - return 0 - - -if __name__ == "__main__": - sys.exit(main()) +_TARGET = Path(__file__).resolve().parents[2] / "sources/denmark/stages/parse-denmark-smiley.py" +exec(compile(_TARGET.read_text(encoding="utf-8"), str(_TARGET), "exec"), globals(), globals()) diff --git a/pipeline/scripts/stages/validate-denmark.py b/pipeline/scripts/stages/validate-denmark.py index bfd0cf4..327d6ba 100644 --- a/pipeline/scripts/stages/validate-denmark.py +++ b/pipeline/scripts/stages/validate-denmark.py @@ -1,90 +1,4 @@ -#!/usr/bin/env python3 -"""Validate classified Denmark records without deleting invalid rows.""" - -import argparse -import collections -import json -import logging -from datetime import datetime +"""Compatibility shim for the Denmark source-owned stage.""" from pathlib import Path - -LOGGER = logging.getLogger("uec.denmark.validate") - - -def validate_file(input_path: Path, output_dir: Path, expected_rows: int | None = None, progress_every: int = 10000) -> Path: - output_dir.mkdir(parents=True, exist_ok=True) - report_path = output_dir / "validation-report.json" - quarantine_path = output_dir / "validation-findings.jsonl" - counts = collections.Counter() - seen_keys = set() - findings = [] - total = 0 - LOGGER.info("stage=validate status=started input=%s", input_path) - with input_path.open(encoding="utf-8") as source, quarantine_path.open("w", encoding="utf-8", newline="\n") as rejected: - for line in source: - if not line.strip(): - continue - total += 1 - record = json.loads(line) - key = record.get("source_record_key") - row_findings = [] - if not key: - row_findings.append(("error", "missing_source_record_key")) - elif key in seen_keys: - row_findings.append(("error", "duplicate_source_record_key")) - else: - seen_keys.add(key) - address = record.get("address", {}) - if not record.get("name"): - row_findings.append(("warning", "missing_name")) - if not address.get("street") and not address.get("city") and not address.get("postal_code"): - row_findings.append(("error", "missing_address")) - if address.get("country_code") != "DK": - row_findings.append(("error", "unexpected_country")) - date_value = record.get("latest_inspection_date") - if date_value: - try: - datetime.strptime(date_value, "%Y-%m-%d") - except ValueError: - row_findings.append(("error", "invalid_normalized_date")) - classification = record.get("classification", {}) - if not classification.get("rule_id"): - row_findings.append(("error", "missing_classification_rule")) - if classification.get("review_status") == "review_required": - row_findings.append(("review", "classification_requires_review")) - coordinate_status = record.get("coordinates", {}).get("review_status") - counts["coordinates_" + str(coordinate_status)] += 1 - if row_findings: - finding = {"source_record_key": key, "findings": [{"severity": s, "code": c} for s, c in row_findings]} - rejected.write(json.dumps({"record": record, "findings": finding["findings"]}, ensure_ascii=False, sort_keys=True) + "\n") - findings.append(finding) - for _, code in row_findings: - counts[code] += 1 - if total % progress_every == 0: - LOGGER.info("stage=validate records=%d findings=%d", total, len(findings)) - if expected_rows is not None and total != expected_rows: - counts["unexpected_row_count"] += 1 - report = { - "status": "success", - "input_path": input_path.as_posix(), - "records_checked": total, - "unique_source_record_keys": len(seen_keys), - "finding_records": len(findings), - "expected_rows": expected_rows, - "counts": dict(counts), - "findings_path": quarantine_path.as_posix(), - "policy": "findings are reported and preserved; no records are deleted", - } - report_path.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") - LOGGER.info("stage=validate status=success records=%d finding_records=%d report=%s", total, len(findings), report_path) - return report_path - - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("input", type=Path) - parser.add_argument("--output-dir", type=Path, required=True) - parser.add_argument("--expected-rows", type=int) - args = parser.parse_args() - logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s", datefmt="%Y-%m-%dT%H:%M:%SZ") - validate_file(args.input, args.output_dir, args.expected_rows) +_TARGET = Path(__file__).resolve().parents[2] / "sources/denmark/stages/validate-denmark.py" +exec(compile(_TARGET.read_text(encoding="utf-8"), str(_TARGET), "exec"), globals(), globals()) diff --git a/pipeline/sources/denmark/README.md b/pipeline/sources/denmark/README.md new file mode 100644 index 0000000..885d448 --- /dev/null +++ b/pipeline/sources/denmark/README.md @@ -0,0 +1,18 @@ +# Denmark source entry points + +Denmark-specific commands are exposed from this directory so country logic has +a stable home as more sources are added. The historical commands under +`pipeline/scripts/stages/` and `pipeline/run-denmark-pipeline.py` remain valid +compatibility paths and retain their argument behavior. + +The current launchers intentionally delegate to the established implementations +to avoid duplicating acquisition, parsing, normalization, classification, and +validation logic. New source-specific behavior should be added here first; +generic release, geocoding, restriction, and maintenance commands remain in +their shared locations. + +Example: + +```powershell +python pipeline/sources/denmark/run-denmark-pipeline.py --help +``` diff --git a/pipeline/sources/denmark/__init__.py b/pipeline/sources/denmark/__init__.py new file mode 100644 index 0000000..a25960c --- /dev/null +++ b/pipeline/sources/denmark/__init__.py @@ -0,0 +1,6 @@ +"""Denmark source entry points. + +The source-specific commands live here as compatibility-preserving launchers +while their mature implementations remain available at the historical paths. +This lets downstream automation migrate one command at a time. +""" diff --git a/pipeline/sources/denmark/cli.py b/pipeline/sources/denmark/cli.py new file mode 100644 index 0000000..4511d52 --- /dev/null +++ b/pipeline/sources/denmark/cli.py @@ -0,0 +1,21 @@ +"""Compatibility launcher for Denmark's existing stage implementations.""" +from pathlib import Path +import runpy +import sys + +ROOT = Path(__file__).resolve().parents[3] +LEGACY_STAGES = ROOT / "pipeline" / "scripts" / "stages" +LEGACY_PIPELINE = ROOT / "pipeline" + + +def run_stage(script_name: str) -> None: + """Execute a historical stage with unchanged argv semantics.""" + runpy.run_path(str(LEGACY_STAGES / script_name), run_name="__main__") + + +def main(script_name: str) -> None: + base = LEGACY_PIPELINE if script_name == "run-denmark-pipeline.py" else LEGACY_STAGES + script = (base / script_name).resolve() + if not script.is_file(): + raise FileNotFoundError(script_name) + runpy.run_path(str(script), run_name="__main__") diff --git a/pipeline/sources/denmark/run-denmark-pipeline.py b/pipeline/sources/denmark/run-denmark-pipeline.py new file mode 100644 index 0000000..3181404 --- /dev/null +++ b/pipeline/sources/denmark/run-denmark-pipeline.py @@ -0,0 +1,12 @@ +"""Source-owned entry point for the Denmark vertical slice.""" +import sys +from pathlib import Path + +PROJECT_ROOT = Path(__file__).resolve().parents[3] +sys.path.insert(0, str(PROJECT_ROOT)) + +from pipeline.sources.denmark.cli import main + + +if __name__ == "__main__": + main("run-denmark-pipeline.py") diff --git a/pipeline/sources/denmark/stages/README.md b/pipeline/sources/denmark/stages/README.md new file mode 100644 index 0000000..58b9b02 --- /dev/null +++ b/pipeline/sources/denmark/stages/README.md @@ -0,0 +1,7 @@ +# Denmark stages + +These are the canonical Denmark-specific stage implementations. Historical +paths under pipeline/scripts/stages remain thin compatibility shims. + +Generic geocoding workers, release validation/promotion, restriction +maintenance, and migration tools remain in shared locations. diff --git a/pipeline/sources/denmark/stages/__init__.py b/pipeline/sources/denmark/stages/__init__.py new file mode 100644 index 0000000..49f9c7d --- /dev/null +++ b/pipeline/sources/denmark/stages/__init__.py @@ -0,0 +1 @@ +"""Denmark-specific acquisition, transformation, and validation stages.""" diff --git a/pipeline/sources/denmark/stages/acquire-denmark-smiley.py b/pipeline/sources/denmark/stages/acquire-denmark-smiley.py new file mode 100644 index 0000000..8320509 --- /dev/null +++ b/pipeline/sources/denmark/stages/acquire-denmark-smiley.py @@ -0,0 +1,198 @@ +#!/usr/bin/env python3 +"""Archive a Denmark Find Smiley artifact without importing or publishing it.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import sys +import tempfile +import urllib.error +import urllib.request +import uuid +from datetime import datetime, timezone +from pathlib import Path + + +SOURCE_ID = "dk.smiley" +ADAPTER_VERSION = "denmark-smiley-acquisition-v1" +DEFAULT_URL = "https://pub.fvst.dk/publikationer/Smileydata.xml" +DEFAULT_MAX_BYTES = 128 * 1024 * 1024 +APPROVED_DECISION = "approved" +SAFE_CONTENT_TYPES = {"application/xml", "text/xml", "application/octet-stream"} + + +class AcquisitionError(ValueError): + """An acquisition was not authorized, valid, or complete.""" + + +def utc_now() -> str: + return datetime.now(timezone.utc).isoformat(timespec="seconds").replace("+00:00", "Z") + + +def default_run_id() -> str: + return datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + "-" + uuid.uuid4().hex[:8] + + +def require_terms_review(path: Path) -> dict: + try: + review = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as error: + raise AcquisitionError(f"terms review cannot be read: {error}") from error + required = {"reviewer", "reference", "reviewed_at", "decision", "notes"} + if not isinstance(review, dict) or required - review.keys(): + raise AcquisitionError("terms review requires reviewer, reference, reviewed_at, decision, and notes") + for field in required: + if not isinstance(review[field], str) or not review[field].strip(): + raise AcquisitionError(f"terms review {field} must be a non-empty string") + try: + datetime.fromisoformat(review["reviewed_at"].replace("Z", "+00:00")) + except ValueError as error: + raise AcquisitionError("terms review reviewed_at must be ISO-8601") from error + if review["decision"] != APPROVED_DECISION: + raise AcquisitionError(f"terms review decision must be {APPROVED_DECISION!r}") + return {field: review[field] for field in sorted(required)} + + +def selected_headers(headers) -> dict[str, str]: + """Persist response metadata relevant to reproducibility, not credentials.""" + return { + name: headers[name] + for name in ("Content-Type", "Content-Length", "ETag", "Last-Modified") + if headers.get(name) is not None + } + + +def content_type_is_safe(headers) -> bool: + raw = headers.get("Content-Type") + return raw is None or raw.split(";", 1)[0].strip().lower() in SAFE_CONTENT_TYPES + + +def archive_stream(stream, artifact_path: Path, *, max_bytes: int) -> tuple[str, int]: + artifact_path.parent.mkdir(parents=True, exist_ok=True) + temp_path: Path | None = None + digest = hashlib.sha256() + total = 0 + try: + with tempfile.NamedTemporaryFile("wb", delete=False, dir=artifact_path.parent, prefix=".download-", suffix=".part") as handle: + temp_path = Path(handle.name) + while chunk := stream.read(1024 * 1024): + total += len(chunk) + if total > max_bytes: + raise AcquisitionError(f"download exceeds max_bytes={max_bytes}") + digest.update(chunk) + handle.write(chunk) + os.replace(temp_path, artifact_path) + return digest.hexdigest(), total + except Exception: + if temp_path is not None: + temp_path.unlink(missing_ok=True) + raise + + +def write_metadata(path: Path, metadata: dict) -> None: + path.write_text(json.dumps(metadata, ensure_ascii=False, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + +def archive_local_file(local_file: Path, output_root: Path, *, run_id: str, retrieved_at: str | None = None) -> dict: + local_file = local_file.resolve() + if not local_file.is_file(): + raise AcquisitionError(f"local file does not exist: {local_file}") + run_dir = output_root / SOURCE_ID / run_id + artifact_path = run_dir / "Smileydata.xml" + retrieved_at = retrieved_at or utc_now() + with local_file.open("rb") as stream: + sha256, byte_size = archive_stream(stream, artifact_path, max_bytes=DEFAULT_MAX_BYTES) + metadata = { + "acquisition_method": "local_file", + "adapter_version": ADAPTER_VERSION, + "artifact": artifact_path.name, + "byte_size": byte_size, + "config_version": ADAPTER_VERSION, + "final_url": "unknown", + "publication_metadata": {}, + "requested_url": "unknown", + "response_headers": {}, + "retrieved_at_utc": retrieved_at, + "run_id": run_id, + "sha256": sha256, + "source_id": SOURCE_ID, + "source_local_path": str(local_file), + "terms_review": "not_required_for_local_file", + } + write_metadata(run_dir / "acquisition-metadata.json", metadata) + return metadata + + +def fetch(url: str, output_root: Path, *, run_id: str, terms_review_path: Path | None, timeout_seconds: float, max_bytes: int) -> dict: + if terms_review_path is None: + raise AcquisitionError("--terms-review is required with --fetch") + if timeout_seconds <= 0 or max_bytes <= 0: + raise AcquisitionError("timeout_seconds and max_bytes must be positive") + terms_review = require_terms_review(terms_review_path) + run_dir = output_root / SOURCE_ID / run_id + artifact_path = run_dir / "Smileydata.xml" + requested_at = utc_now() + try: + request = urllib.request.Request(url, headers={"User-Agent": "UntilEveryCage/controlled-acquisition"}) + with urllib.request.urlopen(request, timeout=timeout_seconds) as response: + if not 200 <= response.status < 300: + raise AcquisitionError(f"source returned HTTP {response.status}") + if not content_type_is_safe(response.headers): + raise AcquisitionError(f"unexpected content type: {response.headers.get('Content-Type')}") + sha256, byte_size = archive_stream(response, artifact_path, max_bytes=max_bytes) + headers = selected_headers(response.headers) + final_url = response.geturl() + except urllib.error.HTTPError as error: + raise AcquisitionError(f"source returned HTTP {error.code}") from error + except urllib.error.URLError as error: + raise AcquisitionError(f"network error: {error.reason}") from error + metadata = { + "acquisition_method": "network_fetch", + "adapter_version": ADAPTER_VERSION, + "artifact": artifact_path.name, + "byte_size": byte_size, + "config_version": ADAPTER_VERSION, + "final_url": final_url, + "publication_metadata": {key: headers[key] for key in ("ETag", "Last-Modified") if key in headers}, + "requested_at_utc": requested_at, + "requested_url": url, + "response_headers": headers, + "retrieved_at_utc": utc_now(), + "run_id": run_id, + "sha256": sha256, + "source_id": SOURCE_ID, + "terms_review": terms_review, + } + write_metadata(run_dir / "acquisition-metadata.json", metadata) + return metadata + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + mode = parser.add_mutually_exclusive_group(required=True) + mode.add_argument("--fetch", action="store_true", help="Fetch the registry URL; requires a reviewed terms file.") + mode.add_argument("--local-file", type=Path, help="Archive a local/synthetic XML file without network access.") + parser.add_argument("--url", default=DEFAULT_URL, help="Requested URL for --fetch.") + parser.add_argument("--terms-review", type=Path, help="Approved JSON terms-review record; required for --fetch.") + parser.add_argument("--output-root", type=Path, default=Path("data/raw")) + parser.add_argument("--run-id", default=default_run_id()) + parser.add_argument("--timeout-seconds", type=float, default=30.0) + parser.add_argument("--max-bytes", type=int, default=DEFAULT_MAX_BYTES) + args = parser.parse_args() + try: + if args.fetch: + metadata = fetch(args.url, args.output_root, run_id=args.run_id, terms_review_path=args.terms_review, timeout_seconds=args.timeout_seconds, max_bytes=args.max_bytes) + else: + metadata = archive_local_file(args.local_file, args.output_root, run_id=args.run_id) + except AcquisitionError as error: + print(json.dumps({"status": "failed", "error": str(error)}, sort_keys=True), file=sys.stderr) + return 2 + print(json.dumps({"status": "archived", "metadata": metadata}, ensure_ascii=False, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/pipeline/sources/denmark/stages/catalog-denmark-categories.py b/pipeline/sources/denmark/stages/catalog-denmark-categories.py new file mode 100644 index 0000000..d722937 --- /dev/null +++ b/pipeline/sources/denmark/stages/catalog-denmark-categories.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +"""Build a complete category catalog from normalized Denmark staging data.""" + +import argparse +import collections +import json +from pathlib import Path + + +def build_catalog(input_path: Path, output_path: Path) -> None: + categories = collections.defaultdict(lambda: {"count": 0, "examples": []}) + with input_path.open(encoding="utf-8") as source: + for line in source: + if not line.strip(): + continue + record = json.loads(line) + activity = record["activity"] + key = ( + activity.get("code"), + activity.get("label"), + activity.get("category"), + ) + item = categories[key] + item["count"] += 1 + if len(item["examples"]) < 3: + item["examples"].append({ + "source_record_key": record.get("source_record_key"), + "name": record.get("name"), + "source_url": record.get("source_url"), + }) + + rows = [ + {"industry_code": key[0], "industry_label": key[1], "category_label": key[2], **value} + for key, value in categories.items() + ] + rows.sort(key=lambda row: (-row["count"], row["industry_code"] or "", row["category_label"] or "")) + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(json.dumps({"status": "success", "categories": rows}, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + print(f"Catalogued {len(rows)} distinct category combinations from {sum(row['count'] for row in rows)} records") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("input", type=Path) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + build_catalog(args.input, args.output) diff --git a/pipeline/sources/denmark/stages/classify-denmark.py b/pipeline/sources/denmark/stages/classify-denmark.py new file mode 100644 index 0000000..3e666f0 --- /dev/null +++ b/pipeline/sources/denmark/stages/classify-denmark.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +"""Apply an explicit, visibility-aware Denmark classification ruleset.""" + +import argparse +import collections +import json +import logging +from pathlib import Path + +LOGGER = logging.getLogger("uec.denmark.classify") + + +def load_rules(path: Path) -> dict: + return json.loads(path.read_text(encoding="utf-8")) + + +def classify_record(record: dict, ruleset: dict) -> dict: + code = record.get("activity", {}).get("code") + decision = ruleset["fallback"] + for rule in ruleset["rules"]: + if code in rule["codes"]: + decision = rule + break + result = dict(record) + result["classification"] = { + "ruleset_id": ruleset["ruleset_id"], + "rule_id": decision["rule_id"], + "category": decision["classification"], + "review_status": decision["review_status"], + "default_visible": decision["default_visible"], + "optional_filter": decision.get("optional_filter"), + } + return result + + +def classify_file(input_path: Path, rules_path: Path, output_dir: Path, progress_every: int = 10000) -> Path: + ruleset = load_rules(rules_path) + output_dir.mkdir(parents=True, exist_ok=True) + output_path = output_dir / "classified-records.jsonl" + report_path = output_dir / "classification-report.json" + counts = collections.Counter() + LOGGER.info("stage=classify status=started input=%s ruleset=%s", input_path, ruleset["ruleset_id"]) + with input_path.open(encoding="utf-8") as source, output_path.open("w", encoding="utf-8", newline="\n") as output: + for count, line in enumerate(source, start=1): + if not line.strip(): + continue + record = classify_record(json.loads(line), ruleset) + classification = record["classification"] + counts.update([classification["category"], classification["review_status"]]) + output.write(json.dumps(record, ensure_ascii=False, sort_keys=True) + "\n") + if count % progress_every == 0: + LOGGER.info("stage=classify records=%d", count) + report = { + "status": "success", + "ruleset_id": ruleset["ruleset_id"], + "input_path": input_path.as_posix(), + "output_path": output_path.as_posix(), + "records_classified": sum(counts[key] for key in set(counts) if key in {r["classification"] for r in ruleset["rules"]} | {ruleset["fallback"]["classification"]}), + "counts_by_classification": {key: value for key, value in counts.items() if key not in {"approved", "review_required"}}, + "counts_by_review_status": {key: value for key, value in counts.items() if key in {"approved", "review_required"}}, + "ruleset_path": rules_path.as_posix(), + } + report_path.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + LOGGER.info("stage=classify status=success records=%d report=%s", report["records_classified"], report_path) + return output_path + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("input", type=Path) + parser.add_argument("--rules", type=Path, required=True) + parser.add_argument("--output-dir", type=Path, required=True) + args = parser.parse_args() + logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s", datefmt="%Y-%m-%dT%H:%M:%SZ") + classify_file(args.input, args.rules, args.output_dir) diff --git a/pipeline/sources/denmark/stages/fetch-denmark-city-references.py b/pipeline/sources/denmark/stages/fetch-denmark-city-references.py new file mode 100644 index 0000000..5f6f900 --- /dev/null +++ b/pipeline/sources/denmark/stages/fetch-denmark-city-references.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 +"""Fetch and normalize DAWA's official Denmark city reference points.""" + +import argparse +import hashlib +import json +import logging +import urllib.request +from datetime import datetime, timezone +from pathlib import Path + +LOGGER = logging.getLogger("uec.denmark.city-references") +URL = "https://api.dataforsyningen.dk/steder?hovedtype=Bebyggelse&undertype=by" +HEADERS = {"User-Agent": "UntilEveryCage/2.0 data-pipeline; contact: untileverycageproject@protonmail.com"} + + +def normalize(payload: object, retrieved_at: str, source_url: str) -> list[dict]: + """Convert DAWA records into stable, database-loadable reference records.""" + if not isinstance(payload, list): + raise ValueError("DAWA city response must be a JSON array") + records = [] + for item in payload: + center = item.get("visueltcenter") + if not isinstance(center, list) or len(center) != 2: + continue + records.append({ + "source_reference_id": item.get("id"), + "country_code": "DK", + "city_name": item.get("primærtnavn"), + "postal_code": None, + "reference_longitude": center[0], + "reference_latitude": center[1], + "reference_source": source_url, + "source_retrieved_at": retrieved_at, + }) + return records + + +def run(output_root: Path, url: str = URL) -> Path: + retrieved = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") + run_id = retrieved.replace("-", "").replace(":", "").replace(".", "") + run_dir = output_root / run_id + run_dir.mkdir(parents=True, exist_ok=False) + request = urllib.request.Request(url, headers=HEADERS) + LOGGER.info("stage=city-references status=fetching source=%s", url) + with urllib.request.urlopen(request, timeout=60) as response: + raw = response.read() + payload = json.loads(raw.decode("utf-8")) + artifact = run_dir / "steder.json" + artifact.write_bytes(raw) + records = normalize(payload, retrieved, url) + staging = run_dir / "city-reference-points.jsonl" + with staging.open("w", encoding="utf-8", newline="\n") as handle: + for record in records: + handle.write(json.dumps(record, ensure_ascii=False, sort_keys=True) + "\n") + metadata = {"source_url": url, "retrieved_at": retrieved, "artifact": artifact.name, + "sha256": hashlib.sha256(raw).hexdigest(), "bytes": len(raw), + "records": len(records), "status": "fetched"} + (run_dir / "metadata.json").write_text(json.dumps(metadata, indent=2) + "\n", encoding="utf-8") + LOGGER.info("stage=city-references status=complete records=%d artifact=%s", len(records), artifact) + return run_dir + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output-root", type=Path, default=Path("data/raw/denmark-city-references")) + args = parser.parse_args() + logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") + print(run(args.output_root)) diff --git a/pipeline/sources/denmark/stages/geocode-denmark-dawa.py b/pipeline/sources/denmark/stages/geocode-denmark-dawa.py new file mode 100644 index 0000000..5ff4f98 --- /dev/null +++ b/pipeline/sources/denmark/stages/geocode-denmark-dawa.py @@ -0,0 +1,190 @@ +#!/usr/bin/env python3 +"""Resumable, cached DAWA geocoder for the Denmark queue.""" + +import argparse +import json +import logging +import re +import hashlib +import os +import time +import urllib.parse +import urllib.request +from datetime import datetime, timezone +from pathlib import Path + +LOGGER = logging.getLogger("uec.denmark.geocode") +MAX_BATCH = 100 + + +def query_params(address: dict) -> dict: + street = address.get("street") or "" + match = re.match(r"^(.*?\s+\d+[A-Za-z]?(?:[-/]\d+[A-Za-z]?)?)(?:\s+(?:st\.?|\d+\.?\s*(?:th|tv|mf|sal)?))?\s*$", street, re.IGNORECASE) + if match: + address_part = match.group(1) + street_name = address_part.rsplit(" ", 1)[0] + house_number = address_part.rsplit(" ", 1)[1].split("-", 1)[0].split("/", 1)[0] + else: + street_name, house_number = street, None + params = {"vejnavn": street_name, "postnr": address.get("postal_code"), "struktur": "mini", "fuzzy": "true"} + if house_number: + params["husnr"] = house_number + return {key: value for key, value in params.items() if value} + + +def fetch(params: dict, base_url: str = "https://api.dataforsyningen.dk/adresser") -> tuple[int, object]: + url = base_url + "?" + urllib.parse.urlencode(params) + request = urllib.request.Request(url, headers={"User-Agent": "UntilEveryCage/2.0 data-pipeline; contact: untileverycageproject@protonmail.com"}) + with urllib.request.urlopen(request, timeout=30) as response: + return response.status, json.loads(response.read().decode("utf-8")) + + +def acceptance(results: object) -> str: + if not isinstance(results, list) or not results: + return "unresolved" + points = {(item.get("x"), item.get("y")) for item in results if item.get("x") is not None and item.get("y") is not None} + if len(points) == 1: + return "accepted_single_point" + if len(points) > 1: + return "review_multiple_points" + return "unresolved" + + +def load_suppression_keys(path: Path | None) -> set[tuple[str, str]]: + if not path: + return set() + keys = set() + for line in path.read_text(encoding="utf-8").splitlines(): + if not line.strip(): + continue + item = json.loads(line) + source_id, source_key = item.get("source_id"), item.get("source_record_key") + if not isinstance(source_id, str) or not source_id.strip() or not isinstance(source_key, str) or not source_key.strip(): + raise ValueError("suppression entries require non-empty source_id and source_record_key") + keys.add((source_id, source_key)) + return keys + + +def require_terms_review(path: Path | None) -> dict: + if path is None: + raise ValueError("network mode requires an approved terms review") + try: + review = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as error: + raise ValueError("terms review cannot be read") from error + required = {"reviewer", "reference", "reviewed_at", "decision", "notes"} + if not isinstance(review, dict) or required - review.keys(): + raise ValueError("terms review requires reviewer, reference, reviewed_at, decision, and notes") + if any(not isinstance(review[field], str) or not review[field].strip() for field in required): + raise ValueError("terms review fields must be non-empty strings") + try: + datetime.fromisoformat(review["reviewed_at"].replace("Z", "+00:00")) + except ValueError as error: + raise ValueError("terms review reviewed_at must be ISO-8601") from error + if review["decision"] != "approved": + raise ValueError("terms review decision must be 'approved'") + return {field: review[field] for field in sorted(required)} + + +def validate_provider_config(path: Path, network: bool, terms_review_path: Path | None) -> dict: + config = json.loads(path.read_text(encoding="utf-8")) + required = ("provider_id", "base_url", "mode", "status", "rate_limit_requests_per_second") + if any(key not in config for key in required): + raise ValueError("provider config is missing required approval/terms fields") + if network: + if config["status"] != "approved_for_development" or config["mode"] != "development_only": + raise ValueError("network mode requires an explicitly development-approved provider") + config["terms_review"] = require_terms_review(terms_review_path) + config["terms_review_sha256"] = hashlib.sha256(terms_review_path.read_bytes()).hexdigest() + if not isinstance(config["rate_limit_requests_per_second"], (int, float)) or config["rate_limit_requests_per_second"] <= 0: + raise ValueError("provider rate limit must be positive") + return config + + +def acquire_lock(output_path: Path) -> Path: + lock = output_path.with_suffix(output_path.suffix + ".lock") + try: + fd = os.open(lock, os.O_CREAT | os.O_EXCL | os.O_WRONLY) + with os.fdopen(fd, "w", encoding="utf-8") as handle: + handle.write(json.dumps({"pid": os.getpid()}) + "\n") + except FileExistsError as error: + raise RuntimeError(f"geocode output is locked: {lock}; inspect and remove it only after confirming the owner is gone") from error + return lock + + +def run(queue_path: Path, output_path: Path, limit: int, delay: float, retries: int, provider_config: Path, suppression_path: Path | None, terms_review_path: Path | None, network: bool = False) -> None: + lock = acquire_lock(output_path) + try: + _run_locked(queue_path, output_path, limit, delay, retries, provider_config, suppression_path, terms_review_path, network) + finally: + lock.unlink(missing_ok=True) + + +def _run_locked(queue_path: Path, output_path: Path, limit: int, delay: float, retries: int, provider_config: Path, suppression_path: Path | None, terms_review_path: Path | None, network: bool = False) -> None: + if limit <= 0 or limit > MAX_BATCH: + raise ValueError(f"limit must be between 1 and {MAX_BATCH}; full-queue runs are not permitted") + if not network: + raise ValueError("network mode must be explicitly enabled; no provider requests were made") + config = validate_provider_config(provider_config, network, terms_review_path) + suppressed = load_suppression_keys(suppression_path) + queue = [json.loads(line) for line in queue_path.read_text(encoding="utf-8").splitlines() if line.strip()] + if limit: + queue = queue[:limit] + completed = {} + if output_path.exists(): + for line in output_path.read_text(encoding="utf-8").splitlines(): + if line.strip(): + item = json.loads(line) + if item["queue_key"] in completed: + raise RuntimeError(f"duplicate queue entries detected in existing output; refusing resume: {output_path}") + completed[item["queue_key"]] = item + output_path.parent.mkdir(parents=True, exist_ok=True) + pending = [item for item in queue if item["queue_key"] not in completed and (item.get("source_id"), item.get("source_record_key")) not in suppressed] + skipped = len(queue) - len(pending) - len([item for item in queue if item["queue_key"] in completed]) + LOGGER.info("stage=geocode status=started batch=%d completed=%d suppressed=%d pending=%d provider=%s", len(queue), len(completed), skipped, len(pending), config["provider_id"]) + with output_path.open("a", encoding="utf-8", newline="\n") as output: + for index, item in enumerate(pending, start=1): + params = query_params(item["original_address"]) + queried_at = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") + result = {**item, "provider": config["provider_id"], "queried_at_utc": queried_at, "query_parameters": params, "response": None} + for attempt in range(1, retries + 1): + try: + status, payload = fetch(params, config["base_url"]) + result["http_status"] = status + result["response"] = payload + result["response_sha256"] = hashlib.sha256(json.dumps(payload, sort_keys=True, ensure_ascii=False).encode()).hexdigest() + result["acceptance"] = acceptance(payload) + result["status"] = "review_required" if result["acceptance"] == "accepted_single_point" else result["acceptance"] + result["precision"] = "review_required" + result["coordinate_review_status"] = "review_required" + break + except Exception as error: + result["status"] = "failed" + result["error"] = str(error) + LOGGER.warning("stage=geocode status=failed attempt=%d/%d", attempt, retries) + if attempt < retries: + time.sleep(min(30, 2 ** attempt)) + output.write(json.dumps(result, ensure_ascii=False, sort_keys=True) + "\n") + output.flush() + LOGGER.info("stage=geocode item=%d/%d status=%s acceptance=%s", index, len(pending), result["status"], result.get("acceptance")) + if index < len(pending): + time.sleep(delay) + report = output_path.with_name("geocode-review-report.json") + report.write_text(json.dumps({"status": "complete", "provider": config["provider_id"], "provider_base_url": config["base_url"], "terms_review_sha256": config["terms_review_sha256"], "batch_size": len(queue), "newly_processed": len(pending), "suppressed": skipped, "output_sha256": hashlib.sha256(output_path.read_bytes()).hexdigest()}, indent=2) + "\n", encoding="utf-8") + LOGGER.info("stage=geocode status=complete batch=%d newly_processed=%d suppressed=%d", len(queue), len(pending), skipped) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("queue", type=Path) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--limit", type=int, required=True) + parser.add_argument("--provider-config", type=Path, default=Path("pipeline/config/geocoding-dev.json")) + parser.add_argument("--suppression-keys", type=Path) + parser.add_argument("--terms-review", type=Path, help="Approved per-run terms review required with --network.") + parser.add_argument("--network", action="store_true", help="Permit provider requests after config approval validation") + parser.add_argument("--delay", type=float, default=1.0) + parser.add_argument("--retries", type=int, default=3) + args = parser.parse_args() + logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s", datefmt="%Y-%m-%dT%H:%M:%SZ") + run(args.queue, args.output, args.limit, args.delay, args.retries, args.provider_config, args.suppression_keys, args.terms_review, args.network) diff --git a/pipeline/sources/denmark/stages/import-denmark.py b/pipeline/sources/denmark/stages/import-denmark.py new file mode 100644 index 0000000..cff8185 --- /dev/null +++ b/pipeline/sources/denmark/stages/import-denmark.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 +"""Import classified Denmark staging data into PostgreSQL/PostGIS transactionally.""" + +import argparse +import json +import os +import uuid +from datetime import datetime, timezone +from pathlib import Path + +import psycopg + + +def now(): + return datetime.now(timezone.utc) + + +def metadata_value(metadata, *names): + for name in names: + if metadata.get(name) is not None: + return metadata[name] + return None + + +def point_from_geocode(item): + # A provider's one-point response is evidence for review, not permission + # to store/display a precise point. Only an explicit review decision can + # advance it to an accepted coordinate. + if item.get("acceptance") != "accepted_single_point" or item.get("coordinate_review_status") != "approved": + return None + response = item.get("response", []) + results = response if isinstance(response, list) else response.get("results", []) + if not results: + return None + return results[0].get("x"), results[0].get("y") + + +def geocode_status(item): + review_status = item.get("coordinate_review_status") + if review_status == "approved" and item.get("acceptance") == "accepted_single_point": + return "accepted" + if review_status == "review_required": + return "review_required" + return { + "accepted_single_point": "review_required", + "review_multiple_points": "review_required", + "unresolved": "unresolved", + }.get(item.get("acceptance"), "failed" if item.get("status") == "failed" else "unresolved") + + +def run(classified_path: Path, artifact_metadata_path: Path, geocode_path: Path | None, database_url: str, release_id: str): + artifact = json.loads(artifact_metadata_path.read_text(encoding="utf-8")) + geocodes = {} + if geocode_path and geocode_path.exists(): + geocodes = {item["queue_key"]: item for item in (json.loads(line) for line in geocode_path.read_text(encoding="utf-8").splitlines() if line.strip())} + run_id = uuid.uuid4() + artifact_id = uuid.uuid4() + checked_at = now() + with psycopg.connect(database_url) as connection: + with connection.transaction(): + connection.execute(""" + INSERT INTO uec.sources(source_id, country_code, name, official_url, access_method, cadence, status, attribution) + VALUES ('dk.smiley', 'DK', 'Find Smiley', %s, 'bulk_xml', 'weekly', 'active', 'Fødevarestyrelsen') + ON CONFLICT (source_id) DO NOTHING + """, (metadata_value(artifact, "source_url", "final_url", "requested_url"),)) + connection.execute(""" + INSERT INTO uec.acquisition_runs(run_id, source_id, checked_at, retrieved_at, ingested_at, status, source_url, code_version, config_version) + VALUES (%s, 'dk.smiley', %s, %s, %s, 'changed', %s, 'import-denmark.py', 'denmark-classification-v1') + """, (run_id, checked_at, artifact.get("retrieved_at_utc"), checked_at, metadata_value(artifact, "source_url", "final_url", "requested_url"))) + connection.execute(""" + INSERT INTO uec.raw_artifacts(artifact_id, storage_key, sha256, byte_size, media_type, retrieved_at) + VALUES (%s, %s, %s, %s, 'application/xml', %s) + ON CONFLICT (sha256) DO NOTHING + """, (artifact_id, metadata_value(artifact, "artifact_path", "artifact"), artifact["sha256"], metadata_value(artifact, "bytes", "byte_size"), artifact["retrieved_at_utc"])) + artifact_id = connection.execute("SELECT artifact_id FROM uec.raw_artifacts WHERE sha256=%s", (artifact["sha256"],)).fetchone()[0] + connection.execute("INSERT INTO uec.acquisition_run_artifacts(run_id, artifact_id) VALUES (%s, %s)", (run_id, artifact_id)) + connection.execute("INSERT INTO uec.releases(release_id, status, ruleset_version, summary) VALUES (%s, 'candidate', 'denmark-classification-v1', %s) ON CONFLICT DO NOTHING", (release_id, json.dumps({"source": "dk.smiley", "run_id": str(run_id)}))) + for line in classified_path.read_text(encoding="utf-8").splitlines(): + if not line.strip(): + continue + record = json.loads(line) + source_record_id = uuid.uuid5(uuid.NAMESPACE_URL, f"urn:uec:source-record:dk.smiley:{record['source_record_key']}:{artifact['sha256']}") + facility_id = uuid.uuid5(uuid.NAMESPACE_URL, f"urn:uec:facility:dk.smiley:{record['source_record_key']}") + observed_at = checked_at + source_fields = record["source_fields"] + address = record["address"] + classification = record["classification"] + geocode = geocodes.get(f"dk.smiley:{record['source_record_key']}") + point = point_from_geocode(geocode) if geocode else None + connection.execute(""" + INSERT INTO uec.source_records(source_record_id, source_id, source_record_key, artifact_id, raw_fields, parsed_at) + VALUES (%s, 'dk.smiley', %s, %s, %s, %s) + ON CONFLICT DO NOTHING + """, (source_record_id, record["source_record_key"], artifact_id, json.dumps(source_fields, ensure_ascii=False), checked_at)) + existing = connection.execute("SELECT source_record_id FROM uec.source_records WHERE source_id='dk.smiley' AND source_record_key=%s AND artifact_id=%s", (record["source_record_key"], artifact_id)).fetchone() + source_record_id = existing[0] + if geocode: + queried_at = geocode.get("queried_at_utc") or checked_at.isoformat() + geocode_id = uuid.uuid5(uuid.NAMESPACE_URL, f"urn:uec:geocode:{geocode['queue_key']}:{geocode.get('provider', 'unknown')}:{queried_at}") + connection.execute(""" + INSERT INTO uec.geocode_results(geocode_result_id, source_record_id, provider_id, query, provider_address_id, result, match_method, status, attempt_number, retryable, response, queried_at) + VALUES (%s, %s, %s, %s, %s, ST_SetSRID(ST_MakePoint(%s, %s), 4326)::geography, %s, %s, 1, %s, %s, %s) + ON CONFLICT (geocode_result_id) DO NOTHING + """, (geocode_id, source_record_id, geocode.get("provider", "unknown"), geocode.get("geocoder_query", ""), (geocode.get("response") or [{}])[0].get("id") if isinstance(geocode.get("response"), list) and geocode.get("response") else None, point[0] if point else None, point[1] if point else None, geocode.get("acceptance", "address"), geocode_status(geocode), geocode_status(geocode) == "failed", json.dumps(geocode, ensure_ascii=False), queried_at)) + connection.execute(""" + INSERT INTO uec.facilities(facility_id, canonical_name, country_code, street_address, postal_code, city, location) + VALUES (%s, %s, 'DK', %s, %s, %s, ST_SetSRID(ST_MakePoint(%s, %s), 4326)::geography) + ON CONFLICT DO NOTHING + """, (facility_id, record.get("name"), address.get("street"), address.get("postal_code"), address.get("city"), point[0] if point else None, point[1] if point else None)) + connection.execute(""" + INSERT INTO uec.facility_source_links(facility_id, source_record_id, match_method, review_status) + VALUES (%s, %s, 'first_source_observation', 'automatic') ON CONFLICT DO NOTHING + """, (facility_id, source_record_id)) + connection.execute(""" + INSERT INTO uec.observations(observation_id, facility_id, source_record_id, observed_at, observation, classification, ruleset_id, rule_id, classification_category, classification_review_status, default_visible, optional_filter, coordinate, coordinate_method, coordinate_precision, coordinate_review_status, first_observed_at) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, ST_SetSRID(ST_MakePoint(%s, %s), 4326)::geography, %s, %s, %s, %s) + ON CONFLICT DO NOTHING + """, (uuid.uuid5(uuid.NAMESPACE_URL, f"urn:uec:observation:dk.smiley:{record['source_record_key']}:{artifact['sha256']}"), facility_id, source_record_id, observed_at, json.dumps(record, ensure_ascii=False), json.dumps(classification), classification["ruleset_id"], classification["rule_id"], classification["category"], classification["review_status"], classification["default_visible"], classification.get("optional_filter"), point[0] if point else None, point[1] if point else None, "dawa" if point else None, "address_point" if point else None, "accepted" if point else "unresolved", observed_at)) + connection.execute(""" + INSERT INTO uec.release_members(release_id, facility_id, observation_id, default_visible) + VALUES (%s, %s, %s, %s) + ON CONFLICT DO NOTHING + """, (release_id, facility_id, uuid.uuid5(uuid.NAMESPACE_URL, f"urn:uec:observation:dk.smiley:{record['source_record_key']}:{artifact['sha256']}"), classification["default_visible"] and not connection.execute("SELECT EXISTS (SELECT 1 FROM uec.public_access_restricted WHERE source_record_id = %s)", (source_record_id,)).fetchone()[0])) + print(f"Imported Denmark run {run_id} as candidate release {release_id}") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("classified", type=Path) + parser.add_argument("--artifact-metadata", type=Path, required=True) + parser.add_argument("--geocodes", type=Path) + parser.add_argument("--database-url", default=os.environ.get("UEC_DATABASE_URL", "postgresql://uec:uec-local-development-only@localhost:5433/uec")) + parser.add_argument("--release-id", default="dk-2026-09-13-candidate") + args = parser.parse_args() + run(args.classified, args.artifact_metadata, args.geocodes, args.database_url, args.release_id) diff --git a/pipeline/sources/denmark/stages/load-city-references.py b/pipeline/sources/denmark/stages/load-city-references.py new file mode 100644 index 0000000..3da7f65 --- /dev/null +++ b/pipeline/sources/denmark/stages/load-city-references.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 +"""Append official city reference points to PostGIS.""" +import argparse, json, os +from pathlib import Path +import psycopg + +DEFAULT_DB = "postgresql://uec:uec-local-development-only@localhost:5433/uec" + +def load(path: Path, database_url: str) -> int: + rows = [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line.strip()] + with psycopg.connect(database_url) as connection: + with connection.transaction(): + inserted = 0 + for row in rows: + result = connection.execute(""" + INSERT INTO uec.city_reference_points + (country_code, city_name, postal_code, reference_location, + reference_source, source_retrieved_at, source_reference_id) + VALUES (%s, %s, %s, ST_SetSRID(ST_MakePoint(%s, %s), 4326)::geography, + %s, %s, %s) + ON CONFLICT DO NOTHING + """, (row["country_code"], row["city_name"], row.get("postal_code"), + row["reference_longitude"], row["reference_latitude"], + row["reference_source"], row["source_retrieved_at"], row["source_reference_id"])) + inserted += result.rowcount + return inserted + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("staging", type=Path) + parser.add_argument("--database-url", default=os.environ.get("UEC_DATABASE_URL", DEFAULT_DB)) + args = parser.parse_args() + print(f"Inserted {load(args.staging, args.database_url)} city reference points") diff --git a/pipeline/sources/denmark/stages/normalize-denmark-smiley.py b/pipeline/sources/denmark/stages/normalize-denmark-smiley.py new file mode 100644 index 0000000..687c721 --- /dev/null +++ b/pipeline/sources/denmark/stages/normalize-denmark-smiley.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python3 +"""Normalize staged Find Smiley rows while preserving source values.""" + +from __future__ import annotations + +import argparse +import json +import logging +import sys +from collections import Counter +from datetime import datetime +from pathlib import Path + +LOGGER = logging.getLogger("uec.denmark.normalize") +SOURCE_TO_CANONICAL = { + "ID_nummer": "source_record_key", + "CVR_nummer": "organization_registration_id", + "P_nummer": "production_unit_id", + "Virksomhed": "name", + "Adresse": "street_address", + "Postnummer": "postal_code", + "By": "city", + "FVST_branchenummer": "industry_code", + "FVST_branche": "industry_label", + "Smileybranche": "category_label", + "Virksomhedstype": "business_type", + "URL": "source_url", + "Geo_Lat": "source_latitude", + "Geo_Lng": "source_longitude", +} + + +def iso_date(value: str | None) -> str | None: + if not value: + return None + for fmt in ("%d/%m/%Y", "%d-%m-%Y %H:%M:%S"): + try: + return datetime.strptime(value, fmt).date().isoformat() + except ValueError: + pass + return None + + +def normalize_record(envelope: dict) -> dict: + source = envelope.get("fields", {}) + normalized = { + "source_id": envelope.get("source_id", "dk.smiley"), + "source_record_key": source.get("ID_nummer") or source.get("navnelbnr"), + "source_artifact_sha256": envelope.get("source_artifact_sha256"), + "name": source.get("Virksomhed") or source.get("navn1"), + "organization_registration_id": source.get("CVR_nummer") or source.get("cvrnr"), + "production_unit_id": source.get("P_nummer") or source.get("pnr"), + "address": { + "street": source.get("Adresse") or source.get("adresse1"), + "postal_code": source.get("Postnummer") or source.get("postnr"), + "city": source.get("By"), + "country_code": "DK", + }, + "activity": { + "code": source.get("FVST_branchenummer") or source.get("brancheKode"), + "label": source.get("FVST_branche") or source.get("branche"), + "category": source.get("Smileybranche") or source.get("Pixibranche"), + }, + "business_type": source.get("Virksomhedstype") or source.get("virksomhedstype"), + "coordinates": { + "latitude": source.get("Geo_Lat"), + "longitude": source.get("Geo_Lng"), + "method": "source" if source.get("Geo_Lat") and source.get("Geo_Lng") else None, + "review_status": "source" if source.get("Geo_Lat") and source.get("Geo_Lng") else "unresolved", + }, + "latest_inspection_date": iso_date(source.get("Seneste_kontrol_dato") or source.get("seneste_kontrol_dato")), + "source_url": source.get("URL"), + "source_fields": source, + } + return normalized + + +def normalize_file(input_path: Path, output_dir: Path, progress_every: int = 10000) -> Path: + output_dir.mkdir(parents=True, exist_ok=True) + output_path = output_dir / "normalized-records.jsonl" + report_path = output_dir / "field-mapping-report.json" + count = 0 + missing_coords = 0 + keys = Counter() + LOGGER.info("stage=normalize status=started input=%s", input_path) + with input_path.open(encoding="utf-8") as source, output_path.open("w", encoding="utf-8", newline="\n") as output: + for line in source: + if not line.strip(): + continue + record = normalize_record(json.loads(line)) + count += 1 + keys.update(record["source_fields"].keys()) + if record["coordinates"]["review_status"] == "unresolved": + missing_coords += 1 + output.write(json.dumps(record, ensure_ascii=False, sort_keys=True) + "\n") + if count % progress_every == 0: + LOGGER.info("stage=normalize records=%d missing_coordinates=%d", count, missing_coords) + report = { + "status": "success", + "input_path": input_path.as_posix(), + "output_path": output_path.as_posix(), + "records_normalized": count, + "records_missing_coordinates": missing_coords, + "source_to_canonical_mapping": SOURCE_TO_CANONICAL, + "observed_source_fields": sorted(keys), + "unmapped_source_fields": sorted(set(keys) - set(SOURCE_TO_CANONICAL)), + "date_policy": "recognized dates are emitted as ISO dates; original values remain in source_fields", + } + report_path.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + LOGGER.info("stage=normalize status=success records=%d missing_coordinates=%d report=%s", count, missing_coords, report_path) + return output_path + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("input", type=Path) + parser.add_argument("--output-dir", type=Path, required=True) + args = parser.parse_args() + logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s", datefmt="%Y-%m-%dT%H:%M:%SZ") + try: + normalize_file(args.input, args.output_dir) + except Exception as error: + LOGGER.error("stage=normalize status=failed error=%s", error) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/pipeline/sources/denmark/stages/parse-denmark-smiley.py b/pipeline/sources/denmark/stages/parse-denmark-smiley.py new file mode 100644 index 0000000..e2f7938 --- /dev/null +++ b/pipeline/sources/denmark/stages/parse-denmark-smiley.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python3 +"""Parse an archived Find Smiley XML file into auditable JSONL staging output.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import logging +import sys +import uuid +import xml.etree.ElementTree as ET +from datetime import datetime, timezone +from pathlib import Path +from typing import Iterator + + +LOGGER = logging.getLogger("uec.denmark.parse") +def utc_now() -> str: + return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as source: + for chunk in iter(lambda: source.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def iter_rows(path: Path) -> Iterator[dict[str, str | None]]: + for _, element in ET.iterparse(path, events=("end",)): + if element.tag.lower() != "row": + continue + values = {child.tag: (child.text or "").strip() or None for child in element} + yield values + element.clear() + + +def parse_file(input_path: Path, output_dir: Path, source_url: str, progress_every: int = 1000) -> Path: + if not input_path.is_file(): + raise FileNotFoundError(input_path) + output_dir.mkdir(parents=True, exist_ok=True) + run_id = f"{datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ')}-{uuid.uuid4().hex[:8]}" + output_path = output_dir / "parsed-rows.jsonl" + metadata_path = output_dir / "run-metadata.json" + started_at = utc_now() + source_hash = sha256_file(input_path) + LOGGER.info("run_id=%s stage=parse status=started input=%s", run_id, input_path) + LOGGER.info("run_id=%s source_sha256=%s", run_id, source_hash) + + row_count = 0 + missing_coordinates = 0 + try: + with output_path.open("w", encoding="utf-8", newline="\n") as output: + for row_count, row in enumerate(iter_rows(input_path), start=1): + latitude = row.get("Geo_Lat") or row.get("Geo_Latitude") + longitude = row.get("Geo_Lng") or row.get("Geo_Longitude") + if not latitude or not longitude: + missing_coordinates += 1 + output.write(json.dumps({ + "source_id": "dk.smiley", + "source_record_key": row.get("ID_nummer") or row.get("navnelbnr"), + "source_artifact_sha256": source_hash, + "fields": row, + }, ensure_ascii=False, sort_keys=True) + "\n") + if row_count % progress_every == 0: + LOGGER.info("run_id=%s stage=parse rows=%d missing_coordinates=%d", run_id, row_count, missing_coordinates) + except ET.ParseError: + LOGGER.exception("run_id=%s stage=parse status=failed reason=invalid_xml", run_id) + raise + + metadata = { + "run_id": run_id, + "source_id": "dk.smiley", + "source_url": source_url, + "input_path": input_path.as_posix(), + "input_sha256": source_hash, + "started_at_utc": started_at, + "completed_at_utc": utc_now(), + "status": "success", + "rows_parsed": row_count, + "rows_missing_coordinates": missing_coordinates, + "output_path": output_path.as_posix(), + "parser": "parse-denmark-smiley.py", + } + metadata_path.write_text(json.dumps(metadata, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + LOGGER.info("run_id=%s stage=parse status=success rows=%d missing_coordinates=%d output=%s", run_id, row_count, missing_coordinates, output_path) + return output_path + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("input", type=Path) + parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument("--source-url", default="https://pub.fvst.dk/publikationer/Smileydata.xml") + parser.add_argument("--progress-every", type=int, default=1000) + args = parser.parse_args() + logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s", datefmt="%Y-%m-%dT%H:%M:%SZ") + try: + parse_file(args.input, args.output_dir, args.source_url, args.progress_every) + except Exception as error: + LOGGER.error("stage=parse status=failed error=%s", error) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/pipeline/sources/denmark/stages/validate-denmark.py b/pipeline/sources/denmark/stages/validate-denmark.py new file mode 100644 index 0000000..bfd0cf4 --- /dev/null +++ b/pipeline/sources/denmark/stages/validate-denmark.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python3 +"""Validate classified Denmark records without deleting invalid rows.""" + +import argparse +import collections +import json +import logging +from datetime import datetime +from pathlib import Path + +LOGGER = logging.getLogger("uec.denmark.validate") + + +def validate_file(input_path: Path, output_dir: Path, expected_rows: int | None = None, progress_every: int = 10000) -> Path: + output_dir.mkdir(parents=True, exist_ok=True) + report_path = output_dir / "validation-report.json" + quarantine_path = output_dir / "validation-findings.jsonl" + counts = collections.Counter() + seen_keys = set() + findings = [] + total = 0 + LOGGER.info("stage=validate status=started input=%s", input_path) + with input_path.open(encoding="utf-8") as source, quarantine_path.open("w", encoding="utf-8", newline="\n") as rejected: + for line in source: + if not line.strip(): + continue + total += 1 + record = json.loads(line) + key = record.get("source_record_key") + row_findings = [] + if not key: + row_findings.append(("error", "missing_source_record_key")) + elif key in seen_keys: + row_findings.append(("error", "duplicate_source_record_key")) + else: + seen_keys.add(key) + address = record.get("address", {}) + if not record.get("name"): + row_findings.append(("warning", "missing_name")) + if not address.get("street") and not address.get("city") and not address.get("postal_code"): + row_findings.append(("error", "missing_address")) + if address.get("country_code") != "DK": + row_findings.append(("error", "unexpected_country")) + date_value = record.get("latest_inspection_date") + if date_value: + try: + datetime.strptime(date_value, "%Y-%m-%d") + except ValueError: + row_findings.append(("error", "invalid_normalized_date")) + classification = record.get("classification", {}) + if not classification.get("rule_id"): + row_findings.append(("error", "missing_classification_rule")) + if classification.get("review_status") == "review_required": + row_findings.append(("review", "classification_requires_review")) + coordinate_status = record.get("coordinates", {}).get("review_status") + counts["coordinates_" + str(coordinate_status)] += 1 + if row_findings: + finding = {"source_record_key": key, "findings": [{"severity": s, "code": c} for s, c in row_findings]} + rejected.write(json.dumps({"record": record, "findings": finding["findings"]}, ensure_ascii=False, sort_keys=True) + "\n") + findings.append(finding) + for _, code in row_findings: + counts[code] += 1 + if total % progress_every == 0: + LOGGER.info("stage=validate records=%d findings=%d", total, len(findings)) + if expected_rows is not None and total != expected_rows: + counts["unexpected_row_count"] += 1 + report = { + "status": "success", + "input_path": input_path.as_posix(), + "records_checked": total, + "unique_source_record_keys": len(seen_keys), + "finding_records": len(findings), + "expected_rows": expected_rows, + "counts": dict(counts), + "findings_path": quarantine_path.as_posix(), + "policy": "findings are reported and preserved; no records are deleted", + } + report_path.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + LOGGER.info("stage=validate status=success records=%d finding_records=%d report=%s", total, len(findings), report_path) + return report_path + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("input", type=Path) + parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument("--expected-rows", type=int) + args = parser.parse_args() + logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s", datefmt="%Y-%m-%dT%H:%M:%SZ") + validate_file(args.input, args.output_dir, args.expected_rows) diff --git a/pipeline/tests/test_denmark_entrypoints.py b/pipeline/tests/test_denmark_entrypoints.py new file mode 100644 index 0000000..c13bff6 --- /dev/null +++ b/pipeline/tests/test_denmark_entrypoints.py @@ -0,0 +1,33 @@ +import subprocess +import sys +import unittest +from pathlib import Path + + +ROOT = Path(__file__).parents[1] + + +class DenmarkEntrypointTests(unittest.TestCase): + def test_new_entrypoint_preserves_help_contract(self): + result = subprocess.run( + [sys.executable, str(ROOT / "sources/denmark/run-denmark-pipeline.py"), "--help"], + cwd=ROOT.parent, + capture_output=True, + text=True, + ) + self.assertEqual(result.returncode, 0) + self.assertIn("Denmark", result.stdout) + + def test_legacy_entrypoint_remains_available(self): + result = subprocess.run( + [sys.executable, str(ROOT / "run-denmark-pipeline.py"), "--help"], + cwd=ROOT.parent, + capture_output=True, + text=True, + ) + self.assertEqual(result.returncode, 0) + self.assertIn("Denmark", result.stdout) + + +if __name__ == "__main__": + unittest.main() From cffb10866fe240a8dcce9a96ef84a41a716459c3 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sun, 13 Sep 2026 20:28:03 -0700 Subject: [PATCH 040/311] fix(frontend): stabilize V2 local query lifecycle --- docs/V2-SPRINT-2026-09-13.md | 44 +++++++++++++++ frontend/src/app/App.svelte | 54 +++++++++++++------ frontend/tests/e2e/local-backend.spec.ts | 20 +++++-- .../unit/localLocationRepository.test.ts | 15 ++++++ 4 files changed, 113 insertions(+), 20 deletions(-) create mode 100644 docs/V2-SPRINT-2026-09-13.md diff --git a/docs/V2-SPRINT-2026-09-13.md b/docs/V2-SPRINT-2026-09-13.md new file mode 100644 index 0000000..0fa6c41 --- /dev/null +++ b/docs/V2-SPRINT-2026-09-13.md @@ -0,0 +1,44 @@ +# V2 sprint integration brief — 2026-09-13 + +This is a development-branch integration record, not publication approval or a +claim that V2 is deployed. Real raw and derived records remain private and are +not included here. + +## Verified local evidence + +- The canonical disposable standard runner passed 63 Rust library tests, 9 + Rust binary tests, and 69 Python tests; 5 Python tests were expected skips. +- Sequential disposable API E2E passed public 8/8, community 6/6, and seeded + 19/19. The real-backend browser journey passed 1/1 against synthetic data. +- The portless backup/restore drill passed: an old backup was rejected before + service start, current restrictions were replayed, and restricted records + stayed out of both public projections. +- Frontend checks, 42 unit tests, build, lint, and boundary checks passed; the + full one-worker Playwright matrix passed 42/42 with 3 expected skips. +- Root Jest passed 19/19. Moved Denmark stages compiled explicitly and + `git diff --check` passed. + +## Scope and limitations + +The source inventory covers nine non-US country codes (CA, DE, DK, ES, FR, IT, +MX, NZ, and GB); US has separate source identities and is listed separately. +Germany and UK adapters remain synthetic/restricted foundations. UK FSS/FSA +composition is identity-preserving and release creation remains human-gated. +Any private UK fetch requires a maintainer-approved terms decision, provenance, +and source-specific release checks; no live fetch or public release is implied. + +The backend ledger gate and restore replay contract are implemented for +disposable/local verification only. Production startup is not wired to an +independent restriction operator, ledger retention policy, or release authority. +Those decisions remain open under `docs/ETHICS.md`. + +V1↔V2 crosswalk mappings remain unresolved unless a deterministic shared source +identifier is evidenced. Name, address, or geocoder similarity is not a mapping +and must not be used to expose or suppress a record silently. + +The frontend page-local `q`/search behavior operates on the loaded page and is +not a complete release-wide query. V2 remains a gated development capability; +source terms, privacy/removal operations, human approval, deployment controls, +provider logging, and cross-system suppression remain release gates. + +Remote CI was not verified by this local run and remains pending. diff --git a/frontend/src/app/App.svelte b/frontend/src/app/App.svelte index 3bdad0d..79b7cd7 100644 --- a/frontend/src/app/App.svelte +++ b/frontend/src/app/App.svelte @@ -17,6 +17,7 @@ let profile: Profile = 'curated'; let selected: Location | undefined = locations[0]; let search = ''; let region = 'all'; let category = 'all'; + let sourceType = 'all'; let displayPrecision = 'all'; let lifecycleStatus = 'all'; let filters: FilterState = initialFilters; let showMap = false; let showExport = false; let showGuidance = false; let localMode = false; @@ -28,8 +29,11 @@ let repo = new LocalLocationRepository(); let csvRepo = new LocalCsvExportRepository(); let metadata: FilterMetadata | undefined; let metadataStatus: 'idle' | 'loading' | 'ready' | 'error' = 'idle'; let nextCursor: string | null = null; let coverageNote = ''; + let paging = false; let listGeneration = 0; let detailGeneration = 0; let listAbort: AbortController | undefined; let detailAbort: AbortController | undefined; + let detailRequestKey = ''; + let activeListKey = ''; let detailErrorHeading: HTMLHeadingElement; $: filters = { search, region, category }; $: source = localMode ? loaded : (profile === 'community' ? locations.slice(0, 1) : locations); @@ -38,14 +42,30 @@ $: eligibleExport = localMode && profile === 'curated' && localStatus === 'ready' && Boolean(release); $: profileLabel = profile === 'curated' ? 'Curated release' : 'Community claims'; let lastRemoteQuery = ''; - $: remoteQuery = `${profile}|${region}|${category}|${search}`; - $: if (localMode && (localStatus === 'ready' || localStatus === 'loading') && remoteQuery !== lastRemoteQuery) { lastRemoteQuery = remoteQuery; const url = new URL(window.location.href); for (const key of ['country_code', 'category', 'q']) url.searchParams.delete(key); if (region !== 'all') url.searchParams.set('country_code', region); if (category !== 'all') url.searchParams.set('category', category); if (search.trim()) url.searchParams.set('q', search.trim()); history.pushState(null, '', url); void loadLocal(); } + $: remoteQuery = `${profile}|${region}|${category}|${sourceType}|${displayPrecision}|${lifecycleStatus}|${search}`; + const currentRemoteQuery = () => `${profile}|${region}|${category}|${sourceType}|${displayPrecision}|${lifecycleStatus}|${search}`; + $: if (localMode && (localStatus === 'ready' || localStatus === 'loading') && remoteQuery !== lastRemoteQuery) { pushFilterUrl(); void loadLocal(); } + + const pushFilterUrl = () => { + const url = new URL(window.location.href); + for (const key of ['country_code', 'category', 'source_type', 'display_precision', 'lifecycle_status', 'q']) url.searchParams.delete(key); + if (region !== 'all') url.searchParams.set('country_code', region); + if (category !== 'all') url.searchParams.set('category', category); + if (sourceType !== 'all') url.searchParams.set('source_type', sourceType); + if (displayPrecision !== 'all') url.searchParams.set('display_precision', displayPrecision); + if (lifecycleStatus !== 'all') url.searchParams.set('lifecycle_status', lifecycleStatus); + if (search.trim()) url.searchParams.set('q', search.trim()); + history.pushState(null, '', url); + }; const invalidateDetail = () => { detailGeneration += 1; detailAbort?.abort(); detailAbort = undefined; detailStatus = 'idle'; }; const syncRoute = async () => { const route = parseRoute(window.location.hash); if (route.kind === 'location' && localMode) { + const requestKey = `${route.facilityId}|${route.profile}|${profile}`; + if (detailRequestKey === requestKey && detailStatus === 'loading') return; + detailRequestKey = requestKey; invalidateDetail(); const generation = detailGeneration; const requestedProfile = profile === 'community' ? 'community' : 'official'; @@ -54,15 +74,18 @@ try { const result = await repo.detail(route.facilityId, requestedProfile, controller.signal); const currentRoute = parseRoute(window.location.hash); if (generation !== detailGeneration || currentRoute.kind !== 'location' || currentRoute.facilityId !== route.facilityId || currentRoute.profile !== route.profile || profile !== route.profile) return; if (result.releaseId !== release) throw new Error('Local V2 detail belongs to a different release.'); selected = result.location; detailStatus = 'idle'; } catch (error) { if (generation !== detailGeneration) return; selected = undefined; detailStatus = 'error'; localError = error instanceof Error ? error.message : 'The local detail response was rejected safely.'; await tick(); detailErrorHeading?.focus(); } } else if (route.kind === 'location') selected = source.find((item) => item.id === route.facilityId) ?? selected; - else if (route.kind === 'home') { invalidateDetail(); selected = localMode ? loaded[0] : source[0]; } + else if (route.kind === 'home') { detailRequestKey = ''; invalidateDetail(); selected = localMode ? loaded[0] : source[0]; } }; - const loadLocal = async () => { + const loadLocal = async (cursor?: string, append = false) => { + const queryKey = currentRemoteQuery(); + if (!append && activeListKey === queryKey) return; + if (!append) activeListKey = queryKey; listGeneration += 1; const generation = listGeneration; listAbort?.abort(); const controller = new AbortController(); listAbort = controller; - invalidateDetail(); lastRemoteQuery = `${profile}|${region}|${category}|${search}`; - localStatus = 'loading'; localError = ''; manifestSha256 = undefined; loaded = []; selected = undefined; nextCursor = null; coverageNote = ''; - try { const result = await repo.list(profile === 'community' ? 'community' : 'official', { country_code: region === 'all' ? undefined : region, category: category === 'all' ? undefined : category }, controller.signal); if (generation !== listGeneration) return; loaded = result.locations; selected = result.locations[0]; release = result.releaseId; ruleset = result.ruleset; nextCursor = result.nextCursor; coverageNote = result.coverageNote; localStatus = 'ready'; await syncRoute(); } - catch (error) { if (generation !== listGeneration) return; const kind = error && typeof error === 'object' && 'kind' in error ? (error as { kind: string }).kind : 'error'; localStatus = kind === 'no-release' ? 'no-release' : 'error'; localError = error instanceof Error ? error.message : 'Local V2 response was rejected safely.'; loaded = []; selected = undefined; } + lastRemoteQuery = queryKey; + if (!append) { invalidateDetail(); localStatus = 'loading'; localError = ''; manifestSha256 = undefined; loaded = []; selected = undefined; nextCursor = null; coverageNote = ''; } else paging = true; + try { const result = await repo.list(profile === 'community' ? 'community' : 'official', { country_code: region === 'all' ? undefined : region, category: category === 'all' ? undefined : category, source_type: sourceType === 'all' ? undefined : sourceType, display_precision: displayPrecision === 'all' ? undefined : displayPrecision, lifecycle_status: lifecycleStatus === 'all' ? undefined : lifecycleStatus, cursor }, controller.signal); if (generation !== listGeneration) return; loaded = append ? [...loaded, ...result.locations] : result.locations; if (!selected) selected = result.locations[0]; release = result.releaseId; ruleset = result.ruleset; nextCursor = result.nextCursor; coverageNote = result.coverageNote; localStatus = 'ready'; paging = false; if (!append) await syncRoute(); } + catch (error) { paging = false; if (!append) activeListKey = ''; if (generation !== listGeneration) return; const kind = error && typeof error === 'object' && 'kind' in error ? (error as { kind: string }).kind : 'error'; localStatus = kind === 'no-release' ? 'no-release' : 'error'; localError = error instanceof Error ? error.message : 'Local V2 response was rejected safely.'; loaded = []; selected = undefined; } }; const downloadCsv = async () => { if (!eligibleExport || exportBusy) return; exportBusy = true; exportError = ''; @@ -72,19 +95,20 @@ }; const select = (id: string) => { selected = source.find((item) => item.id === id) ?? selected; window.location.hash = `/locations/${id}?profile=${profile}`; }; const profileChanged = () => { history.pushState(null, '', `#/?profile=${profile}`); if (localMode) void loadLocal(); else void syncRoute(); }; - const clearFilters = () => { search = ''; region = 'all'; category = 'all'; }; + const clearFilters = () => { search = ''; region = 'all'; category = 'all'; sourceType = 'all'; displayPrecision = 'all'; lifecycleStatus = 'all'; }; + const searchChanged = () => { if (localMode) pushFilterUrl(); }; onMount(() => { const params = new URLSearchParams(window.location.search); localMode = params.get('mode') === 'local-v2'; const route = parseRoute(window.location.hash); if (route.kind !== 'not-found') profile = route.profile; if (localMode) { - search = params.get('q') ?? ''; region = params.get('country_code') ?? 'all'; category = params.get('category') ?? 'all'; + search = params.get('q') ?? ''; region = params.get('country_code') ?? 'all'; category = params.get('category') ?? 'all'; sourceType = params.get('source_type') ?? 'all'; displayPrecision = params.get('display_precision') ?? 'all'; lifecycleStatus = params.get('lifecycle_status') ?? 'all'; try { const api = params.get('api') ?? undefined; repo = new LocalLocationRepository(globalThis.fetch, api); csvRepo = new LocalCsvExportRepository(globalThis.fetch, api ?? ''); metadataStatus = 'loading'; void new FilterMetadataRepository(globalThis.fetch, api ?? '').get().then((value) => { metadata = value; metadataStatus = 'ready'; }).catch(() => { metadataStatus = 'error'; }); } catch (error) { localStatus = 'error'; localError = error instanceof Error ? error.message : 'Local API origin was rejected safely.'; } } if (localMode && localStatus !== 'error') void loadLocal(); else if (!localMode) void syncRoute(); const onHashChange = () => { const next = parseRoute(window.location.hash); if (next.kind !== 'not-found' && next.profile !== profile) { profile = next.profile; if (localMode) void loadLocal(); else void syncRoute(); } else void syncRoute(); }; - const onPopState = () => { const current = new URLSearchParams(window.location.search); if (localMode) { search = current.get('q') ?? ''; region = current.get('country_code') ?? 'all'; category = current.get('category') ?? 'all'; } onHashChange(); }; + const onPopState = () => { const current = new URLSearchParams(window.location.search); if (localMode) { search = current.get('q') ?? ''; region = current.get('country_code') ?? 'all'; category = current.get('category') ?? 'all'; sourceType = current.get('source_type') ?? 'all'; displayPrecision = current.get('display_precision') ?? 'all'; lifecycleStatus = current.get('lifecycle_status') ?? 'all'; } onHashChange(); }; window.addEventListener('hashchange', onHashChange); window.addEventListener('popstate', onPopState); return () => { listAbort?.abort(); detailAbort?.abort(); window.removeEventListener('hashchange', onHashChange); window.removeEventListener('popstate', onPopState); }; }); @@ -94,13 +118,13 @@
UNTIL EVERY CAGE V2 / FIELD NOTE

EVIDENCE DESK · {localMode ? 'LOCAL V2 API' : 'SYNTHETIC PREVIEW'}

See what a record can—and cannot—tell us.

Start with the source profile, narrow the visible evidence, then inspect what a record can—and cannot—tell us.

-

01 / DISCOVER

Choose the evidence lane

Profiles stay separate. A community claim is never silently promoted into a curated result.

{#if localMode && metadataStatus === 'loading'}

Loading available filter values…

{:else if localMode && metadataStatus === 'error'}

Filter metadata is unavailable; only unscoped discovery is shown.

{/if}
-
{search || region !== 'all' || category !== 'all' ? `Filters applied: ${[search && `name “${search}”`, region !== 'all' && region, category !== 'all' && category].filter(Boolean).join(' · ')}` : 'No additional filters applied'}
+

01 / DISCOVER

Choose the evidence lane

Profiles stay separate. A community claim is never silently promoted into a curated result.

{#if localMode && metadataStatus === 'loading'}

Loading available filter values…

{:else if localMode && metadataStatus === 'error'}

Filter metadata is unavailable; only unscoped discovery is shown.

{/if}
+
{search || region !== 'all' || category !== 'all' || sourceType !== 'all' || displayPrecision !== 'all' || lifecycleStatus !== 'all' ? `Filters applied: ${[search && `name “${search}”`, region !== 'all' && region, category !== 'all' && category, sourceType !== 'all' && sourceType, displayPrecision !== 'all' && displayPrecision, lifecycleStatus !== 'all' && lifecycleStatus].filter(Boolean).join(' · ')}` : 'No additional filters applied'}
{#if profile === 'community'}
Community claimsUnreviewed community claims: Not verified by Until Every Cage. Check each record’s factual review status before relying on it.
{/if} - {#if localMode && localStatus === 'loading'}
Loading the {profileLabel.toLowerCase()}…
{:else if localMode && (localStatus === 'error' || localStatus === 'no-release')}{:else if localMode && detailStatus === 'loading'}
Loading the selected local record…
{:else if localMode && detailStatus === 'error'}{:else} + {#if localMode && localStatus === 'loading'}
Loading the {profileLabel.toLowerCase()}…
{:else if localMode && (localStatus === 'error' || localStatus === 'no-release')}{:else if localMode && detailStatus === 'loading'}
Loading the selected local record…
{:else if localMode && detailStatus === 'error'}{:else}

02 / FILTER & COMPARE

Results {visibleLocations.length}{localMode && nextCursor ? ' on first page' : ''}

{localMode ? 'Current eligible response' : 'Fictional demonstration data'} · {profileLabel}

{#if localMode}

{coverageNote} {nextCursor ? 'Only the first page is loaded. Search and filters below may miss later records; counts and map points are partial.' : 'All records in this response are loaded; search applies to those records.'}

{/if} -
{#if selected}

03 / RECORD DETAIL

{selected.name}

{selected.region} · {selected.category}

{#if selected.evidence?.publicationProfile === 'community' && selected.evidence.factualReviewStatus === 'unreviewed'}

{selected.evidence.publicationWarning ?? 'Unreviewed community claim — not verified by Until Every Cage'}

{/if}
OBSERVED{selected.observed}
MAP STATUS{selected.lat === null ? 'No publishable map location' : selected.evidence?.displayPrecision === 'exact' ? 'Exact display point' : 'Approximate display point'}
{#if selected.evidence}
Source origin
{selected.evidence.sourceType === 'user_submitted' ? 'Community-submitted' : selected.evidence.sourceType === 'official' ? 'Government-sourced' : 'Secondary-sourced'}
Factual review
{selected.evidence.factualReviewStatus}{selected.evidence.reviewerRole ? ` · ${selected.evidence.reviewerRole}` : ' · reviewer role unavailable'}
Privacy screening
{selected.evidence.privacyScreeningStatus}
Project approval
{selected.evidence.projectApproval}
Published profile
{selected.evidence.publicationProfile} · release {release}
Source
{selected.source} · {selected.evidence.sourceId}
Retrieved
{selected.evidence.retrievedAt}
{/if}

READ WITH CARE

This is an observation in a particular release, not a guarantee of current operation. Source origin, factual review, privacy screening, and project approval are separate signals.

{/if}
+
{#if selected}

03 / RECORD DETAIL

{selected.name}

{selected.region} · {selected.category}

{#if selected.evidence?.publicationProfile === 'community' && selected.evidence.factualReviewStatus === 'unreviewed'}

{selected.evidence.publicationWarning ?? 'Unreviewed community claim — not verified by Until Every Cage'}

{/if}
OBSERVED{selected.observed}
MAP STATUS{selected.lat === null ? 'No publishable map location' : selected.evidence?.displayPrecision === 'exact' ? 'Exact display point' : 'Approximate display point'}
{#if selected.evidence}
Source origin
{selected.evidence.sourceType === 'user_submitted' ? 'Community-submitted' : selected.evidence.sourceType === 'official' ? 'Government-sourced' : 'Secondary-sourced'}
Factual review
{selected.evidence.factualReviewStatus}{selected.evidence.reviewerRole ? ` · ${selected.evidence.reviewerRole}` : ' · reviewer role unavailable'}
Privacy screening
{selected.evidence.privacyScreeningStatus}
Project approval
{selected.evidence.projectApproval}
Published profile
{selected.evidence.publicationProfile} · release {release}
Source
{selected.source} · {selected.evidence.sourceId}
Retrieved
{selected.evidence.retrievedAt}
{/if}

READ WITH CARE

This is an observation in a particular release, not a guarantee of current operation. Source origin, factual review, privacy screening, and project approval are separate signals.

{/if}
{#if selected}

RECORD / {selected.id}

{/if} {/if} {#if localMode && localStatus === 'ready'}{/if} diff --git a/frontend/tests/e2e/local-backend.spec.ts b/frontend/tests/e2e/local-backend.spec.ts index 172da47..b9d8bc2 100644 --- a/frontend/tests/e2e/local-backend.spec.ts +++ b/frontend/tests/e2e/local-backend.spec.ts @@ -3,17 +3,27 @@ import { test, expect } from '@playwright/test'; test.skip(process.env.LOCAL_V2_E2E !== '1', 'Set LOCAL_V2_E2E=1 to run against the real local backend'); test('renders the real seeded local V2 record and opens its detail route', async ({ page }) => { - const fixtureNames = ['North Star Cooperative', 'River Meadow Foods', 'Quiet Field Holdings']; + const apiUrl = process.env.UEC_E2E_API_URL ?? process.env.LOCAL_V2_API_URL ?? 'http://127.0.0.1:8000'; let list: { data?: Array<{ facility_id: string; canonical_name: string }> } = {}; - const response = await fetch('http://127.0.0.1:8000/api/v2/locations?profile=official&limit=1'); + const response = await fetch(`${apiUrl}/api/v2/locations?profile=official&limit=1`); expect(response.ok).toBeTruthy(); list = await response.json() as typeof list; const record = list.data?.[0]; expect(record?.facility_id).toBeTruthy(); expect(record?.canonical_name).toBeTruthy(); - await page.goto('./?mode=local-v2&api=http%3A%2F%2F127.0.0.1%3A8000#/'); + await page.route('**/api/v2/**', async route => { + const requestUrl = new URL(route.request().url()); + const upstream = await fetch(`${apiUrl}${requestUrl.pathname}${requestUrl.search}`); + await route.fulfill({ status: upstream.status, headers: { 'content-type': upstream.headers.get('content-type') ?? 'application/json' }, body: await upstream.text() }); + }); + await page.goto('./?mode=local-v2#/'); + const resultButton = page.getByRole('button', { name: new RegExp(record?.canonical_name ?? '') }); + await expect(resultButton).toBeVisible(); + await resultButton.click(); await expect(page.getByRole('heading', { name: record?.canonical_name ?? '' })).toBeVisible(); - for (const name of fixtureNames) await expect(page.getByText(name, { exact: true })).toHaveCount(0); - await page.goto(`./?mode=local-v2&api=http%3A%2F%2F127.0.0.1%3A8000#/locations/${record?.facility_id}?profile=curated`); + await expect(page.getByText('Fictional demonstration data', { exact: true })).toHaveCount(0); + await expect(page.locator('article')).toContainText('Government-sourced'); + await page.goto(`./?mode=local-v2#/locations/${record?.facility_id}?profile=curated`); await expect(page.getByRole('heading', { name: record?.canonical_name ?? '' })).toBeVisible(); + await expect(page.locator('article')).toContainText('Project approval'); }); diff --git a/frontend/tests/unit/localLocationRepository.test.ts b/frontend/tests/unit/localLocationRepository.test.ts index af4046a..fa89f00 100644 --- a/frontend/tests/unit/localLocationRepository.test.ts +++ b/frontend/tests/unit/localLocationRepository.test.ts @@ -2,6 +2,21 @@ import{describe,expect,it,vi}from'vitest';import{LocalLocationRepository}from'.. const row={facility_id:'550e8400-e29b-41d4-a716-446655440000',canonical_name:'Local V2 Fixture',city:'North Coast',country_code:'DK',category:'dairy',source_type:'official',publication_profile:'official',factual_review_status:'reviewed',privacy_screening_status:'passed',project_approval:'approved',reviewer_role:null,publication_warning:null,display_precision:'city',latitude:55,longitude:10,first_observed_at:null,last_observed_at:'2026-01-01T00:00:00Z',observation_count:1,lifecycle_status:'active_observed',provenance_source_id:'source-1',provenance_source_name:'Synthetic local source',provenance_source_url:'https://example.test/source',provenance_retrieved_at:'2026-01-01T00:00:00Z',release_id:'rel-1',release_ruleset_version:'rules-1'}; const response=(body:unknown,status=200)=>new Response(JSON.stringify(body),{status,headers:{'content-type':'application/json'}});const envelope=(data=[row],meta={release_id:'rel-1',ruleset_version:'rules-1',profile:'official',next_cursor:null,coverage_note:'Local promoted release.'})=>({data,api_version:'v2',meta}); describe('LocalLocationRepository',()=>{it('maps a valid Rust-shaped envelope',async()=>{const result=await new LocalLocationRepository(vi.fn().mockResolvedValue(response(envelope()))).list();expect(result.locations[0]).toMatchObject({id:row.facility_id,name:'Local V2 Fixture',lat:55});});it('fails closed when no release is promoted',async()=>{await expect(new LocalLocationRepository(vi.fn().mockResolvedValue(response(envelope([],{release_id:null,profile:'official',coverage_note:'No promoted release.'})))).list()).rejects.toMatchObject({kind:'no-release'});});it('classifies HTTP failures',async()=>{await expect(new LocalLocationRepository(vi.fn().mockResolvedValue(response({},503))).list()).rejects.toMatchObject({kind:'http',status:503});});it('rejects malformed or restricted payloads',async()=>{await expect(new LocalLocationRepository(vi.fn().mockResolvedValue(response({...envelope(),api_version:'v1'}))).list()).rejects.toMatchObject({kind:'invalid-contract'});await expect(new LocalLocationRepository(vi.fn().mockResolvedValue(response(envelope([{...row,privacy_screening_status:'failed'}])))).list()).rejects.toMatchObject({kind:'invalid-contract'});});}); +describe('LocalLocationRepository query contract', () => { + it('passes supported filters and the opaque cursor without inventing search semantics', async () => { + const fetcher = vi.fn().mockResolvedValue(response(envelope([], { ...envelope().meta, next_cursor: 'cursor-2' }))); + const result = await new LocalLocationRepository(fetcher).list('official', { country_code: 'DK', category: 'dairy', source_type: 'official', display_precision: 'city', lifecycle_status: 'active_observed', cursor: 'cursor-1' }); + const request = String(fetcher.mock.calls[0]?.[0]); + expect(request).toContain('profile=official'); + expect(request).toContain('country_code=DK'); + expect(request).toContain('category=dairy'); + expect(request).toContain('source_type=official'); + expect(request).toContain('display_precision=city'); + expect(request).toContain('lifecycle_status=active_observed'); + expect(request).toContain('cursor=cursor-1'); + expect(result.nextCursor).toBe('cursor-2'); + }); +}); describe('community list safety', () => { const community = { ...row, source_type: 'user_submitted', publication_profile: 'community', factual_review_status: 'unreviewed', project_approval: 'pending', publication_warning: 'Unreviewed community claim — not verified by Until Every Cage' }; From e6f542ecc993e6e98765383418700763c745411c Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sun, 13 Sep 2026 19:56:35 -0700 Subject: [PATCH 041/311] Add UK source reconnaissance handoff --- docs/country-recon-uk.md | 80 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 docs/country-recon-uk.md diff --git a/docs/country-recon-uk.md b/docs/country-recon-uk.md new file mode 100644 index 0000000..c953302 --- /dev/null +++ b/docs/country-recon-uk.md @@ -0,0 +1,80 @@ +# UK and country source reconnaissance + +This is a read-only source assessment and private-artifact record, not source approval, +legal clearance, or publication authorization. Real artifacts remain under ignored +restricted staging. No pipeline code or public data was changed. + +## V1 country set verified + +The V1 `static_data` directory contains nine country codes with `locations.csv`: + +| Code | Rows | Columns | Existing V1 source/processing evidence | +|---|---:|---:|---| +| ca | 1,323 | 100 | Ontario/federal CSVs and cleaners | +| de | 9,254 | 269 | BVL workflow comments and migration script | +| dk | 1,561 | 100 | Smiley XML and Danish converter history | +| es | 4,222 | 100 | legacy Spain CSV/converters | +| fr | 3,201 | 100 | legacy locations only in current checkout | +| mx | 14,836 | 101 | DENUE/aquaculture processing history | +| nz | 1,652 | 17 | bespoke source/process history | +| uk | 7,600 | 100 | `Old CSVs/uk-data.csv`, legacy converter | +| us | 7,101 | 269 | APHIS files/reports | + +The user estimate of ten is not supported by the current V1 `static_data` inventory; +the verified count is nine. This is a code/data inventory, not a claim of complete +historical country coverage. + +## Readiness matrix + +| Source/country | Responsible body and canonical evidence | Type/coverage/refresh/format | Terms/privacy | Acquisition and adapter readiness | Blockers | +|---|---|---|---|---|---| +| UK England + Wales approved establishments | FSA catalogue and CSV: | Animal-origin establishments under Reg. 853/2004; monthly snapshots; current catalogue entry dated 2026-09-01; CSV | Catalogue says UK OGL v3.0; 104 rows marked `AddressWithheld=Yes` in the private snapshot, so public field minimization/review remains required | Strong: direct CSV and stable application number; reuse BLtU positional/activity preservation pattern, with withheld-address suppression | England/Wales only; duplicate application identifiers observed; coordinate semantics and missingness need review; no automatic publication | +| UK Scotland approved establishments | FSS page: | Scotland approvals; monthly; CSV; page published 2026-08/09 | Page states OGL v3; exact CSV terms and field-level privacy still require project review | Good separate adapter; source-specific schema and activity codes required | Separate authority/file; cannot merge with FSA without identity/schema review | +| UK Northern Ireland approved establishments | FSA data catalogue link from national entry: | Separate NI approved-establishment source; format/update needs metadata inspection | Terms and privacy not yet verified in this reconnaissance | Unknown until metadata/resource inspection | Separate source and terms/coverage confirmation | +| UK FHRS/FHIS | FSA API: | Food businesses and ratings across England, Scotland, Wales, NI; API JSON/XML; open data files nightly/daily; not limited to animal agriculture | FSA says OGL generally, but API terms/brand conditions apply; businesses can include small premises and potentially personal-address exposure | Technically strong API/open-data adapter; not interchangeable with approved-establishment source | Scope mismatch with facility map; privacy/minimization and local-authority coverage review | +| Canada | Legacy Ontario/federal artifacts; no current authoritative endpoint verified here | Partial, mixed federal/provincial; CSV | Unverified | Hold; existing cleaner is source-specific | Partial coverage, terms, coordinates | +| Denmark | Existing Smiley XML/V1 source | National food businesses; XML; current refresh/license not reassessed in this lane | Existing source review elsewhere; do not infer clearance | Existing adapter foundation | Terms/privacy/source freshness | +| Spain | MAPA REGA/SITRAN and regional derivatives assessed previously | National register privacy-restricted; regional open derivatives | National bulk reuse unresolved; regional licenses vary | Hold pending national/regional source selection | Access/coverage/license fragmentation | +| France | No authoritative current source evidence in V1 checkout | Legacy data only | Unverified | Hold | Provenance and terms | +| Mexico | INEGI DENUE-based historical processing | Legacy economic directory extract; source-specific | Existing historical claims need review | Hold | Current endpoint, scope, personal/address risk | +| New Zealand | Historical bespoke source/process | 17-column legacy file; 665 missing coordinates in prior inventory | Unverified | Hold | Source access/terms and completeness | +| United States | USDA APHIS historical files/reports | Source-specific wide schema and reports | Requires current terms/privacy review | Reference only | Not a clean expansion template | + +## UK deeper assessment + +The FSA England/Wales catalogue is the clearest near-term candidate: it names the +publisher, states OGL, provides dated monthly CSV snapshots, and separates Scotland +and Northern Ireland. The current private snapshot was acquired from the catalog-linked +official blob URL at 2026-09-14T02:47:04.5428766Z; its manifest records 1,774,417 bytes, +SHA-256 `d5cfec048b0f4dc4a8594b0597982f3788f10eb1b4270f9593ead8abce33b61f`, and an +effective snapshot date of 2026-09-01. + +Private aggregate inspection found 5,342 rows and 71 columns, no missing application +numbers, 2,475 rows with both X/Y values, and 104 rows with `AddressWithheld=Yes`. +Duplicate application-number values were observed and require quarantine/identity +review; they must not be silently deduplicated. The file contains England, Wales, +Jersey, Isle of Man, and Guernsey values despite the catalogue's England/Wales title, +which requires a coverage decision before ingestion. No geocoding was performed. + +Scotland is a good second UK source because FSS explicitly states monthly cadence, +CSV, and OGL v3. Northern Ireland remains metadata-first until its separate catalogue +resource and terms are inspected. FHRS/FHIS is technically highly automatable, with +documented JSON/XML API and nightly open-data files, but its broad food-business scope +and local-authority coverage make it a separate product/source decision rather than a +drop-in approved-establishment feed. + +## Recommendation + +1. Proceed first with an England/Wales FSA approved-establishment synthetic adapter and + private restricted staging, using `AppNo` only as a candidate identifier after the + duplicate review; preserve all source values, activity strings, withheld-address + flags, source coordinates, and coverage values. +2. Add Scotland as the next candidate after inspecting its CSV schema and exact terms. +3. Keep Northern Ireland metadata-gated and do not represent the four UK feeds as one + source until identity, coverage, update, terms, and suppression behavior are + explicitly reconciled. + +The FSA England/Wales source is a stronger near-term candidate than Spain's national +REGA for source clarity and automation, but it is not cleared for public release. +Human terms, privacy, safety, suppression, legal, data-quality, and publication gates +remain mandatory. From e0ad302ad32f56da862165bec8c09ed99adf0c75 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sun, 13 Sep 2026 21:51:18 -0700 Subject: [PATCH 042/311] Clarify V2 public facility count semantics --- docs/api/v2-contract.json | 6 +++--- docs/api/v2-contract.md | 6 ++++-- pipeline/tests/e2e/test_seeded_api.py | 3 +++ pipeline/tests/test_database_contract.py | 12 ++++++++---- src/lib.rs | 18 ++++++++++++++---- 5 files changed, 32 insertions(+), 13 deletions(-) diff --git a/docs/api/v2-contract.json b/docs/api/v2-contract.json index 29c7928..7c98ef3 100644 --- a/docs/api/v2-contract.json +++ b/docs/api/v2-contract.json @@ -4,11 +4,11 @@ "endpoints": { "GET /health/live": {"success": {"status": "ok", "service": "uec-api"}}, "GET /health/ready": {"success": {"status": "ready", "database": "ok"}, "unavailable_status": 503}, - "GET /api/v2/locations": {"success": {"api_version": "v2", "data": [], "meta": {"profile": "official", "next_cursor": null}}, "no_release": 200}, - "GET /api/v2/locations/{facility_id}": {"success": {"api_version": "v2", "data": {}, "meta": {}}, "not_found": 404}, + "GET /api/v2/locations": {"success": {"api_version": "v2", "data": [], "meta": {"profile": "official", "next_cursor": null, "coverage_scope": "selected_promoted_release_public_facilities", "count_semantics": "public facility projection rows, not animal counts"}}, "no_release": 200}, + "GET /api/v2/locations/{facility_id}": {"success": {"api_version": "v2", "data": {}, "meta": {"coverage_scope": "selected_promoted_release_public_facilities", "count_semantics": "public facility projection, not an animal count"}}, "not_found": 404}, "GET /api/v2/locations.csv": {"success_content_type": "text/csv", "bounded_rows": 1000, "profile_required_for_nonofficial": true, "not_found_without_promoted_manifest": 404}, "GET /api/v2/discovery/filters": {"success": {"api_version": "v2", "contract_version": "v1", "dimensions": {"country_code": "allowlist", "category": "allowlist", "source_type": "allowlist", "profile": "allowlist", "display_precision": "allowlist", "lifecycle_status": "allowlist"}}} - ,"GET /api/v2/discovery/facets": {"success": {"api_version": "v2", "meta": {"profile": "official", "release_id": "string", "filters": {}}, "dimensions": {}}, "max_values_per_dimension": 20, "counts_are_from": "selected eligible promoted public projection"} + ,"GET /api/v2/discovery/facets": {"success": {"api_version": "v2", "meta": {"profile": "official", "release_id": "string", "ruleset_version": "string", "release_created_at": "timestamp", "coverage_scope": "selected_promoted_release_public_facilities", "count_semantics": "eligible public facility projection rows after current suppression; not story-wide or animal counts", "filters": {}}, "dimensions": {}}, "max_values_per_dimension": 20, "counts_are_from": "selected eligible promoted public projection"} }, "error": {"shape": {"api_version": "v2", "error": {"code": "string", "message": "string"}}, "codes": ["invalid_filter", "invalid_profile", "invalid_limit", "invalid_offset", "invalid_cursor", "invalid_pagination", "database_not_configured", "database_pool_unavailable", "database_transaction_unavailable", "release_query_failed", "location_query_failed", "location_not_found", "rate_limited"]}, "privacy": "Public responses contain reviewed projection fields only; restricted records and raw evidence are never returned." diff --git a/docs/api/v2-contract.md b/docs/api/v2-contract.md index fc2d107..04307b7 100644 --- a/docs/api/v2-contract.md +++ b/docs/api/v2-contract.md @@ -1,6 +1,6 @@ # V2 API contract -The machine-readable contract is [v2-contract.json](v2-contract.json). Successful list/detail response shapes remain unchanged. Errors use one additive, stable envelope: +The machine-readable contract is [v2-contract.json](v2-contract.json). Successful list/detail response shapes remain unchanged; additive metadata identifies release coverage and prevents facility rows from being mistaken for story-wide or animal totals. Errors use one additive, stable envelope: ```json {"api_version":"v2","error":{"code":"invalid_profile","message":"profile is unsupported"}} @@ -12,4 +12,6 @@ Researchers may request `GET /api/v2/locations.csv?profile=official` (or another Clients can discover the current controlled vocabularies at `GET /api/v2/discovery/filters`. Filter values are allowlisted and versioned; clients must not invent category/source/profile/status values or send arbitrary free-text search. `country_code` is validated as an uppercase ISO alpha-2 code and returns zero rows when the project has no capability/source coverage for that country. New country adapters should register capabilities and vocabularies in the contract before becoming public. -`GET /api/v2/discovery/facets` returns deterministic value/count pairs for the same controlled dimensions, scoped to the selected promoted profile and current public projection. It applies the supplied filters before counting, caps each dimension at 20 values, and returns no addresses, queries, raw payloads, inactive releases, or restricted records. +`GET /api/v2/discovery/facets` returns deterministic value/count pairs for the same controlled dimensions, scoped to the selected promoted profile and current public projection. Its metadata includes the selected release, ruleset, release creation time, and an explicit coverage scope. Counts are eligible public facility-projection rows after current suppression; they are not story-wide totals or animal counts. It applies the supplied filters before counting, caps each dimension at 20 values, and returns no addresses, queries, raw payloads, inactive releases, or restricted records. + +List and detail metadata use the same coverage scope. Their source identifiers, source URL, retrieval timestamp, review state, release, and ruleset remain record-level provenance; they do not establish a story-wide denominator or an animal count. Narrative aggregate claims must come from a separately sourced, dated editorial ledger. diff --git a/pipeline/tests/e2e/test_seeded_api.py b/pipeline/tests/e2e/test_seeded_api.py index 2c2aab4..c9fde64 100644 --- a/pipeline/tests/e2e/test_seeded_api.py +++ b/pipeline/tests/e2e/test_seeded_api.py @@ -102,6 +102,9 @@ def test_provenance_and_precision_are_returned_for_each_public_record(self): def test_facets_apply_filters_and_never_include_restricted_record(self): body = self.get('/api/v2/discovery/facets?profile=official&category=slaughter') self.assertEqual(body['meta']['release_id'], 'e2e-promoted') + self.assertEqual(body['meta']['ruleset_version'], 'e2e-v1') + self.assertEqual(body['meta']['coverage_scope'], 'selected_promoted_release_public_facilities') + self.assertIn('not story-wide or animal counts', body['meta']['count_semantics']) self.assertEqual(body['dimensions']['category'], [{'value': 'slaughter', 'count': 1}]) self.assertNotIn('restricted', json.dumps(body)) empty = self.get('/api/v2/discovery/facets?country_code=ZZ') diff --git a/pipeline/tests/test_database_contract.py b/pipeline/tests/test_database_contract.py index 752a3f2..a92957c 100644 --- a/pipeline/tests/test_database_contract.py +++ b/pipeline/tests/test_database_contract.py @@ -100,10 +100,14 @@ def test_map_projection_is_read_only_and_requires_accepted_geocode(self): "SELECT relkind FROM pg_class WHERE oid = 'uec.map_facilities_public'::regclass" ).fetchone()[0] self.assertEqual(public_relation, "v") - self.assertEqual( - self.connection.execute("SELECT count(*) FROM uec.map_facilities_public").fetchone()[0], - 0, - ) + # This contract test may run against a persistent developer database + # that contains unrelated promoted fixtures. Assert the view invariant + # rather than assuming the whole shared database is empty. + invalid_public_rows = self.connection.execute( + "SELECT count(*) FROM uec.map_facilities_public " + "WHERE release_status <> 'promoted' OR geocoded_location IS NULL" + ).fetchone()[0] + self.assertEqual(invalid_public_rows, 0) display_relation = self.connection.execute( "SELECT relkind FROM pg_class WHERE oid = 'uec.map_facilities_display'::regclass" ).fetchone()[0] diff --git a/src/lib.rs b/src/lib.rs index 24f2e21..003b9a8 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -410,7 +410,7 @@ pub async fn get_v2_facets_handler( ); } }; - let release = match client.query_opt("SELECT release_id FROM uec.releases WHERE status='promoted' AND profile=$1 ORDER BY created_at DESC, release_id DESC LIMIT 1", &[&profile]).await { Ok(row) => row, Err(_) => return v2_error(StatusCode::SERVICE_UNAVAILABLE, "release_query_failed", "release query failed") }; + let release = match client.query_opt("SELECT release_id, ruleset_version, created_at FROM uec.releases WHERE status='promoted' AND profile=$1 ORDER BY created_at DESC, release_id DESC LIMIT 1", &[&profile]).await { Ok(row) => row, Err(_) => return v2_error(StatusCode::SERVICE_UNAVAILABLE, "release_query_failed", "release query failed") }; let Some(release) = release else { return v2_error( StatusCode::NOT_FOUND, @@ -419,6 +419,8 @@ pub async fn get_v2_facets_handler( ); }; let release_id: String = release.get(0); + let ruleset_version: String = release.get(1); + let release_created_at: chrono::DateTime = release.get(2); let rows = match client.query("SELECT country_code, classification_category, display_precision, lifecycle_status, provenance_origin_type FROM uec.map_facilities_display_history WHERE release_id=$1 AND ($2::text IS NULL OR country_code=$2) AND ($3::text IS NULL OR classification_category=$3) AND ($4::text IS NULL OR provenance_origin_type=$4) AND ($5::text IS NULL OR display_precision=$5) AND ($6::text IS NULL OR lifecycle_status=$6)", &[&release_id, ¶ms.country_code, ¶ms.category, ¶ms.source_type, ¶ms.display_precision, ¶ms.lifecycle_status]).await { Ok(rows) => rows, Err(_) => return v2_error(StatusCode::SERVICE_UNAVAILABLE, "facets_query_failed", "public facets unavailable") }; let mut dimensions = serde_json::Map::new(); for (name, values) in [ @@ -460,7 +462,7 @@ pub async fn get_v2_facets_handler( ), ); } - Json(json!({"api_version":"v2", "meta":{"profile":profile,"release_id":release_id,"filters":{"country_code":params.country_code,"category":params.category,"source_type":params.source_type,"display_precision":params.display_precision,"lifecycle_status":params.lifecycle_status}}, "dimensions":dimensions})).into_response() + Json(json!({"api_version":"v2", "meta":{"profile":profile,"release_id":release_id,"ruleset_version":ruleset_version,"release_created_at":release_created_at,"coverage_scope":"selected_promoted_release_public_facilities","count_semantics":"Counts are eligible public facility projection rows after current suppression; they are not story-wide or animal counts.","filters":{"country_code":params.country_code,"category":params.category,"source_type":params.source_type,"display_precision":params.display_precision,"lifecycle_status":params.lifecycle_status}}, "dimensions":dimensions})).into_response() } #[derive(Deserialize)] @@ -741,7 +743,9 @@ pub async fn get_v2_locations_handler( "release_created_at": promoted_created_at, "profile": promoted_profile, "next_cursor": next_cursor, - "coverage_note": "Results are limited to the selected promoted release and public-access policy." + "coverage_note": "Results are eligible public facility projection rows from the selected promoted release after current suppression; they are not story-wide or animal counts.", + "coverage_scope": "selected_promoted_release_public_facilities", + "count_semantics": "Each row represents a public facility projection, not an animal count." }); if transaction.commit().await.is_err() { return ( @@ -880,7 +884,7 @@ pub async fn get_v2_location_detail_handler( ) .into_response(); } - Json(serde_json::json!({"data": item, "api_version": "v2", "meta": {"release_id": release_id, "ruleset_version": ruleset, "release_created_at": created_at, "profile": profile}})).into_response() + Json(serde_json::json!({"data": item, "api_version": "v2", "meta": {"release_id": release_id, "ruleset_version": ruleset, "release_created_at": created_at, "profile": profile, "coverage_scope": "selected_promoted_release_public_facilities", "count_semantics": "This record is a public facility projection, not an animal count."}})).into_response() } #[derive(Deserialize)] @@ -920,6 +924,12 @@ mod v2_api_tests { assert_eq!(contract["version"], "v2"); assert_eq!(contract["profiles"].as_array().unwrap().len(), 3); assert_eq!(contract["error"]["shape"]["api_version"], "v2"); + assert_eq!( + contract["endpoints"]["GET /api/v2/discovery/facets"]["success"]["meta"]["coverage_scope"], + "selected_promoted_release_public_facilities" + ); + assert!(contract["endpoints"]["GET /api/v2/discovery/facets"]["success"]["meta"]["count_semantics"] + .as_str().unwrap().contains("not story-wide")); } #[tokio::test] From 7b8a2daae44133d7baef9876938ebfb558f9a772 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sun, 13 Sep 2026 21:51:43 -0700 Subject: [PATCH 043/311] Add private V2 scale narrative prototype --- frontend/src/app/App.svelte | 6 +- .../src/features/scale/ScaleNarrative.svelte | 66 +++++++++++++++++++ frontend/src/features/scale/scaleModel.ts | 20 ++++++ frontend/src/map/MapView.svelte | 2 +- frontend/tests/e2e/fixture-platform.spec.ts | 12 ++++ frontend/tests/e2e/local-safety.spec.ts | 15 ++--- frontend/tests/unit/scaleModel.test.ts | 15 +++++ 7 files changed, 122 insertions(+), 14 deletions(-) create mode 100644 frontend/src/features/scale/ScaleNarrative.svelte create mode 100644 frontend/src/features/scale/scaleModel.ts create mode 100644 frontend/tests/unit/scaleModel.test.ts diff --git a/frontend/src/app/App.svelte b/frontend/src/app/App.svelte index 79b7cd7..ed8ba1f 100644 --- a/frontend/src/app/App.svelte +++ b/frontend/src/app/App.svelte @@ -13,6 +13,7 @@ import { FilterMetadataRepository, type FilterMetadata } from '../api/FilterMetadataRepository'; import ReleaseContext from '../ui/ReleaseContext.svelte'; import ExportControl from '../ui/ExportControl.svelte'; + import ScaleNarrative from '../features/scale/ScaleNarrative.svelte'; let profile: Profile = 'curated'; let selected: Location | undefined = locations[0]; @@ -42,7 +43,7 @@ $: eligibleExport = localMode && profile === 'curated' && localStatus === 'ready' && Boolean(release); $: profileLabel = profile === 'curated' ? 'Curated release' : 'Community claims'; let lastRemoteQuery = ''; - $: remoteQuery = `${profile}|${region}|${category}|${sourceType}|${displayPrecision}|${lifecycleStatus}|${search}`; + $: remoteQuery = `${profile}|${region}|${category}|${sourceType}|${displayPrecision}|${lifecycleStatus}`; const currentRemoteQuery = () => `${profile}|${region}|${category}|${sourceType}|${displayPrecision}|${lifecycleStatus}|${search}`; $: if (localMode && (localStatus === 'ready' || localStatus === 'loading') && remoteQuery !== lastRemoteQuery) { pushFilterUrl(); void loadLocal(); } @@ -96,7 +97,7 @@ const select = (id: string) => { selected = source.find((item) => item.id === id) ?? selected; window.location.hash = `/locations/${id}?profile=${profile}`; }; const profileChanged = () => { history.pushState(null, '', `#/?profile=${profile}`); if (localMode) void loadLocal(); else void syncRoute(); }; const clearFilters = () => { search = ''; region = 'all'; category = 'all'; sourceType = 'all'; displayPrecision = 'all'; lifecycleStatus = 'all'; }; - const searchChanged = () => { if (localMode) pushFilterUrl(); }; + const searchChanged = () => { if (localMode) { const url = new URL(window.location.href); if (search.trim()) url.searchParams.set('q', search.trim()); else url.searchParams.delete('q'); history.replaceState(null, '', url); } }; onMount(() => { const params = new URLSearchParams(window.location.search); localMode = params.get('mode') === 'local-v2'; @@ -118,6 +119,7 @@
UNTIL EVERY CAGE V2 / FIELD NOTE

EVIDENCE DESK · {localMode ? 'LOCAL V2 API' : 'SYNTHETIC PREVIEW'}

See what a record can—and cannot—tell us.

Start with the source profile, narrow the visible evidence, then inspect what a record can—and cannot—tell us.

+

01 / DISCOVER

Choose the evidence lane

Profiles stay separate. A community claim is never silently promoted into a curated result.

{#if localMode && metadataStatus === 'loading'}

Loading available filter values…

{:else if localMode && metadataStatus === 'error'}

Filter metadata is unavailable; only unscoped discovery is shown.

{/if}
{search || region !== 'all' || category !== 'all' || sourceType !== 'all' || displayPrecision !== 'all' || lifecycleStatus !== 'all' ? `Filters applied: ${[search && `name “${search}”`, region !== 'all' && region, category !== 'all' && category, sourceType !== 'all' && sourceType, displayPrecision !== 'all' && displayPrecision, lifecycleStatus !== 'all' && lifecycleStatus].filter(Boolean).join(' · ')}` : 'No additional filters applied'}
{#if profile === 'community'}
Community claimsUnreviewed community claims: Not verified by Until Every Cage. Check each record’s factual review status before relying on it.
{/if} diff --git a/frontend/src/features/scale/ScaleNarrative.svelte b/frontend/src/features/scale/ScaleNarrative.svelte new file mode 100644 index 0000000..5331ad9 --- /dev/null +++ b/frontend/src/features/scale/ScaleNarrative.svelte @@ -0,0 +1,66 @@ + + +
+
+

A SCALE CHECK · SYNTHETIC MODEL

+

Start with one individual.

+

A record can point to a place. It cannot turn the lives represented by a count into anonymous dots.

+

This small interaction is a model for making scale easier to hold in mind. It is not a biography, a live counter, or a measured total.

+
+ +
+ + + + +
+ MODEL ESTIMATE · SYNTHETIC + {formatCount(selected.estimate)} + {selected.label}: individuals represented in this bounded example +
+

Uncertainty: {selected.uncertainty}. Scope: a fictional interaction, not a published facility or geographic total. Date: model authored 2026-09-13.

+
+ +
+

MEASURED VS MODELLED

+

Measured observations belong to a dated source record. Modelled values are arithmetic chosen to explain a relationship. This page uses the second kind only.

+

Synthetic scale example ends here. The next section contains source-linked facility records.

+ Continue to the source-linked explorer ↓ +
+
+ + diff --git a/frontend/src/features/scale/scaleModel.ts b/frontend/src/features/scale/scaleModel.ts new file mode 100644 index 0000000..e496aed --- /dev/null +++ b/frontend/src/features/scale/scaleModel.ts @@ -0,0 +1,20 @@ +export type ScaleStep = Readonly<{ + id: string; + label: string; + multiplier: number; + estimate: number; + uncertainty: string; +}>; + +// Synthetic interaction values only. They are intentionally not presented as +// an estimate of a country, company, facility, or real-world annual total. +export const SYNTHETIC_BASE = 1; + +export const scaleSteps: readonly [ScaleStep, ...ScaleStep[]] = [ + { id: 'one', label: 'One individual', multiplier: 1, estimate: SYNTHETIC_BASE, uncertainty: 'Illustrative range: 1–1' }, + { id: 'small', label: 'A small group', multiplier: 10, estimate: SYNTHETIC_BASE * 10, uncertainty: 'Illustrative range: 8–12' }, + { id: 'large', label: 'A large group', multiplier: 100, estimate: SYNTHETIC_BASE * 100, uncertainty: 'Illustrative range: 80–120' }, + { id: 'system', label: 'A system-scale example', multiplier: 1000, estimate: SYNTHETIC_BASE * 1000, uncertainty: 'Illustrative range: 800–1,200' } +]; + +export const formatCount = (value: number): string => new Intl.NumberFormat('en-US').format(value); diff --git a/frontend/src/map/MapView.svelte b/frontend/src/map/MapView.svelte index b570518..4b225e9 100644 --- a/frontend/src/map/MapView.svelte +++ b/frontend/src/map/MapView.svelte @@ -19,5 +19,5 @@ $: adapter?.update(features, selectedId); -
{#if hasUnreviewedClaims}

Unreviewed community claims — not verified by Until Every Cage

{/if}

Blank local background · {features.length} display points · no external tiles

+
{#if hasUnreviewedClaims}

Unreviewed community claims — not verified by Until Every Cage

{/if}

Facility pins only, not animal counts. The results list is the accessible equivalent. Blank local background · {features.length} display points · no external tiles

diff --git a/frontend/tests/e2e/fixture-platform.spec.ts b/frontend/tests/e2e/fixture-platform.spec.ts index ddfaca2..1e67066 100644 --- a/frontend/tests/e2e/fixture-platform.spec.ts +++ b/frontend/tests/e2e/fixture-platform.spec.ts @@ -17,6 +17,18 @@ test('renders the synthetic evidence desk', async ({ page }) => { await expect(page.getByText('SYNTHETIC PREVIEW')).toBeVisible(); }); +test('moves through the accessible synthetic scale slice into the explorer', async ({ page }) => { + await page.goto('./#/'); + await expect(page.getByRole('heading', { name: 'Start with one individual.' })).toBeVisible(); + await expect(page.getByText('MODEL ESTIMATE · SYNTHETIC')).toBeVisible(); + const scale = page.getByLabel('How large is the example?'); + await scale.press('ArrowRight'); + await expect(page.getByText('10')).toBeVisible(); + await expect(page.getByText(/fictional interaction, not a published facility/)).toBeVisible(); + await page.getByRole('link', { name: /Continue to the source-linked explorer/ }).click(); + await expect(page.getByRole('heading', { name: 'Choose the evidence lane' })).toBeVisible(); +}); + test('selecting the community profile shows persistent warning context', async ({ page }) => { await page.goto('./#/'); await page.getByLabel('Profile').selectOption('community'); diff --git a/frontend/tests/e2e/local-safety.spec.ts b/frontend/tests/e2e/local-safety.spec.ts index c4d04f8..14dc57c 100644 --- a/frontend/tests/e2e/local-safety.spec.ts +++ b/frontend/tests/e2e/local-safety.spec.ts @@ -69,22 +69,15 @@ test('late list response cannot replace a newer community selection', async ({ p await expect(page.getByRole('note')).toContainText('Not verified'); }); -test('changing search while a list is pending invalidates the old response', async ({ page }) => { +test('search filters the loaded page without refetching or claiming global completeness', async ({ page }) => { await mockMetadata(page); - let releaseFirst: (() => void) | undefined; - const firstHeld = new Promise(resolve => { releaseFirst = resolve; }); let listRequests = 0; - await page.route('**/api/v2/locations**', async route => { - const index = ++listRequests; - if (index === 1) await firstHeld; - try { await route.fulfill({ json: list('official', [row(firstId, index === 1 ? 'Stale initial result' : 'Current filtered result')]) }); } catch { /* Aborted request. */ } - }); + await page.route('**/api/v2/locations**', async route => { listRequests += 1; await route.fulfill({ json: list('official', [row(firstId, 'Current filtered result')], secondId) }); }); await page.goto('./?mode=local-v2#/'); await page.getByLabel('Search locations').fill('current'); await expect(page.getByRole('heading', { name: 'Current filtered result' })).toBeVisible(); - releaseFirst?.(); - await expect(page.getByRole('heading', { name: 'Stale initial result' })).toHaveCount(0); - expect(listRequests).toBeGreaterThanOrEqual(2); + expect(listRequests).toBe(1); + await expect(page).toHaveURL(/q=current/); }); test('late detail response cannot replace a newer route', async ({ page }) => { diff --git a/frontend/tests/unit/scaleModel.test.ts b/frontend/tests/unit/scaleModel.test.ts new file mode 100644 index 0000000..f365e7a --- /dev/null +++ b/frontend/tests/unit/scaleModel.test.ts @@ -0,0 +1,15 @@ +import { describe, expect, it } from 'vitest'; +import { SYNTHETIC_BASE, formatCount, scaleSteps } from '../../src/features/scale/scaleModel'; + +describe('synthetic scale model', () => { + it('keeps bounded values explicit and derived from the base example', () => { + expect(scaleSteps).toHaveLength(4); + expect(scaleSteps.map((step) => step.estimate)).toEqual([1, 10, 100, 1000]); + expect(scaleSteps.every((step) => step.uncertainty.includes('Illustrative range'))).toBe(true); + expect(scaleSteps[0]?.estimate).toBe(SYNTHETIC_BASE); + }); + + it('formats model values without implying precision beyond the example', () => { + expect(formatCount(1000)).toBe('1,000'); + }); +}); From d634574357ed92bcc0f80e706771012c4eaed1ed Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sun, 13 Sep 2026 21:52:19 -0700 Subject: [PATCH 044/311] Document story claims and country source reconnaissance --- docs/country-recon-fr.md | 73 ++++++++++++++++++++++++++++++++++++ docs/country-recon-it.md | 52 +++++++++++++++++++++++++ docs/country-recon-mx.md | 42 +++++++++++++++++++++ docs/country-recon-nz.md | 33 ++++++++++++++++ docs/country-recon-uk.md | 21 +++++++++++ docs/story/README.md | 27 +++++++++++++ docs/story/claim-ledger.md | 41 ++++++++++++++++++++ docs/story/storyboard.md | 50 ++++++++++++++++++++++++ docs/story/user-test-plan.md | 33 ++++++++++++++++ 9 files changed, 372 insertions(+) create mode 100644 docs/country-recon-fr.md create mode 100644 docs/country-recon-it.md create mode 100644 docs/country-recon-mx.md create mode 100644 docs/country-recon-nz.md create mode 100644 docs/story/README.md create mode 100644 docs/story/claim-ledger.md create mode 100644 docs/story/storyboard.md create mode 100644 docs/story/user-test-plan.md diff --git a/docs/country-recon-fr.md b/docs/country-recon-fr.md new file mode 100644 index 0000000..0a0e642 --- /dev/null +++ b/docs/country-recon-fr.md @@ -0,0 +1,73 @@ +# France source reconnaissance + +Status: reconnaissance only. No adapter, release, publication, or row-level fixture was created. No row-level records, personal names, addresses, contacts, coordinates, or private artifacts are retained here. + +Last checked: 2026-09-14 UTC under `docs/ETHICS.md`, policy version 1.0, last reviewed 2026-09-12. This document is source-status evidence, not publication approval or a healthy-pipeline claim. + +## Readiness + +| Source | Discovered / verified | Acquisition | Adapter / validation | Terms / privacy | Blocker / next action | +|---|---|---|---|---|---| +| DGAL approved CE lists | Official Ministry page and Section I/II TXT routes verified | HTTP 200 bounded retrieval; bytes discarded | Not started | Etalab attribution appears on Ministry page; confirm file-specific terms; screen names, addresses, and precise geocodes | Freeze category dictionary and daily snapshot/provenance handling, then human review | +| Alim’confiance | Official DGAL Opendatasoft dataset/API/CSV routes verified | Bounded API query HTTP 200; bytes discarded | Not started | Licence Ouverte 2.0 in data.gouv metadata; screen address/coordinate and farm/person fields | Define qualifying activity/agreement labels; do not ingest all food establishments | +| INSEE SIRENE | Official open-data page, bulk route, and API terms verified | Not attempted; API account/subscription and multi-GB bulk | Not started | Licence Ouverte 2.0; diffusion-partielle and personal-data rules are material | Authorized access, partitioned import, NAF mapping, and privacy rules | +| HVE directory | Official Ministry dataset/current CSV verified | HTTP 200 bounded retrieval; bytes discarded | Not started | Licence Ouverte 2.0; voluntary opt-in, head-office address, possible individual farm names | Treat only as labeled HVE subset, never exhaustive farm source | +| Agence Bio professionals API | Official API route, terms/rate limit, and CGU verified | Bounded lookup HTTP 200; response discarded | Not started | Published CGU; manager/address fields require minimization/screening | Confirm nested schema and certificate-link handling; organic subset only | +| Géorisques ICPE | Official download/WMS/WFS/API entry points verified | Not attempted; national export dynamic/tokenized | Not started | Mirror licence unspecified; precise coordinates require review | Confirm schema, licence, token/rate rules, and livestock/food rubric coverage | + +## Ranked source notes + +### DGAL approved CE establishment lists + +Official index: . + +Verified files: [Section I domestic ungulate TXT](https://fichiers-publics.agriculture.gouv.fr/dgal/ListesOfficielles/SSA1_VIAN_ONG_DOM.txt) and [Section II poultry/lagomorph TXT](https://fichiers-publics.agriculture.gouv.fr/dgal/ListesOfficielles/SSA1_VIAN_COL_LAGO.txt). The observed header contains department number, approval number, SIRET, legal/trade name, address, postal code, commune, category, associated activities, and species. `SH` was observed as a slaughterhouse category code and `CP` as a cutting/processing category; freeze the codebook and preserve source values. + +These are approved food-establishment lists, not general farm registers. Daily replacement, undocumented abbreviations, encoding/quoting variation, and absent source checksums require deterministic snapshots, schema quarantine, and retrieval provenance. Addresses may involve individuals, agricultural entities, schools, or mixed sites; geocoding must remain a separate enrichment with provider/query/time/precision/review and privacy screening. + +### Alim’confiance + +Official dataset: . Verified routes include metadata at , records at , and CSV at . + +The dataset covers abattoirs, retail, restaurants, farm sales, catering, and other food businesses. Filter using activity/agreement fields; do not classify every record as a facility in scope. Observed metadata reported a 2026-09-12 update, weekly frequency, and 72,594 records at the checked processing timestamp. This is a live query surface, not an immutable release; snapshot query parameters, retrieval time, response hash, and processing date. SIRETs, addresses, coordinates, farm/person fields, and geospatial fields require minimization and privacy screening. + +### INSEE SIRENE + +Official dataset: ; API service: . + +SIRENE provides broad establishment discovery and crosswalk identifiers, not a facility-specific register. NAF codes are declared administrative activity, not proof of animals, slaughter, or current operation. API access requires account/subscription; the advertised limit is 30 requests/minute. The current stock bulk route is multi-gigabyte, and a versioned 2026-09-01 redirect was observed but not downloaded. Preserve SIRET/SIREN, active/closed dates, diffusion status, and source variables; treat partial-diffusion (`P`) as a hard privacy input and plan for the NAF transition. + +### HVE directory + +Official dataset: . Verified [July 2026 CSV](https://static.data.gouv.fr/resources/annuaire-des-exploitations-certifiees-haute-valeur-environnementale/20260903-130258/annuaire-des-exploitations-hve-juillet-2026.csv). It is a voluntary, non-exhaustive directory. Head-office SIRET/address is not automatically an operating livestock site; use only as a labeled HVE subset and screen possible individual farm names/addresses. + +### Agence Bio professionals API + +Official records/terms: , [CGU](https://api.gouv.fr/resources/CGU%20API%20Professionnels%20du%20bio.pdf). Verified route: . It covers organic operators, including farms, processors, distributors, and importers, with active/stopped certification information. Preserve nested source values; minimize `manager`, addresses, and social/contact fields. Use only as a labeled organic subset, not a complete farm inventory. + +### Géorisques ICPE + +Official pages: , , and . ICPE classifications and rubrics are regulatory evidence, not proof of current animal use, capacity used, or complete farm coverage. The current download schema, licence, token/rate rules, and livestock/food rubric mapping remain unresolved; do not silently substitute the narrower data.gouv mirror. + +## Private retrieval provenance (no raw artifact retained) + +All listed requests were bounded and read-only; response bytes were discarded. No row-level values were written to the repository, fixtures, logs, or releases. + +| Artifact/query | UTC retrieval | HTTP | Bytes | SHA-256 | +|---|---|---:|---:|---| +| DGAL Section I TXT | 2026-09-14T04:35:06.9602154Z | 200 | 204,211 | `b1171561865ab664ddf18adeeed7b6993224cc2275277fdaa6e4d411dd062649` | +| DGAL Section II TXT | 2026-09-14T04:35:09.6616214Z | 200 | 136,506 | `50af4ba9e7227d876cb89c369dfc8d7c3fb329039cbb82b4985748ec09595643` | +| HVE July 2026 CSV | 2026-09-14T04:35:11.6594779Z | 200 | 2,831,601 | `0498f6804b43ae76f95166258371fe4c31a25e24cff8ee17301d44403cec41cd` | +| Alim’confiance bounded API query | 2026-09-14T04:35:36.5998785Z | 200 | 364 | `cb861a4bb49c6f27b4fc8a630d491f4d865526e634f9c51912b1fa4e3d7727da` | +| Alim’confiance metadata | 2026-09-14T04:36:15.7272614Z | 200 | 19,305 | `b028a4f6fe456a190436fcd03dcadac7c8f54e75c70e17f6cc2a39c51ad43312` | +| Alim’confiance aggregate category query | 2026-09-14T04:36:49.5329067Z | 200 | 320 | `f045dba990abdf588ac2db65f758cfdb1a393c2e5d4f166ab8d4b60ded1f2488` | +| Agence Bio bounded lookup | 2026-09-14T04:35:37.3045285Z | 200 | 24 | `ff530172cf6b5992874e8bfada4002be95aed7f635bd09dee8fed513bbfc1edb` | + +## Recommended sequence + +1. Start with DGAL Section I/II TXT as a direct slaughterhouse layer, preserving approval number, SIRET, category, activity, species, and raw values. +2. Use Alim’confiance as a separately labeled inspection/corroboration layer and selected producer-farm records. +3. Add HVE and Agence Bio only as labeled voluntary/certification subsets. +4. Use SIRENE as a cross-source backbone only after authorized access and diffusion/privacy handling. +5. Treat Géorisques as a later regulatory complement after schema/licence/token/rubric review. +6. Obtain authorized project approval before any release; acquisition success and government origin are not publication authorization. diff --git a/docs/country-recon-it.md b/docs/country-recon-it.md new file mode 100644 index 0000000..2ede651 --- /dev/null +++ b/docs/country-recon-it.md @@ -0,0 +1,52 @@ +# Italy source reconnaissance (V2) + +Status: reconnaissance only. No adapter, release, publication, row-level fixture, or downloaded artifact was created. No row-level real data, personal names, addresses, contacts, coordinates, or private artifacts are retained here. + +Checked 2026-09-13. This document is source-status evidence, not publication approval or a healthy-pipeline claim. + +## Executive finding + +The strongest candidate is the Italian Ministry of Health open-data catalog rather than the session-bound `ConsultazioneStabilimentiServlet` interface. The catalog describes daily CSV data, with JSON/XML alternatives, for establishments approved under Regulation (EC) 853/2004, plus a separate 1069/2009 animal-by-products dataset. The servlet and general Ministry landing page presented a JavaScript/cookie security challenge during the read-only check; no bypass or undocumented API inference was attempted. + +## Sources and evidence + +- Maintainer page: +- Ministry landing page: +- 853/2004 catalog: +- 1069/2009 catalog: +- 853 schema dictionary: +- 1069 schema dictionary: + +The catalog reported 853 data last updated 2026-09-12 and daily frequency; the 1069 catalog reported last updated 2026-09-11 and daily frequency. Category pages may have independent amendment dates. The catalog identifies the Ministry of Health/DGSAN Office 2 and Italian Open Data Licence v2.0. It warns that some coordinates came from OpenStreetMap contributors; this is source metadata, not permission to publish precise points. + +## Meaning and schema + +The 853/2004 sections are regulatory product/activity sections, not animal species or a simple facility type. Observed concepts include approval number, name, VAT/tax identifiers, town/region, category, associated activities, species, remarks, recognition number, activity/status fields, codes, products, export countries, coordinates, geolocation status, and last-update date. The separate 1069/2009 dataset covers animal by-products with its own recognition number, plant/activity/product codes, coordinates, status, and an optional 853 recognition link. + +Keep the datasets separate and preserve original source values. Treat activity/status and coordinates as evidence requiring interpretation and review, not proof of current animal use, safety, completeness, or permission to expose a precise location. + +## Acquisition and access + +No real-row artifact was acquired or retained. Catalog-linked download routes were documented, but direct retrieval was refused by the environment and the Ministry interface displayed a JS/cookie challenge. This is an acquisition blocker, not evidence that downloads are unavailable. No official API, bulk endpoint, rate limit, or authentication contract was verified; servlet query parameters must not be treated as an API. Prefer catalog-linked downloads or an authorized export, recording HTTP metadata, UTC retrieval, SHA-256, byte size, update date, and adapter/config version. Keep raw artifacts outside Git. + +## Historical boundary + +The repository’s historical Italy CSV and scraper are legacy/unverified inputs, not evidence of a supported API or current source. Historical transformations, name truncation, province mapping, and third-party geocoding must not be reused as source truth. Legacy rows remain visibly historical. + +## Per-source readiness + +| Source | Discovered | Acquisition | Adapter / validation | Terms / privacy / publication | Blocker / next action | +|---|---|---|---|---|---| +| Ministry 853/2004 food establishments | Official catalog and regulatory sections verified | Not acquired; direct fetch refused/challenged | Feasible via catalog formats; not implemented/tested | Italian Open Data Licence v2.0; coordinate provenance partly OSM; ETHICS privacy/approval gates apply | Authorized maintainer acquires bounded catalog sample with provenance/hash/size, then synthetic adapter test | +| Ministry 1069/2009 by-products | Separate official catalog/dictionary verified | Not acquired | Separate schema/scope; not implemented | Same licence and privacy/approval gates | Decide whether scope belongs in project, then acquire/validate separately | +| Servlet HTML interface | Official interface identified | Not acquired; JS/cookie challenge | Historical HTML parser is brittle; no API claim | No export/terms contract verified; do not scrape through challenge | Prefer catalog downloads or request authorized export/documented endpoint | + +## Integration recommendation + +Build a deterministic catalog-download adapter with an explicit dataset variant and format. Validate encoding, delimiter/header, recognition identifiers, status vocabulary, category/activity codes, dates, coordinate ranges, duplicate identifiers, and count changes. Quarantine schema drift and malformed rows. Geocoding, if approved later, must be a separate derived event with provider/query/time/precision/review fields. + +Do not publish names, addresses, tax identifiers, or precise coordinates merely because the Ministry publishes them. Apply residential/private-location screening, source-origin labels, project approval, and publication profile independently. Government-sourced does not mean current, complete, project-approved, or safe to expose. + +## Limitations + +This reconnaissance did not acquire current rows, verify download responses by HTTP, establish rate limits/authentication, or certify completeness/current accuracy. It makes no healthy-pipeline claim. diff --git a/docs/country-recon-mx.md b/docs/country-recon-mx.md new file mode 100644 index 0000000..de8099b --- /dev/null +++ b/docs/country-recon-mx.md @@ -0,0 +1,42 @@ +# Mexico source reconnaissance + +Status: sanitized metadata handoff; no row-level records or downloaded artifacts are retained here. Reconnaissance is not publication approval or evidence of a healthy pipeline. Observed 2026-09-14 UTC under `docs/ETHICS.md`. + +## Sources and findings + +### INEGI DENUE + +Official documentation: . + +INEGI documents JSON endpoints under `https://www.inegi.org.mx/app/api/denue/v1/consulta/`, including `Buscar`, `Ficha`, `Nombre`, `BuscarEntidad`, `BuscarAreaAct`, `BuscarAreaActEstr`, and `Cuantificar`. Responses may include identification, SCIAN activity, size stratum, administrative location, coordinates, stable `Id`/`CLEE`, geographic keys, and establishment start date. + +DENUE is a broad economic directory, not proof of TIF certification, operating status, animal treatment, or completeness. Animal-agriculture candidates require explicit SCIAN mapping and review rather than free-text matching. The API requires a registered token; exact quota/rate-limit terms remain unresolved. Its precise addresses, coordinates, phones, emails, and websites require field minimization and privacy review before any publication. + +### SENASICA TIF + +Official pages: and . + +The official pages define TIF facilities as regulated installations where animals may be slaughtered and animal-origin goods processed, packed, refrigerated, or industrialized. A public ArcGIS FeatureServer route was verified at . Its observed metadata exposed only `fid`, `estado`, and `municipio`; it must not be assumed to equal the full directory or to provide certification details. + +The current full directory artifact, fields, stable identifier, publication date, cadence, and redistribution terms remain unresolved. A secondary 2024 PDF mirror was noted during reconnaissance but must not substitute for a current official release. + +### DGSIAP/SIAP + +Landing pages: , , and . + +These annual municipal/state production series provide agricultural context, not named farms, slaughterhouses, or facility coordinates. The portal describes reuse subject to attribution and continuity/share-alike-style conditions. A bounded 2025 download attempt failed with connection refusal; no bytes, hash, or artifact are claimed. + +## Readiness block (last checked 2026-09-14 UTC) + +| Source | Source verification | Private acquisition | Adapter / validation | Terms, privacy, publication | Blocker | Next action | +|---|---|---|---|---|---|---| +| DENUE | Official documentation and API URL patterns verified | Not performed; no token/account | Not implemented or tested | Token terms, contact-field minimization, coordinate policy, and completeness unresolved | No token and SCIAN mapping unresolved | Maintainer-approved bounded API sample or bulk metadata check with URL, timestamp, hash/size where applicable | +| SENASICA/TIF | Official TIF pages and public ArcGIS metadata verified; full directory not verified | Not performed; no row-level query/account | Not implemented; layer completeness and identifiers unknown | Government origin only; precise locations/contact fields need privacy review; terms unresolved | Current directory, schema, cadence, certification semantics | Obtain current official directory privately and compare with GIS layer | +| DGSIAP/SIAP | Official landing pages and current release listing verified | Download attempted and blocked; no artifact retained | Not implemented; source is not a facility registry | Aggregate-only role; attribution/continuity conditions need confirmation | Transport failure and no dictionary/hash | Retry from approved environment and capture artifact provenance privately | + +## Open verification gaps + +- Confirm DENUE token quotas and bulk-download terms. +- Obtain and verify the current SENASICA TIF directory and compare it with the GIS layer. +- Preserve source identifiers and original values; do not infer closure from disappearance. +- Obtain maintainer decisions for privacy eligibility, project approval, and publication profile. Government origin alone does not establish approval. diff --git a/docs/country-recon-nz.md b/docs/country-recon-nz.md new file mode 100644 index 0000000..0551dde --- /dev/null +++ b/docs/country-recon-nz.md @@ -0,0 +1,33 @@ +# New Zealand source reconnaissance + +Status: reconnaissance only; no row-level records, names, addresses, coordinates, or private artifacts are retained here. This is not publication approval or a healthy live-pipeline claim. Last checked 2026-09-13. + +## MPI approved premises and country listings + +Primary routes: + +- Register index: +- Country-listing search example: + +The portal exposes market/product-specific lists of approved or listed premises. Depending on the list, fields may include an MPI identifier, premises name, address, operation/type, product, species, and expiry or audit date. Relevant categories include slaughterhouse, boning/cutting, processing, cold store, and related animal-product operations. + +No stable public API or bulk URL was verified. MPI pages returned HTTP 403 during automated retrieval. Lists have market-specific scope, overlapping records, and regular updates. Treat disappearance as “not observed,” not closure. Rights and attribution must be confirmed for the specific download/page. Rural/RD and mixed residential/business addresses require privacy and safety review. Do not infer ownership, residence, welfare status, current operation, or activity beyond the source statement. + +## Stats NZ Agricultural Production Statistics + +Primary metadata: . + +The observed latest metadata described **Agricultural Production Statistics: June 2025 (Final)**, identifier `a9908d00-4827-46b2-a38a-310c0c75b029`, version 2, DDI 3.3, version date `2026-04-08T02:42:33.9384122Z`. This is regional aggregate context, not a named-farm or facility-point source. Coverage includes livestock and farm practices; releases distinguish provisional/final results, census years, sample surveys, confidentiality protections, perturbation, suppression, and imputation. + +The metadata states CC BY 4.0 with attribution to Statistics New Zealand and restrictions on misuse of government emblems. A stable statistical table API or bulk endpoint was not verified. A bounded metadata JSON fetch failed with connection refusal; no artifact, bytes, or hash are claimed. + +## Readiness block + +| Source | Source verification | Private acquisition | Adapter / validation | Terms, privacy, publication | Blocker | Next action | +|---|---|---|---|---|---|---| +| MPI listings | Official routes identified; download/API not verified | Blocked by HTTP 403; no artifact or rows | No adapter or validation | Confirm list-specific terms; screen rural/mixed-use addresses; preserve source origin separately | Portal access, endpoint, licensing, and schema variation | Authorized maintainer confirms permitted route/terms, then stage one bounded artifact with URL, UTC timestamp, hash, size, and version | +| Stats NZ aggregates | Official metadata route and release metadata observed | Local fetch failed; no artifact or hash | No adapter; aggregate data must not become facility points | CC BY 4.0 attribution; protect confidentiality and preserve suppression/perturbation/imputation limits | No verified table/bulk endpoint or reproducible artifact | Confirm official table/API route, privately capture one aggregate artifact, and validate release/version/schema | + +## Safety boundary + +Neither source is ready for public facility publication. MPI lists may contain precise premises information and require current terms/privacy review. Stats NZ is a contextual aggregate source and must not be transformed into named farms or facility coordinates. Government origin does not establish project review, approval, or current operation. diff --git a/docs/country-recon-uk.md b/docs/country-recon-uk.md index c953302..cb6f127 100644 --- a/docs/country-recon-uk.md +++ b/docs/country-recon-uk.md @@ -78,3 +78,24 @@ The FSA England/Wales source is a stronger near-term candidate than Spain's nati REGA for source clarity and automation, but it is not cleared for public release. Human terms, privacy, safety, suppression, legal, data-quality, and publication gates remain mandatory. + +## Adapter reconciliation + +The canonical implementation is `pipeline/sources/uk/fsa_approved/`. It currently +provides a pinned synthetic schema, ordered source-value preservation, strict +encoding/schema checks, duplicate-identifier quarantine, activity/status and +authority/nation checks, address privacy-risk quarantine, deterministic manifests, +and a human publication gate. + +The isolated experiment in commit `f73299b` assumes a different CSV contract using +`AppNo`, `X/Y`, and `AddressWithheld`, with additional coordinate and coverage +diagnostics. It must not be cherry-picked as a second UK adapter: doing so would +create two incompatible interpretations of the same country source. Its unique +behaviors are useful requirements for a future canonical schema review, especially +explicit withheld-address semantics, source-coordinate CRS/axis/range validation, +coverage counts for out-of-scope nations, and the related provenance fingerprints. + +This reconciliation is a design record, not source approval or legal clearance. +No live row data is included, and no schema-dependent gap should be implemented +until the actual source contract, terms, privacy treatment, and publication scope +are reviewed together. diff --git a/docs/story/README.md b/docs/story/README.md new file mode 100644 index 0000000..b17cdc2 --- /dev/null +++ b/docs/story/README.md @@ -0,0 +1,27 @@ +# Narrative evidence lane + +This directory contains internal narrative and evidence-planning materials for V2. It is not a publication approval, a source archive, or a substitute for `docs/ETHICS.md`. No claim in this directory is public copy until its source, scope, calculation, date, and review status are recorded. + +## Working narrative contract + +The experience may take a clear moral position: animal suffering matters and deserves attention. It must still separate observed facts, calculated estimates, interpretations, and moral judgments. A facility row is not an animal, a map pin is not a death count, and a modeled clock is not a live measurement. + +The first prototype should move through: + +1. **One life:** establish that a counted animal is an individual, without inventing a biography or assigning a facility-specific animal to a named story. +2. **Scale:** show a dated, sourced annual range and explain its denominator, exclusions, and uncertainty. +3. **Time:** derive a transparent per-day/per-second rate from the annual range; label it as a model. +4. **Proximity:** invite an optional region search and show only public-eligible, coarse or approved locations; never use proximity to expose a home or private person. +5. **Evidence:** let visitors inspect source, date, calculation, release/profile, and limitations at the point of use. + +The mechanism is inspired by scale demonstrations that use scrolling distance and familiar comparisons to make very large quantities legible. The reference mechanism is not evidence for any animal claim and should not be copied as public factual support. + +See [claim-ledger.md](claim-ledger.md), [storyboard.md](storyboard.md), and [user-test-plan.md](user-test-plan.md). + +## Initial source-verification result + +FAOSTAT is a defensible primary source for a bounded land-animal slaughtering figure. A private retrieval of the 2024 normalized bulk release on 2026-09-13 found 17 direct meat items with head units for Area=World and Element=Producing Animals/Slaughtered. After converting `1000 An` rows to heads and excluding aggregate rows and duplicate meat by-products, the selected items sum to **87,896,729,120 heads**. The modeled average is **240,653,071 heads/day** or **2,785 heads/second**, using 365.2425 days/year. This is not a universal total or live counter: it excludes items not in the selected set, includes FAO estimates where present, and spreads annual observations evenly across a calendar-year convention. The source archive hash was `C5835418C18F9322E7DECBD6800F93A216EAAE3CDFA31ACB08F0518C0C6D6853`; raw data remains outside the repository. + +The exact selected item list, units, exclusions, formula, retrieval date, source links, and limitations are recorded in claim C-16/C-17. The FAO catalog currently identifies the dataset license as CC BY 4.0, but the release/terms should be rechecked before redistribution or embedding raw data. Before public use, the maintainer should confirm that the selected scope is described prominently enough not to be mistaken for all-animal mortality. + +No aquatic individual-count range is approved in this lane. Biomass, capture/landing totals, and farmed production cannot be converted to individuals without species- and size-specific assumptions. A striking familiar-scale comparison is also blocked until its denominator is independently sourced and made commensurate with the chosen animal scope. diff --git a/docs/story/claim-ledger.md b/docs/story/claim-ledger.md new file mode 100644 index 0000000..22b2db9 --- /dev/null +++ b/docs/story/claim-ledger.md @@ -0,0 +1,41 @@ +# Narrative claim ledger + +Status vocabulary: **candidate** means a source still needs review; **ready-for-prototype** means the method and source scope are documented but not public approval; **blocked** means do not use in public copy. + +| ID | Candidate claim/copy | Type | Required evidence and scope | Calculation / uncertainty | Status | +|---|---|---|---|---|---| +| C-01 | “Animals are individuals, not database rows.” | Moral/editorial framing | No factual source required; must not be presented as a measured statistic. | Avoids anthropomorphic biography and facility attribution. | ready-for-prototype | +| C-02 | “This map records places and evidence; it does not count animals at each pin.” | Product limitation | V2 API release-scoped public facility projections, release/profile metadata, source/provenance fields. | API list/detail/facet counts are facilities/records in a selected public release, not animals or worldwide totals. Directly state that facility rows are not animal counts. | ready-for-prototype | +| C-03 | “A source record can be outdated, incomplete, inaccurate, or ambiguous.” | Policy-backed factual limitation | `docs/ETHICS.md` §§4-5, dated policy version. | No numeric estimate. | ready-for-prototype | +| C-04 | “The annual total is an estimate for [defined population/scope] in [year/range], not a live count.” | Modeled quantitative claim | Primary statistical source(s), exact species, geography, production/use category, year, unit, denominator, coverage. | `annual_low <= annual_estimate <= annual_high`; publish source range and exclusions. | blocked until source review | +| C-05 | “On the same assumptions, the modeled rate is [range] per second.” | Derived quantitative claim | Approved C-04 inputs and formula. | `rate_per_second = annual_count / 365.2425 / 24 / 60 / 60`; show leap-year convention and propagated range. | blocked until C-04 | +| C-06 | “One pixel represents [N] animals in this view.” | Visualization contract | C-04 approved range and chosen scale. | Pixel/bin rounding must not imply precision below source uncertainty; expose scale and total. | blocked until C-04 | +| C-07 | “You may be near a mapped facility.” | Proximity framing | User-entered region only; approved public release; current coordinate/display precision and safety review. | No claim that the nearest facility is operating or that a visitor’s home is near it. | blocked until privacy/location audit | +| C-08 | “This facility was observed in [date].” | Record-level observed fact | API `last_observed_at`, source retrieval/observation date, release ID. | Do not translate to “operating now” or “closed.” | ready when API fields present | +| C-09 | “Government-sourced” | Source-origin label | Source registry identity, URL, retrieval date. | Does not imply truth, approval, review, or currentness. | ready with provenance UI | +| C-10 | “Project-approved in release [ID]” | Publication decision | Publication review event, release membership, approval actor/scope. | Approval is version/release scoped; changed records do not inherit automatically. | blocked until release evidence | +| C-11 | “An individual animal’s day looked like [specific biography].” | Narrative factual claim | Specific welfare/behavior source and lawful, non-identifying provenance. | Must not be invented or inferred from a facility row. | **blocked** | +| C-12 | “Most animals live in conditions of [specific welfare assertion].” | Welfare/generalization | Primary species-specific welfare research, population/scope, definition, date. | Separate observed conditions from moral interpretation; avoid universal wording. | blocked until review | +| C-13 | “Death is much closer than you imagined.” | Interpretive/moral thesis | No empirical source as written; must be framed as the project’s interpretation. | Do not imply a tested psychological effect. | candidate editorial line | +| C-14 | “The industry hides this from you.” | Intentionality claim | Evidence of intentional concealment and defined actor/scope. | Not supported by current repository evidence. | **blocked** | +| C-15 | “Over 56,000 locations” | Legacy aggregate claim | Dated release manifest, country/scope, inclusion rules, suppression state. | Must distinguish facilities/records from animals and current from legacy. | blocked until release audit | +| C-16 | “FAOSTAT reports an annual number of animals slaughtered for a defined set of land-animal meat items.” | Observed/statistical-source claim | FAOSTAT **Livestock primary (Global, National - Annual)** bulk release, 2024, Area=World, Element=Producing Animals/Slaughtered. Private retrieval 2026-09-13 from [FAO bulk data](https://bulks-faostat.fao.org/production/Production_Crops_Livestock_E_All_Data_(Normalized).zip); source catalog [FAO catalog](https://data.fao.org/catalog/iso/55375b1e-51d0-47db-ac9b-536ac8a1c738); [FAO methodology](https://files-faostat.fao.org/production/QCL/QCL_methodology_e.pdf). | Selected direct meat items with head units (converting `1000 An` to heads) sum to **87,896,729,120 heads in 2024**: cattle, buffalo, sheep, goat, pig, chicken, duck, goose, pigeon/other birds, rabbit/hare, turkey, ass, horse/equine, mule, camel, other camelid, and other domestic rodents. Excludes aggregate rows (beef/buffalo, sheep/goat, poultry), hides/fat/offal duplicates, and species/items not represented in this selected head-count set. FAO states national values concern slaughter within national boundaries and aggregates may include estimates. This is a selected-scope total, not “all animals killed.” | **ready-for-prototype with scope label; not a universal total** | +| C-17 | “The selected 2024 FAOSTAT scope averages about 240.7 million heads per day, or 2,785 heads per second.” | Derived calculation | C-16 exact selected item list and 2024 value. | `daily = 87,896,729,120 / 365.2425 = 240,653,070.55`; `per_second = 87,896,729,120 / (365.2425 * 86,400) = 2,785.34`. Use “modeled average” and explain 365.2425 is a calendar-year convention; do not call it live or event-timed. | **ready-for-prototype; public release needs maintainer/source review** | +| C-18 | “A separate global aquatic-animal annual range is [X–Y].” | Quantitative estimate | Must use a primary fish/aquatic source with explicit species, wild/captured vs farmed scope, unit, year, and conversion from biomass to individuals. | Biomass-to-individual conversion requires species/size assumptions and should be a range; fish-count methods are not interchangeable with FAOSTAT land-animal heads. | **blocked: no reviewed primary estimate yet** | +| C-19 | “This is comparable to [familiar object/person/lifespan].” | Scale comparison | Comparison denominator must be independently sourced, dated, and commensurate with the same unit and scope. | Avoid comparisons that imply equivalence or certainty; show both original values and conversion. | **blocked pending independent source packet** | + +## Required source packet for any numeric claim + +Before a quantitative line moves beyond internal prototype, attach: + +- primary source URL or artifact ID; +- publication/effective year and retrieval timestamp; +- exact geography, species, category, and inclusion/exclusion rules; +- original unit and any conversion; +- low/high or confidence interval, with method; +- formula and constants used; +- code/configuration version; +- reviewer and review outcome; +- known gaps, likely undercount/overcount direction, and a correction path. + +Candidate sources such as FAOSTAT, national competent-authority statistics, and peer-reviewed welfare research are leads only until the exact table/version and license are reviewed. Do not use a source family name as if it were a checked citation. diff --git a/docs/story/storyboard.md b/docs/story/storyboard.md new file mode 100644 index 0000000..10633a5 --- /dev/null +++ b/docs/story/storyboard.md @@ -0,0 +1,50 @@ +# Candidate storyboard: individual → scale → proximity + +This is an internal prototype, not a launch specification. + +## 1. Opening: one life, without a fabricated story + +**Screen:** A quiet, accessible statement: “Every number in this story refers to individual animals, even when the number is too large to picture.” + +**Evidence boundary:** This is moral/editorial framing, not a claim about a particular animal. Do not show an invented name, face, facility, or biography. + +**Interaction:** “How this works” opens the claim ledger and distinguishes observed, modeled, and interpretive language. + +## 2. Scale: a dated range, not a magic total + +**Screen:** A scroll/animation where visual distance is proportional to a reviewed annual estimate. The scale label remains fixed and readable. + +**Required copy:** “Modeled annual estimate for [scope], [year]: [low–high]. This is not a live count.” + +**Interaction:** Source, denominator, exclusions, formula, and uncertainty stay visible. Users can pause, reduce motion, jump by keyboard, and switch to a table. + +**Guardrail:** If no reviewed numeric source packet exists, show the method with synthetic placeholder values labeled “prototype only,” or omit the quantitative animation. + +## 3. Time: derived rate with visible assumptions + +**Screen:** Convert the annual range into per-day and per-second ranges. + +**Formula:** `per_second = annual / days_per_year / 86,400`. Use an explicitly named year convention; do not silently imply real-time events. + +**Required copy:** “This clock is a mathematical translation of the annual estimate. It does not measure events as they happen.” + +## 4. Proximity: optional, coarse, and safety-screened + +**Screen:** “Where is this documented?” with optional town/region entry. Device geolocation is not required. + +**Interaction:** A user can choose a country/region and see only the selected public release/profile. Exact points are shown only when eligible; otherwise use city/coarse/unmapped states. + +**Guardrail:** Never imply that a facility is operating now, that a home is near a facility, or that a person is associated with a record. A removed/restricted location must disappear from this narrative path too. + +## 5. Evidence return path + +Every claim card needs: source origin, source name/URL where safe, retrieval/observation date, release/profile, review/approval state when actually available, uncertainty, and a correction/privacy link. “Government-sourced” must not be rendered as “verified.” + +## Editorial and accessibility cautions + +- Avoid historical-atrocity analogies in public copy; describe the project’s moral position directly. +- Avoid gore, graphic imagery, countdown pressure, autoplay audio, and language that shames visitors. +- Do not rely on color, motion, hover, or proximity alone to communicate status. +- Provide reduced-motion behavior, pause/step controls, keyboard navigation, screen-reader text, a text/table equivalent, sufficient contrast, and a clear “what this number means” explanation. +- Do not put precise user-entered locations in URLs, analytics, shared links, or error messages. +- Keep activist urgency separate from factual certainty; emotional effect is a test outcome, not evidence of truth. diff --git a/docs/story/user-test-plan.md b/docs/story/user-test-plan.md new file mode 100644 index 0000000..de85b41 --- /dev/null +++ b/docs/story/user-test-plan.md @@ -0,0 +1,33 @@ +# Narrative prototype user-test plan + +Test a non-public prototype with synthetic or already approved aggregate inputs. Do not use restricted records, real residential examples, or unreviewed community submissions. + +## Questions + +1. **Comprehension:** Can participants explain the difference between a source observation, a modeled estimate, a facility row, and a moral interpretation? +2. **Emotional effect:** Does the sequence make scale and proximity feel concrete without causing panic, numbness, or spectacle? +3. **Manipulation perception:** Do participants feel the animation hides assumptions, exaggerates precision, pressures them, or implies unsupported intent? +4. **Source checkability:** Can participants find the source, date, scope, formula, uncertainty, and correction route without facilitator help? +5. **Accessibility:** Can keyboard and screen-reader users pause, skip, understand the equivalent table, and recover from motion or dense content? + +## Method + +- Recruit a small mixed group of journalists/researchers, advocates, and general visitors; record role, not unnecessary personal data. +- Compare two variants: scroll visualization versus static/table-first presentation. +- Use task prompts rather than leading questions: “What does this number describe?” “What would you verify before citing it?” “What does this pin prove?” +- Measure task success, source lookup time, incorrect interpretations, voluntary emotional descriptors, perceived pressure/manipulation, and accessibility failures. +- Ask participants to paraphrase the strongest claim and identify what remains unknown. +- Keep a change log of copy revisions and unresolved objections. + +## Stop conditions + +Pause the prototype if participants commonly infer that: + +- a facility row equals a number of animals; +- a modeled rate is a live counter; +- government origin means project verification; +- a mapped place is operating now; +- proximity identifies a private person or home; +- the story proves intentional concealment or a specific biography. + +These are editorial safety failures, not merely usability issues. From 8c4fccb383e572680e6eebc40fdc44860b7b6d33 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sun, 13 Sep 2026 21:52:45 -0700 Subject: [PATCH 045/311] Track evidence-backed source pipeline readiness --- docs/source-status.json | 24 ++++++++++ docs/source-status.md | 31 +++++++++++++ pipeline/tests/test_source_status_baseline.py | 46 +++++++++++++++++++ 3 files changed, 101 insertions(+) create mode 100644 docs/source-status.json create mode 100644 docs/source-status.md create mode 100644 pipeline/tests/test_source_status_baseline.py diff --git a/docs/source-status.json b/docs/source-status.json new file mode 100644 index 0000000..07eccd2 --- /dev/null +++ b/docs/source-status.json @@ -0,0 +1,24 @@ +{ + "schema_version": "1.0", + "purpose": "Evidence-backed source readiness baseline; not a runtime health monitor or publication approval register.", + "status_vocabulary": { + "metadata": ["verified", "partial", "unknown"], + "acquisition": ["not_run", "blocked", "artifact_private_only", "verified"], + "runtime_health": ["not_run", "unknown", "healthy"], + "publication_eligibility": ["not_assessed", "blocked", "eligible_pending_release_approval"] + }, + "sources": [ + {"source_id":"fr.locations","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-fr.md","pipeline/source_registry.json"],"next_action":"Acquire and snapshot an official DGAL Section I/II artifact with terms, schema, and privacy review."}, + {"source_id":"it.locations","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-it.md","pipeline/source_registry.json"],"next_action":"Use catalog-linked Ministry downloads; record provenance and validate 853/2004 separately from 1069/2009."}, + {"source_id":"mx.locations","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-mx.md","pipeline/source_registry.json"],"next_action":"Resolve DENUE token/terms and SENASICA current directory/schema before bounded acquisition."}, + {"source_id":"nz.locations","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-nz.md","pipeline/source_registry.json"],"next_action":"Confirm MPI permitted acquisition route, list-specific terms, and schema before private staging."}, + {"source_id":"uk.locations","metadata":"partial","acquisition":"artifact_private_only","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-uk.md","pipeline/source_registry.json"],"next_action":"Keep national feeds separate; review withheld addresses, duplicates, coverage, terms, and source-specific adapters."}, + {"source_id":"dk.smiley","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["pipeline/README.md","pipeline/source_registry.json"],"next_action":"Verify current endpoint, publication date semantics, licence, coverage, and release approval."}, + {"source_id":"de.locations","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["pipeline/source_registry.json"],"next_action":"Replace session-bound legacy URL with a verified stable BVL endpoint and confirm terms/schema."}, + {"source_id":"ca.locations","metadata":"unknown","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["pipeline/source_registry.json"],"next_action":"Split federal and Ontario candidates, identify authoritative endpoints, and verify terms/schema before acquisition."}, + {"source_id":"es.locations","metadata":"unknown","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["pipeline/source_registry.json"],"next_action":"Identify the current competent-authority publication and distinguish original source from legacy transformations."}, + {"source_id":"us.fsis","metadata":"unknown","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["pipeline/source_registry.json"],"next_action":"Trace legacy rows to a specific FSIS artifact and verify current terms, schema, and coverage."}, + {"source_id":"us.aphis","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["pipeline/source_registry.json"],"next_action":"Verify current export workflow and keep licences, registrants, annual reports, and exception reports separately attributed."}, + {"source_id":"us.inspections","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["pipeline/source_registry.json"],"next_action":"Verify current inspection export and use explicit, reviewable identity matching rather than treating observations as a facility master."} + ] +} diff --git a/docs/source-status.md b/docs/source-status.md new file mode 100644 index 0000000..74a7e35 --- /dev/null +++ b/docs/source-status.md @@ -0,0 +1,31 @@ +# Source status baseline + +This is the canonical human-readable view of [`source-status.json`](source-status.json). It records repository evidence and reconnaissance state; it is not a live monitor, acquisition log, pipeline health dashboard, release approval, or publication authorization. + +## How to read it + +- `metadata` describes whether a source route/scope was documented from repository or reconnaissance evidence. +- `acquisition` describes only bounded acquisition evidence. `not_run` and `blocked` do not mean zero rows or source failure. +- `runtime_health` is `not_run` unless a repeatable current pipeline check is recorded. No source here is claimed healthy. +- `publication_eligibility` is separate from source origin and metadata. These sources remain `blocked` until privacy, terms, validation, review, release, and maintainer approval gates are satisfied. + +No last-success timestamp is invented. Private artifacts are not proof of a public release. “Government-sourced” does not mean current, complete, project-approved, or safe to expose. + +## Current baseline + +| Source ID | Metadata | Acquisition | Runtime health | Publication | Evidence / next action | +|---|---|---|---|---|---| +| `fr.locations` | verified | blocked | not_run | blocked | DGAL/Alim’confiance/SIRENE/HVE/Agence Bio/Géorisques reconnaissance; acquire permitted official artifact and review terms/privacy | +| `it.locations` | verified | blocked | not_run | blocked | Ministry 853/2004 and 1069/2009 catalogs identified; use catalog downloads, not challenged servlet | +| `mx.locations` | verified | blocked | not_run | blocked | DENUE/SENASICA/DGSIAP reconnaissance; resolve token, directory, terms, and schema | +| `nz.locations` | verified | blocked | not_run | blocked | MPI/Stats NZ reconnaissance; resolve 403/access and aggregate-vs-facility boundaries | +| `uk.locations` | partial | artifact_private_only | not_run | blocked | National-feed reconciliation and withheld-address/duplicate/coverage review remain open | +| `dk.smiley` | partial | not_run | not_run | blocked | Existing partial pipeline; current endpoint/licence/coverage/release verification remains open | +| `de.locations` | partial | not_run | not_run | blocked | Legacy session URL and current BVL endpoint/schema/terms remain unresolved | +| `ca.locations` | unknown | not_run | not_run | blocked | Federal/Ontario candidates must be split and verified before acquisition | +| `es.locations` | unknown | not_run | not_run | blocked | Current competent-authority source and legacy/source boundary unresolved | +| `us.fsis` | unknown | not_run | not_run | blocked | Legacy rows are not traced to a verified current FSIS artifact | +| `us.aphis` | partial | not_run | not_run | blocked | Export workflow and separate report/license provenance require review | +| `us.inspections` | partial | not_run | not_run | blocked | Inspection observations require current export and explicit identity matching | + +The machine-readable file is the source of truth for these statuses. `.locations` IDs may represent composite legacy coverage rather than one upstream source. Candidate feeds mentioned in the France, Mexico, New Zealand, and Italy reconnaissance documents are not silently conflated into a single healthy source; source splitting remains a next action. Country reconnaissance documents provide evidence and next actions; they do not override this status vocabulary or authorize publication. diff --git a/pipeline/tests/test_source_status_baseline.py b/pipeline/tests/test_source_status_baseline.py new file mode 100644 index 0000000..e79e94f --- /dev/null +++ b/pipeline/tests/test_source_status_baseline.py @@ -0,0 +1,46 @@ +"""CI-discovered consistency checks for the source-status evidence baseline.""" + +import json +import unittest +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[2] + + +class SourceStatusBaselineTests(unittest.TestCase): + def load_status(self): + return json.loads((ROOT / "docs" / "source-status.json").read_text(encoding="utf-8")) + + def load_registry_ids(self): + registry = json.loads((ROOT / "pipeline" / "source_registry.json").read_text(encoding="utf-8")) + return {source["source_id"] for source in registry["sources"]} + + def test_status_covers_exactly_the_source_registry_ids(self): + payload = self.load_status() + ids = {source["source_id"] for source in payload["sources"]} + self.assertEqual(ids, self.load_registry_ids()) + + def test_status_values_are_explicit_and_conservative(self): + payload = self.load_status() + vocab = payload["status_vocabulary"] + for source in payload["sources"]: + self.assertIn(source["metadata"], vocab["metadata"]) + self.assertIn(source["acquisition"], vocab["acquisition"]) + self.assertIn(source["runtime_health"], vocab["runtime_health"]) + self.assertIn(source["publication_eligibility"], vocab["publication_eligibility"]) + self.assertTrue(source["evidence"]) + self.assertTrue(source["next_action"]) + self.assertTrue(all(source["runtime_health"] != "healthy" for source in payload["sources"])) + self.assertTrue(all(source["publication_eligibility"] != "eligible" for source in payload["sources"])) + + def test_evidence_paths_exist_and_exclude_private_roots(self): + payload = self.load_status() + for source in payload["sources"]: + for evidence in source["evidence"]: + self.assertFalse(evidence.startswith((".codex/", "data/terms-reviews/", "philosophy/"))) + self.assertTrue((ROOT / evidence).exists(), evidence) + + +if __name__ == "__main__": + unittest.main() From 1a06e202bbe9b306fb30f0768111624fed87407e Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sun, 13 Sep 2026 22:02:35 -0700 Subject: [PATCH 046/311] Consolidate US and Canada source reconnaissance --- docs/country-recon-ca.md | 39 +++++++++++++++++++++++++++++++++++++++ docs/country-recon-fr.md | 8 +++++++- docs/country-recon-us.md | 27 +++++++++++++++++++++++++++ docs/source-status.json | 8 ++++---- docs/source-status.md | 8 ++++---- 5 files changed, 81 insertions(+), 9 deletions(-) create mode 100644 docs/country-recon-ca.md create mode 100644 docs/country-recon-us.md diff --git a/docs/country-recon-ca.md b/docs/country-recon-ca.md new file mode 100644 index 0000000..540f53d --- /dev/null +++ b/docs/country-recon-ca.md @@ -0,0 +1,39 @@ +# Canada source reconnaissance + +Status: sanitized metadata handoff; no facility rows, names, addresses, coordinates, or downloaded artifacts are retained here. Last checked 2026-09-14 UTC. This is not publication approval or a healthy-pipeline claim. + +## Comparison + +| Source | Role | Access / coverage | Difficulty | +|---|---|---|---| +| CFIA federally registered/export-eligible meat lists | Slaughter and processing evidence | Official HTML with CSV/XML on some lists; federal/export subsets, not complete domestic coverage | Medium-high | +| Provincial meat-plant datasets | Provincial abattoir/meat-plant coverage | Province-specific open data; Ontario verified | Medium | +| AAFC / Statistics Canada | Livestock, slaughter, farm and production aggregates | Open-data/statistical tables; context only, not facility points | Low for context; unsuitable as facility registry | + +## CFIA + +Official guidance: . CFIA distinguishes federally licensed and provincially registered establishments, so a federal list is not complete Canadian coverage. + +Verified export-list route: . It exposes CSV/XML, establishment number, country/program filters, activities, and version information, but primarily lists foreign establishments eligible to export to Canada. Country-specific export lists, such as , contain Canadian establishment information but are export-eligibility subsets, not necessarily the domestic federal registry. + +Establishment numbers are candidate identifiers; semantics across lists require confirmation. Currency is list-specific. Auth, rate limits, redistribution terms, and complete federal coverage remain unresolved. Names, business addresses, phones, and precise locations require privacy screening. + +## Provincial meat plants + +Verified Ontario dataset: . It covers provincially licensed meat plants, including abattoirs and processing plants, with plant identifiers, contact fields, coordinates, and animal classes. It is Ontario-only, not national. Other provinces require separate source, terms, identifier, cadence, and adapter review. Confirm current licence/attribution and apply coordinate/contact privacy review before use. + +## Agriculture and Statistics Canada + +AAFC livestock/slaughter context: . Additional farm/statistical context includes and the Statistics Canada Business Register overview at . These support aggregate context and validation, not named farms or facility points; preserve suppression, quality, licence, and privacy notes. + +## Readiness block + +| Source | Verification | Acquisition | Adapter / validation | Terms/privacy/publication | Blocker / next action | +|---|---|---|---|---|---| +| CFIA | Official guidance/export routes verified; complete federal route unresolved | Not performed; no account/rows | Not implemented/tested | Government-sourced only; distinguish export eligibility/domestic registration; minimize fields | Locate current federal list and privately capture URL/date/size/hash | +| Ontario | Open Government Portal dataset/resource structure verified | Not performed | Not implemented/tested | Confirm current licence/attribution and coordinate privacy | Acquire metadata/bounded schema check; assess other provinces separately | +| AAFC/Statistics Canada | Official aggregate sources verified | Not performed | Not implemented/tested | Aggregate role only; table-specific licences/suppression notes required | Select aggregate tables and keep separate from facility totals | + +## Unresolved gaps + +Verify the complete CFIA federal registry, verify provinces individually, confirm licences/attribution/rate limits/update cadence/stable identifiers, and make project approval, privacy eligibility, and publication-profile decisions separately from government source origin. diff --git a/docs/country-recon-fr.md b/docs/country-recon-fr.md index 0a0e642..af61372 100644 --- a/docs/country-recon-fr.md +++ b/docs/country-recon-fr.md @@ -37,18 +37,24 @@ Official dataset: . Verified [July 2026 CSV](https://static.data.gouv.fr/resources/annuaire-des-exploitations-certifiees-haute-valeur-environnementale/20260903-130258/annuaire-des-exploitations-hve-juillet-2026.csv). It is a voluntary, non-exhaustive directory. Head-office SIRET/address is not automatically an operating livestock site; use only as a labeled HVE subset and screen possible individual farm names/addresses. ### Agence Bio professionals API -Official records/terms: , [CGU](https://api.gouv.fr/resources/CGU%20API%20Professionnels%20du%20bio.pdf). Verified route: . It covers organic operators, including farms, processors, distributors, and importers, with active/stopped certification information. Preserve nested source values; minimize `manager`, addresses, and social/contact fields. Use only as a labeled organic subset, not a complete farm inventory. +Official records/terms: , [CGU](https://api.gouv.fr/resources/CGU%20API%20Professionnels%20du%20bio.pdf). Verified route: . It covers organic operators, including farms, processors, distributors, and importers, with active/stopped certification information. The service description advertises 50 calls/second/IP and no availability SLA; treat this as published service information, not a performance guarantee. Preserve nested source values; minimize `manager`, addresses, and social/contact fields. Use only as a labeled organic subset, not a complete farm inventory. ### Géorisques ICPE Official pages: , , and . ICPE classifications and rubrics are regulatory evidence, not proof of current animal use, capacity used, or complete farm coverage. The current download schema, licence, token/rate rules, and livestock/food rubric mapping remain unresolved; do not silently substitute the narrower data.gouv mirror. +## Sources not suitable as V1 facility rows + +Agreste agricultural census tables are useful for commune-level aggregate context but lack facility identifiers and point locations. BDNI/animal-identification systems are restricted administrative systems rather than public facility downloads. Alim’confiance includes restaurants and retail as well as relevant activities; dataset presence alone is not enough to classify a record as animal agriculture or slaughtering. + ## Private retrieval provenance (no raw artifact retained) All listed requests were bounded and read-only; response bytes were discarded. No row-level values were written to the repository, fixtures, logs, or releases. diff --git a/docs/country-recon-us.md b/docs/country-recon-us.md new file mode 100644 index 0000000..e4c8c49 --- /dev/null +++ b/docs/country-recon-us.md @@ -0,0 +1,27 @@ +# United States source reconnaissance (V2) + +Scope/date: private reconnaissance of checked-in V1 material and current primary routes, observed 2026-09-13 UTC. No row-level names, contacts, addresses, coordinates, or raw artifacts are retained here. This is not publication approval or a healthy-pipeline claim. + +## Finding + +V1’s “USDA” layer is the FSIS Meat, Poultry and Egg Product Inspection (MPI) Directory, not a generic USDA register. The checked-in `static_data/us/locations.csv` contains 7,101 legacy rows and the older file contains 7,099; this is an inventory comparison, not a validated current-vs-current change. The lab concept is APHIS Animal Care research facilities. APHIS annual-use, inspection-report, and older compiled/scraped artifacts are distinct populations and must not be joined or presented as one dataset. + +## Source matrix + +| V1 component | Primary route | Evidence / readiness | Caveats and blocker | +|---|---|---|---| +| FSIS establishments/demographics | [FSIS MPI Directory](https://www.fsis.usda.gov/inspection/establishments) and [inspected establishments](https://www.fsis.usda.gov/inspection/fsis-inspected-establishments) | Official page exposes downloadable CSV directory/demographic files and documentation; not privately fetched and no hash/bytes claimed | Weekly replacement; no API/rate contract verified; FSIS coverage is not all slaughter/processing sites and state programs are separate | +| APHIS research annual use | [Annual Usage Summary](https://www.aphis.usda.gov/awa/research-facility-report/annual-summary), [Public Search Tool](https://direct.aphis.usda.gov/animal-care/awa-services/usda-animal-care-public-search-tool), [annual reports](https://efile.aphis.usda.gov/PublicSearchTool/s/annual-reports) | Official fiscal-year/search routes identified; not privately fetched | Interactive/UI-mediated, no documented bulk API/rate contract; amended annual reports may differ; use sanctioned route only | +| APHIS inspections/registrants | [AWA inspections and annual reports](https://www.aphis.usda.gov/awa/annual-inspection-reports) | Separate public-search/inspection population identified; not reproduced | Redactions/changes and FOIA boundary; absence/presence does not prove operation or violation | + +## V1 aggregate crosswalk + +Legacy FSIS activity strings are multi-valued and overlap: Meat Slaughter 1,103; Poultry Slaughter 420; Meat Processing 5,479; Poultry Processing 4,212; Egg Product 157; Imported Product 215. These are string-presence counts, not mutually exclusive totals or current coverage. APHIS annual-use rows are fiscal-year reports, not a laboratory census and not equivalent to inspection rows. + +## Safe automation boundary + +Acquire only official FSIS downloads or documented APHIS public-search/export workflows with UTC retrieval, effective/publication date, byte size, SHA-256, URL, and adapter/config version. Validate content type, signatures, headers, IDs, dates, coordinates, duplicates, and category vocabulary; quarantine HTML/login responses and sharp changes. Preserve raw/parsed layers separately in ignored restricted staging, keep source values/identifiers, avoid names/phones in logs, and never fuzzy-merge FSIS, APHIS annual reports, and inspections. Suppress personal names, direct contacts, residential/private locations, and precise points where ETHICS.md requires. A successful fetch is not publication approval. + +## Blockers and recommendation + +No safe bounded private fetch was performed, so current hashes/bytes and deterministic reproduction are intentionally unavailable. FSIS is the strongest automation candidate because recurring CSV downloads and source descriptions are available. APHIS is secondary/manual/UI-mediated and should be an explicitly versioned, human-reviewed annual-report adapter or restricted manual input. Do not build a laboratory-supplier layer from APHIS records without a separately identified, licensed source. Existing Selenium/compiler code is not production-grade: obsolete selectors, no provenance manifest, quarantine, terms/schema/privacy gates, and unsafe duplicate handling. diff --git a/docs/source-status.json b/docs/source-status.json index 07eccd2..cb16470 100644 --- a/docs/source-status.json +++ b/docs/source-status.json @@ -15,10 +15,10 @@ {"source_id":"uk.locations","metadata":"partial","acquisition":"artifact_private_only","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-uk.md","pipeline/source_registry.json"],"next_action":"Keep national feeds separate; review withheld addresses, duplicates, coverage, terms, and source-specific adapters."}, {"source_id":"dk.smiley","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["pipeline/README.md","pipeline/source_registry.json"],"next_action":"Verify current endpoint, publication date semantics, licence, coverage, and release approval."}, {"source_id":"de.locations","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["pipeline/source_registry.json"],"next_action":"Replace session-bound legacy URL with a verified stable BVL endpoint and confirm terms/schema."}, - {"source_id":"ca.locations","metadata":"unknown","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["pipeline/source_registry.json"],"next_action":"Split federal and Ontario candidates, identify authoritative endpoints, and verify terms/schema before acquisition."}, + {"source_id":"ca.locations","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ca.md","pipeline/source_registry.json"],"next_action":"Keep federal/export and Ontario candidates separate; verify an authoritative permitted artifact, terms, schema, coverage, and privacy before acquisition."}, {"source_id":"es.locations","metadata":"unknown","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["pipeline/source_registry.json"],"next_action":"Identify the current competent-authority publication and distinguish original source from legacy transformations."}, - {"source_id":"us.fsis","metadata":"unknown","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["pipeline/source_registry.json"],"next_action":"Trace legacy rows to a specific FSIS artifact and verify current terms, schema, and coverage."}, - {"source_id":"us.aphis","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["pipeline/source_registry.json"],"next_action":"Verify current export workflow and keep licences, registrants, annual reports, and exception reports separately attributed."}, - {"source_id":"us.inspections","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["pipeline/source_registry.json"],"next_action":"Verify current inspection export and use explicit, reviewable identity matching rather than treating observations as a facility master."} + {"source_id":"us.fsis","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-us.md","pipeline/source_registry.json"],"next_action":"Acquire and snapshot a permitted current FSIS MPI artifact; reconcile legacy inventory only as a comparison and verify terms, schema, coverage, and privacy."}, + {"source_id":"us.aphis","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-us.md","pipeline/source_registry.json"],"next_action":"Verify the current APHIS export workflow and keep licences, registrants, annual reports, and exception reports separately attributed before acquisition."}, + {"source_id":"us.inspections","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-us.md","pipeline/source_registry.json"],"next_action":"Verify a current inspection export and use explicit, reviewable identity matching rather than treating observations as a facility master."} ] } diff --git a/docs/source-status.md b/docs/source-status.md index 74a7e35..84b59dc 100644 --- a/docs/source-status.md +++ b/docs/source-status.md @@ -22,10 +22,10 @@ No last-success timestamp is invented. Private artifacts are not proof of a publ | `uk.locations` | partial | artifact_private_only | not_run | blocked | National-feed reconciliation and withheld-address/duplicate/coverage review remain open | | `dk.smiley` | partial | not_run | not_run | blocked | Existing partial pipeline; current endpoint/licence/coverage/release verification remains open | | `de.locations` | partial | not_run | not_run | blocked | Legacy session URL and current BVL endpoint/schema/terms remain unresolved | -| `ca.locations` | unknown | not_run | not_run | blocked | Federal/Ontario candidates must be split and verified before acquisition | +| `ca.locations` | verified | not_run | not_run | blocked | Federal/export and Ontario candidates are documented separately; verify permitted artifact, terms, schema, coverage, and privacy before acquisition | | `es.locations` | unknown | not_run | not_run | blocked | Current competent-authority source and legacy/source boundary unresolved | -| `us.fsis` | unknown | not_run | not_run | blocked | Legacy rows are not traced to a verified current FSIS artifact | -| `us.aphis` | partial | not_run | not_run | blocked | Export workflow and separate report/license provenance require review | -| `us.inspections` | partial | not_run | not_run | blocked | Inspection observations require current export and explicit identity matching | +| `us.fsis` | verified | not_run | not_run | blocked | FSIS MPI route is documented; acquire a permitted current artifact and treat legacy inventory only as a comparison | +| `us.aphis` | partial | not_run | not_run | blocked | APHIS export workflow and separate report/license provenance require review | +| `us.inspections` | partial | not_run | not_run | blocked | Inspection observations require a current export and explicit identity matching | The machine-readable file is the source of truth for these statuses. `.locations` IDs may represent composite legacy coverage rather than one upstream source. Candidate feeds mentioned in the France, Mexico, New Zealand, and Italy reconnaissance documents are not silently conflated into a single healthy source; source splitting remains a next action. Country reconnaissance documents provide evidence and next actions; they do not override this status vocabulary or authorize publication. From d2da3bd8b8ee1bc66a3577449cfb122c5f43ac23 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sun, 13 Sep 2026 22:17:45 -0700 Subject: [PATCH 047/311] Test private candidates remain absent from public API --- pipeline/tests/e2e/README.md | 2 ++ pipeline/tests/e2e/fixture.py | 22 ++++++++++++++++++++++ pipeline/tests/e2e/test_public_api.py | 20 ++++++++++++++++++++ 3 files changed, 44 insertions(+) diff --git a/pipeline/tests/e2e/README.md b/pipeline/tests/e2e/README.md index 7c7dc46..6dcc413 100644 --- a/pipeline/tests/e2e/README.md +++ b/pipeline/tests/e2e/README.md @@ -18,6 +18,8 @@ This requires Docker Desktop, Cargo, and the pinned Python dependencies. The fix `test_seeded_api.py` uses `seed_official_scenario()` to create synthetic exact, city-level, unmapped, lifecycle, category, and safety-restricted records in a promoted release. This is the reusable starting point for future user-submission and moderation scenarios. Tests should assert both positive behavior and absence of disclosure. A record being present in the database is not sufficient to make it public; every public response must pass release, visibility, and safety filtering. + +The public API suite also seeds a synthetic candidate-only record whose review and geocode fields look publishable. It verifies that candidate data is absent from public list, detail, and export routes. This is not a private preview API: any future end-user candidate preview must be a separate dev/test-only server or route, bound to loopback/private access, unavailable in production mode, and fail closed for remote or ambiguous configuration. For a local run that exactly matches the standard GitHub Actions database job, use PowerShell 7 (`pwsh`) and run: diff --git a/pipeline/tests/e2e/fixture.py b/pipeline/tests/e2e/fixture.py index d4dca4f..c66595f 100644 --- a/pipeline/tests/e2e/fixture.py +++ b/pipeline/tests/e2e/fixture.py @@ -159,6 +159,28 @@ def create_failed_candidate(self): db.execute("INSERT INTO uec.releases (release_id,status,ruleset_version,summary) VALUES ('e2e-failed-candidate','candidate','e2e-v2','{}') ON CONFLICT (release_id) DO NOTHING") db.execute("INSERT INTO uec.validation_findings (severity,code,details) VALUES ('error','synthetic_failure','{}')") + def seed_private_candidate_scenario(self): + """Seed candidate-only data; review fields alone must not make it public. + + The fixture is disposable and synthetic. Its accepted coordinate and + approval-shaped event intentionally test that release status remains a + separate publication gate. + """ + now = datetime.now(timezone.utc) + with psycopg.connect(self.database_url) as db: + with db.transaction(): + db.execute("INSERT INTO uec.sources (source_id,country_code,name,official_url,access_method) VALUES ('e2e.private-candidate','DK','Synthetic private candidate source','https://example.invalid/private-candidate','fixture')") + db.execute("INSERT INTO uec.releases (release_id,status,ruleset_version,profile,summary) VALUES ('e2e-private-candidate','candidate','e2e-private-v1','official','{}')") + record, facility, observation, artifact = uuid.uuid4(), uuid.uuid4(), uuid.uuid4(), uuid.uuid4() + db.execute("INSERT INTO uec.raw_artifacts (artifact_id,storage_key,sha256,byte_size,retrieved_at) VALUES (%s,'e2e/private-candidate',%s,1,%s)", (artifact, uuid.uuid4().hex * 2, now)) + db.execute("INSERT INTO uec.source_records (source_record_id,source_id,source_record_key,artifact_id,raw_fields,parsed_at) VALUES (%s,'e2e.private-candidate','candidate-only',%s,'{}',%s)", (record, artifact, now)) + db.execute("INSERT INTO uec.facilities (facility_id,canonical_name,country_code,city) VALUES (%s,'E2E private candidate','DK','Candidateby')", (facility,)) + db.execute("INSERT INTO uec.observations (observation_id,facility_id,source_record_id,observed_at,observation,classification,ruleset_id,rule_id,classification_category,classification_review_status,default_visible,first_observed_at) VALUES (%s,%s,%s,%s,'{}','{}','e2e-private-v1','e2e','slaughter','approved',true,%s)", (observation, facility, record, now, now)) + db.execute("INSERT INTO uec.release_members (release_id,facility_id,observation_id,default_visible) VALUES ('e2e-private-candidate',%s,%s,true)", (facility, observation)) + db.execute("INSERT INTO uec.geocode_results (source_record_id,provider_id,query,match_method,status,attempt_number,result,queried_at) VALUES (%s,'e2e','synthetic candidate','fixture','accepted',1,ST_SetSRID(ST_MakePoint(12,56),4326)::geography,%s)", (record, now)) + db.execute("INSERT INTO uec.publication_review_events (source_record_id,release_id,factual_review_status,privacy_screening_status,maintainer_approval,publication_eligible,reviewer_role) VALUES (%s,'e2e-private-candidate','reviewed','passed','approved',true,'maintainer')", (record,)) + self.private_candidate_facility_id = facility + def restore_restricted_record(self): """Append a restoration event; the original evidence is unchanged.""" with psycopg.connect(self.database_url) as db: diff --git a/pipeline/tests/e2e/test_public_api.py b/pipeline/tests/e2e/test_public_api.py index 87a2e5f..6fe6d27 100644 --- a/pipeline/tests/e2e/test_public_api.py +++ b/pipeline/tests/e2e/test_public_api.py @@ -15,6 +15,7 @@ def setUpClass(cls): if os.environ.get("UEC_RUN_E2E") != "1": raise unittest.SkipTest("set UEC_RUN_E2E=1 to run Docker-backed E2E tests") cls.env = E2EEnvironment().start() + cls.env.seed_private_candidate_scenario() @classmethod def tearDownClass(cls): @@ -28,6 +29,25 @@ def test_default_is_empty_before_promotion(self): status, body = self.get("/api/v2/locations?country_code=DK") self.assertEqual(status, 200); self.assertEqual(body["api_version"], "v2"); self.assertEqual(body["data"], []) + def test_candidate_only_record_is_absent_from_every_public_surface(self): + # A candidate can contain otherwise publishable-looking evidence, but + # public APIs must select promoted releases only. The future private + # preview path must remain a separate, explicitly gated interface. + candidate = self.env.private_candidate_facility_id + _, body = self.get("/api/v2/locations?profile=official&limit=100") + self.assertNotIn(str(candidate), json.dumps(body)) + try: + with urllib.request.urlopen( + f"http://localhost:{self.env.api_port}/api/v2/locations.csv?profile=official", + timeout=10, + ) as response: + self.assertNotIn("E2E private candidate", response.read().decode()) + except urllib.error.HTTPError as error: + self.assertEqual(error.code, 404) + with self.assertRaises(urllib.error.HTTPError) as error: + self.get(f"/api/v2/locations/{candidate}?profile=official") + self.assertEqual(error.exception.code, 404) + def test_filters_do_not_bypass_publication_gate(self): for path in ("?category=retail_and_prepared_food", "?display_precision=city", "?lifecycle_status=explicitly_closed"): _, body = self.get("/api/v2/locations" + path) From f5a9a5fb7b18f90cb71826cbeedb14e0b9ad6d20 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sun, 13 Sep 2026 22:19:30 -0700 Subject: [PATCH 048/311] Clarify V2 facility evidence and count semantics --- frontend/README.md | 2 ++ frontend/src/api/LocalLocationRepository.ts | 8 +++++--- frontend/src/api/wireSchema.ts | 4 +++- frontend/src/app/App.svelte | 10 +++++++--- frontend/src/domain/location.ts | 4 ++++ frontend/src/features/scale/ScaleNarrative.svelte | 1 + frontend/src/features/scale/scaleModel.ts | 1 + 7 files changed, 23 insertions(+), 7 deletions(-) diff --git a/frontend/README.md b/frontend/README.md index 10d68ba..8414dcc 100644 --- a/frontend/README.md +++ b/frontend/README.md @@ -23,4 +23,6 @@ Local V2 integration (opt-in only): run the backend with `cargo run` (port 8000) The opt-in local view loads one API page at a time. When the response includes `next_cursor`, the UI labels its counts, search, and map as partial; search runs only against loaded records. Full pagination and backend search remain future integration work. Record context is limited to fields in the current V2 wire response and does not imply that review events, evidence hashes, or scoped approvals are available. +The private scale/story prototype begins with a neutral individual-animal representation and uses only bounded synthetic values. It labels model arithmetic separately from measured facility evidence; no biography, live counter, global animal total, or sourced aggregate is embedded in the production build. Candidate sourced scale figures remain outside this UI until maintainer publication approval. + Persistent local two-port workflow (never uses `down -v`): from the repository root run `powershell -ExecutionPolicy Bypass -File pipeline/scripts/maintenance/local-v2.ps1 start`. It starts the named Postgres stack on `5433`, applies migrations, seeds and promotes the synthetic contract release, and starts Axum on `8000`. Run `npm --prefix frontend run dev` in another terminal and open `http://127.0.0.1:5173/v2-preview/?mode=local-v2#/`. Check ownership and health with `... local-v2.ps1 status`; probe list/detail with `... local-v2.ps1 probe`; stop both services with `... local-v2.ps1 stop`. The helper is local-only and does not alter V1, production, or unrelated data. diff --git a/frontend/src/api/LocalLocationRepository.ts b/frontend/src/api/LocalLocationRepository.ts index ec16f7a..99b87ca 100644 --- a/frontend/src/api/LocalLocationRepository.ts +++ b/frontend/src/api/LocalLocationRepository.ts @@ -5,7 +5,7 @@ import type { Location } from '../domain/location'; export type FetchLike = (input: RequestInfo | URL, init?: RequestInit) => Promise; export type LocalProfile = 'official' | 'secondary' | 'community'; export type LocationFilters = Readonly<{ country_code?: string | undefined; category?: string | undefined; source_type?: string | undefined; display_precision?: string | undefined; lifecycle_status?: string | undefined; cursor?: string | undefined }>; -export type LocalListResult = Readonly<{ locations: readonly Location[]; releaseId: string; profile: LocalProfile; coverageNote: string; nextCursor: string | null; ruleset?: string }>; +export type LocalListResult = Readonly<{ locations: readonly Location[]; releaseId: string; profile: LocalProfile; coverageNote: string; coverageScope?: string; countSemantics?: string; nextCursor: string | null; ruleset?: string }>; export const localOrigin = (value: string | undefined): string | undefined => { if (!value) return undefined; const url = new URL(value); if (url.protocol !== 'http:' || !['127.0.0.1', 'localhost', '::1'].includes(url.hostname)) throw new Error('Local API origin must be loopback HTTP.'); return url.origin; }; const fail = (kind: ApiError['kind'], message: string, status?: number): ApiError => Object.assign(new Error(message), status === undefined ? { kind } : { kind, status }); const map = (r: WireLocation): Location => ({ @@ -16,10 +16,12 @@ const map = (r: WireLocation): Location => ({ privacyScreeningStatus: r.privacy_screening_status, projectApproval: r.project_approval, publicationProfile: r.publication_profile, publicationWarning: r.publication_warning, sourceId: r.provenance_source_id, sourceUrl: r.provenance_source_url, - retrievedAt: r.provenance_retrieved_at, displayPrecision: r.display_precision, + retrievedAt: r.provenance_retrieved_at, displayPrecision: r.display_precision, lifecycleStatus: r.lifecycle_status, observationCount: r.observation_count, }, }); const query = (profile: LocalProfile, filters: LocationFilters) => { const params = new URLSearchParams({ profile }); for (const [key, value] of Object.entries(filters)) if (value) params.set(key, value); return `/api/v2/locations?${params}`; }; +// Eligibility is a conservative client-side check, not a publication decision; +// the server's current public projection and suppression rules remain authoritative. const eligible = (row: WireLocation, profile: LocalProfile, releaseId: string, ruleset: string): boolean => row.publication_profile === profile && row.release_id === releaseId && row.release_ruleset_version === ruleset && row.privacy_screening_status === 'passed' && row.factual_review_status !== 'rejected' && @@ -36,7 +38,7 @@ export class LocalLocationRepository { if (b.data.meta.release_id === null) throw fail('no-release', b.data.meta.coverage_note); const { release_id, ruleset_version } = b.data.meta; if (ruleset_version === undefined || b.data.data.some(row => !eligible(row, profile, release_id, ruleset_version))) throw fail('invalid-contract', 'Local V2 list snapshot was rejected.'); - return { locations: b.data.data.map(map), releaseId: release_id, profile, coverageNote: b.data.meta.coverage_note, nextCursor: b.data.meta.next_cursor ?? null, ruleset: ruleset_version }; + return { locations: b.data.data.map(map), releaseId: release_id, profile, coverageNote: b.data.meta.coverage_note, coverageScope: b.data.meta.coverage_scope ?? 'selected promoted release public facilities', countSemantics: b.data.meta.count_semantics ?? 'Eligible public facility projection rows, not animals or a story-wide total.', nextCursor: b.data.meta.next_cursor ?? null, ruleset: ruleset_version }; } catch (e) { if (e && typeof e === 'object' && 'kind' in e) throw e; if (e instanceof DOMException && e.name === 'AbortError') throw fail('aborted', 'Local V2 request was aborted.'); if (e instanceof TypeError) throw fail('network', 'Local V2 request could not connect.'); throw fail('invalid-contract', 'Local V2 response could not be read safely.'); } } async detail(id: string, profile: LocalProfile = 'official', signal?: AbortSignal) { diff --git a/frontend/src/api/wireSchema.ts b/frontend/src/api/wireSchema.ts index d9c37ae..5fa2370 100644 --- a/frontend/src/api/wireSchema.ts +++ b/frontend/src/api/wireSchema.ts @@ -1,7 +1,9 @@ import { z } from 'zod'; const textOrNull=z.string().nullable(); +// This is the Rust-shaped boundary. Optional coverage/count metadata is additive; +// older valid list envelopes remain readable with safe UI fallbacks. export const locationSchema=z.object({facility_id:z.string().uuid(),canonical_name:z.string().min(1),city:textOrNull,country_code:z.string().regex(/^[A-Z]{2}$/),category:z.string(),source_type:z.enum(['official','secondary','user_submitted']),publication_profile:z.enum(['official','secondary','community']),factual_review_status:z.enum(['unreviewed','reviewed','rejected']),privacy_screening_status:z.literal('passed'),project_approval:z.enum(['pending','approved']),reviewer_role:textOrNull,publication_warning:textOrNull,display_precision:z.enum(['exact','city','unmapped']),latitude:z.number().finite().nullable(),longitude:z.number().finite().nullable(),first_observed_at:textOrNull,last_observed_at:textOrNull,observation_count:z.number().int().nonnegative().nullable(),lifecycle_status:z.enum(['active_observed','explicitly_closed','not_seen_recently','status_unknown']),provenance_source_id:z.string(),provenance_source_name:z.string(),provenance_source_url:z.string().url().refine(value=>{try{return ['http:','https:'].includes(new URL(value).protocol);}catch{return false;}},'source URL must use HTTP or HTTPS'),provenance_retrieved_at:z.string(),release_id:z.string(),release_ruleset_version:z.string()}).superRefine((row,ctx)=>{if((row.latitude===null)!==(row.longitude===null))ctx.addIssue({code:z.ZodIssueCode.custom,message:'coordinate pair must be complete'});if(row.display_precision==='unmapped'&&(row.latitude!==null||row.longitude!==null))ctx.addIssue({code:z.ZodIssueCode.custom,message:'unmapped record cannot have coordinates'});if(row.display_precision!=='unmapped'&&(row.latitude===null||row.longitude===null))ctx.addIssue({code:z.ZodIssueCode.custom,message:'mapped record requires coordinates'});}); -export const envelopeSchema=z.object({data:z.array(locationSchema),api_version:z.literal('v2'),meta:z.object({release_id:z.string().nullable(),ruleset_version:z.string().optional(),release_created_at:z.string().optional(),profile:z.enum(['official','secondary','community']),next_cursor:z.string().nullable().optional(),coverage_note:z.string().min(1)})}); +export const envelopeSchema=z.object({data:z.array(locationSchema),api_version:z.literal('v2'),meta:z.object({release_id:z.string().nullable(),ruleset_version:z.string().optional(),release_created_at:z.string().optional(),profile:z.enum(['official','secondary','community']),next_cursor:z.string().nullable().optional(),coverage_note:z.string().min(1),coverage_scope:z.string().optional(),count_semantics:z.string().optional()})}); export type WireEnvelope=z.infer;export type WireLocation=z.infer; export const detailEnvelopeSchema=z.object({data:locationSchema,api_version:z.literal('v2'),meta:z.object({release_id:z.string(),ruleset_version:z.string(),release_created_at:z.string(),profile:z.enum(['official','secondary','community'])})}); export type DetailEnvelope=z.infer; diff --git a/frontend/src/app/App.svelte b/frontend/src/app/App.svelte index ed8ba1f..6f8cd86 100644 --- a/frontend/src/app/App.svelte +++ b/frontend/src/app/App.svelte @@ -29,7 +29,7 @@ let ruleset: string | undefined; let manifestSha256: string | undefined; let repo = new LocalLocationRepository(); let csvRepo = new LocalCsvExportRepository(); let metadata: FilterMetadata | undefined; let metadataStatus: 'idle' | 'loading' | 'ready' | 'error' = 'idle'; - let nextCursor: string | null = null; let coverageNote = ''; + let nextCursor: string | null = null; let coverageNote = ''; let coverageScope = ''; let countSemantics = ''; let paging = false; let listGeneration = 0; let detailGeneration = 0; let listAbort: AbortController | undefined; let detailAbort: AbortController | undefined; @@ -43,6 +43,8 @@ $: eligibleExport = localMode && profile === 'curated' && localStatus === 'ready' && Boolean(release); $: profileLabel = profile === 'curated' ? 'Curated release' : 'Community claims'; let lastRemoteQuery = ''; + // Search is intentionally excluded: the API has no q contract, so it filters + // only the loaded page while controlled dimensions refetch a new snapshot. $: remoteQuery = `${profile}|${region}|${category}|${sourceType}|${displayPrecision}|${lifecycleStatus}`; const currentRemoteQuery = () => `${profile}|${region}|${category}|${sourceType}|${displayPrecision}|${lifecycleStatus}|${search}`; $: if (localMode && (localStatus === 'ready' || localStatus === 'loading') && remoteQuery !== lastRemoteQuery) { pushFilterUrl(); void loadLocal(); } @@ -84,8 +86,8 @@ listGeneration += 1; const generation = listGeneration; listAbort?.abort(); const controller = new AbortController(); listAbort = controller; lastRemoteQuery = queryKey; - if (!append) { invalidateDetail(); localStatus = 'loading'; localError = ''; manifestSha256 = undefined; loaded = []; selected = undefined; nextCursor = null; coverageNote = ''; } else paging = true; - try { const result = await repo.list(profile === 'community' ? 'community' : 'official', { country_code: region === 'all' ? undefined : region, category: category === 'all' ? undefined : category, source_type: sourceType === 'all' ? undefined : sourceType, display_precision: displayPrecision === 'all' ? undefined : displayPrecision, lifecycle_status: lifecycleStatus === 'all' ? undefined : lifecycleStatus, cursor }, controller.signal); if (generation !== listGeneration) return; loaded = append ? [...loaded, ...result.locations] : result.locations; if (!selected) selected = result.locations[0]; release = result.releaseId; ruleset = result.ruleset; nextCursor = result.nextCursor; coverageNote = result.coverageNote; localStatus = 'ready'; paging = false; if (!append) await syncRoute(); } + if (!append) { invalidateDetail(); localStatus = 'loading'; localError = ''; manifestSha256 = undefined; loaded = []; selected = undefined; nextCursor = null; coverageNote = ''; coverageScope = ''; countSemantics = ''; } else paging = true; + try { const result = await repo.list(profile === 'community' ? 'community' : 'official', { country_code: region === 'all' ? undefined : region, category: category === 'all' ? undefined : category, source_type: sourceType === 'all' ? undefined : sourceType, display_precision: displayPrecision === 'all' ? undefined : displayPrecision, lifecycle_status: lifecycleStatus === 'all' ? undefined : lifecycleStatus, cursor }, controller.signal); if (generation !== listGeneration) return; loaded = append ? [...loaded, ...result.locations] : result.locations; if (!selected) selected = result.locations[0]; release = result.releaseId; ruleset = result.ruleset; nextCursor = result.nextCursor; coverageNote = result.coverageNote; coverageScope = result.coverageScope ?? ''; countSemantics = result.countSemantics ?? ''; localStatus = 'ready'; paging = false; if (!append) await syncRoute(); } catch (error) { paging = false; if (!append) activeListKey = ''; if (generation !== listGeneration) return; const kind = error && typeof error === 'object' && 'kind' in error ? (error as { kind: string }).kind : 'error'; localStatus = kind === 'no-release' ? 'no-release' : 'error'; localError = error instanceof Error ? error.message : 'Local V2 response was rejected safely.'; loaded = []; selected = undefined; } }; const downloadCsv = async () => { @@ -125,6 +127,8 @@ {#if profile === 'community'}
Community claimsUnreviewed community claims: Not verified by Until Every Cage. Check each record’s factual review status before relying on it.
{/if} {#if localMode && localStatus === 'loading'}
Loading the {profileLabel.toLowerCase()}…
{:else if localMode && (localStatus === 'error' || localStatus === 'no-release')}{:else if localMode && detailStatus === 'loading'}
Loading the selected local record…
{:else if localMode && detailStatus === 'error'}{:else}

02 / FILTER & COMPARE

Results {visibleLocations.length}{localMode && nextCursor ? ' on first page' : ''}

{localMode ? 'Current eligible response' : 'Fictional demonstration data'} · {profileLabel}

+ + {#if localMode}
VISIBLE FACILITY RECORDS{visibleLocations.length}{nextCursor ? '+' : ''}
DENOMINATORNot available

{countSemantics || 'Counts refer to eligible public facility projection rows, not animals or a story-wide total.'} Scope: {coverageScope || 'selected promoted release public facilities'}. {nextCursor ? 'This is a partial page.' : 'This response has no further page.'} Legacy status is not inferred: the current V2 record contract has no legacy field.

{/if} {#if localMode}

{coverageNote} {nextCursor ? 'Only the first page is loaded. Search and filters below may miss later records; counts and map points are partial.' : 'All records in this response are loaded; search applies to those records.'}

{/if}
{#if selected}

03 / RECORD DETAIL

{selected.name}

{selected.region} · {selected.category}

{#if selected.evidence?.publicationProfile === 'community' && selected.evidence.factualReviewStatus === 'unreviewed'}

{selected.evidence.publicationWarning ?? 'Unreviewed community claim — not verified by Until Every Cage'}

{/if}
OBSERVED{selected.observed}
MAP STATUS{selected.lat === null ? 'No publishable map location' : selected.evidence?.displayPrecision === 'exact' ? 'Exact display point' : 'Approximate display point'}
{#if selected.evidence}
Source origin
{selected.evidence.sourceType === 'user_submitted' ? 'Community-submitted' : selected.evidence.sourceType === 'official' ? 'Government-sourced' : 'Secondary-sourced'}
Factual review
{selected.evidence.factualReviewStatus}{selected.evidence.reviewerRole ? ` · ${selected.evidence.reviewerRole}` : ' · reviewer role unavailable'}
Privacy screening
{selected.evidence.privacyScreeningStatus}
Project approval
{selected.evidence.projectApproval}
Published profile
{selected.evidence.publicationProfile} · release {release}
Source
{selected.source} · {selected.evidence.sourceId}
Retrieved
{selected.evidence.retrievedAt}
{/if}

READ WITH CARE

This is an observation in a particular release, not a guarantee of current operation. Source origin, factual review, privacy screening, and project approval are separate signals.

{/if}
{#if selected}

RECORD / {selected.id}

{/if} diff --git a/frontend/src/domain/location.ts b/frontend/src/domain/location.ts index ed2e203..56734c2 100644 --- a/frontend/src/domain/location.ts +++ b/frontend/src/domain/location.ts @@ -1,4 +1,6 @@ export type LocationId = string; +// Evidence stays separate from the display name: origin, review, privacy, +// approval, precision, and lifecycle are independent signals. export type LocationEvidence = Readonly<{ sourceType: 'official' | 'secondary' | 'user_submitted'; factualReviewStatus: 'unreviewed' | 'reviewed' | 'rejected'; @@ -11,5 +13,7 @@ export type LocationEvidence = Readonly<{ sourceUrl: string; retrievedAt: string; displayPrecision: 'exact' | 'city' | 'unmapped'; + lifecycleStatus: 'active_observed' | 'explicitly_closed' | 'not_seen_recently' | 'status_unknown'; + observationCount: number | null; }>; export type Location = Readonly<{id:LocationId,name:string,region:string,category:string,lat:number|null,lon:number|null,observed:string,source:string,evidence?:LocationEvidence}>; diff --git a/frontend/src/features/scale/ScaleNarrative.svelte b/frontend/src/features/scale/ScaleNarrative.svelte index 5331ad9..d759256 100644 --- a/frontend/src/features/scale/ScaleNarrative.svelte +++ b/frontend/src/features/scale/ScaleNarrative.svelte @@ -5,6 +5,7 @@ $: selected = scaleSteps[selectedIndex] ?? scaleSteps[0]; +

A SCALE CHECK · SYNTHETIC MODEL

diff --git a/frontend/src/features/scale/scaleModel.ts b/frontend/src/features/scale/scaleModel.ts index e496aed..831a9d2 100644 --- a/frontend/src/features/scale/scaleModel.ts +++ b/frontend/src/features/scale/scaleModel.ts @@ -8,6 +8,7 @@ export type ScaleStep = Readonly<{ // Synthetic interaction values only. They are intentionally not presented as // an estimate of a country, company, facility, or real-world annual total. +// Keep this private/dev-only model separate from any sourced aggregate ledger. export const SYNTHETIC_BASE = 1; export const scaleSteps: readonly [ScaleStep, ...ScaleStep[]] = [ From 21e0dd1415da7bc95499f7a0b62946e7de43ce51 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sun, 13 Sep 2026 22:10:09 -0700 Subject: [PATCH 049/311] Add reusable source adapter contract for Denmark --- pipeline/contracts/README.md | 13 +++++ pipeline/contracts/adapter_contract.py | 25 +++++++++ pipeline/sources/denmark/adapter.py | 67 ++++++++++++++++++++++++ pipeline/sources/denmark/test_adapter.py | 26 +++++++++ 4 files changed, 131 insertions(+) create mode 100644 pipeline/contracts/README.md create mode 100644 pipeline/sources/denmark/adapter.py create mode 100644 pipeline/sources/denmark/test_adapter.py diff --git a/pipeline/contracts/README.md b/pipeline/contracts/README.md new file mode 100644 index 0000000..17539c6 --- /dev/null +++ b/pipeline/contracts/README.md @@ -0,0 +1,13 @@ +# Source-adapter contract + +Adapters receive a preserved raw artifact and `SourceArtifact` facts: source URL, +UTC retrieval time, SHA-256, byte size, supplied publication/effective dates, +code/config versions, rights/privacy caveats, and coverage. They must retain +source values, make uncertainty explicit, quarantine malformed or unresolved +records, and write deterministic parsed/normalized/quarantined JSONL plus a +manifest. The manifest is private staging (`release_state: not-created`); +passing validation is not approval, health, or publication authorization. + +Denmark's `DenmarkSmileyAdapter` is the proving implementation. Network +acquisition remains the existing reviewed acquisition command; raw/private +artifacts are intentionally not fixtures or committed data. diff --git a/pipeline/contracts/adapter_contract.py b/pipeline/contracts/adapter_contract.py index f330acb..4b7b7e5 100644 --- a/pipeline/contracts/adapter_contract.py +++ b/pipeline/contracts/adapter_contract.py @@ -2,6 +2,31 @@ import hashlib from pathlib import Path import json +from dataclasses import dataclass +from typing import Any, Protocol + + +@dataclass(frozen=True) +class SourceArtifact: + """Immutable acquisition facts supplied to a source adapter.""" + source_url: str + retrieved_at_utc: str + sha256: str + byte_size: int + publication_date: str | None = None + effective_date: str | None = None + code_version: str = "unknown" + config_version: str = "unknown" + rights_caveat: str | None = None + privacy_caveat: str | None = None + coverage: str | None = None + + +class SourceAdapter(Protocol): + source_id: str + adapter_version: str + + def run(self, raw_path: str | Path, run_dir: str | Path, artifact: SourceArtifact) -> dict[str, Any]: ... def assert_manifest(manifest: dict, raw: bytes, schema_version: str) -> None: diff --git a/pipeline/sources/denmark/adapter.py b/pipeline/sources/denmark/adapter.py new file mode 100644 index 0000000..3fe8bc3 --- /dev/null +++ b/pipeline/sources/denmark/adapter.py @@ -0,0 +1,67 @@ +"""Contract adapter for the Denmark Find Smiley XML source. + +Parsing is private staging only. This adapter never creates a release or +publishes coordinates; records with no stable source key are quarantined. +""" +from __future__ import annotations +import hashlib, json, os +import xml.etree.ElementTree as ET +from pathlib import Path +from typing import Any +from pipeline.contracts.adapter_contract import SourceArtifact + +SOURCE_ID = "dk.smiley" +ADAPTER_VERSION = "denmark-smiley-contract-v1" + +def _atomic(path: Path, payload: bytes) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_name(path.name + ".tmp") + temporary.write_bytes(payload) + os.replace(temporary, path) + +def _jsonl(path: Path, rows: list[dict[str, Any]]) -> str: + payload = b"".join((json.dumps(row, ensure_ascii=False, sort_keys=True, default=list) + "\n").encode() for row in rows) + _atomic(path, payload) + return hashlib.sha256(payload).hexdigest() + +class DenmarkSmileyAdapter: + source_id = SOURCE_ID + adapter_version = ADAPTER_VERSION + + def run(self, raw_path: str | Path, run_dir: str | Path, artifact: SourceArtifact) -> dict[str, Any]: + raw = Path(raw_path).read_bytes() + actual = hashlib.sha256(raw).hexdigest() + if actual != artifact.sha256 or len(raw) != artifact.byte_size: + raise ValueError("acquisition metadata does not match raw artifact") + rows: list[dict[str, Any]] = [] + quarantined: list[dict[str, Any]] = [] + try: + root = ET.fromstring(raw) + for number, element in enumerate(root.iter(), 1): + if element.tag.lower() != "row": + continue + fields = {child.tag: (child.text or "").strip() or None for child in element} + key = fields.get("ID_nummer") or fields.get("navnelbnr") + record = {"source_id": SOURCE_ID, "source_record_key": key, + "source_artifact_sha256": actual, "source_fields": fields, + "normalized": {"name": fields.get("Virksomhed"), + "address": fields.get("Adresse"), + "postcode": fields.get("Postnummer"), + "city": fields.get("By"), "country_code": "DK", + "coordinates": None}} + (quarantined if not key else rows).append({"reasons": ["missing_source_key"], "record": record} if not key else record) + except ET.ParseError as exc: + raise ValueError("invalid Denmark XML") from exc + root = Path(run_dir) + parsed_hash = _jsonl(root / "parsed" / "records.jsonl", rows + [item["record"] for item in quarantined]) + normalized_hash = _jsonl(root / "normalized" / "records.jsonl", rows) + _jsonl(root / "quarantined" / "records.jsonl", quarantined) + manifest = {"source_id": SOURCE_ID, "adapter_version": ADAPTER_VERSION, + "schema_version": ADAPTER_VERSION, "checksum_sha256": actual, + "byte_size": len(raw), "input_rows": len(rows) + len(quarantined), + "normalized_rows": len(rows), "quarantined_rows": len(quarantined), + "parsed_sha256": parsed_hash, "normalized_sha256": normalized_hash, + "release_state": "not-created", "publication_state": "private-candidate", + "acquisition": artifact.__dict__} + _atomic(root / "manifest.json", (json.dumps(manifest, ensure_ascii=False, sort_keys=True, indent=2) + "\n").encode()) + return manifest diff --git a/pipeline/sources/denmark/test_adapter.py b/pipeline/sources/denmark/test_adapter.py new file mode 100644 index 0000000..24a9b8c --- /dev/null +++ b/pipeline/sources/denmark/test_adapter.py @@ -0,0 +1,26 @@ +import hashlib, tempfile, unittest +from pathlib import Path +from .adapter import DenmarkSmileyAdapter +from pipeline.contracts.adapter_contract import SourceArtifact + +XML = b'1TestUnkeyed' + +class DenmarkAdapterTests(unittest.TestCase): + def artifact(self, data=XML): + return SourceArtifact("https://example.test/smiley.xml", "2026-01-01T00:00:00Z", hashlib.sha256(data).hexdigest(), len(data), code_version="test", config_version="test") + + def test_rerun_is_deterministic_and_private(self): + with tempfile.TemporaryDirectory() as d: + root = Path(d); raw = root / "raw.xml"; raw.write_bytes(XML) + a = DenmarkSmileyAdapter().run(raw, root / "one", self.artifact()) + b = DenmarkSmileyAdapter().run(raw, root / "two", self.artifact()) + self.assertEqual(a, b); self.assertEqual(a["quarantined_rows"], 1) + self.assertFalse((root / "one" / "released").exists()) + + def test_failed_acquisition_does_not_write_staging(self): + with tempfile.TemporaryDirectory() as d: + root = Path(d); raw = root / "raw.xml"; raw.write_bytes(XML) + with self.assertRaises(ValueError): DenmarkSmileyAdapter().run(raw, root / "run", self.artifact(b"wrong")) + self.assertFalse((root / "run").exists()) + +if __name__ == "__main__": unittest.main() From edb76d5f52681d69422e32f3e484d441e3bc2d46 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sun, 13 Sep 2026 22:12:45 -0700 Subject: [PATCH 050/311] Bridge Denmark adapter to registered staging --- pipeline/contracts/README.md | 11 +++++++++++ pipeline/sources/denmark/adapter.py | 18 ++++++++++++++++++ pipeline/sources/denmark/test_adapter.py | 13 +++++++++++++ 3 files changed, 42 insertions(+) diff --git a/pipeline/contracts/README.md b/pipeline/contracts/README.md index 17539c6..477fab5 100644 --- a/pipeline/contracts/README.md +++ b/pipeline/contracts/README.md @@ -11,3 +11,14 @@ passing validation is not approval, health, or publication authorization. Denmark's `DenmarkSmileyAdapter` is the proving implementation. Network acquisition remains the existing reviewed acquisition command; raw/private artifacts are intentionally not fixtures or committed data. + +The shared registered-input runner can call Denmark's `run_registered` bridge. +It requires recorded URL, UTC retrieval time, hash, and byte size; missing or +mismatched provenance fails closed. Database import, geocoding, release approval, +and publication remain separately gated. + +For disposable development teardown, remove only the selected +`data/staging/denmark-smiley//` directory after checking retention duties, +then recreate the local database through the existing maintenance script with +an explicitly local development URL. Never point teardown at production and do +not delete retained research evidence without an authorized decision. diff --git a/pipeline/sources/denmark/adapter.py b/pipeline/sources/denmark/adapter.py index 3fe8bc3..1d24833 100644 --- a/pipeline/sources/denmark/adapter.py +++ b/pipeline/sources/denmark/adapter.py @@ -28,6 +28,24 @@ class DenmarkSmileyAdapter: source_id = SOURCE_ID adapter_version = ADAPTER_VERSION + def run_registered(self, raw_path: str | Path, run_dir: str | Path, config: dict[str, Any]) -> dict[str, Any]: + """Bridge the shared registered-input runner using recorded evidence.""" + required = ("source_url", "retrieved_at_utc", "checksum_sha256", "byte_size") + missing = [key for key in required if not config.get(key)] + if missing: + raise ValueError("missing acquisition provenance: " + ", ".join(missing)) + raw = Path(raw_path).read_bytes() + digest = hashlib.sha256(raw).hexdigest() + if digest != config["checksum_sha256"] or len(raw) != config["byte_size"]: + raise ValueError("registered acquisition integrity mismatch") + artifact = SourceArtifact( + source_url=str(config["source_url"]), retrieved_at_utc=str(config["retrieved_at_utc"]), + sha256=digest, byte_size=len(raw), publication_date=config.get("publication_date"), + effective_date=config.get("effective_date"), code_version=str(config.get("code_version", ADAPTER_VERSION)), + config_version=str(config.get("config_version", "unknown")), rights_caveat=config.get("rights_caveat"), + privacy_caveat=config.get("privacy_caveat"), coverage=config.get("coverage")) + return self.run(raw_path, run_dir, artifact) + def run(self, raw_path: str | Path, run_dir: str | Path, artifact: SourceArtifact) -> dict[str, Any]: raw = Path(raw_path).read_bytes() actual = hashlib.sha256(raw).hexdigest() diff --git a/pipeline/sources/denmark/test_adapter.py b/pipeline/sources/denmark/test_adapter.py index 24a9b8c..ed21689 100644 --- a/pipeline/sources/denmark/test_adapter.py +++ b/pipeline/sources/denmark/test_adapter.py @@ -23,4 +23,17 @@ def test_failed_acquisition_does_not_write_staging(self): with self.assertRaises(ValueError): DenmarkSmileyAdapter().run(raw, root / "run", self.artifact(b"wrong")) self.assertFalse((root / "run").exists()) + def test_registered_bridge_requires_and_checks_provenance(self): + with tempfile.TemporaryDirectory() as d: + root = Path(d); raw = root / "raw.xml"; raw.write_bytes(XML) + adapter = DenmarkSmileyAdapter() + with self.assertRaisesRegex(ValueError, "missing acquisition provenance"): + adapter.run_registered(raw, root / "missing", {}) + config = {"source_url": "https://example.test/source.xml", "retrieved_at_utc": "2026-01-01T00:00:00Z", "checksum_sha256": "0" * 64, "byte_size": len(XML)} + with self.assertRaisesRegex(ValueError, "integrity mismatch"): + adapter.run_registered(raw, root / "bad", config) + config["checksum_sha256"] = hashlib.sha256(XML).hexdigest() + manifest = adapter.run_registered(raw, root / "good", config) + self.assertEqual(manifest["acquisition"]["source_url"], config["source_url"]) + if __name__ == "__main__": unittest.main() From a1d799ed35a75af060edd6a821405027e549637c Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sun, 13 Sep 2026 22:14:34 -0700 Subject: [PATCH 051/311] Clarify Denmark adapter contract safeguards --- pipeline/contracts/adapter_contract.py | 3 +++ pipeline/sources/denmark/adapter.py | 5 +++++ 2 files changed, 8 insertions(+) diff --git a/pipeline/contracts/adapter_contract.py b/pipeline/contracts/adapter_contract.py index 4b7b7e5..6ba8f7c 100644 --- a/pipeline/contracts/adapter_contract.py +++ b/pipeline/contracts/adapter_contract.py @@ -23,6 +23,7 @@ class SourceArtifact: class SourceAdapter(Protocol): + """Minimal boundary between acquisition evidence and private staging.""" source_id: str adapter_version: str @@ -30,6 +31,7 @@ def run(self, raw_path: str | Path, run_dir: str | Path, artifact: SourceArtifac def assert_manifest(manifest: dict, raw: bytes, schema_version: str) -> None: + """Assert safety invariants shared by adapter tests; not a release gate.""" assert manifest["checksum_sha256"] == hashlib.sha256(raw).hexdigest() assert manifest["byte_size"] == len(raw) assert manifest["schema_version"] == schema_version @@ -38,4 +40,5 @@ def assert_manifest(manifest: dict, raw: bytes, schema_version: str) -> None: def read_jsonl(path: Path) -> list[dict]: + """Read deterministic staging records for test and review tooling.""" return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line] diff --git a/pipeline/sources/denmark/adapter.py b/pipeline/sources/denmark/adapter.py index 1d24833..0c32dff 100644 --- a/pipeline/sources/denmark/adapter.py +++ b/pipeline/sources/denmark/adapter.py @@ -14,12 +14,14 @@ ADAPTER_VERSION = "denmark-smiley-contract-v1" def _atomic(path: Path, payload: bytes) -> None: + """Publish one complete staging file, never a partially written artifact.""" path.parent.mkdir(parents=True, exist_ok=True) temporary = path.with_name(path.name + ".tmp") temporary.write_bytes(payload) os.replace(temporary, path) def _jsonl(path: Path, rows: list[dict[str, Any]]) -> str: + """Serialize with stable ordering so reruns can be compared byte-for-byte.""" payload = b"".join((json.dumps(row, ensure_ascii=False, sort_keys=True, default=list) + "\n").encode() for row in rows) _atomic(path, payload) return hashlib.sha256(payload).hexdigest() @@ -47,6 +49,7 @@ def run_registered(self, raw_path: str | Path, run_dir: str | Path, config: dict return self.run(raw_path, run_dir, artifact) def run(self, raw_path: str | Path, run_dir: str | Path, artifact: SourceArtifact) -> dict[str, Any]: + """Parse one preserved artifact into private, human-gated candidate data.""" raw = Path(raw_path).read_bytes() actual = hashlib.sha256(raw).hexdigest() if actual != artifact.sha256 or len(raw) != artifact.byte_size: @@ -67,6 +70,7 @@ def run(self, raw_path: str | Path, run_dir: str | Path, artifact: SourceArtifac "postcode": fields.get("Postnummer"), "city": fields.get("By"), "country_code": "DK", "coordinates": None}} + # Without a stable source identity, normalization must not invent one. (quarantined if not key else rows).append({"reasons": ["missing_source_key"], "record": record} if not key else record) except ET.ParseError as exc: raise ValueError("invalid Denmark XML") from exc @@ -74,6 +78,7 @@ def run(self, raw_path: str | Path, run_dir: str | Path, artifact: SourceArtifac parsed_hash = _jsonl(root / "parsed" / "records.jsonl", rows + [item["record"] for item in quarantined]) normalized_hash = _jsonl(root / "normalized" / "records.jsonl", rows) _jsonl(root / "quarantined" / "records.jsonl", quarantined) + # This state is deliberately private: validation cannot authorize release. manifest = {"source_id": SOURCE_ID, "adapter_version": ADAPTER_VERSION, "schema_version": ADAPTER_VERSION, "checksum_sha256": actual, "byte_size": len(raw), "input_rows": len(rows) + len(quarantined), From 522615624643a77cdc3148644aadd5c7efd82169 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sun, 13 Sep 2026 22:33:03 -0700 Subject: [PATCH 052/311] Reconcile UK FSA monthly adapter --- docs/country-recon-uk.md | 37 +-- pipeline/sources/uk/approved/test_compose.py | 10 +- pipeline/sources/uk/fsa_approved/README.md | 45 +-- pipeline/sources/uk/fsa_approved/adapter.py | 262 ++++++++---------- .../sources/uk/fsa_approved/test_adapter.py | 33 ++- 5 files changed, 202 insertions(+), 185 deletions(-) diff --git a/docs/country-recon-uk.md b/docs/country-recon-uk.md index cb6f127..6dfe8ba 100644 --- a/docs/country-recon-uk.md +++ b/docs/country-recon-uk.md @@ -81,21 +81,22 @@ remain mandatory. ## Adapter reconciliation -The canonical implementation is `pipeline/sources/uk/fsa_approved/`. It currently -provides a pinned synthetic schema, ordered source-value preservation, strict -encoding/schema checks, duplicate-identifier quarantine, activity/status and -authority/nation checks, address privacy-risk quarantine, deterministic manifests, -and a human publication gate. - -The isolated experiment in commit `f73299b` assumes a different CSV contract using -`AppNo`, `X/Y`, and `AddressWithheld`, with additional coordinate and coverage -diagnostics. It must not be cherry-picked as a second UK adapter: doing so would -create two incompatible interpretations of the same country source. Its unique -behaviors are useful requirements for a future canonical schema review, especially -explicit withheld-address semantics, source-coordinate CRS/axis/range validation, -coverage counts for out-of-scope nations, and the related provenance fingerprints. - -This reconciliation is a design record, not source approval or legal clearance. -No live row data is included, and no schema-dependent gap should be implemented -until the actual source contract, terms, privacy treatment, and publication scope -are reviewed together. +The canonical implementation is `pipeline/sources/uk/fsa_approved/`. It supports +the existing synthetic contract and the observed monthly FSA CSV profile through +one adapter. Registered runs require source URL, retrieval UTC, checksum, and byte +size and fail closed before staging on missing or mismatched metadata. They preserve +source values, emit deterministic parsed/normalized/quarantined states, record +normalized-output checksums and schema fingerprints, and always use +`release_state=not-created` with a private-candidate publication state. + +The monthly profile treats England and Wales as the current source scope and keeps +Northern Ireland as a separate future feed. It quarantines duplicate IDs within a +nation, unknown jurisdictions, malformed rows, missing activities, remarks and +address-risk rows; suppresses `AddressWithheld` addresses and coordinates; validates +X/Y as source longitude/latitude without geocoding; and reports aggregate coverage +and anomaly counts. The synthetic profile retains its authority/status/activity +vocabulary tests for the canonical composition contract. + +This reconciliation is a design and test record, not source approval or legal +clearance. No live row data is included, and the private artifact remains outside +Git and public outputs. diff --git a/pipeline/sources/uk/approved/test_compose.py b/pipeline/sources/uk/approved/test_compose.py index 3c488a7..699d14a 100644 --- a/pipeline/sources/uk/approved/test_compose.py +++ b/pipeline/sources/uk/approved/test_compose.py @@ -1,4 +1,5 @@ import json +import hashlib import tempfile import unittest from pathlib import Path @@ -17,8 +18,11 @@ def _runs(self, root): fss_dir, fsa_dir = root / "fss", root / "fsa" fss = FssApprovedEstablishmentsAdapter() fsa = FsaApprovedEstablishmentsAdapter() - fss_manifest = fss.run(FSS_FIXTURE, fss_dir) - fsa_manifest = fsa.run(FSA_FIXTURE, fsa_dir) + def artifact(path): + raw = path.read_bytes() + return {"source_url": "https://example.invalid/fsa", "retrieved_at_utc": "2026-09-14T00:00:00Z", "checksum_sha256": hashlib.sha256(raw).hexdigest(), "byte_size": len(raw), "effective_date": "2026-09-01"} + fss_manifest = fss.run(FSS_FIXTURE, fss_dir, {"source_url": "https://example.invalid/fss", "retrieved_at": "2026-09-14T00:00:00Z"}) + fsa_manifest = fsa.run(FSA_FIXTURE, fsa_dir, artifact(FSA_FIXTURE)) return [ {"source_id": fss.source_id, "manifest": fss_manifest, "normalized_path": fss_dir / "normalized/records.jsonl", "terms_state": "unresolved", "review_state": "human-review-required"}, {"source_id": fsa.source_id, "manifest": fsa_manifest, "normalized_path": fsa_dir / "normalized/records.jsonl", "terms_state": "unresolved", "review_state": "human-review-required"}, @@ -56,7 +60,7 @@ def test_exact_cross_source_name_postcode_is_signal_not_merge(self): inputs = self._runs(root) modified_raw = root / "modified-fss.csv" modified_raw.write_bytes(FSS_FIXTURE.read_bytes().replace(b"North Star Foods", b"East March Foods").replace(b"AB1 2CD", b"PE1 2AB")) - inputs[0]["manifest"] = FssApprovedEstablishmentsAdapter().run(modified_raw, root / "fss") + inputs[0]["manifest"] = FssApprovedEstablishmentsAdapter().run(modified_raw, root / "fss", {"source_url": "https://example.invalid/fss", "retrieved_at": "2026-09-14T00:00:00Z"}) manifest = compose_sources(inputs, root / "country") signals = [json.loads(line) for line in (root / "country/reviewable/possible-match-signals.jsonl").read_text().splitlines()] self.assertEqual(manifest["possible_match_signals"], 1) diff --git a/pipeline/sources/uk/fsa_approved/README.md b/pipeline/sources/uk/fsa_approved/README.md index c38cb1a..ddb7a20 100644 --- a/pipeline/sources/uk/fsa_approved/README.md +++ b/pipeline/sources/uk/fsa_approved/README.md @@ -1,25 +1,28 @@ -# FSA England, Wales and Northern Ireland approved establishments +# FSA approved establishments -This adapter is synthetic-fixture-only. Its CSV columns and authority/nation -mapping are explicit test assumptions, not an inferred live FSA artifact -schema. FSS Scotland remains a separate source and adapter; no FSS records or -configuration are merged into this capability. +This adapter supports the pinned synthetic contract and the observed monthly FSA +CSV profile. The monthly profile requires a recorded SourceArtifact and validates +the AppNo, TradingName, Country, CompetentAuthority, X, Y, AddressWithheld, and +activity headers before staging. FSS Scotland and Northern Ireland remain separate +source feeds; they are not merged into this capability. -The adapter preserves every source cell and identifier string, including -leading zeroes, and emits no guessed coordinates. Establishment IDs are -unique only within a nation, allowing authority datasets to be isolated even -when identifiers repeat across nations. Authority/nation mismatches, schema -drift, duplicate or missing IDs, unknown activities/statuses, malformed rows, -remarks and privacy-risk addresses are quarantined or fail closed. +Source values and identifiers are preserved. Duplicate IDs are quarantined within +nation, unknown jurisdictions are quarantined, and malformed rows, missing or +unresolved activity, remarks, authority mismatches, and address-risk values remain +explicit review outcomes. `AddressWithheld=Yes` emits no address or coordinates; +X/Y are validated as source longitude/latitude without geocoding. Registered runs +write deterministic parsed, normalized, and quarantined states with a manifest +whose `release_state` is always `not-created` and whose publication state is +private-candidate. -## FSA-specific acquisition gates +The adapter does not download or automate acquisition. Before a registered run, +maintainers must verify the current official URL, effective/publication date, +ownership, terms/licence, attribution, rate limits, retention/removal rules, and +redistribution status. Privacy/suppression review, factual review, project approval, +and publication remain independent gates. -Before any acquisition, a maintainer must verify separately for England, -Wales and Northern Ireland: the current artifact URL and format, publication -and effective dates, FSA/department ownership, terms/licence, attribution -requirements, update automation/rate limits, raw-artifact retention and -removal rules, and whether the source permits redistribution. No live schema, -download, automation, geocoding, release, public API/export, or external -contact is authorized by this fixture contract. Privacy/suppression review, -human factual review, project approval and publication remain independent -gates. +Run focused tests from the repository root: + +```text +python -m unittest -q pipeline.sources.uk.fsa_approved.test_adapter pipeline.sources.uk.approved.test_compose +``` diff --git a/pipeline/sources/uk/fsa_approved/adapter.py b/pipeline/sources/uk/fsa_approved/adapter.py index e2efc7e..7f7e454 100644 --- a/pipeline/sources/uk/fsa_approved/adapter.py +++ b/pipeline/sources/uk/fsa_approved/adapter.py @@ -1,155 +1,133 @@ -"""Synthetic-only FSA England/Wales/Northern Ireland adapter. - -The CSV shape is a pinned test contract, not a claim about a live FSA file. -""" +"""FSA approved-establishments adapter for synthetic and monthly source profiles.""" from __future__ import annotations -import csv -import hashlib -import json -import os -import re +import csv, hashlib, json, os, re, tempfile from dataclasses import asdict, dataclass from pathlib import Path from typing import Any -ROOT = Path(__file__).parent -CONFIG = json.loads((ROOT / "config.json").read_text(encoding="utf-8")) -REQUIRED_COLUMNS = tuple(CONFIG["required_columns"]) -ALLOWED_ACTIVITIES = frozenset(CONFIG["allowed_activities"]) -ALLOWED_STATUSES = frozenset(CONFIG["allowed_statuses"]) -AUTHORITY_BY_NATION = CONFIG["authority_by_nation"] -ADDRESS_RISK = re.compile(r"\b(flat|apartment|house|home|residential|c/o|care of|caravan|lodge)\b", re.I) - +ROOT=Path(__file__).parent +CONFIG=json.loads((ROOT/"config.json").read_text(encoding="utf-8")) +REQUIRED_COLUMNS=tuple(CONFIG["required_columns"]) +ALLOWED_ACTIVITIES=frozenset(CONFIG["allowed_activities"]) +ALLOWED_STATUSES=frozenset(CONFIG["allowed_statuses"]) +AUTHORITY_BY_NATION=CONFIG["authority_by_nation"] +ADDRESS_RISK=re.compile(r"\b(flat|apartment|house|home|residential|c/o|care of|caravan|lodge)\b",re.I) +MONTHLY_REQUIRED=frozenset({"AppNo","TradingName","Country","CompetentAuthority","X","Y","AddressWithheld","All_Activities"}) +MONTHLY_COUNTRIES=frozenset({"England","Wales"}) class FsaContractError(ValueError): - """The supplied artifact cannot be interpreted under the assumed contract.""" - + """The artifact does not satisfy a supported source contract.""" @dataclass(frozen=True) class ValidationResult: - accepted: tuple[dict[str, Any], ...] - quarantined: tuple[dict[str, Any], ...] + accepted: tuple[dict[str,Any],...] + quarantined: tuple[dict[str,Any],...] source_sha256: str - contract_version: str = CONFIG["contract_version"] - release_allowed: bool = False - - def as_dict(self) -> dict[str, Any]: - return asdict(self) - - -def _clean(value: str | None) -> str | None: - if value is None: - return None - value = value.strip() - return value or None - - -def _split(value: str | None) -> tuple[str, ...]: - return tuple(part for part in (_clean(item) for item in (value or "").split(";")) if part) - - -def _record(row: dict[str, str], line: int) -> dict[str, Any]: - nation = _clean(row.get("nation")) - return {"source_id": CONFIG["source_id"], "source_row": line, - "source_values": dict(row), "normalized": { - "establishment_id": _clean(row.get("establishment_id")), - "trading_name": _clean(row.get("trading_name")), - "address_lines": tuple(_clean(row.get(f"address_line_{n}")) for n in range(1, 4)), - "postcode": _clean(row.get("postcode")), "activities": _split(row.get("activities")), - "species": _clean(row.get("species")), - "competent_authority": _clean(row.get("competent_authority")), - "nation": nation, "authority_nation_key": nation, - "status": _clean(row.get("status")), "remarks": _clean(row.get("remarks")), - "published_date": _clean(row.get("published_date")), "coordinates": None}} - - -class FsaApprovedEstablishmentsAdapter: - source_id = CONFIG["source_id"] - schema_version = CONFIG["contract_version"] - adapter_version = CONFIG["adapter_version"] - - def parse_bytes(self, content: bytes) -> ValidationResult: - digest = hashlib.sha256(content).hexdigest() + contract_version: str=CONFIG["contract_version"] + release_allowed: bool=False + profile: str="synthetic" + schema_fingerprint: str="" + coverage_counts: dict[str,int]|None=None + anomaly_counts: dict[str,int]|None=None + def as_dict(self): return asdict(self) + +def _clean(v): + if v is None:return None + v=v.strip();return v or None +def _split(v): return tuple(x for x in (_clean(i) for i in (v or "").split(";")) if x) +def _atomic(path,payload): + path.parent.mkdir(parents=True,exist_ok=True);fd,tmp=tempfile.mkstemp(prefix=f".{path.name}.",dir=path.parent) + try: + with os.fdopen(fd,"wb") as h:h.write(payload) + os.replace(tmp,path) + except Exception: + try:os.unlink(tmp) + except FileNotFoundError:pass + raise +def _jsonl(path,rows): + payload=b"".join((json.dumps(r,ensure_ascii=False,sort_keys=True,default=list)+"\n").encode() for r in rows);_atomic(path,payload);return hashlib.sha256(payload).hexdigest() +def _csv(content): + for enc in ("utf-8-sig","cp1252"): try: - text = content.decode("utf-8-sig") - reader = csv.DictReader(text.splitlines(), strict=True) - if tuple(reader.fieldnames or ()) != REQUIRED_COLUMNS: - raise FsaContractError("schema drift: expected pinned synthetic FSA columns in exact order") - rows = list(reader) - except UnicodeDecodeError as exc: - raise FsaContractError("source is not UTF-8 CSV") from exc - except csv.Error as exc: - raise FsaContractError("malformed CSV") from exc - if any(None in row for row in rows): - raise FsaContractError("schema drift: a row has extra columns") - keys = [(_clean(row.get("nation")), _clean(row.get("establishment_id"))) for row in rows] - duplicates = {key for key in keys if key[0] and key[1] and keys.count(key) > 1} - accepted: list[dict[str, Any]] = [] - quarantined: list[dict[str, Any]] = [] - for line, row in enumerate(rows, 2): - reasons: list[str] = [] - nation = _clean(row.get("nation")) - identifier = _clean(row.get("establishment_id")) - if any(value is None for value in row.values()): - reasons.append("malformed_row") - if not identifier: - reasons.append("missing_establishment_id") - if (nation, identifier) in duplicates: - reasons.append("duplicate_id_within_nation") - if nation not in CONFIG["covered_nations"]: - reasons.append("unknown_nation") - authority = _clean(row.get("competent_authority")) - if nation in AUTHORITY_BY_NATION and authority != AUTHORITY_BY_NATION[nation]: - reasons.append("authority_nation_mismatch") - activities = _split(row.get("activities")) - if not activities: - reasons.append("missing_activity") - elif any(activity not in ALLOWED_ACTIVITIES for activity in activities): - reasons.append("unknown_activity") - status = _clean(row.get("status")) - if status and status.lower() not in ALLOWED_STATUSES: - reasons.append("unknown_status") - if _clean(row.get("remarks")): - reasons.append("remarks_present") - address = " ".join(_clean(row.get(f"address_line_{n}")) or "" for n in range(1, 4)) - if ADDRESS_RISK.search(address): - reasons.append("address_privacy_risk") - record = _record(row, line) - (quarantined if reasons else accepted).append({"reasons": tuple(reasons), "record": record} if reasons else record) - return ValidationResult(tuple(accepted), tuple(quarantined), digest) - - def parse_file(self, path: str | Path) -> ValidationResult: - return self.parse_bytes(Path(path).read_bytes()) + rows=list(csv.reader(content.decode(enc).splitlines(),strict=True)) + if not rows or not rows[0]:raise FsaContractError("missing source header") + return rows[0],rows[1:],enc + except UnicodeDecodeError:continue + except csv.Error as exc:raise FsaContractError("malformed CSV") from exc + raise FsaContractError("unsupported CSV encoding") +def _synthetic_record(row,line): + nation=_clean(row.get("nation"));return {"source_id":CONFIG["source_id"],"source_row":line,"source_values":dict(row),"normalized":{"establishment_id":_clean(row.get("establishment_id")),"trading_name":_clean(row.get("trading_name")),"address_lines":tuple(_clean(row.get(f"address_line_{n}")) for n in range(1,4)),"postcode":_clean(row.get("postcode")),"activities":_split(row.get("activities")),"species":_clean(row.get("species")),"competent_authority":_clean(row.get("competent_authority")),"nation":nation,"authority_nation_key":nation,"status":_clean(row.get("status")),"remarks":_clean(row.get("remarks")),"published_date":_clean(row.get("published_date")),"coordinates":None}} +def _coords(row): + try:x,y=float(row.get("X","").strip()),float(row.get("Y","").strip()) + except ValueError:return None,None,"unresolved-nonnumeric" + if not(-8.5<=x<=2.5 and 49.0<=y<=61.5):return None,None,"unresolved-out-of-range" + return x,y,"source-x-lon-y-lat" +def _monthly_record(row,line): + withheld=(_clean(row.get("AddressWithheld")) or "").lower()=="yes";x=y=None;status="withheld" if withheld else "unavailable" + if not withheld:x,y,status=_coords(row) + acts=tuple(x for x in (_clean(row.get("All_Activities")),_clean(row.get("Part_A__All_sections_")),_clean(row.get("Part B All sections "))) if x) + return {"source_id":CONFIG["source_id"],"source_row":line,"source_values":dict(row),"normalized":{"establishment_id":_clean(row.get("AppNo")),"trading_name":_clean(row.get("TradingName")),"address_lines":None if withheld else tuple(_clean(row.get(k)) for k in ("Address1","Address2","Address3")),"postcode":_clean(row.get("Postcode")),"activities":acts,"species":_clean(row.get("Species")),"competent_authority":_clean(row.get("CompetentAuthority")),"nation":_clean(row.get("Country")),"authority_nation_key":_clean(row.get("Country")),"status":None,"remarks":_clean(row.get("Remarks")),"published_date":None,"coordinates":None if withheld else {"longitude":x,"latitude":y,"status":status},"privacy_gate":"restricted-withheld-address" if withheld else "pending-review","publication_gate":"blocked"}} - def run(self, raw_path: str | Path, run_dir: str | Path, config: dict[str, Any] | None = None) -> dict[str, Any]: - raw = Path(raw_path).read_bytes() - result = self.parse_bytes(raw) - root = Path(run_dir) - _write_jsonl(root / "parsed" / "records.jsonl", list(result.accepted) + [item["record"] for item in result.quarantined]) - normalized_sha256 = _write_jsonl(root / "normalized" / "records.jsonl", list(result.accepted)) - _write_jsonl(root / "quarantined" / "records.jsonl", list(result.quarantined)) - (root / "released").mkdir(parents=True, exist_ok=True) - manifest = {"source_id": self.source_id, "adapter_version": self.adapter_version, - "schema_version": self.schema_version, "schema_status": CONFIG["schema_status"], - "checksum_sha256": result.source_sha256, "byte_size": len(raw), - "input_rows": len(result.accepted) + len(result.quarantined), - "normalized_rows": len(result.accepted), "normalized_sha256": normalized_sha256, - "quarantined_rows": len(result.quarantined), - "release_state": "not-created", "publication_state": "human-gate-required", - "acquisition": CONFIG["acquisition"], "source_url": (config or {}).get("source_url"), - "retrieved_at": (config or {}).get("retrieved_at")} - _atomic(root / "manifest.json", (json.dumps(manifest, ensure_ascii=False, sort_keys=True, indent=2) + "\n").encode()) - return manifest - - -def _write_jsonl(path: Path, rows: list[dict[str, Any]]) -> str: - payload = b"".join((json.dumps(row, ensure_ascii=False, sort_keys=True, default=list) + "\n").encode() for row in rows) - _atomic(path, payload) - return hashlib.sha256(payload).hexdigest() - - -def _atomic(path: Path, payload: bytes) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - temporary = path.with_name(path.name + ".tmp") - temporary.write_bytes(payload) - os.replace(temporary, path) +class FsaApprovedEstablishmentsAdapter: + source_id=CONFIG["source_id"];schema_version=CONFIG["contract_version"];adapter_version=CONFIG["adapter_version"] + def parse_bytes(self,content): + digest=hashlib.sha256(content).hexdigest();headers,rows,_=_csv(content);fp=hashlib.sha256(json.dumps(headers,ensure_ascii=False,separators=(",",":")).encode()).hexdigest();synthetic=tuple(headers)==REQUIRED_COLUMNS;monthly=MONTHLY_REQUIRED.issubset(headers) + if not synthetic and not monthly:raise FsaContractError("schema drift: unsupported FSA header profile") + return self._synthetic(headers,rows,digest,fp) if synthetic else self._monthly(headers,rows,digest,fp) + def _synthetic(self,headers,rows,digest,fp): + accepted=[];quarantined=[];parsed=[];keys=[] + for line,row in enumerate(rows,2): + v={h:row[i] if i1} + for line,v in parsed: + reasons=[];nation,ident=_clean(v.get("nation")),_clean(v.get("establishment_id")) + if len(v)!=len(headers) or any(x is None for x in v.values()):reasons.append("malformed_row") + if not ident:reasons.append("missing_establishment_id") + if (nation,ident) in duplicates:reasons.append("duplicate_id_within_nation") + if nation not in CONFIG["covered_nations"]:reasons.append("unknown_nation") + authority=_clean(v.get("competent_authority")) + if nation in AUTHORITY_BY_NATION and authority!=AUTHORITY_BY_NATION[nation]:reasons.append("authority_nation_mismatch") + acts=_split(v.get("activities")) + if not acts:reasons.append("missing_activity") + elif any(a not in ALLOWED_ACTIVITIES for a in acts):reasons.append("unknown_activity") + status=_clean(v.get("status")) + if status and status.lower() not in ALLOWED_STATUSES:reasons.append("unknown_status") + if _clean(v.get("remarks")):reasons.append("remarks_present") + if ADDRESS_RISK.search(" ".join(_clean(v.get(f"address_line_{n}")) or "" for n in range(1,4))):reasons.append("address_privacy_risk") + record=_synthetic_record(v,line);(quarantined if reasons else accepted).append({"reasons":tuple(dict.fromkeys(reasons)),"record":record} if reasons else record) + return ValidationResult(tuple(accepted),tuple(quarantined),digest,profile="synthetic",schema_fingerprint=fp) + def _monthly(self,headers,rows,digest,fp): + accepted=[];quarantined=[];parsed=[];keys=[];coverage={};anomalies={} + for line,row in enumerate(rows,2): + v={h:row[i] if i1} + for line,v,reasons in parsed: + country,ident=_clean(v.get("Country")),_clean(v.get("AppNo"));coverage[country or ""]=coverage.get(country or "",0)+1 + if not ident:reasons.append("missing_establishment_id") + if (country,ident) in duplicates:reasons.append("duplicate_id_within_nation") + if country not in MONTHLY_COUNTRIES:reasons.append("unknown_nation") + if not any(_clean(v.get(k)) for k in ("All_Activities","Part_A__All_sections_","Part B All sections ")):reasons.append("missing_activity") + if _clean(v.get("Remarks")):reasons.append("remarks_present") + withheld=(_clean(v.get("AddressWithheld")) or "").lower()=="yes" + if not withheld and ADDRESS_RISK.search(" ".join(_clean(v.get(k)) or "" for k in ("Address1","Address2","Address3","Town","Postcode"))):reasons.append("address_privacy_risk") + record=_monthly_record(v,line) + if reasons: + for reason in reasons:anomalies[reason]=anomalies.get(reason,0)+1 + quarantined.append({"reasons":tuple(dict.fromkeys(reasons)),"record":record}) + else:accepted.append(record) + return ValidationResult(tuple(accepted),tuple(quarantined),digest,profile="monthly",schema_fingerprint=fp,coverage_counts=coverage,anomaly_counts=anomalies) + def parse_file(self,path):return self.parse_bytes(Path(path).read_bytes()) + def run(self,raw_path,run_dir,artifact=None): + if artifact is None:raise FsaContractError("SourceArtifact metadata is required") + required={"source_url","retrieved_at_utc","checksum_sha256","byte_size"};missing=sorted(required-set(artifact)) + if missing:raise FsaContractError(f"missing SourceArtifact fields: {', '.join(missing)}") + raw=Path(raw_path).read_bytes();digest=hashlib.sha256(raw).hexdigest() + if digest!=artifact["checksum_sha256"] or len(raw)!=int(artifact["byte_size"]):raise FsaContractError("source checksum or byte size mismatch") + result=self.parse_bytes(raw);accepted=list(result.accepted);quarantined=list(result.quarantined);root=Path(run_dir);parsed=accepted+[x["record"] for x in quarantined] + normalized_sha=_jsonl(root/"normalized"/"records.jsonl",accepted);_jsonl(root/"parsed"/"records.jsonl",parsed);_jsonl(root/"quarantined"/"records.jsonl",quarantined);(root/"released").mkdir(parents=True,exist_ok=True) + manifest={**artifact,"source_id":self.source_id,"adapter_version":self.adapter_version,"schema_version":self.schema_version,"checksum_sha256":digest,"byte_size":len(raw),"input_rows":len(parsed),"normalized_rows":len(accepted),"normalized_sha256":normalized_sha,"quarantined_rows":len(quarantined),"profile":result.profile,"schema_fingerprint":result.schema_fingerprint,"coverage_counts":result.coverage_counts or {},"anomaly_counts":result.anomaly_counts or {},"geocoding":"disabled","release_state":"not-created","publication_state":"private-candidate"} + _atomic(root/"manifest.json",(json.dumps(manifest,ensure_ascii=False,sort_keys=True,indent=2,default=list)+"\n").encode());return manifest + +def run_registered(raw_path,run_dir,config):return FsaApprovedEstablishmentsAdapter().run(raw_path,run_dir,config) diff --git a/pipeline/sources/uk/fsa_approved/test_adapter.py b/pipeline/sources/uk/fsa_approved/test_adapter.py index 2aefe7d..b3ef8c5 100644 --- a/pipeline/sources/uk/fsa_approved/test_adapter.py +++ b/pipeline/sources/uk/fsa_approved/test_adapter.py @@ -1,4 +1,6 @@ import tempfile +import hashlib +import json import unittest from pathlib import Path @@ -38,10 +40,39 @@ def test_schema_drift_fails_closed(self): def test_manifest_rerun_is_deterministic_and_release_is_absent(self): with tempfile.TemporaryDirectory() as directory: first, second = Path(directory) / "one", Path(directory) / "two" - self.assertEqual(self.adapter.run(FIXTURES / "valid.csv", first), self.adapter.run(FIXTURES / "valid.csv", second)) + raw = (FIXTURES / "valid.csv").read_bytes() + artifact = {"source_url": "https://example.invalid/fsa", "retrieved_at_utc": "2026-09-14T00:00:00Z", "checksum_sha256": hashlib.sha256(raw).hexdigest(), "byte_size": len(raw), "effective_date": "2026-09-01"} + self.assertEqual(self.adapter.run(FIXTURES / "valid.csv", first, artifact), self.adapter.run(FIXTURES / "valid.csv", second, artifact)) self.assertFalse((first / "released" / "records.jsonl").exists()) self.assertEqual((first / "manifest.json").read_bytes(), (second / "manifest.json").read_bytes()) + def test_source_artifact_is_required_and_mismatch_fails_closed(self): + with tempfile.TemporaryDirectory() as directory: + source = FIXTURES / "valid.csv" + with self.assertRaises(FsaContractError): + self.adapter.run(source, Path(directory) / "missing-artifact") + raw = source.read_bytes() + artifact = {"source_url": "https://example.invalid/fsa", "retrieved_at_utc": "2026-09-14T00:00:00Z", "checksum_sha256": "0" * 64, "byte_size": len(raw)} + with self.assertRaises(FsaContractError): + self.adapter.run(source, Path(directory) / "bad-hash", artifact) + + def test_monthly_profile_enforces_withholding_coverage_duplicates_and_coordinates(self): + monthly = "AppNo,TradingName,Country,CompetentAuthority,X,Y,AddressWithheld,All_Activities,Address1,Town,Postcode\nA-1,Example,Wales,Food Standards Agency,-3.18,51.48,No,CP,Industrial Road,Cardiff,CF1\nA-2,Withheld,England,Food Standards Agency,-0.12,51.50,Yes,SH,Private Road,London,SW1\nA-2,Duplicate,England,Food Standards Agency,-0.11,51.51,No,CP,Second Road,London,SW2\nA-3,Out of scope,Jersey,Food Standards Agency,-2.1,49.2,No,CP,Island Road,St Helier,JE1\n" + with tempfile.TemporaryDirectory() as directory: + source = Path(directory) / "monthly.csv" + raw = monthly.encode("cp1252") + source.write_bytes(raw) + artifact = {"source_url": "https://example.invalid/monthly.csv", "retrieved_at_utc": "2026-09-14T00:00:00Z", "checksum_sha256": hashlib.sha256(raw).hexdigest(), "byte_size": len(raw), "effective_date": "2026-09-01"} + result = self.adapter.parse_bytes(raw) + self.assertEqual(result.profile, "monthly") + self.assertEqual(len(result.accepted), 1) + self.assertEqual(len(result.quarantined), 3) + withheld = next(item for item in result.quarantined if item["record"]["normalized"]["privacy_gate"] == "restricted-withheld-address") + self.assertIsNone(withheld["record"]["normalized"]["coordinates"]) + manifest = self.adapter.run(source, Path(directory) / "run", artifact) + self.assertEqual(manifest["release_state"], "not-created") + self.assertEqual(manifest["coverage_counts"], {"England": 2, "Jersey": 1, "Wales": 1}) + if __name__ == "__main__": unittest.main() From 9a76ddfadaf18c008712a27ca8d6f1b4150b23ae Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sun, 13 Sep 2026 22:35:23 -0700 Subject: [PATCH 053/311] Record private UK and Denmark pipeline evidence --- docs/source-status.json | 4 ++-- docs/source-status.md | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/source-status.json b/docs/source-status.json index cb16470..fe2045b 100644 --- a/docs/source-status.json +++ b/docs/source-status.json @@ -12,8 +12,8 @@ {"source_id":"it.locations","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-it.md","pipeline/source_registry.json"],"next_action":"Use catalog-linked Ministry downloads; record provenance and validate 853/2004 separately from 1069/2009."}, {"source_id":"mx.locations","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-mx.md","pipeline/source_registry.json"],"next_action":"Resolve DENUE token/terms and SENASICA current directory/schema before bounded acquisition."}, {"source_id":"nz.locations","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-nz.md","pipeline/source_registry.json"],"next_action":"Confirm MPI permitted acquisition route, list-specific terms, and schema before private staging."}, - {"source_id":"uk.locations","metadata":"partial","acquisition":"artifact_private_only","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-uk.md","pipeline/source_registry.json"],"next_action":"Keep national feeds separate; review withheld addresses, duplicates, coverage, terms, and source-specific adapters."}, - {"source_id":"dk.smiley","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["pipeline/README.md","pipeline/source_registry.json"],"next_action":"Verify current endpoint, publication date semantics, licence, coverage, and release approval."}, + {"source_id":"uk.locations","metadata":"partial","acquisition":"artifact_private_only","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-uk.md","docs/countries/uk/fss-approved-establishments-source-assessment.md","pipeline/sources/uk/fsa_approved/README.md","pipeline/source_registry.json"],"next_action":"Keep the UK ID composite: validate the private England/Wales monthly candidate while keeping NI and Scotland separate; review withheld addresses, duplicates, coverage, terms, and release approval."}, + {"source_id":"dk.smiley","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["pipeline/README.md","pipeline/contracts/README.md","pipeline/source_registry.json"],"next_action":"Use the SourceArtifact/registered-adapter contract for a permitted official acquisition, then verify current endpoint, publication date semantics, licence, coverage, and release approval."}, {"source_id":"de.locations","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["pipeline/source_registry.json"],"next_action":"Replace session-bound legacy URL with a verified stable BVL endpoint and confirm terms/schema."}, {"source_id":"ca.locations","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ca.md","pipeline/source_registry.json"],"next_action":"Keep federal/export and Ontario candidates separate; verify an authoritative permitted artifact, terms, schema, coverage, and privacy before acquisition."}, {"source_id":"es.locations","metadata":"unknown","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["pipeline/source_registry.json"],"next_action":"Identify the current competent-authority publication and distinguish original source from legacy transformations."}, diff --git a/docs/source-status.md b/docs/source-status.md index 84b59dc..5852cc5 100644 --- a/docs/source-status.md +++ b/docs/source-status.md @@ -19,8 +19,8 @@ No last-success timestamp is invented. Private artifacts are not proof of a publ | `it.locations` | verified | blocked | not_run | blocked | Ministry 853/2004 and 1069/2009 catalogs identified; use catalog downloads, not challenged servlet | | `mx.locations` | verified | blocked | not_run | blocked | DENUE/SENASICA/DGSIAP reconnaissance; resolve token, directory, terms, and schema | | `nz.locations` | verified | blocked | not_run | blocked | MPI/Stats NZ reconnaissance; resolve 403/access and aggregate-vs-facility boundaries | -| `uk.locations` | partial | artifact_private_only | not_run | blocked | National-feed reconciliation and withheld-address/duplicate/coverage review remain open | -| `dk.smiley` | partial | not_run | not_run | blocked | Existing partial pipeline; current endpoint/licence/coverage/release verification remains open | +| `uk.locations` | partial | artifact_private_only | not_run | blocked | Private England/Wales monthly candidate is validated; NI and Scotland remain separate, with withheld-address/duplicate/coverage/terms/release review open | +| `dk.smiley` | partial | not_run | not_run | blocked | SourceArtifact and registered-adapter contracts are documented; official live acquisition, endpoint/licence/coverage/release verification remain open | | `de.locations` | partial | not_run | not_run | blocked | Legacy session URL and current BVL endpoint/schema/terms remain unresolved | | `ca.locations` | verified | not_run | not_run | blocked | Federal/export and Ontario candidates are documented separately; verify permitted artifact, terms, schema, coverage, and privacy before acquisition | | `es.locations` | unknown | not_run | not_run | blocked | Current competent-authority source and legacy/source boundary unresolved | From c864d4ad290f616db4d63e91590ec47ca4edd5e7 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sun, 13 Sep 2026 22:46:18 -0700 Subject: [PATCH 054/311] Document Spain source reconnaissance and access blocker --- docs/country-recon-es.md | 27 +++++++++++++++++++++++++++ docs/source-status.json | 2 +- docs/source-status.md | 2 +- 3 files changed, 29 insertions(+), 2 deletions(-) create mode 100644 docs/country-recon-es.md diff --git a/docs/country-recon-es.md b/docs/country-recon-es.md new file mode 100644 index 0000000..5904a1d --- /dev/null +++ b/docs/country-recon-es.md @@ -0,0 +1,27 @@ +# Spain source reconnaissance + +Status: reconnaissance only; no adapter, release, publication, or row-level fixture. Current V2 source baseline marks `es.locations` as partial/blocked until a permitted current artifact and rights review exist. + +## Assessment + +The strongest competent-authority route identified is AESAN's Registro General Sanitario de Empresas Alimentarias y Alimentos (RGSEAA), especially its subset of Spanish establishments authorised to produce and market products of animal origin under EU rules. Official entry points are [RGSEAA](https://aesan.gob.es/registro-sanitario/empresas-alimentarias), [Spanish establishments authorised in the EU](https://www.aesan.gob.es/registro-sanitario/empresas-espa-olas-ue), and the linked [EU authorised-establishments list](https://food.ec.europa.eu/food-safety/food-hygiene/establishments_en). + +AESAN describes RGSEAA as an administrative register covering food operators and establishments in production, transformation, preparation, packing, storage, distribution, transport, and import. It is broader than animal agriculture and includes retail, restaurants, and other sectors; a later adapter must filter by authorised activity/category and keep the source identity separate from any geocoding or legacy transformation. The RGSEAA number is an administrative identifier, not proof of current operational status or public-release permission. + +MAPA also exposes sector-specific public searches, including [SILUM animal-feed establishments](https://servicio.mapa.gob.es/es/ganaderia/temas/alimentacion-animal/acceso-publico/registro_general_establecimientos) and [SANDACH establishments](https://servicio.mapa.gob.es/sandachcorebuspub/). These are complements, not substitutes for a food-establishment master: their sector boundaries, current export/schema, and terms require separate verification. + +## Access, terms, and privacy + +The official pages are publicly reachable in web search, but a bounded direct fetch from this environment was refused by the target host; no artifact was retained and no access control was bypassed. No file-specific licence or redistribution grant was established. Treat official origin as provenance only. Establishment names, postal addresses, operator identifiers, and possible sole-trader/farm information require minimisation, privacy screening, and human publication approval. Do not infer facility completeness from RGSEAA, SILUM, SANDACH, or the EU list. + +## Acquisition recipe (private, repeatable) + +1. From an approved runner, retrieve only the official AESAN page and its explicitly linked current authorised-establishments export or query route; record final URL, UTC retrieval, HTTP status, content type, byte count, SHA-256, supplied publication/effective date, and terms text. +2. Store the raw response only in access-controlled ignored private storage. Never place row-level values in Git, logs, screenshots, fixtures, or handoff messages. +3. Fingerprint format/encoding, headers, delimiter or JSON schema, identifier fields, activity/category vocabulary, status/date fields, and geographic scope. Quarantine if the route is interactive/session-bound or schema changes. +4. Validate animal-facility coverage separately from feed, SANDACH, retail, restaurant, and general food sectors. Keep any legacy CSV as comparison-only until source identity and transformation history are proven. +5. Apply privacy/terms review, coarse-location policy, suppression controls, and maintainer publication approval before any adapter or release work. + +## Recommendation + +HOLD. Spain has a credible official discovery route, but current acquisition, export/schema fingerprint, file-specific rights, effective-date semantics, coverage boundaries, and privacy handling are unresolved. A later adapter can proceed only after one permitted current artifact is privately staged and reviewed; no pipeline implementation is authorized by this reconnaissance. diff --git a/docs/source-status.json b/docs/source-status.json index fe2045b..52bc49d 100644 --- a/docs/source-status.json +++ b/docs/source-status.json @@ -16,7 +16,7 @@ {"source_id":"dk.smiley","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["pipeline/README.md","pipeline/contracts/README.md","pipeline/source_registry.json"],"next_action":"Use the SourceArtifact/registered-adapter contract for a permitted official acquisition, then verify current endpoint, publication date semantics, licence, coverage, and release approval."}, {"source_id":"de.locations","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["pipeline/source_registry.json"],"next_action":"Replace session-bound legacy URL with a verified stable BVL endpoint and confirm terms/schema."}, {"source_id":"ca.locations","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ca.md","pipeline/source_registry.json"],"next_action":"Keep federal/export and Ontario candidates separate; verify an authoritative permitted artifact, terms, schema, coverage, and privacy before acquisition."}, - {"source_id":"es.locations","metadata":"unknown","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["pipeline/source_registry.json"],"next_action":"Identify the current competent-authority publication and distinguish original source from legacy transformations."}, + {"source_id":"es.locations","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-es.md","pipeline/source_registry.json"],"next_action":"Resolve a permitted current AESAN/MAPA artifact or query route, then verify export/schema, rights, effective-date semantics, sector coverage, privacy, and legacy/source boundaries before acquisition."}, {"source_id":"us.fsis","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-us.md","pipeline/source_registry.json"],"next_action":"Acquire and snapshot a permitted current FSIS MPI artifact; reconcile legacy inventory only as a comparison and verify terms, schema, coverage, and privacy."}, {"source_id":"us.aphis","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-us.md","pipeline/source_registry.json"],"next_action":"Verify the current APHIS export workflow and keep licences, registrants, annual reports, and exception reports separately attributed before acquisition."}, {"source_id":"us.inspections","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-us.md","pipeline/source_registry.json"],"next_action":"Verify a current inspection export and use explicit, reviewable identity matching rather than treating observations as a facility master."} diff --git a/docs/source-status.md b/docs/source-status.md index 5852cc5..d501174 100644 --- a/docs/source-status.md +++ b/docs/source-status.md @@ -23,7 +23,7 @@ No last-success timestamp is invented. Private artifacts are not proof of a publ | `dk.smiley` | partial | not_run | not_run | blocked | SourceArtifact and registered-adapter contracts are documented; official live acquisition, endpoint/licence/coverage/release verification remain open | | `de.locations` | partial | not_run | not_run | blocked | Legacy session URL and current BVL endpoint/schema/terms remain unresolved | | `ca.locations` | verified | not_run | not_run | blocked | Federal/export and Ontario candidates are documented separately; verify permitted artifact, terms, schema, coverage, and privacy before acquisition | -| `es.locations` | unknown | not_run | not_run | blocked | Current competent-authority source and legacy/source boundary unresolved | +| `es.locations` | partial | blocked | not_run | blocked | AESAN RGSEAA and MAPA sector routes are documented, but direct acquisition was refused; rights, export/schema, effective dates, sector coverage, privacy, and legacy/source boundaries remain unresolved | | `us.fsis` | verified | not_run | not_run | blocked | FSIS MPI route is documented; acquire a permitted current artifact and treat legacy inventory only as a comparison | | `us.aphis` | partial | not_run | not_run | blocked | APHIS export workflow and separate report/license provenance require review | | `us.inspections` | partial | not_run | not_run | blocked | Inspection observations require a current export and explicit identity matching | From 5ed88f44b566c81f44065b1e4cc8e43f7bd3c4f2 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sun, 13 Sep 2026 22:47:38 -0700 Subject: [PATCH 055/311] Gate private candidate preview to local development --- docs/api/v2-contract.json | 1 + docs/deployment/dev-preview.md | 27 ++++ pipeline/tests/e2e/fixture.py | 4 +- pipeline/tests/e2e/test_public_api.py | 19 +++ src/lib.rs | 195 ++++++++++++++++++++++++++ src/main.rs | 154 +++++++++++++++++++- 6 files changed, 395 insertions(+), 5 deletions(-) create mode 100644 docs/deployment/dev-preview.md diff --git a/docs/api/v2-contract.json b/docs/api/v2-contract.json index 7c98ef3..c7212fe 100644 --- a/docs/api/v2-contract.json +++ b/docs/api/v2-contract.json @@ -8,6 +8,7 @@ "GET /api/v2/locations/{facility_id}": {"success": {"api_version": "v2", "data": {}, "meta": {"coverage_scope": "selected_promoted_release_public_facilities", "count_semantics": "public facility projection, not an animal count"}}, "not_found": 404}, "GET /api/v2/locations.csv": {"success_content_type": "text/csv", "bounded_rows": 1000, "profile_required_for_nonofficial": true, "not_found_without_promoted_manifest": 404}, "GET /api/v2/discovery/filters": {"success": {"api_version": "v2", "contract_version": "v1", "dimensions": {"country_code": "allowlist", "category": "allowlist", "source_type": "allowlist", "profile": "allowlist", "display_precision": "allowlist", "lifecycle_status": "allowlist"}}} + ,"GET /api/dev/preview/candidates": {"success": {"api_version": "dev-preview-v1", "data": [], "meta": {"test_only": true, "private_preview": true, "profile": null, "coverage_scope": "candidate_release_only", "next_cursor": null}}, "auth_header": "X-UEC-Dev-Preview-Token", "production": "unavailable"} ,"GET /api/v2/discovery/facets": {"success": {"api_version": "v2", "meta": {"profile": "official", "release_id": "string", "ruleset_version": "string", "release_created_at": "timestamp", "coverage_scope": "selected_promoted_release_public_facilities", "count_semantics": "eligible public facility projection rows after current suppression; not story-wide or animal counts", "filters": {}}, "dimensions": {}}, "max_values_per_dimension": 20, "counts_are_from": "selected eligible promoted public projection"} }, "error": {"shape": {"api_version": "v2", "error": {"code": "string", "message": "string"}}, "codes": ["invalid_filter", "invalid_profile", "invalid_limit", "invalid_offset", "invalid_cursor", "invalid_pagination", "database_not_configured", "database_pool_unavailable", "database_transaction_unavailable", "release_query_failed", "location_query_failed", "location_not_found", "rate_limited"]}, diff --git a/docs/deployment/dev-preview.md b/docs/deployment/dev-preview.md new file mode 100644 index 0000000..f4a3c06 --- /dev/null +++ b/docs/deployment/dev-preview.md @@ -0,0 +1,27 @@ +# Private candidate preview + +The candidate preview is a development-only operator surface at +`GET /api/dev/preview/candidates`. It is not a V2 profile and never reads the +public promoted-release routes. + +The server starts the preview only when all of these are true: + +- `UEC_RUNTIME_MODE=development`; +- `UEC_DEV_PREVIEW=true`; +- `UEC_BIND_HOST` is exactly `127.0.0.1` or `::1`; +- `UEC_DEV_PREVIEW_TOKEN` is supplied as a non-empty process secret; and +- configured CORS origins are loopback-only. + +Every request must send the token in the `X-UEC-Dev-Preview-Token` header. +Never place it in a URL, source file, browser bundle, or log. Production mode, +non-loopback binding, missing authentication, remote CORS, and ambiguous +configuration fail closed. A local dev proxy may hold the token server-side; +browser clients may hold it only in memory for the current session. + +Preview responses contain only privacy-screened, non-withheld candidate fields +and source/retrieval metadata. They omit raw payloads, addresses, and geocoder +queries. Every row is labeled `Private development candidate — not +project-approved or published`; `project_approval` remains `false` and the +release remains `candidate`. Candidate seed/reset/rebuild belongs to a +disposable E2E database only. This path is not production deployment or +publication authorization. diff --git a/pipeline/tests/e2e/fixture.py b/pipeline/tests/e2e/fixture.py index c66595f..a95c575 100644 --- a/pipeline/tests/e2e/fixture.py +++ b/pipeline/tests/e2e/fixture.py @@ -28,6 +28,8 @@ def __init__(self): self.backend = None self.backend_log = None self.start_attempts = 0 + # Synthetic only: this token is scoped to the disposable test server. + self.dev_preview_token = "uec-e2e-preview-token" def command(self, *args): return ["docker", "compose", "-p", self.project, "-f", str(COMPOSE), *args] @@ -73,7 +75,7 @@ def start(self): raise print("[e2e] building backend", flush=True) subprocess.run(["cargo", "build", "--quiet"], cwd=ROOT, check=True, timeout=180) - env = os.environ.copy(); env.update({"UEC_DATABASE_URL": self.database_url, "PORT": str(self.api_port)}) + env = os.environ.copy(); env.update({"UEC_DATABASE_URL": self.database_url, "PORT": str(self.api_port), "UEC_RUNTIME_MODE": "development", "UEC_BIND_HOST": "127.0.0.1", "UEC_DEV_PREVIEW": "true", "UEC_DEV_PREVIEW_TOKEN": self.dev_preview_token}) binary = ROOT / "target/debug/uec-api.exe" if not binary.exists(): binary = ROOT / "target/debug/uec-api" diff --git a/pipeline/tests/e2e/test_public_api.py b/pipeline/tests/e2e/test_public_api.py index 6fe6d27..bb3a620 100644 --- a/pipeline/tests/e2e/test_public_api.py +++ b/pipeline/tests/e2e/test_public_api.py @@ -48,6 +48,25 @@ def test_candidate_only_record_is_absent_from_every_public_surface(self): self.get(f"/api/v2/locations/{candidate}?profile=official") self.assertEqual(error.exception.code, 404) + def test_private_candidate_preview_requires_auth_and_is_explicitly_labeled(self): + endpoint = f"http://localhost:{self.env.api_port}/api/dev/preview/candidates?limit=10" + missing = urllib.request.Request(endpoint) + with self.assertRaises(urllib.error.HTTPError) as error: + urllib.request.urlopen(missing, timeout=10) + self.assertEqual(error.exception.code, 401) + wrong = urllib.request.Request(endpoint, headers={"X-UEC-Dev-Preview-Token": "wrong"}) + with self.assertRaises(urllib.error.HTTPError) as error: + urllib.request.urlopen(wrong, timeout=10) + self.assertEqual(error.exception.code, 401) + request = urllib.request.Request(endpoint, headers={"X-UEC-Dev-Preview-Token": self.env.dev_preview_token}) + with urllib.request.urlopen(request, timeout=10) as response: + body = json.loads(response.read()) + self.assertEqual(body["api_version"], "dev-preview-v1") + self.assertEqual(body["meta"]["test_only"], True) + self.assertEqual(body["data"][0]["release_status"], "candidate") + self.assertEqual(body["data"][0]["project_approval"], False) + self.assertIn("not project-approved", body["data"][0]["preview_label"]) + def test_filters_do_not_bypass_publication_gate(self): for path in ("?category=retail_and_prepared_food", "?display_precision=city", "?lifecycle_status=explicitly_closed"): _, body = self.get("/api/v2/locations" + path) diff --git a/src/lib.rs b/src/lib.rs index 003b9a8..eda933b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -16,6 +16,7 @@ // Contact the developer directly at untileverycageproject@protonmail.com use axum::extract::{Path, Query, State}; +use axum::http::HeaderMap; use axum::{ Json, http::{Response, StatusCode}, @@ -289,6 +290,185 @@ pub async fn get_v2_locations_export_handler( #[derive(Clone)] pub struct ApiState { pub database: Option, + pub dev_preview_token: Option, +} + +const DEV_PREVIEW_TOKEN_HEADER: &str = "x-uec-dev-preview-token"; + +fn constant_time_token_matches(expected: &str, provided: &str) -> bool { + let mut difference = expected.len() ^ provided.len(); + for (left, right) in expected.bytes().zip(provided.bytes()) { + difference |= usize::from(left ^ right); + } + difference == 0 +} + +fn is_loopback_host(value: &str) -> bool { + let Ok(uri) = format!("http://{value}").parse::() else { + return false; + }; + let Some(host) = uri.host() else { return false }; + host == "localhost" + || host + .parse::() + .is_ok_and(|ip| ip.is_loopback()) +} + +fn preview_request_is_local(headers: &HeaderMap) -> bool { + let Some(host) = headers + .get(axum::http::header::HOST) + .and_then(|value| value.to_str().ok()) + else { + return false; + }; + if !is_loopback_host(host) { + return false; + } + headers + .get(axum::http::header::ORIGIN) + .map(|origin| { + origin + .to_str() + .ok() + .and_then(|value| value.parse::().ok()) + .and_then(|uri| uri.host().map(str::to_owned)) + .is_some_and(|host| is_loopback_host(&host)) + }) + .unwrap_or(true) +} + +#[derive(Deserialize)] +pub struct DevPreviewParams { + pub limit: Option, +} + +/// Candidate preview is deliberately separate from `/api/v2`: it is a local +/// operator tool, not a release/profile or project-approval mechanism. +pub async fn get_dev_candidate_preview_handler( + State(state): State, + headers: HeaderMap, + Query(params): Query, +) -> impl IntoResponse { + let Some(expected_token) = state.dev_preview_token.as_deref() else { + return v2_error( + StatusCode::NOT_FOUND, + "dev_preview_unavailable", + "candidate preview unavailable", + ); + }; + if !preview_request_is_local(&headers) { + return v2_error( + StatusCode::FORBIDDEN, + "dev_preview_origin_rejected", + "candidate preview requires loopback host and origin", + ); + } + let Some(provided_token) = headers + .get(DEV_PREVIEW_TOKEN_HEADER) + .and_then(|value| value.to_str().ok()) + else { + return v2_error( + StatusCode::UNAUTHORIZED, + "dev_preview_auth_required", + "candidate preview requires operator authentication", + ); + }; + if !constant_time_token_matches(expected_token, provided_token) { + return v2_error( + StatusCode::UNAUTHORIZED, + "dev_preview_auth_failed", + "candidate preview authentication failed", + ); + } + let limit = match params.limit.as_deref().unwrap_or("100").parse::() { + Ok(value) if (1..=1000).contains(&value) => value, + _ => { + return v2_error( + StatusCode::BAD_REQUEST, + "invalid_limit", + "limit must be between 1 and 1000", + ); + } + }; + let Some(pool) = state.database else { + return v2_error( + StatusCode::SERVICE_UNAVAILABLE, + "database_not_configured", + "candidate preview database is not configured", + ); + }; + let client = match pool.get().await { + Ok(client) => client, + Err(_) => { + return v2_error( + StatusCode::SERVICE_UNAVAILABLE, + "database_pool_unavailable", + "candidate preview database unavailable", + ); + } + }; + let rows = match client.query(r#" + SELECT r.release_id, r.status, o.source_record_id, o.facility_id, + f.canonical_name, f.country_code, f.city, o.classification_category, + ST_Y(g.result::geometry), ST_X(g.result::geometry), + source.origin_type, source.source_id, source.name, source.official_url, + artifact.retrieved_at, review.factual_review_status, + review.privacy_screening_status, review.maintainer_approval + FROM uec.release_members member + JOIN uec.releases r ON r.release_id = member.release_id + JOIN uec.observations o ON o.observation_id = member.observation_id + JOIN uec.facilities f ON f.facility_id = member.facility_id + JOIN uec.source_records record ON record.source_record_id = o.source_record_id + JOIN uec.sources source ON source.source_id = record.source_id + JOIN uec.raw_artifacts artifact ON artifact.artifact_id = record.artifact_id + JOIN uec.publication_review_release_current review + ON review.source_record_id = o.source_record_id AND review.release_id = member.release_id + JOIN LATERAL ( + SELECT result FROM uec.geocode_results + WHERE source_record_id = o.source_record_id AND status = 'accepted' AND result IS NOT NULL + ORDER BY queried_at DESC, geocode_result_id DESC LIMIT 1 + ) g ON true + WHERE r.status = 'candidate' + AND member.default_visible = true + AND record.source_state NOT IN ('rejected', 'superseded') + AND review.privacy_screening_status = 'passed' + AND review.factual_review_status <> 'rejected' + AND NOT EXISTS (SELECT 1 FROM uec.public_access_restricted restricted WHERE restricted.source_record_id = o.source_record_id) + ORDER BY r.release_id, o.facility_id + LIMIT $1 + "#, &[&limit]).await { + Ok(rows) => rows, + Err(_) => return v2_error(StatusCode::SERVICE_UNAVAILABLE, "dev_preview_query_failed", "candidate preview unavailable"), + }; + let data = rows + .into_iter() + .map(|row| { + json!({ + "candidate_id": row.get::<_, uuid::Uuid>(2), + "source_record_id": row.get::<_, uuid::Uuid>(2), + "facility_id": row.get::<_, uuid::Uuid>(3), + "canonical_name": row.get::<_, Option>(4), + "country_code": row.get::<_, String>(5), + "city": row.get::<_, Option>(6), + "category": row.get::<_, String>(7), + "display_precision": "exact", + "latitude": row.get::<_, Option>(8), + "longitude": row.get::<_, Option>(9), + "source_type": row.get::<_, String>(10), + "provenance_source_id": row.get::<_, String>(11), + "provenance_source_name": row.get::<_, String>(12), + "provenance_source_url": row.get::<_, String>(13), + "provenance_retrieved_at": row.get::<_, chrono::DateTime>(14), + "factual_review_status": row.get::<_, String>(15), + "privacy_screening_status": row.get::<_, String>(16), + "project_approval": false, + "release_id": row.get::<_, String>(0), + "release_status": row.get::<_, String>(1), + "preview_label": "Private development candidate — not project-approved or published" + }) + }) + .collect::>(); + Json(json!({"api_version":"dev-preview-v1", "data":data, "meta":{"test_only":true,"private_preview":true,"profile":null,"coverage_scope":"candidate_release_only","next_cursor":null}})).into_response() } mod location; @@ -914,6 +1094,7 @@ mod v2_api_tests { .create_pool(Some(deadpool_postgres::Runtime::Tokio1), NoTls) .unwrap(), ), + dev_preview_token: None, } } @@ -932,6 +1113,20 @@ mod v2_api_tests { .as_str().unwrap().contains("not story-wide")); } + #[test] + fn candidate_preview_rejects_non_loopback_host_and_origin() { + let mut local = HeaderMap::new(); + local.insert("host", "127.0.0.1:8000".parse().unwrap()); + assert!(preview_request_is_local(&local)); + local.insert("origin", "https://localhost:3000".parse().unwrap()); + assert!(preview_request_is_local(&local)); + local.insert("origin", "https://attacker.example".parse().unwrap()); + assert!(!preview_request_is_local(&local)); + local.insert("host", "preview.example:8000".parse().unwrap()); + local.remove("origin"); + assert!(!preview_request_is_local(&local)); + } + #[tokio::test] async fn filter_metadata_is_versioned_and_allowlisted() { let response = get_v2_filter_metadata_handler().await.into_response(); diff --git a/src/main.rs b/src/main.rs index 6b0c836..a014d61 100644 --- a/src/main.rs +++ b/src/main.rs @@ -59,6 +59,10 @@ pub fn app(state: uec_api::ApiState) -> Router { "/api/v2/locations/{facility_id}", get(uec_api::get_v2_location_detail_handler), ) + .route( + "/api/dev/preview/candidates", + get(uec_api::get_dev_candidate_preview_handler), + ) .route( "/api/aphis-reports", get(uec_api::get_aphis_reports_handler), @@ -131,7 +135,66 @@ fn cors_layer() -> Result { Ok(CorsLayer::new() .allow_origin(AllowOrigin::list(origins)) .allow_methods([Method::GET, Method::OPTIONS]) - .allow_headers([axum::http::header::CONTENT_TYPE, axum::http::header::ACCEPT])) + .allow_headers([ + axum::http::header::CONTENT_TYPE, + axum::http::header::ACCEPT, + axum::http::HeaderName::from_static("x-uec-dev-preview-token"), + ])) +} + +fn preview_config( + mode: &str, + opt_in: Option<&str>, + bind_host: &str, + token: Option<&str>, + cors_origins: Option<&str>, + legacy_cors_origin: Option<&str>, +) -> Result, &'static str> { + let enabled = match opt_in.unwrap_or("false") { + "true" => true, + "false" | "" => false, + _ => return Err("UEC_DEV_PREVIEW must be true or false"), + }; + let host = bind_host + .parse::() + .map_err(|_| "UEC_BIND_HOST must be a valid IP address")?; + let token = token.filter(|value| !value.is_empty()); + if mode == "production" && (enabled || token.is_some()) { + return Err("development candidate preview is unavailable in production"); + } + if !enabled { + if token.is_some() { + return Err("UEC_DEV_PREVIEW_TOKEN requires UEC_DEV_PREVIEW=true"); + } + return Ok(None); + } + if mode != "development" { + return Err("candidate preview requires development runtime mode"); + } + if !host.is_loopback() { + return Err("candidate preview requires an exact loopback bind host"); + } + let token = token.ok_or("UEC_DEV_PREVIEW_TOKEN is required when preview is enabled")?; + let cors = cors_origins + .or(legacy_cors_origin) + .unwrap_or("http://localhost:3000"); + for origin in cors + .split(',') + .map(str::trim) + .filter(|value| !value.is_empty()) + { + let uri = origin + .parse::() + .map_err(|_| "candidate preview CORS origin is invalid")?; + let origin_host = uri + .host() + .ok_or("candidate preview CORS origin must have a host")?; + let origin_ip = origin_host.parse::().ok(); + if !matches!(origin_host, "localhost") && !origin_ip.is_some_and(|ip| ip.is_loopback()) { + return Err("candidate preview CORS origins must be loopback only"); + } + } + Ok(Some(token.to_owned())) } const RATE_WINDOW: Duration = Duration::from_secs(60); @@ -253,6 +316,13 @@ async fn main() { let mode = std::env::var("UEC_RUNTIME_MODE").unwrap_or_else(|_| "development".to_string()); let database_url = std::env::var("UEC_DATABASE_URL").ok(); let port = std::env::var("PORT").unwrap_or_else(|_| "8000".to_string()); + let bind_host = std::env::var("UEC_BIND_HOST").unwrap_or_else(|_| { + if mode == "development" { + "127.0.0.1".into() + } else { + "0.0.0.0".into() + } + }); let port = validate_runtime(&mode, database_url.as_deref(), &port).unwrap_or_else(|error| { eprintln!( "{{\"event\":\"configuration_error\",\"reason\":\"{}\"}}", @@ -271,6 +341,21 @@ async fn main() { ); std::process::exit(2); } + let dev_preview_token = preview_config( + mode.as_str(), + std::env::var("UEC_DEV_PREVIEW").ok().as_deref(), + &bind_host, + std::env::var("UEC_DEV_PREVIEW_TOKEN").ok().as_deref(), + std::env::var("UEC_CORS_ORIGINS").ok().as_deref(), + std::env::var("UEC_CORS_ORIGIN").ok().as_deref(), + ) + .unwrap_or_else(|error| { + eprintln!( + "{{\"event\":\"configuration_error\",\"reason\":\"{}\"}}", + error + ); + std::process::exit(2) + }); let database = database_url.and_then(|url| { let mut config = Config::new(); config.url = Some(url); @@ -303,7 +388,7 @@ async fn main() { } } }); - let addr = format!("0.0.0.0:{}", port); + let addr = format!("{}:{}", bind_host, port); println!( "{{\"event\":\"server_starting\",\"service\":\"uec-api\",\"mode\":\"{}\",\"port\":{}}}", mode, port @@ -312,7 +397,11 @@ async fn main() { let listener = tokio::net::TcpListener::bind(addr).await.unwrap(); axum::serve( listener, - app(uec_api::ApiState { database }).into_make_service_with_connect_info::(), + app(uec_api::ApiState { + database, + dev_preview_token, + }) + .into_make_service_with_connect_info::(), ) .await .unwrap(); @@ -320,7 +409,7 @@ async fn main() { #[cfg(test)] mod config_tests { - use super::{parse_cors_origins, validate_runtime}; + use super::{parse_cors_origins, preview_config, validate_runtime}; #[test] fn development_allows_local_defaults() { assert_eq!(validate_runtime("development", None, "8000"), Ok(8000)); @@ -358,6 +447,63 @@ mod config_tests { 1 ); } + + #[test] + fn candidate_preview_requires_loopback_opt_in_and_token() { + assert!( + preview_config("development", Some("true"), "127.0.0.1", None, None, None).is_err() + ); + assert!( + preview_config( + "development", + Some("true"), + "0.0.0.0", + Some("secret"), + None, + None + ) + .is_err() + ); + assert!( + preview_config( + "production", + Some("true"), + "127.0.0.1", + Some("secret"), + None, + None + ) + .is_err() + ); + assert!( + preview_config( + "development", + Some("true"), + "127.0.0.1", + Some("secret"), + Some("https://remote.example"), + None + ) + .is_err() + ); + assert_eq!( + preview_config( + "development", + Some("true"), + "127.0.0.1", + Some("secret"), + None, + None + ) + .unwrap() + .as_deref(), + Some("secret") + ); + assert_eq!( + preview_config("development", None, "127.0.0.1", None, None, None).unwrap(), + None + ); + } } #[cfg(test)] From 0467f8ac642aac01673cfd8b7af33be05e722f21 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sun, 13 Sep 2026 22:50:39 -0700 Subject: [PATCH 056/311] Record private Italy source acquisitions --- docs/country-recon-it.md | 14 ++++++++------ docs/source-status.json | 2 +- docs/source-status.md | 2 +- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/docs/country-recon-it.md b/docs/country-recon-it.md index 2ede651..6792385 100644 --- a/docs/country-recon-it.md +++ b/docs/country-recon-it.md @@ -1,6 +1,6 @@ # Italy source reconnaissance (V2) -Status: reconnaissance only. No adapter, release, publication, row-level fixture, or downloaded artifact was created. No row-level real data, personal names, addresses, contacts, coordinates, or private artifacts are retained here. +Status: reconnaissance only. Two current catalog artifacts were acquired privately for schema/provenance inspection; no adapter, release, publication, or row-level fixture was created. No row-level values are retained in this document. Checked 2026-09-13. This document is source-status evidence, not publication approval or a healthy-pipeline claim. @@ -17,7 +17,9 @@ The strongest candidate is the Italian Ministry of Health open-data catalog rath - 853 schema dictionary: - 1069 schema dictionary: -The catalog reported 853 data last updated 2026-09-12 and daily frequency; the 1069 catalog reported last updated 2026-09-11 and daily frequency. Category pages may have independent amendment dates. The catalog identifies the Ministry of Health/DGSAN Office 2 and Italian Open Data Licence v2.0. It warns that some coordinates came from OpenStreetMap contributors; this is source metadata, not permission to publish precise points. +The catalog reported 853 data last updated 2026-09-13 and daily frequency; the 1069 catalog reported last updated 2026-09-11 and daily frequency. Private retrieval was 2026-09-14T05:44:54.7132759Z UTC. The 853 artifact is 49,927,230 bytes (SHA-256 `af1ec6eb7b530fef8dd420cdd08215b36d1b29cb202cf355b87a95fc938d6fea`) with 47,369 data rows; the separate 1069 artifact is 7,708,707 bytes (SHA-256 `4071c10f00f59070f75435988c7b22858bbd24abdc456613d448ff873bd9a2ce`) with 9,935 data rows. Raw files and sanitized metadata remain under ignored `data/raw/italy/` and are not release inputs. + +The catalog identifies the Ministry of Health/DGSAN Office 2 and Italian Open Data Licence v2.0. It warns that some coordinates came from OpenStreetMap contributors; this is source metadata, not permission to publish precise points. ## Meaning and schema @@ -27,7 +29,7 @@ Keep the datasets separate and preserve original source values. Treat activity/s ## Acquisition and access -No real-row artifact was acquired or retained. Catalog-linked download routes were documented, but direct retrieval was refused by the environment and the Ministry interface displayed a JS/cookie challenge. This is an acquisition blocker, not evidence that downloads are unavailable. No official API, bulk endpoint, rate limit, or authentication contract was verified; servlet query parameters must not be treated as an API. Prefer catalog-linked downloads or an authorized export, recording HTTP metadata, UTC retrieval, SHA-256, byte size, update date, and adapter/config version. Keep raw artifacts outside Git. +Catalog-linked downloads succeeded through an authorized direct HTTPS route; the session-bound servlet still displayed a JS/cookie challenge and was not scraped. No official API, bulk endpoint, rate limit, or authentication contract was verified; servlet query parameters must not be treated as an API. Raw artifacts remain outside Git. The recorded artifacts establish acquisition integrity only, not permission to publish rows. ## Historical boundary @@ -37,8 +39,8 @@ The repository’s historical Italy CSV and scraper are legacy/unverified inputs | Source | Discovered | Acquisition | Adapter / validation | Terms / privacy / publication | Blocker / next action | |---|---|---|---|---|---| -| Ministry 853/2004 food establishments | Official catalog and regulatory sections verified | Not acquired; direct fetch refused/challenged | Feasible via catalog formats; not implemented/tested | Italian Open Data Licence v2.0; coordinate provenance partly OSM; ETHICS privacy/approval gates apply | Authorized maintainer acquires bounded catalog sample with provenance/hash/size, then synthetic adapter test | -| Ministry 1069/2009 by-products | Separate official catalog/dictionary verified | Not acquired | Separate schema/scope; not implemented | Same licence and privacy/approval gates | Decide whether scope belongs in project, then acquire/validate separately | +| Ministry 853/2004 food establishments | Official catalog and regulatory sections verified | Private current CSV acquired; provenance recorded | Adapter not implemented; mapping requires dictionary review | Italian Open Data Licence v2.0; coordinate provenance partly OSM; ETHICS privacy/approval gates apply | Review dictionary and implement synthetic-only adapter | +| Ministry 1069/2009 by-products | Separate official catalog/dictionary verified | Private current CSV acquired; provenance recorded | Kept separate; no adapter | Same licence and privacy/approval gates | Decide whether scope belongs in project, then validate separately | | Servlet HTML interface | Official interface identified | Not acquired; JS/cookie challenge | Historical HTML parser is brittle; no API claim | No export/terms contract verified; do not scrape through challenge | Prefer catalog downloads or request authorized export/documented endpoint | ## Integration recommendation @@ -49,4 +51,4 @@ Do not publish names, addresses, tax identifiers, or precise coordinates merely ## Limitations -This reconnaissance did not acquire current rows, verify download responses by HTTP, establish rate limits/authentication, or certify completeness/current accuracy. It makes no healthy-pipeline claim. +This reconnaissance does not certify completeness/current accuracy, rate limits/authentication, or publication eligibility. Both artifacts are private candidates only; no release or healthy-pipeline claim is made. No source-local adapter was added because the current dictionary-to-contract mapping and safe publication treatment of addresses, identifiers, and OSM-derived coordinates remain to be reviewed. diff --git a/docs/source-status.json b/docs/source-status.json index 52bc49d..bb0c4d7 100644 --- a/docs/source-status.json +++ b/docs/source-status.json @@ -9,7 +9,7 @@ }, "sources": [ {"source_id":"fr.locations","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-fr.md","pipeline/source_registry.json"],"next_action":"Acquire and snapshot an official DGAL Section I/II artifact with terms, schema, and privacy review."}, - {"source_id":"it.locations","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-it.md","pipeline/source_registry.json"],"next_action":"Use catalog-linked Ministry downloads; record provenance and validate 853/2004 separately from 1069/2009."}, + {"source_id":"it.locations","metadata":"verified","acquisition":"artifact_private_only","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-it.md","pipeline/source_registry.json"],"next_action":"Review the 853/2004 and 1069/2009 source dictionaries and privacy mappings, then implement and validate separate adapters without treating private artifacts as release inputs."}, {"source_id":"mx.locations","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-mx.md","pipeline/source_registry.json"],"next_action":"Resolve DENUE token/terms and SENASICA current directory/schema before bounded acquisition."}, {"source_id":"nz.locations","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-nz.md","pipeline/source_registry.json"],"next_action":"Confirm MPI permitted acquisition route, list-specific terms, and schema before private staging."}, {"source_id":"uk.locations","metadata":"partial","acquisition":"artifact_private_only","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-uk.md","docs/countries/uk/fss-approved-establishments-source-assessment.md","pipeline/sources/uk/fsa_approved/README.md","pipeline/source_registry.json"],"next_action":"Keep the UK ID composite: validate the private England/Wales monthly candidate while keeping NI and Scotland separate; review withheld addresses, duplicates, coverage, terms, and release approval."}, diff --git a/docs/source-status.md b/docs/source-status.md index d501174..8c4f0b8 100644 --- a/docs/source-status.md +++ b/docs/source-status.md @@ -16,7 +16,7 @@ No last-success timestamp is invented. Private artifacts are not proof of a publ | Source ID | Metadata | Acquisition | Runtime health | Publication | Evidence / next action | |---|---|---|---|---|---| | `fr.locations` | verified | blocked | not_run | blocked | DGAL/Alim’confiance/SIRENE/HVE/Agence Bio/Géorisques reconnaissance; acquire permitted official artifact and review terms/privacy | -| `it.locations` | verified | blocked | not_run | blocked | Ministry 853/2004 and 1069/2009 catalogs identified; use catalog downloads, not challenged servlet | +| `it.locations` | verified | artifact_private_only | not_run | blocked | Current 853/2004 and 1069/2009 CSVs were privately acquired with provenance; schemas remain distinct and require dictionary/privacy mapping plus separate adapters | | `mx.locations` | verified | blocked | not_run | blocked | DENUE/SENASICA/DGSIAP reconnaissance; resolve token, directory, terms, and schema | | `nz.locations` | verified | blocked | not_run | blocked | MPI/Stats NZ reconnaissance; resolve 403/access and aggregate-vs-facility boundaries | | `uk.locations` | partial | artifact_private_only | not_run | blocked | Private England/Wales monthly candidate is validated; NI and Scotland remain separate, with withheld-address/duplicate/coverage/terms/release review open | From f8ea7cbdaca44c1c6de8f4164064dec695340dfd Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sun, 13 Sep 2026 22:51:19 -0700 Subject: [PATCH 057/311] Add gated private candidate preview UI --- frontend/README.md | 2 ++ .../src/api/DevCandidatePreviewRepository.ts | 36 +++++++++++++++++++ frontend/src/app/App.svelte | 25 +++++++++++-- .../features/devPreview/devPreviewContract.ts | 16 +++++++++ .../tests/unit/devPreviewContract.test.ts | 18 ++++++++++ 5 files changed, 94 insertions(+), 3 deletions(-) create mode 100644 frontend/src/api/DevCandidatePreviewRepository.ts create mode 100644 frontend/src/features/devPreview/devPreviewContract.ts create mode 100644 frontend/tests/unit/devPreviewContract.test.ts diff --git a/frontend/README.md b/frontend/README.md index 8414dcc..f6cf82d 100644 --- a/frontend/README.md +++ b/frontend/README.md @@ -26,3 +26,5 @@ The opt-in local view loads one API page at a time. When the response includes ` The private scale/story prototype begins with a neutral individual-animal representation and uses only bounded synthetic values. It labels model arithmetic separately from measured facility evidence; no biography, live counter, global animal total, or sourced aggregate is embedded in the production build. Candidate sourced scale figures remain outside this UI until maintainer publication approval. Persistent local two-port workflow (never uses `down -v`): from the repository root run `powershell -ExecutionPolicy Bypass -File pipeline/scripts/maintenance/local-v2.ps1 start`. It starts the named Postgres stack on `5433`, applies migrations, seeds and promotes the synthetic contract release, and starts Axum on `8000`. Run `npm --prefix frontend run dev` in another terminal and open `http://127.0.0.1:5173/v2-preview/?mode=local-v2#/`. Check ownership and health with `... local-v2.ps1 status`; probe list/detail with `... local-v2.ps1 probe`; stop both services with `... local-v2.ps1 stop`. The helper is local-only and does not alter V1, production, or unrelated data. + +Private candidate preview is a separate development-only shell. Its API is `/api/dev/preview/candidates?limit=100`, never `/api/v2/*`; it requires loopback, explicit development opt-in, and an operator token in `X-UEC-Dev-Preview-Token`. The token must stay in memory/session-only state or a local proxy, never a URL, committed source, log, or production bundle. Candidate rows must remain visibly marked `PRIVATE TEST DATA — NOT REVIEWED OR PUBLISHED`, with source/date/coverage/uncertainty context on every list, detail, map, and export surface. No real candidate rows or credentials belong in this repository. diff --git a/frontend/src/api/DevCandidatePreviewRepository.ts b/frontend/src/api/DevCandidatePreviewRepository.ts new file mode 100644 index 0000000..c37ce7d --- /dev/null +++ b/frontend/src/api/DevCandidatePreviewRepository.ts @@ -0,0 +1,36 @@ +import { z } from 'zod'; +import type { FetchLike } from './LocalLocationRepository'; +import type { Location } from '../domain/location'; +import { DEV_PREVIEW_PATH, DEV_PREVIEW_TOKEN_HEADER } from '../features/devPreview/devPreviewContract'; + +const rowSchema = z.object({ + candidate_id: z.string().min(1), source_record_id: z.string().min(1), facility_id: z.string().min(1), canonical_name: z.string().min(1), + country_code: z.string().min(1), city: z.string().nullable(), category: z.string().min(1), display_precision: z.enum(['exact', 'city', 'unmapped']), + latitude: z.number().finite().nullable(), longitude: z.number().finite().nullable(), source_type: z.enum(['official', 'secondary', 'user_submitted']), + provenance_source_id: z.string().min(1), provenance_source_name: z.string().min(1), provenance_source_url: z.string().url(), provenance_retrieved_at: z.string().min(1), + factual_review_status: z.enum(['unreviewed', 'reviewed', 'rejected']), privacy_screening_status: z.literal('passed'), project_approval: z.literal(false), + release_id: z.string().nullable(), release_status: z.literal('candidate'), preview_label: z.string().min(1), +}).superRefine((row, ctx) => { if ((row.latitude === null) !== (row.longitude === null)) ctx.addIssue({ code: z.ZodIssueCode.custom, message: 'coordinate pair must be complete' }); }); +const envelopeSchema = z.object({ api_version: z.literal('dev-preview-v1'), data: z.array(rowSchema), meta: z.object({ test_only: z.literal(true), private_preview: z.literal(true), profile: z.null(), coverage_scope: z.literal('candidate_release_only'), next_cursor: z.null() }) }); + +export type DevCandidate = Location & Readonly<{ candidateId: string; sourceRecordId: string; previewLabel: string; releaseStatus: 'candidate'; coverageScope: string }>; + +/** Token is accepted only as an in-memory argument; callers must not persist or put it in a URL. */ +export class DevCandidatePreviewRepository { + constructor(private readonly fetcher: FetchLike = globalThis.fetch, private readonly baseUrl = '') {} + async list(token: string, signal?: AbortSignal): Promise { + if (!token.trim()) throw new Error('An operator token is required for the private candidate preview.'); + const init: RequestInit = { cache: 'no-store', headers: { [DEV_PREVIEW_TOKEN_HEADER]: token } }; + if (signal) init.signal = signal; + const response = await this.fetcher.call(globalThis, `${this.baseUrl}${DEV_PREVIEW_PATH}?limit=100`, init); + if (!response.ok) throw new Error(response.status === 401 ? 'Private candidate preview authentication failed.' : response.status === 404 ? 'Private candidate preview is disabled.' : `Private candidate preview failed with status ${response.status}.`); + const parsed = envelopeSchema.safeParse(await response.json()); + if (!parsed.success) throw new Error('Private candidate preview response was rejected safely.'); + return parsed.data.data.map((row) => ({ + id: row.facility_id, name: row.canonical_name, region: row.city ?? row.country_code, category: row.category, + lat: row.latitude, lon: row.longitude, observed: 'candidate observation date unavailable', source: row.provenance_source_name, + candidateId: row.candidate_id, sourceRecordId: row.source_record_id, previewLabel: row.preview_label, releaseStatus: row.release_status, coverageScope: parsed.data.meta.coverage_scope, + evidence: { sourceType: row.source_type, factualReviewStatus: row.factual_review_status, reviewerRole: null, privacyScreeningStatus: row.privacy_screening_status, projectApproval: 'pending', publicationProfile: 'community', publicationWarning: row.preview_label, sourceId: row.provenance_source_id, sourceUrl: row.provenance_source_url, retrievedAt: row.provenance_retrieved_at, displayPrecision: row.display_precision, lifecycleStatus: 'status_unknown', observationCount: null }, + })); + } +} diff --git a/frontend/src/app/App.svelte b/frontend/src/app/App.svelte index 6f8cd86..87cdcfa 100644 --- a/frontend/src/app/App.svelte +++ b/frontend/src/app/App.svelte @@ -14,6 +14,8 @@ import ReleaseContext from '../ui/ReleaseContext.svelte'; import ExportControl from '../ui/ExportControl.svelte'; import ScaleNarrative from '../features/scale/ScaleNarrative.svelte'; + import { canOpenDevPreview, DEV_PREVIEW_LABEL } from '../features/devPreview/devPreviewContract'; + import type { DevCandidate } from '../api/DevCandidatePreviewRepository'; let profile: Profile = 'curated'; let selected: Location | undefined = locations[0]; @@ -22,6 +24,11 @@ let filters: FilterState = initialFilters; let showMap = false; let showExport = false; let showGuidance = false; let localMode = false; + let devPreviewMode = false; + let previewToken = ''; + let previewStatus: 'idle' | 'loading' | 'ready' | 'error' | 'blocked' = 'idle'; + let previewError = ''; + let previewRows: readonly DevCandidate[] = []; let localStatus: 'idle' | 'loading' | 'ready' | 'error' | 'no-release' = 'idle'; let detailStatus: 'idle' | 'loading' | 'error' = 'idle'; let localError = ''; let exportError = ''; let exportBusy = false; @@ -37,7 +44,7 @@ let activeListKey = ''; let detailErrorHeading: HTMLHeadingElement; $: filters = { search, region, category }; - $: source = localMode ? loaded : (profile === 'community' ? locations.slice(0, 1) : locations); + $: source = devPreviewMode ? previewRows : localMode ? loaded : (profile === 'community' ? locations.slice(0, 1) : locations); $: visibleLocations = filterLocations(source, filters); $: exportPreview = previewExport(makeExportModel(visibleLocations, profile, release)); $: eligibleExport = localMode && profile === 'curated' && localStatus === 'ready' && Boolean(release); @@ -98,18 +105,27 @@ }; const select = (id: string) => { selected = source.find((item) => item.id === id) ?? selected; window.location.hash = `/locations/${id}?profile=${profile}`; }; const profileChanged = () => { history.pushState(null, '', `#/?profile=${profile}`); if (localMode) void loadLocal(); else void syncRoute(); }; + const loadDevPreview = async () => { + if (!canOpenDevPreview(import.meta.env.DEV, devPreviewMode ? 'dev-candidates' : null)) { previewStatus = 'blocked'; previewError = 'Private candidate preview is unavailable in production builds.'; return; } + if (!previewToken.trim()) { previewStatus = 'error'; previewError = 'Enter the operator token for this development session.'; return; } + previewStatus = 'loading'; previewError = ''; + try { const { DevCandidatePreviewRepository } = await import('../api/DevCandidatePreviewRepository'); previewRows = await new DevCandidatePreviewRepository().list(previewToken); previewStatus = 'ready'; selected = previewRows[0]; } + catch (error) { previewStatus = 'error'; previewError = error instanceof Error ? error.message : 'Private candidate preview was rejected safely.'; previewRows = []; selected = undefined; } + }; const clearFilters = () => { search = ''; region = 'all'; category = 'all'; sourceType = 'all'; displayPrecision = 'all'; lifecycleStatus = 'all'; }; const searchChanged = () => { if (localMode) { const url = new URL(window.location.href); if (search.trim()) url.searchParams.set('q', search.trim()); else url.searchParams.delete('q'); history.replaceState(null, '', url); } }; onMount(() => { const params = new URLSearchParams(window.location.search); localMode = params.get('mode') === 'local-v2'; + devPreviewMode = params.get('preview') === 'dev-candidates'; const route = parseRoute(window.location.hash); if (route.kind !== 'not-found') profile = route.profile; if (localMode) { search = params.get('q') ?? ''; region = params.get('country_code') ?? 'all'; category = params.get('category') ?? 'all'; sourceType = params.get('source_type') ?? 'all'; displayPrecision = params.get('display_precision') ?? 'all'; lifecycleStatus = params.get('lifecycle_status') ?? 'all'; try { const api = params.get('api') ?? undefined; repo = new LocalLocationRepository(globalThis.fetch, api); csvRepo = new LocalCsvExportRepository(globalThis.fetch, api ?? ''); metadataStatus = 'loading'; void new FilterMetadataRepository(globalThis.fetch, api ?? '').get().then((value) => { metadata = value; metadataStatus = 'ready'; }).catch(() => { metadataStatus = 'error'; }); } catch (error) { localStatus = 'error'; localError = error instanceof Error ? error.message : 'Local API origin was rejected safely.'; } } - if (localMode && localStatus !== 'error') void loadLocal(); else if (!localMode) void syncRoute(); + if (devPreviewMode) { previewStatus = import.meta.env.DEV ? 'idle' : 'blocked'; if (!import.meta.env.DEV) previewError = 'Private candidate preview is unavailable in production builds.'; } + else if (localMode && localStatus !== 'error') void loadLocal(); else if (!localMode) void syncRoute(); const onHashChange = () => { const next = parseRoute(window.location.hash); if (next.kind !== 'not-found' && next.profile !== profile) { profile = next.profile; if (localMode) void loadLocal(); else void syncRoute(); } else void syncRoute(); }; const onPopState = () => { const current = new URLSearchParams(window.location.search); if (localMode) { search = current.get('q') ?? ''; region = current.get('country_code') ?? 'all'; category = current.get('category') ?? 'all'; sourceType = current.get('source_type') ?? 'all'; displayPrecision = current.get('display_precision') ?? 'all'; lifecycleStatus = current.get('lifecycle_status') ?? 'all'; } onHashChange(); }; window.addEventListener('hashchange', onHashChange); window.addEventListener('popstate', onPopState); @@ -120,7 +136,9 @@ Until Every Cage · evidence desk
UNTIL EVERY CAGE V2 / FIELD NOTE
-

EVIDENCE DESK · {localMode ? 'LOCAL V2 API' : 'SYNTHETIC PREVIEW'}

See what a record can—and cannot—tell us.

Start with the source profile, narrow the visible evidence, then inspect what a record can—and cannot—tell us.

+

EVIDENCE DESK · {devPreviewMode ? 'PRIVATE CANDIDATE PREVIEW' : localMode ? 'LOCAL V2 API' : 'SYNTHETIC PREVIEW'}

See what a record can—and cannot—tell us.

Start with the source profile, narrow the visible evidence, then inspect what a record can—and cannot—tell us.

+ {#if devPreviewMode}{/if} + {#if !devPreviewMode || previewStatus === 'ready'}

01 / DISCOVER

Choose the evidence lane

Profiles stay separate. A community claim is never silently promoted into a curated result.

{#if localMode && metadataStatus === 'loading'}

Loading available filter values…

{:else if localMode && metadataStatus === 'error'}

Filter metadata is unavailable; only unscoped discovery is shown.

{/if}
{search || region !== 'all' || category !== 'all' || sourceType !== 'all' || displayPrecision !== 'all' || lifecycleStatus !== 'all' ? `Filters applied: ${[search && `name “${search}”`, region !== 'all' && region, category !== 'all' && category, sourceType !== 'all' && sourceType, displayPrecision !== 'all' && displayPrecision, lifecycleStatus !== 'all' && lifecycleStatus].filter(Boolean).join(' · ')}` : 'No additional filters applied'}
@@ -136,4 +154,5 @@ {#if localMode && localStatus === 'ready'}{/if}
{#if showGuidance}

Do not infer closure, identity, or permission from a map point. For a correction, privacy concern, or suppression request, preserve the record ID and contact the project maintainer through the reporting channel on the ethics page. Do not include sensitive personal details in a public issue.

Read reporting guidance ↗
{/if}
{#if showMap}{/if}{#if showExport}

IN-MEMORY EXPORT PREVIEW

{profile} · {release}

Loaded results only. This preview is not a live or complete export.

{exportPreview}
{/if}
{localMode ? 'Local V2 mode · no fixture fallback' : 'Method preview · no live requests'}
+ {/if}
diff --git a/frontend/src/features/devPreview/devPreviewContract.ts b/frontend/src/features/devPreview/devPreviewContract.ts new file mode 100644 index 0000000..8077a9f --- /dev/null +++ b/frontend/src/features/devPreview/devPreviewContract.ts @@ -0,0 +1,16 @@ +/** + * Shape agreed for the private candidate-preview boundary. The client never + * supplies credentials in a URL and never treats these rows as publication. + */ +export type DevPreviewState = 'disabled' | 'unavailable' | 'loading' | 'ready' | 'error'; + +export const DEV_PREVIEW_PATH = '/api/dev/preview/candidates'; +export const DEV_PREVIEW_QUERY = 'dev-candidates'; +export const DEV_PREVIEW_TOKEN_HEADER = 'X-UEC-Dev-Preview-Token'; + +/** Keep a manually typed preview URL inert in production builds. */ +export const canOpenDevPreview = (isDevelopment: boolean, requestedMode: string | null): boolean => + isDevelopment && requestedMode === DEV_PREVIEW_QUERY; + +/** Persistent copy used on every private-preview surface. */ +export const DEV_PREVIEW_LABEL = 'PRIVATE TEST DATA — NOT REVIEWED OR PUBLISHED'; diff --git a/frontend/tests/unit/devPreviewContract.test.ts b/frontend/tests/unit/devPreviewContract.test.ts new file mode 100644 index 0000000..854baee --- /dev/null +++ b/frontend/tests/unit/devPreviewContract.test.ts @@ -0,0 +1,18 @@ +import { describe, expect, it } from 'vitest'; +import { canOpenDevPreview, DEV_PREVIEW_LABEL, DEV_PREVIEW_PATH, DEV_PREVIEW_QUERY, DEV_PREVIEW_TOKEN_HEADER } from '../../src/features/devPreview/devPreviewContract'; + +describe('dev preview boundary', () => { + it('requires both a development build and the explicit mode', () => { + expect(canOpenDevPreview(true, DEV_PREVIEW_QUERY)).toBe(true); + expect(canOpenDevPreview(false, DEV_PREVIEW_QUERY)).toBe(false); + expect(canOpenDevPreview(true, null)).toBe(false); + expect(canOpenDevPreview(true, 'local-v2')).toBe(false); + }); + + it('keeps the route and label distinct from public V2', () => { + expect(DEV_PREVIEW_PATH).toBe('/api/dev/preview/candidates'); + expect(DEV_PREVIEW_PATH).not.toContain('/api/v2/'); + expect(DEV_PREVIEW_TOKEN_HEADER).toBe('X-UEC-Dev-Preview-Token'); + expect(DEV_PREVIEW_LABEL).toContain('NOT REVIEWED OR PUBLISHED'); + }); +}); From 5e5db9869fbcb2b0e2525f3222cbe9fe785a14eb Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sun, 13 Sep 2026 22:52:23 -0700 Subject: [PATCH 058/311] Require approved coordinates in private preview --- docs/deployment/dev-preview.md | 4 +++- pipeline/tests/e2e/fixture.py | 12 +++++++++++- pipeline/tests/e2e/test_public_api.py | 2 ++ src/lib.rs | 3 +++ 4 files changed, 19 insertions(+), 2 deletions(-) diff --git a/docs/deployment/dev-preview.md b/docs/deployment/dev-preview.md index f4a3c06..945dd1b 100644 --- a/docs/deployment/dev-preview.md +++ b/docs/deployment/dev-preview.md @@ -19,7 +19,9 @@ configuration fail closed. A local dev proxy may hold the token server-side; browser clients may hold it only in memory for the current session. Preview responses contain only privacy-screened, non-withheld candidate fields -and source/retrieval metadata. They omit raw payloads, addresses, and geocoder +and source/retrieval metadata. Exact coordinates require a separate explicit +`coordinate_review_status=approved` decision; an accepted geocoder result alone +is not permission to expose a point. They omit raw payloads, addresses, and geocoder queries. Every row is labeled `Private development candidate — not project-approved or published`; `project_approval` remains `false` and the release remains `candidate`. Candidate seed/reset/rebuild belongs to a diff --git a/pipeline/tests/e2e/fixture.py b/pipeline/tests/e2e/fixture.py index a95c575..f96c40f 100644 --- a/pipeline/tests/e2e/fixture.py +++ b/pipeline/tests/e2e/fixture.py @@ -177,10 +177,20 @@ def seed_private_candidate_scenario(self): db.execute("INSERT INTO uec.raw_artifacts (artifact_id,storage_key,sha256,byte_size,retrieved_at) VALUES (%s,'e2e/private-candidate',%s,1,%s)", (artifact, uuid.uuid4().hex * 2, now)) db.execute("INSERT INTO uec.source_records (source_record_id,source_id,source_record_key,artifact_id,raw_fields,parsed_at) VALUES (%s,'e2e.private-candidate','candidate-only',%s,'{}',%s)", (record, artifact, now)) db.execute("INSERT INTO uec.facilities (facility_id,canonical_name,country_code,city) VALUES (%s,'E2E private candidate','DK','Candidateby')", (facility,)) - db.execute("INSERT INTO uec.observations (observation_id,facility_id,source_record_id,observed_at,observation,classification,ruleset_id,rule_id,classification_category,classification_review_status,default_visible,first_observed_at) VALUES (%s,%s,%s,%s,'{}','{}','e2e-private-v1','e2e','slaughter','approved',true,%s)", (observation, facility, record, now, now)) + db.execute("INSERT INTO uec.observations (observation_id,facility_id,source_record_id,observed_at,observation,classification,ruleset_id,rule_id,classification_category,classification_review_status,default_visible,coordinate_review_status,first_observed_at) VALUES (%s,%s,%s,%s,'{}','{}','e2e-private-v1','e2e','slaughter','approved',true,'approved',%s)", (observation, facility, record, now, now)) db.execute("INSERT INTO uec.release_members (release_id,facility_id,observation_id,default_visible) VALUES ('e2e-private-candidate',%s,%s,true)", (facility, observation)) db.execute("INSERT INTO uec.geocode_results (source_record_id,provider_id,query,match_method,status,attempt_number,result,queried_at) VALUES (%s,'e2e','synthetic candidate','fixture','accepted',1,ST_SetSRID(ST_MakePoint(12,56),4326)::geography,%s)", (record, now)) db.execute("INSERT INTO uec.publication_review_events (source_record_id,release_id,factual_review_status,privacy_screening_status,maintainer_approval,publication_eligible,reviewer_role) VALUES (%s,'e2e-private-candidate','reviewed','passed','approved',true,'maintainer')", (record,)) + # Same privacy/review state, but no coordinate decision: the + # preview must omit it rather than treating geocoder success as clearance. + pending_record, pending_facility, pending_observation, pending_artifact = uuid.uuid4(), uuid.uuid4(), uuid.uuid4(), uuid.uuid4() + db.execute("INSERT INTO uec.raw_artifacts (artifact_id,storage_key,sha256,byte_size,retrieved_at) VALUES (%s,'e2e/private-candidate-pending',%s,1,%s)", (pending_artifact, uuid.uuid4().hex * 2, now)) + db.execute("INSERT INTO uec.source_records (source_record_id,source_id,source_record_key,artifact_id,raw_fields,parsed_at) VALUES (%s,'e2e.private-candidate','candidate-pending',%s,'{}',%s)", (pending_record, pending_artifact, now)) + db.execute("INSERT INTO uec.facilities (facility_id,canonical_name,country_code,city) VALUES (%s,'E2E pending coordinate candidate','DK','Candidateby')", (pending_facility,)) + db.execute("INSERT INTO uec.observations (observation_id,facility_id,source_record_id,observed_at,observation,classification,ruleset_id,rule_id,classification_category,classification_review_status,default_visible,first_observed_at) VALUES (%s,%s,%s,%s,'{}','{}','e2e-private-v1','e2e','slaughter','approved',true,%s)", (pending_observation, pending_facility, pending_record, now, now)) + db.execute("INSERT INTO uec.release_members (release_id,facility_id,observation_id,default_visible) VALUES ('e2e-private-candidate',%s,%s,true)", (pending_facility, pending_observation)) + db.execute("INSERT INTO uec.geocode_results (source_record_id,provider_id,query,match_method,status,attempt_number,result,queried_at) VALUES (%s,'e2e','synthetic candidate','fixture','accepted',1,ST_SetSRID(ST_MakePoint(12,56),4326)::geography,%s)", (pending_record, now)) + db.execute("INSERT INTO uec.publication_review_events (source_record_id,release_id,factual_review_status,privacy_screening_status,maintainer_approval,publication_eligible,reviewer_role) VALUES (%s,'e2e-private-candidate','reviewed','passed','approved',true,'maintainer')", (pending_record,)) self.private_candidate_facility_id = facility def restore_restricted_record(self): diff --git a/pipeline/tests/e2e/test_public_api.py b/pipeline/tests/e2e/test_public_api.py index bb3a620..aeb992d 100644 --- a/pipeline/tests/e2e/test_public_api.py +++ b/pipeline/tests/e2e/test_public_api.py @@ -66,6 +66,8 @@ def test_private_candidate_preview_requires_auth_and_is_explicitly_labeled(self) self.assertEqual(body["data"][0]["release_status"], "candidate") self.assertEqual(body["data"][0]["project_approval"], False) self.assertIn("not project-approved", body["data"][0]["preview_label"]) + self.assertEqual(len(body["data"]), 1) + self.assertEqual(body["data"][0]["canonical_name"], "E2E private candidate") def test_filters_do_not_bypass_publication_gate(self): for path in ("?category=retail_and_prepared_food", "?display_precision=city", "?lifecycle_status=explicitly_closed"): diff --git a/src/lib.rs b/src/lib.rs index eda933b..f547ea2 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -433,6 +433,9 @@ pub async fn get_dev_candidate_preview_handler( AND record.source_state NOT IN ('rejected', 'superseded') AND review.privacy_screening_status = 'passed' AND review.factual_review_status <> 'rejected' + -- An accepted geocoder result is not itself permission to expose a + -- precise point; candidate preview requires explicit coordinate review. + AND o.coordinate_review_status = 'approved' AND NOT EXISTS (SELECT 1 FROM uec.public_access_restricted restricted WHERE restricted.source_record_id = o.source_record_id) ORDER BY r.release_id, o.facility_id LIMIT $1 From 3313c887fd52f8feb84052feb64b920411000b4d Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sun, 13 Sep 2026 22:52:59 -0700 Subject: [PATCH 059/311] Record private Denmark source acquisition --- docs/source-status.json | 2 +- docs/source-status.md | 2 +- pipeline/sources/denmark/README.md | 18 ++++++++++++++++++ 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/docs/source-status.json b/docs/source-status.json index bb0c4d7..bd174b5 100644 --- a/docs/source-status.json +++ b/docs/source-status.json @@ -13,7 +13,7 @@ {"source_id":"mx.locations","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-mx.md","pipeline/source_registry.json"],"next_action":"Resolve DENUE token/terms and SENASICA current directory/schema before bounded acquisition."}, {"source_id":"nz.locations","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-nz.md","pipeline/source_registry.json"],"next_action":"Confirm MPI permitted acquisition route, list-specific terms, and schema before private staging."}, {"source_id":"uk.locations","metadata":"partial","acquisition":"artifact_private_only","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-uk.md","docs/countries/uk/fss-approved-establishments-source-assessment.md","pipeline/sources/uk/fsa_approved/README.md","pipeline/source_registry.json"],"next_action":"Keep the UK ID composite: validate the private England/Wales monthly candidate while keeping NI and Scotland separate; review withheld addresses, duplicates, coverage, terms, and release approval."}, - {"source_id":"dk.smiley","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["pipeline/README.md","pipeline/contracts/README.md","pipeline/source_registry.json"],"next_action":"Use the SourceArtifact/registered-adapter contract for a permitted official acquisition, then verify current endpoint, publication date semantics, licence, coverage, and release approval."}, + {"source_id":"dk.smiley","metadata":"partial","acquisition":"artifact_private_only","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["pipeline/sources/denmark/README.md","pipeline/contracts/README.md","pipeline/source_registry.json"],"next_action":"Keep the privately staged official artifact as validation-only; verify coverage and effective-date semantics, then complete terms/privacy/release review before any publication."}, {"source_id":"de.locations","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["pipeline/source_registry.json"],"next_action":"Replace session-bound legacy URL with a verified stable BVL endpoint and confirm terms/schema."}, {"source_id":"ca.locations","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ca.md","pipeline/source_registry.json"],"next_action":"Keep federal/export and Ontario candidates separate; verify an authoritative permitted artifact, terms, schema, coverage, and privacy before acquisition."}, {"source_id":"es.locations","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-es.md","pipeline/source_registry.json"],"next_action":"Resolve a permitted current AESAN/MAPA artifact or query route, then verify export/schema, rights, effective-date semantics, sector coverage, privacy, and legacy/source boundaries before acquisition."}, diff --git a/docs/source-status.md b/docs/source-status.md index 8c4f0b8..29de6f1 100644 --- a/docs/source-status.md +++ b/docs/source-status.md @@ -20,7 +20,7 @@ No last-success timestamp is invented. Private artifacts are not proof of a publ | `mx.locations` | verified | blocked | not_run | blocked | DENUE/SENASICA/DGSIAP reconnaissance; resolve token, directory, terms, and schema | | `nz.locations` | verified | blocked | not_run | blocked | MPI/Stats NZ reconnaissance; resolve 403/access and aggregate-vs-facility boundaries | | `uk.locations` | partial | artifact_private_only | not_run | blocked | Private England/Wales monthly candidate is validated; NI and Scotland remain separate, with withheld-address/duplicate/coverage/terms/release review open | -| `dk.smiley` | partial | not_run | not_run | blocked | SourceArtifact and registered-adapter contracts are documented; official live acquisition, endpoint/licence/coverage/release verification remain open | +| `dk.smiley` | partial | artifact_private_only | not_run | blocked | Current official artifact and registered adapter are privately staged for validation; coverage/effective-date uncertainty and terms/privacy/release review remain open | | `de.locations` | partial | not_run | not_run | blocked | Legacy session URL and current BVL endpoint/schema/terms remain unresolved | | `ca.locations` | verified | not_run | not_run | blocked | Federal/export and Ontario candidates are documented separately; verify permitted artifact, terms, schema, coverage, and privacy before acquisition | | `es.locations` | partial | blocked | not_run | blocked | AESAN RGSEAA and MAPA sector routes are documented, but direct acquisition was refused; rights, export/schema, effective dates, sector coverage, privacy, and legacy/source boundaries remain unresolved | diff --git a/pipeline/sources/denmark/README.md b/pipeline/sources/denmark/README.md index 885d448..2bc78b2 100644 --- a/pipeline/sources/denmark/README.md +++ b/pipeline/sources/denmark/README.md @@ -16,3 +16,21 @@ Example: ```powershell python pipeline/sources/denmark/run-denmark-pipeline.py --help ``` + +## Current-source evidence (private staging) + +The current bulk XML endpoint is the Fødevarestyrelsen publication linked from +the official [Find Smiley data page](https://www.findsmiley.dk/om-smiley/statistik-og-data/hent-smileydata). +That page says the export covers data available on findsmiley.dk, is reusable +under public-data terms, requires Fødevarestyrelsen attribution, prohibits use +of its logo, and requires displayed smileys to remain current and follow the +design rules. The page does not supply a dataset effective date; the adapter +records supplied HTTP publication metadata separately from retrieval time. + +Coverage is the publisher's Find Smiley dataset (primarily food-service/detail +inspection records); the publisher's statistics page explicitly excludes +wholesale businesses. This is source coverage, not a claim that the project +dataset is complete or that records are current. A reviewed acquisition may be +retained in ignored private storage for research and validation only. The +adapter emits a private candidate with `release_state: not-created`; no health, +approval, geocoding, or publication conclusion follows from a successful run. From 18d25eeb6fcaca637f8e61c58cdea52a38ff428b Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sun, 13 Sep 2026 22:46:30 -0700 Subject: [PATCH 060/311] Narrow UK monthly address quarantine heuristic --- docs/country-recon-uk.md | 34 +++++++++++++++++-- pipeline/sources/uk/fsa_approved/README.md | 6 +++- pipeline/sources/uk/fsa_approved/adapter.py | 5 ++- .../sources/uk/fsa_approved/test_adapter.py | 12 +++++++ 4 files changed, 53 insertions(+), 4 deletions(-) diff --git a/docs/country-recon-uk.md b/docs/country-recon-uk.md index 6dfe8ba..3779253 100644 --- a/docs/country-recon-uk.md +++ b/docs/country-recon-uk.md @@ -94,8 +94,38 @@ Northern Ireland as a separate future feed. It quarantines duplicate IDs within nation, unknown jurisdictions, malformed rows, missing activities, remarks and address-risk rows; suppresses `AddressWithheld` addresses and coordinates; validates X/Y as source longitude/latitude without geocoding; and reports aggregate coverage -and anomaly counts. The synthetic profile retains its authority/status/activity -vocabulary tests for the canonical composition contract. +and anomaly counts. Remarks remain a quarantine reason because their free text is +preserved in restricted source values and has not passed privacy review. The address +heuristic was narrowed after aggregate QA: generic facility-building words such as +`house`, `home`, and `lodge` are not sufficient by themselves, while explicit +residential or intermediary indicators remain review blockers. The synthetic profile +retains its authority/status/activity vocabulary tests for the canonical composition +contract. + +### Aggregate-only monthly QA (2026-09-14) + +The privately retained 2026-09-01 snapshot reproduced 5,342 input rows, 4,090 +normalized rows, and 1,252 quarantined rows before the heuristic correction. The +sanitized reason matrix was: + +| Reason | Rows | Interpretation | Action | +|---|---:|---|---| +| `remarks_present` | 999 | Short free-text source notes; 956 were under 40 characters; only 46 overlapped an address-risk hit | Keep quarantined pending privacy review | +| `address_privacy_risk` | 268 | 227 generic `house` hits, 25 `lodge`, 7 `home`, 8 `c/o`, and 3 `flat` hits; none had `AddressWithheld=Yes` | Narrow heuristic; retain explicit indicators for review | +| `unknown_nation` | 31 | Jersey, Isle of Man, or Guernsey rows outside the England/Wales profile | Keep quarantined; use separate source scope | +| `duplicate_id_within_nation` | 4 | Repeated application identifiers | Keep quarantined; never silently deduplicate | + +This is an aggregate QA result, not source approval. The artifact, row-level values, +coordinates, and derived records remain restricted and were not committed or +published. The automated pipeline requirement remains end-to-end: acquisition, +checksum/metadata validation, parsing, quarantine, manifesting, and downstream +ingestion must be orchestrated before any release gate can open. + +After the narrow heuristic correction, the same private artifact produced 4,300 +normalized rows and 1,042 quarantined rows. The remaining address-risk count was +11; duplicate, unknown-jurisdiction, and remarks counts were unchanged. This +reduction is not a release decision: remarks, out-of-scope jurisdictions, and +duplicate identifiers remain blocked pending their respective reviews. This reconciliation is a design and test record, not source approval or legal clearance. No live row data is included, and the private artifact remains outside diff --git a/pipeline/sources/uk/fsa_approved/README.md b/pipeline/sources/uk/fsa_approved/README.md index ddb7a20..37e3b65 100644 --- a/pipeline/sources/uk/fsa_approved/README.md +++ b/pipeline/sources/uk/fsa_approved/README.md @@ -9,7 +9,11 @@ source feeds; they are not merged into this capability. Source values and identifiers are preserved. Duplicate IDs are quarantined within nation, unknown jurisdictions are quarantined, and malformed rows, missing or unresolved activity, remarks, authority mismatches, and address-risk values remain -explicit review outcomes. `AddressWithheld=Yes` emits no address or coordinates; +explicit review outcomes. Remarks remain quarantined because their free text is +retained in restricted source values and has not passed privacy review. Address +privacy heuristics intentionally exclude generic facility-building names such as +"house", "home", and "lodge"; explicit residential or intermediary indicators +still require review. `AddressWithheld=Yes` emits no address or coordinates; X/Y are validated as source longitude/latitude without geocoding. Registered runs write deterministic parsed, normalized, and quarantined states with a manifest whose `release_state` is always `not-created` and whose publication state is diff --git a/pipeline/sources/uk/fsa_approved/adapter.py b/pipeline/sources/uk/fsa_approved/adapter.py index 7f7e454..e3315d5 100644 --- a/pipeline/sources/uk/fsa_approved/adapter.py +++ b/pipeline/sources/uk/fsa_approved/adapter.py @@ -11,7 +11,10 @@ ALLOWED_ACTIVITIES=frozenset(CONFIG["allowed_activities"]) ALLOWED_STATUSES=frozenset(CONFIG["allowed_statuses"]) AUTHORITY_BY_NATION=CONFIG["authority_by_nation"] -ADDRESS_RISK=re.compile(r"\b(flat|apartment|house|home|residential|c/o|care of|caravan|lodge)\b",re.I) +# Generic facility-building names such as "house", "home", and "lodge" are +# common in legitimate establishment addresses. Keep only terms that are +# stronger indicators of a residential, private, or intermediary address. +ADDRESS_RISK=re.compile(r"\b(flat|apartment|residential|c/o|care of|caravan)\b",re.I) MONTHLY_REQUIRED=frozenset({"AppNo","TradingName","Country","CompetentAuthority","X","Y","AddressWithheld","All_Activities"}) MONTHLY_COUNTRIES=frozenset({"England","Wales"}) diff --git a/pipeline/sources/uk/fsa_approved/test_adapter.py b/pipeline/sources/uk/fsa_approved/test_adapter.py index b3ef8c5..852ca74 100644 --- a/pipeline/sources/uk/fsa_approved/test_adapter.py +++ b/pipeline/sources/uk/fsa_approved/test_adapter.py @@ -73,6 +73,18 @@ def test_monthly_profile_enforces_withholding_coverage_duplicates_and_coordinate self.assertEqual(manifest["release_state"], "not-created") self.assertEqual(manifest["coverage_counts"], {"England": 2, "Jersey": 1, "Wales": 1}) + def test_monthly_generic_facility_building_name_is_not_privacy_quarantine(self): + monthly = "AppNo,TradingName,Country,CompetentAuthority,X,Y,AddressWithheld,All_Activities,Address1,Town,Postcode\nA-1,House Foods,England,Food Standards Agency,-0.12,51.50,No,CP,House Farm,London,SW1\n" + result = self.adapter.parse_bytes(monthly.encode("cp1252")) + self.assertEqual(len(result.accepted), 1) + self.assertEqual(result.accepted[0]["normalized"]["address_lines"], ("House Farm", None, None)) + + def test_monthly_explicit_private_address_indicator_remains_quarantined(self): + monthly = "AppNo,TradingName,Country,CompetentAuthority,X,Y,AddressWithheld,All_Activities,Address1,Town,Postcode\nA-1,Private Foods,England,Food Standards Agency,-0.12,51.50,No,CP,Flat 2,London,SW1\n" + result = self.adapter.parse_bytes(monthly.encode("cp1252")) + self.assertEqual(len(result.accepted), 0) + self.assertEqual(result.quarantined[0]["reasons"], ("address_privacy_risk",)) + if __name__ == "__main__": unittest.main() From 3b1dcdee7f35c2ed1bc6f30c1282d1fffbaf76e7 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sun, 13 Sep 2026 22:50:24 -0700 Subject: [PATCH 061/311] Keep UK monthly coordinates behind privacy review --- docs/country-recon-uk.md | 8 +++++--- pipeline/sources/uk/fsa_approved/README.md | 5 ++++- pipeline/sources/uk/fsa_approved/adapter.py | 4 +++- pipeline/sources/uk/fsa_approved/test_adapter.py | 5 +++++ 4 files changed, 17 insertions(+), 5 deletions(-) diff --git a/docs/country-recon-uk.md b/docs/country-recon-uk.md index 3779253..2712926 100644 --- a/docs/country-recon-uk.md +++ b/docs/country-recon-uk.md @@ -98,9 +98,11 @@ and anomaly counts. Remarks remain a quarantine reason because their free text i preserved in restricted source values and has not passed privacy review. The address heuristic was narrowed after aggregate QA: generic facility-building words such as `house`, `home`, and `lodge` are not sufficient by themselves, while explicit -residential or intermediary indicators remain review blockers. The synthetic profile -retains its authority/status/activity vocabulary tests for the canonical composition -contract. +residential or intermediary indicators remain review blockers. A monthly record not +flagged by that heuristic is still marked `privacy-review-required`; its normalized +coordinates remain suppressed until an authorized privacy decision. A heuristic pass +is not privacy clearance. The synthetic profile retains its authority/status/activity +vocabulary tests for the canonical composition contract. ### Aggregate-only monthly QA (2026-09-14) diff --git a/pipeline/sources/uk/fsa_approved/README.md b/pipeline/sources/uk/fsa_approved/README.md index 37e3b65..44d2b16 100644 --- a/pipeline/sources/uk/fsa_approved/README.md +++ b/pipeline/sources/uk/fsa_approved/README.md @@ -14,7 +14,10 @@ retained in restricted source values and has not passed privacy review. Address privacy heuristics intentionally exclude generic facility-building names such as "house", "home", and "lodge"; explicit residential or intermediary indicators still require review. `AddressWithheld=Yes` emits no address or coordinates; -X/Y are validated as source longitude/latitude without geocoding. Registered runs +X/Y are validated as source longitude/latitude without geocoding, but monthly +normalized coordinates remain suppressed behind an explicit +`privacy-review-required` gate even when no heuristic address-risk token is +present. A heuristic pass is not privacy clearance. Registered runs write deterministic parsed, normalized, and quarantined states with a manifest whose `release_state` is always `not-created` and whose publication state is private-candidate. diff --git a/pipeline/sources/uk/fsa_approved/adapter.py b/pipeline/sources/uk/fsa_approved/adapter.py index e3315d5..3c642d1 100644 --- a/pipeline/sources/uk/fsa_approved/adapter.py +++ b/pipeline/sources/uk/fsa_approved/adapter.py @@ -69,7 +69,9 @@ def _monthly_record(row,line): withheld=(_clean(row.get("AddressWithheld")) or "").lower()=="yes";x=y=None;status="withheld" if withheld else "unavailable" if not withheld:x,y,status=_coords(row) acts=tuple(x for x in (_clean(row.get("All_Activities")),_clean(row.get("Part_A__All_sections_")),_clean(row.get("Part B All sections "))) if x) - return {"source_id":CONFIG["source_id"],"source_row":line,"source_values":dict(row),"normalized":{"establishment_id":_clean(row.get("AppNo")),"trading_name":_clean(row.get("TradingName")),"address_lines":None if withheld else tuple(_clean(row.get(k)) for k in ("Address1","Address2","Address3")),"postcode":_clean(row.get("Postcode")),"activities":acts,"species":_clean(row.get("Species")),"competent_authority":_clean(row.get("CompetentAuthority")),"nation":_clean(row.get("Country")),"authority_nation_key":_clean(row.get("Country")),"status":None,"remarks":_clean(row.get("Remarks")),"published_date":None,"coordinates":None if withheld else {"longitude":x,"latitude":y,"status":status},"privacy_gate":"restricted-withheld-address" if withheld else "pending-review","publication_gate":"blocked"}} + privacy_gate="restricted-withheld-address" if withheld else "privacy-review-required" + coordinate_gate="restricted-withheld-address" if withheld else "privacy-review-required" + return {"source_id":CONFIG["source_id"],"source_row":line,"source_values":dict(row),"normalized":{"establishment_id":_clean(row.get("AppNo")),"trading_name":_clean(row.get("TradingName")),"address_lines":None if withheld else tuple(_clean(row.get(k)) for k in ("Address1","Address2","Address3")),"postcode":_clean(row.get("Postcode")),"activities":acts,"species":_clean(row.get("Species")),"competent_authority":_clean(row.get("CompetentAuthority")),"nation":_clean(row.get("Country")),"authority_nation_key":_clean(row.get("Country")),"status":None,"remarks":_clean(row.get("Remarks")),"published_date":None,"coordinates":None,"coordinate_gate":coordinate_gate,"privacy_gate":privacy_gate,"publication_gate":"blocked"}} class FsaApprovedEstablishmentsAdapter: source_id=CONFIG["source_id"];schema_version=CONFIG["contract_version"];adapter_version=CONFIG["adapter_version"] diff --git a/pipeline/sources/uk/fsa_approved/test_adapter.py b/pipeline/sources/uk/fsa_approved/test_adapter.py index 852ca74..861d2bf 100644 --- a/pipeline/sources/uk/fsa_approved/test_adapter.py +++ b/pipeline/sources/uk/fsa_approved/test_adapter.py @@ -78,12 +78,17 @@ def test_monthly_generic_facility_building_name_is_not_privacy_quarantine(self): result = self.adapter.parse_bytes(monthly.encode("cp1252")) self.assertEqual(len(result.accepted), 1) self.assertEqual(result.accepted[0]["normalized"]["address_lines"], ("House Farm", None, None)) + self.assertEqual(result.accepted[0]["normalized"]["privacy_gate"], "privacy-review-required") + self.assertEqual(result.accepted[0]["normalized"]["coordinate_gate"], "privacy-review-required") + self.assertIsNone(result.accepted[0]["normalized"]["coordinates"]) def test_monthly_explicit_private_address_indicator_remains_quarantined(self): monthly = "AppNo,TradingName,Country,CompetentAuthority,X,Y,AddressWithheld,All_Activities,Address1,Town,Postcode\nA-1,Private Foods,England,Food Standards Agency,-0.12,51.50,No,CP,Flat 2,London,SW1\n" result = self.adapter.parse_bytes(monthly.encode("cp1252")) self.assertEqual(len(result.accepted), 0) self.assertEqual(result.quarantined[0]["reasons"], ("address_privacy_risk",)) + self.assertEqual(result.quarantined[0]["record"]["normalized"]["privacy_gate"], "privacy-review-required") + self.assertIsNone(result.quarantined[0]["record"]["normalized"]["coordinates"]) if __name__ == "__main__": From 3b31952499c0f496105c8ba7ab8910b34521c37a Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sun, 13 Sep 2026 23:00:27 -0700 Subject: [PATCH 062/311] Record current US FSIS acquisition blocker --- docs/country-recon-us.md | 13 +++++++++++++ docs/source-status.json | 2 +- docs/source-status.md | 2 +- 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/docs/country-recon-us.md b/docs/country-recon-us.md index e4c8c49..95f728c 100644 --- a/docs/country-recon-us.md +++ b/docs/country-recon-us.md @@ -22,6 +22,19 @@ Legacy FSIS activity strings are multi-valued and overlap: Meat Slaughter 1,103; Acquire only official FSIS downloads or documented APHIS public-search/export workflows with UTC retrieval, effective/publication date, byte size, SHA-256, URL, and adapter/config version. Validate content type, signatures, headers, IDs, dates, coordinates, duplicates, and category vocabulary; quarantine HTML/login responses and sharp changes. Preserve raw/parsed layers separately in ignored restricted staging, keep source values/identifiers, avoid names/phones in logs, and never fuzzy-merge FSIS, APHIS annual reports, and inspections. Suppress personal names, direct contacts, residential/private locations, and precise points where ETHICS.md requires. A successful fetch is not publication approval. +## 2026-09-14 FSIS access blocker + +The official FSIS MPI Directory route remains the identified source, but the +current CSV links could not be safely acquired on 2026-09-14: ordinary direct +and browser page access returned HTTP 403. Stale 2025 links were not used, and +no artifact, byte count, or hash was retained. This is an access blocker, not +evidence that the source is unavailable or that its terms permit reuse. + +Next step: obtain an authorized current FSIS export route or access context, +then privately record the final URL, retrieval time, effective/publication date, +content type, byte size, SHA-256, terms, and schema before any adapter or +publication decision. + ## Blockers and recommendation No safe bounded private fetch was performed, so current hashes/bytes and deterministic reproduction are intentionally unavailable. FSIS is the strongest automation candidate because recurring CSV downloads and source descriptions are available. APHIS is secondary/manual/UI-mediated and should be an explicitly versioned, human-reviewed annual-report adapter or restricted manual input. Do not build a laboratory-supplier layer from APHIS records without a separately identified, licensed source. Existing Selenium/compiler code is not production-grade: obsolete selectors, no provenance manifest, quarantine, terms/schema/privacy gates, and unsafe duplicate handling. diff --git a/docs/source-status.json b/docs/source-status.json index bd174b5..3436b17 100644 --- a/docs/source-status.json +++ b/docs/source-status.json @@ -17,7 +17,7 @@ {"source_id":"de.locations","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["pipeline/source_registry.json"],"next_action":"Replace session-bound legacy URL with a verified stable BVL endpoint and confirm terms/schema."}, {"source_id":"ca.locations","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ca.md","pipeline/source_registry.json"],"next_action":"Keep federal/export and Ontario candidates separate; verify an authoritative permitted artifact, terms, schema, coverage, and privacy before acquisition."}, {"source_id":"es.locations","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-es.md","pipeline/source_registry.json"],"next_action":"Resolve a permitted current AESAN/MAPA artifact or query route, then verify export/schema, rights, effective-date semantics, sector coverage, privacy, and legacy/source boundaries before acquisition."}, - {"source_id":"us.fsis","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-us.md","pipeline/source_registry.json"],"next_action":"Acquire and snapshot a permitted current FSIS MPI artifact; reconcile legacy inventory only as a comparison and verify terms, schema, coverage, and privacy."}, + {"source_id":"us.fsis","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-us.md","pipeline/source_registry.json"],"next_action":"Obtain authorized access to a current FSIS MPI export after 403 responses; then record URL, retrieval/effective dates, content type, byte size, SHA-256, terms, and schema before adapter or publication review."}, {"source_id":"us.aphis","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-us.md","pipeline/source_registry.json"],"next_action":"Verify the current APHIS export workflow and keep licences, registrants, annual reports, and exception reports separately attributed before acquisition."}, {"source_id":"us.inspections","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-us.md","pipeline/source_registry.json"],"next_action":"Verify a current inspection export and use explicit, reviewable identity matching rather than treating observations as a facility master."} ] diff --git a/docs/source-status.md b/docs/source-status.md index 29de6f1..ce373db 100644 --- a/docs/source-status.md +++ b/docs/source-status.md @@ -24,7 +24,7 @@ No last-success timestamp is invented. Private artifacts are not proof of a publ | `de.locations` | partial | not_run | not_run | blocked | Legacy session URL and current BVL endpoint/schema/terms remain unresolved | | `ca.locations` | verified | not_run | not_run | blocked | Federal/export and Ontario candidates are documented separately; verify permitted artifact, terms, schema, coverage, and privacy before acquisition | | `es.locations` | partial | blocked | not_run | blocked | AESAN RGSEAA and MAPA sector routes are documented, but direct acquisition was refused; rights, export/schema, effective dates, sector coverage, privacy, and legacy/source boundaries remain unresolved | -| `us.fsis` | verified | not_run | not_run | blocked | FSIS MPI route is documented; acquire a permitted current artifact and treat legacy inventory only as a comparison | +| `us.fsis` | verified | blocked | not_run | blocked | Official FSIS MPI route is documented, but current CSV access returned 403; obtain authorized export access and record provenance/schema before adapter or publication review | | `us.aphis` | partial | not_run | not_run | blocked | APHIS export workflow and separate report/license provenance require review | | `us.inspections` | partial | not_run | not_run | blocked | Inspection observations require a current export and explicit identity matching | From 0ff8c5e2299b280912409e984a0277c3442b206f Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sun, 13 Sep 2026 23:01:04 -0700 Subject: [PATCH 063/311] Preserve unapproved private preview status --- frontend/src/api/DevCandidatePreviewRepository.ts | 2 +- frontend/src/domain/location.ts | 4 ++-- frontend/tests/unit/devPreviewContract.test.ts | 7 +++++++ 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/frontend/src/api/DevCandidatePreviewRepository.ts b/frontend/src/api/DevCandidatePreviewRepository.ts index c37ce7d..5080379 100644 --- a/frontend/src/api/DevCandidatePreviewRepository.ts +++ b/frontend/src/api/DevCandidatePreviewRepository.ts @@ -30,7 +30,7 @@ export class DevCandidatePreviewRepository { id: row.facility_id, name: row.canonical_name, region: row.city ?? row.country_code, category: row.category, lat: row.latitude, lon: row.longitude, observed: 'candidate observation date unavailable', source: row.provenance_source_name, candidateId: row.candidate_id, sourceRecordId: row.source_record_id, previewLabel: row.preview_label, releaseStatus: row.release_status, coverageScope: parsed.data.meta.coverage_scope, - evidence: { sourceType: row.source_type, factualReviewStatus: row.factual_review_status, reviewerRole: null, privacyScreeningStatus: row.privacy_screening_status, projectApproval: 'pending', publicationProfile: 'community', publicationWarning: row.preview_label, sourceId: row.provenance_source_id, sourceUrl: row.provenance_source_url, retrievedAt: row.provenance_retrieved_at, displayPrecision: row.display_precision, lifecycleStatus: 'status_unknown', observationCount: null }, + evidence: { sourceType: row.source_type, factualReviewStatus: row.factual_review_status, reviewerRole: null, privacyScreeningStatus: row.privacy_screening_status, projectApproval: false, publicationProfile: null, publicationWarning: row.preview_label, sourceId: row.provenance_source_id, sourceUrl: row.provenance_source_url, retrievedAt: row.provenance_retrieved_at, displayPrecision: row.display_precision, lifecycleStatus: 'status_unknown', observationCount: null }, })); } } diff --git a/frontend/src/domain/location.ts b/frontend/src/domain/location.ts index 56734c2..a73822f 100644 --- a/frontend/src/domain/location.ts +++ b/frontend/src/domain/location.ts @@ -6,8 +6,8 @@ export type LocationEvidence = Readonly<{ factualReviewStatus: 'unreviewed' | 'reviewed' | 'rejected'; reviewerRole: string | null; privacyScreeningStatus: 'passed'; - projectApproval: 'pending' | 'approved'; - publicationProfile: 'official' | 'secondary' | 'community'; + projectApproval: 'pending' | 'approved' | false; + publicationProfile: 'official' | 'secondary' | 'community' | null; publicationWarning: string | null; sourceId: string; sourceUrl: string; diff --git a/frontend/tests/unit/devPreviewContract.test.ts b/frontend/tests/unit/devPreviewContract.test.ts index 854baee..eca9315 100644 --- a/frontend/tests/unit/devPreviewContract.test.ts +++ b/frontend/tests/unit/devPreviewContract.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from 'vitest'; import { canOpenDevPreview, DEV_PREVIEW_LABEL, DEV_PREVIEW_PATH, DEV_PREVIEW_QUERY, DEV_PREVIEW_TOKEN_HEADER } from '../../src/features/devPreview/devPreviewContract'; +import { DevCandidatePreviewRepository } from '../../src/api/DevCandidatePreviewRepository'; describe('dev preview boundary', () => { it('requires both a development build and the explicit mode', () => { @@ -15,4 +16,10 @@ describe('dev preview boundary', () => { expect(DEV_PREVIEW_TOKEN_HEADER).toBe('X-UEC-Dev-Preview-Token'); expect(DEV_PREVIEW_LABEL).toContain('NOT REVIEWED OR PUBLISHED'); }); + + it('preserves candidate unapproved and unpublished semantics', async () => { + const fetcher = async () => new Response(JSON.stringify({ api_version: 'dev-preview-v1', data: [{ candidate_id: 'candidate-1', source_record_id: 'source-row-1', facility_id: 'facility-1', canonical_name: 'Candidate facility', country_code: 'DK', city: null, category: 'dairy', display_precision: 'unmapped', latitude: null, longitude: null, source_type: 'official', provenance_source_id: 'source-1', provenance_source_name: 'Private source', provenance_source_url: 'https://example.test/source', provenance_retrieved_at: '2026-01-01T00:00:00Z', factual_review_status: 'unreviewed', privacy_screening_status: 'passed', project_approval: false, release_id: 'candidate-release', release_status: 'candidate', preview_label: DEV_PREVIEW_LABEL }], meta: { test_only: true, private_preview: true, profile: null, coverage_scope: 'candidate_release_only', next_cursor: null } })); + const result = await new DevCandidatePreviewRepository(fetcher).list('operator-token'); + expect(result[0]?.evidence).toMatchObject({ projectApproval: false, publicationProfile: null, publicationWarning: DEV_PREVIEW_LABEL }); + }); }); From d25d36973d98747729e06feb50a8eab6227ea96a Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sun, 13 Sep 2026 23:04:44 -0700 Subject: [PATCH 064/311] Keep public export controls out of private preview --- frontend/src/app/App.svelte | 10 +++++----- frontend/src/features/devPreview/devPreviewContract.ts | 1 + frontend/src/styles/research.css | 2 ++ frontend/tests/unit/devPreviewContract.test.ts | 5 ++++- 4 files changed, 12 insertions(+), 6 deletions(-) diff --git a/frontend/src/app/App.svelte b/frontend/src/app/App.svelte index 87cdcfa..0c69cd9 100644 --- a/frontend/src/app/App.svelte +++ b/frontend/src/app/App.svelte @@ -14,7 +14,7 @@ import ReleaseContext from '../ui/ReleaseContext.svelte'; import ExportControl from '../ui/ExportControl.svelte'; import ScaleNarrative from '../features/scale/ScaleNarrative.svelte'; - import { canOpenDevPreview, DEV_PREVIEW_LABEL } from '../features/devPreview/devPreviewContract'; + import { canOpenDevPreview, DEV_PREVIEW_LABEL, devPreviewExportLabel } from '../features/devPreview/devPreviewContract'; import type { DevCandidate } from '../api/DevCandidatePreviewRepository'; let profile: Profile = 'curated'; @@ -46,8 +46,8 @@ $: filters = { search, region, category }; $: source = devPreviewMode ? previewRows : localMode ? loaded : (profile === 'community' ? locations.slice(0, 1) : locations); $: visibleLocations = filterLocations(source, filters); - $: exportPreview = previewExport(makeExportModel(visibleLocations, profile, release)); - $: eligibleExport = localMode && profile === 'curated' && localStatus === 'ready' && Boolean(release); + $: exportPreview = devPreviewExportLabel(devPreviewMode) ?? previewExport(makeExportModel(visibleLocations, profile, release)); + $: eligibleExport = !devPreviewMode && localMode && profile === 'curated' && localStatus === 'ready' && Boolean(release); $: profileLabel = profile === 'curated' ? 'Curated release' : 'Community claims'; let lastRemoteQuery = ''; // Search is intentionally excluded: the API has no q contract, so it filters @@ -98,7 +98,7 @@ catch (error) { paging = false; if (!append) activeListKey = ''; if (generation !== listGeneration) return; const kind = error && typeof error === 'object' && 'kind' in error ? (error as { kind: string }).kind : 'error'; localStatus = kind === 'no-release' ? 'no-release' : 'error'; localError = error instanceof Error ? error.message : 'Local V2 response was rejected safely.'; loaded = []; selected = undefined; } }; const downloadCsv = async () => { - if (!eligibleExport || exportBusy) return; exportBusy = true; exportError = ''; + if (devPreviewMode || !eligibleExport || exportBusy) return; exportBusy = true; exportError = ''; try { const result = await csvRepo.download('official'); release = result.releaseId; manifestSha256 = result.manifestSha256; const url = URL.createObjectURL(new Blob([result.body], { type: 'text/csv;charset=utf-8' })); const anchor = document.createElement('a'); anchor.href = url; anchor.download = `uec-v2-${result.releaseId}.csv`; anchor.click(); URL.revokeObjectURL(url); } catch (error) { const status = error && typeof error === 'object' && 'status' in error ? (error as { status: number }).status : undefined; exportError = status === 400 ? 'Choose an explicit supported profile before exporting.' : status === 404 ? 'No eligible promoted release with a manifest is available.' : status === 429 ? 'Export is temporarily rate-limited; try again later.' : error instanceof Error ? error.message : 'The CSV export could not be prepared safely.'; } finally { exportBusy = false; } @@ -134,7 +134,7 @@ Until Every Cage · evidence desk -
+
UNTIL EVERY CAGE V2 / FIELD NOTE

EVIDENCE DESK · {devPreviewMode ? 'PRIVATE CANDIDATE PREVIEW' : localMode ? 'LOCAL V2 API' : 'SYNTHETIC PREVIEW'}

See what a record can—and cannot—tell us.

Start with the source profile, narrow the visible evidence, then inspect what a record can—and cannot—tell us.

{#if devPreviewMode}{/if} diff --git a/frontend/src/features/devPreview/devPreviewContract.ts b/frontend/src/features/devPreview/devPreviewContract.ts index 8077a9f..72394fe 100644 --- a/frontend/src/features/devPreview/devPreviewContract.ts +++ b/frontend/src/features/devPreview/devPreviewContract.ts @@ -14,3 +14,4 @@ export const canOpenDevPreview = (isDevelopment: boolean, requestedMode: string /** Persistent copy used on every private-preview surface. */ export const DEV_PREVIEW_LABEL = 'PRIVATE TEST DATA — NOT REVIEWED OR PUBLISHED'; +export const devPreviewExportLabel = (isPreview: boolean): string | null => isPreview ? 'Private test preview — export unavailable.' : null; diff --git a/frontend/src/styles/research.css b/frontend/src/styles/research.css index 58fe906..75c0578 100644 --- a/frontend/src/styles/research.css +++ b/frontend/src/styles/research.css @@ -1,2 +1,4 @@ .research-bar{display:grid;grid-template-columns:minmax(220px,.8fr) 1.6fr;gap:30px;border-top:1px solid #cfc4b2;border-bottom:1px solid #cfc4b2;padding:24px 0;margin-bottom:22px}.research-bar h2,.results-head h2{font:600 1.8rem Georgia,serif;margin:0 0 8px}.research-bar p:not(.eyebrow){color:#5f5549;line-height:1.5;margin:0}.research-bar .toolbar{border:0;padding:0;margin:0;display:grid;grid-template-columns:repeat(2,minmax(120px,1fr));gap:12px}.research-bar .toolbar label{min-width:0}.research-bar input,.research-bar select{display:block;width:100%;margin-top:7px;background:#fbf8f2;border:1px solid #b9ad9b;border-radius:3px;padding:10px 11px;color:#17283b;font-size:.95rem}.results-head{display:flex;justify-content:space-between;align-items:end;margin:26px 0 12px}.results-head h2 span{font:700 .85rem ui-sans-serif;color:#a34927;vertical-align:middle}.scope{color:#62594e;font-size:.8rem}.release-panel{display:flex;flex-wrap:wrap;gap:18px;border:1px solid #cfc4b2;background:#ece4d8;padding:18px;margin-top:22px}.release-panel div{min-width:130px}.release-panel span{display:block;color:#62594e;font-size:.65rem;letter-spacing:.12em;margin-bottom:5px}.release-panel strong{font-size:.88rem;overflow-wrap:anywhere}.release-panel .digest{font-family:ui-monospace,monospace;font-size:.72rem}.release-panel p{width:100%;margin:0;color:#62594e;font-size:.78rem;line-height:1.45}.export-control{display:flex;align-items:center;gap:18px;padding:15px 0;border-bottom:1px solid #cfc4b2}.export-control button,.state button{width:auto;background:#a34927;color:#fff;padding:11px 15px;border-radius:3px;font-weight:700}.export-control button:disabled{background:#b9ad9b;cursor:not-allowed}.export-control p{margin:0;color:#62594e;font-size:.78rem;line-height:1.4}.export-control .export-error{color:#a34927}.empty{padding:24px 18px;color:#62594e;line-height:1.5}.guidance{margin-top:28px;border-top:1px solid #cfc4b2;border-bottom:1px solid #cfc4b2}.guidance-toggle{align-items:center;justify-content:space-between;padding:18px 0;border:0}.guidance-toggle>span:first-child{display:flex;flex-direction:column;gap:4px}.guidance-toggle .eyebrow{margin:0}.guidance-toggle>span:last-child{color:#a34927;font-size:.8rem;font-weight:700}.guidance-body{max-width:680px;padding:0 0 20px;line-height:1.55;color:#4f5c69}.guidance-body a{color:#a34927;font-weight:700}@media(max-width:680px){.research-bar{display:block}.research-bar .toolbar{margin-top:22px}.results-head{display:block}.scope{margin-top:10px}.export-control{display:block}.export-control p{margin-top:10px}.release-panel{display:block}.release-panel div{margin-bottom:13px}} .page-context{margin:0 0 16px;color:#62594e;line-height:1.5;font-size:.85rem}.claim-warning{color:#8d351c!important;font-weight:700}.evidence{display:grid;grid-template-columns:repeat(2,minmax(0,1fr));gap:12px;border-top:1px solid #cfc4b2;margin:28px 0 0;padding-top:18px;font-size:.82rem}.evidence div{min-width:0}.evidence dt{color:#62594e;font-size:.7rem;text-transform:uppercase;letter-spacing:.08em}.evidence dd{margin:4px 0 0;overflow-wrap:anywhere}.evidence a{color:#a34927}.state.error button{display:inline-flex;margin:12px 12px 0 0}.state.error h2:focus{outline:3px solid #a34927;outline-offset:4px}@media(max-width:680px){.evidence{grid-template-columns:1fr}} +.dev-preview .phase-controls, +.dev-preview .export-preview { display: none !important; } diff --git a/frontend/tests/unit/devPreviewContract.test.ts b/frontend/tests/unit/devPreviewContract.test.ts index eca9315..fa26731 100644 --- a/frontend/tests/unit/devPreviewContract.test.ts +++ b/frontend/tests/unit/devPreviewContract.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import { canOpenDevPreview, DEV_PREVIEW_LABEL, DEV_PREVIEW_PATH, DEV_PREVIEW_QUERY, DEV_PREVIEW_TOKEN_HEADER } from '../../src/features/devPreview/devPreviewContract'; +import { canOpenDevPreview, DEV_PREVIEW_LABEL, DEV_PREVIEW_PATH, DEV_PREVIEW_QUERY, DEV_PREVIEW_TOKEN_HEADER, devPreviewExportLabel } from '../../src/features/devPreview/devPreviewContract'; import { DevCandidatePreviewRepository } from '../../src/api/DevCandidatePreviewRepository'; describe('dev preview boundary', () => { @@ -15,6 +15,9 @@ describe('dev preview boundary', () => { expect(DEV_PREVIEW_PATH).not.toContain('/api/v2/'); expect(DEV_PREVIEW_TOKEN_HEADER).toBe('X-UEC-Dev-Preview-Token'); expect(DEV_PREVIEW_LABEL).toContain('NOT REVIEWED OR PUBLISHED'); + expect(devPreviewExportLabel(true)).toContain('export unavailable'); + expect(devPreviewExportLabel(true)).not.toContain('curated'); + expect(devPreviewExportLabel(false)).toBeNull(); }); it('preserves candidate unapproved and unpublished semantics', async () => { From 8a4e7e78c0e06fe899e3667e5f2aafdeb36a3ed0 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sun, 13 Sep 2026 23:07:42 -0700 Subject: [PATCH 065/311] Unmount public export control in private preview --- frontend/src/app/App.svelte | 4 ++-- frontend/src/features/devPreview/devPreviewContract.ts | 1 + frontend/tests/unit/devPreviewContract.test.ts | 4 +++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/frontend/src/app/App.svelte b/frontend/src/app/App.svelte index 0c69cd9..6fd047a 100644 --- a/frontend/src/app/App.svelte +++ b/frontend/src/app/App.svelte @@ -14,7 +14,7 @@ import ReleaseContext from '../ui/ReleaseContext.svelte'; import ExportControl from '../ui/ExportControl.svelte'; import ScaleNarrative from '../features/scale/ScaleNarrative.svelte'; - import { canOpenDevPreview, DEV_PREVIEW_LABEL, devPreviewExportLabel } from '../features/devPreview/devPreviewContract'; + import { canOpenDevPreview, DEV_PREVIEW_LABEL, devPreviewExportLabel, canMountPublicExport } from '../features/devPreview/devPreviewContract'; import type { DevCandidate } from '../api/DevCandidatePreviewRepository'; let profile: Profile = 'curated'; @@ -151,7 +151,7 @@
{#if selected}

03 / RECORD DETAIL

{selected.name}

{selected.region} · {selected.category}

{#if selected.evidence?.publicationProfile === 'community' && selected.evidence.factualReviewStatus === 'unreviewed'}

{selected.evidence.publicationWarning ?? 'Unreviewed community claim — not verified by Until Every Cage'}

{/if}
OBSERVED{selected.observed}
MAP STATUS{selected.lat === null ? 'No publishable map location' : selected.evidence?.displayPrecision === 'exact' ? 'Exact display point' : 'Approximate display point'}
{#if selected.evidence}
Source origin
{selected.evidence.sourceType === 'user_submitted' ? 'Community-submitted' : selected.evidence.sourceType === 'official' ? 'Government-sourced' : 'Secondary-sourced'}
Factual review
{selected.evidence.factualReviewStatus}{selected.evidence.reviewerRole ? ` · ${selected.evidence.reviewerRole}` : ' · reviewer role unavailable'}
Privacy screening
{selected.evidence.privacyScreeningStatus}
Project approval
{selected.evidence.projectApproval}
Published profile
{selected.evidence.publicationProfile} · release {release}
Source
{selected.source} · {selected.evidence.sourceId}
Retrieved
{selected.evidence.retrievedAt}
{/if}

READ WITH CARE

This is an observation in a particular release, not a guarantee of current operation. Source origin, factual review, privacy screening, and project approval are separate signals.

{/if}
{#if selected}

RECORD / {selected.id}

{/if} {/if} - {#if localMode && localStatus === 'ready'}{/if} + {#if localMode && localStatus === 'ready'}{#if canMountPublicExport(devPreviewMode)}{/if}{/if}
{#if showGuidance}

Do not infer closure, identity, or permission from a map point. For a correction, privacy concern, or suppression request, preserve the record ID and contact the project maintainer through the reporting channel on the ethics page. Do not include sensitive personal details in a public issue.

Read reporting guidance ↗
{/if}
{#if showMap}{/if}{#if showExport}

IN-MEMORY EXPORT PREVIEW

{profile} · {release}

Loaded results only. This preview is not a live or complete export.

{exportPreview}
{/if}
{localMode ? 'Local V2 mode · no fixture fallback' : 'Method preview · no live requests'}
{/if} diff --git a/frontend/src/features/devPreview/devPreviewContract.ts b/frontend/src/features/devPreview/devPreviewContract.ts index 72394fe..6db1605 100644 --- a/frontend/src/features/devPreview/devPreviewContract.ts +++ b/frontend/src/features/devPreview/devPreviewContract.ts @@ -15,3 +15,4 @@ export const canOpenDevPreview = (isDevelopment: boolean, requestedMode: string /** Persistent copy used on every private-preview surface. */ export const DEV_PREVIEW_LABEL = 'PRIVATE TEST DATA — NOT REVIEWED OR PUBLISHED'; export const devPreviewExportLabel = (isPreview: boolean): string | null => isPreview ? 'Private test preview — export unavailable.' : null; +export const canMountPublicExport = (isPreview: boolean): boolean => !isPreview; diff --git a/frontend/tests/unit/devPreviewContract.test.ts b/frontend/tests/unit/devPreviewContract.test.ts index fa26731..d6dabe9 100644 --- a/frontend/tests/unit/devPreviewContract.test.ts +++ b/frontend/tests/unit/devPreviewContract.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import { canOpenDevPreview, DEV_PREVIEW_LABEL, DEV_PREVIEW_PATH, DEV_PREVIEW_QUERY, DEV_PREVIEW_TOKEN_HEADER, devPreviewExportLabel } from '../../src/features/devPreview/devPreviewContract'; +import { canOpenDevPreview, DEV_PREVIEW_LABEL, DEV_PREVIEW_PATH, DEV_PREVIEW_QUERY, DEV_PREVIEW_TOKEN_HEADER, devPreviewExportLabel, canMountPublicExport } from '../../src/features/devPreview/devPreviewContract'; import { DevCandidatePreviewRepository } from '../../src/api/DevCandidatePreviewRepository'; describe('dev preview boundary', () => { @@ -18,6 +18,8 @@ describe('dev preview boundary', () => { expect(devPreviewExportLabel(true)).toContain('export unavailable'); expect(devPreviewExportLabel(true)).not.toContain('curated'); expect(devPreviewExportLabel(false)).toBeNull(); + expect(canMountPublicExport(true)).toBe(false); // includes ?preview=dev-candidates&mode=local-v2 + expect(canMountPublicExport(false)).toBe(true); }); it('preserves candidate unapproved and unpublished semantics', async () => { From 46b4f95f235a07a14673111d1dabb4eda5bbcdc9 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sun, 13 Sep 2026 23:14:04 -0700 Subject: [PATCH 066/311] Add private candidate handoff contract --- pipeline/contracts/CANDIDATE-HANDOFF.md | 14 ++++++++ pipeline/contracts/candidate_handoff.py | 37 ++++++++++++++++++++ pipeline/contracts/test_candidate_handoff.py | 20 +++++++++++ 3 files changed, 71 insertions(+) create mode 100644 pipeline/contracts/CANDIDATE-HANDOFF.md create mode 100644 pipeline/contracts/candidate_handoff.py create mode 100644 pipeline/contracts/test_candidate_handoff.py diff --git a/pipeline/contracts/CANDIDATE-HANDOFF.md b/pipeline/contracts/CANDIDATE-HANDOFF.md new file mode 100644 index 0000000..d60d860 --- /dev/null +++ b/pipeline/contracts/CANDIDATE-HANDOFF.md @@ -0,0 +1,14 @@ +# Private candidate handoff v1 + +`candidate_handoff.write_handoff` emits the importer-compatible `manifest.json` +and `normalized/records.jsonl`. Required provenance is copied from +`SourceArtifact`; rows retain private `source_values` and require an explicit +`normalized.establishment_id`, `source_id`, and `source_row`. The manifest is +always `release_state: not-created`, `publication_state: private-candidate`, +`review_state: review_required`, `privacy_gate: pending`, and +`coordinate_gate: review_required`. It never infers identities or approval. + +The Denmark adapter currently uses `source_record_key` rather than +`normalized.establishment_id`; it must receive a reviewed source-specific +mapping before this handoff can be used for Denmark import. No mapping is +invented by this contract. diff --git a/pipeline/contracts/candidate_handoff.py b/pipeline/contracts/candidate_handoff.py new file mode 100644 index 0000000..c15181d --- /dev/null +++ b/pipeline/contracts/candidate_handoff.py @@ -0,0 +1,37 @@ +"""Versioned, source-agnostic private candidate handoff contract.""" +from __future__ import annotations +import hashlib, json, os +from pathlib import Path +from typing import Any +from .adapter_contract import SourceArtifact + +CONTRACT_VERSION = "candidate-handoff-v1" + +def _atomic(path: Path, payload: bytes) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + tmp = path.with_name(path.name + ".tmp") + tmp.write_bytes(payload); os.replace(tmp, path) + +def write_handoff(run_dir: str | Path, rows: list[dict[str, Any]], artifact: SourceArtifact, + *, source_id: str, profile: str = "default") -> dict[str, Any]: + """Write importer-compatible JSONL/manifest, rejecting guessed identities.""" + for row in rows: + normalized = row.get("normalized") + if not isinstance(row.get("source_id"), str) or row.get("source_id") != source_id: + raise ValueError("candidate row source_id does not match manifest") + if row.get("source_row") is None or not isinstance(normalized, dict) or not normalized.get("establishment_id"): + raise ValueError("candidate row requires source_row and normalized.establishment_id") + payload = b"".join((json.dumps(row, ensure_ascii=False, sort_keys=True, default=list) + "\n").encode() for row in rows) + root = Path(run_dir); normalized_path = root / "normalized" / "records.jsonl"; _atomic(normalized_path, payload) + manifest = {"contract_version": CONTRACT_VERSION, "profile": profile, "source_id": source_id, + "source_url": artifact.source_url, "retrieved_at_utc": artifact.retrieved_at_utc, + "checksum_sha256": artifact.sha256, "byte_size": artifact.byte_size, + "code_version": artifact.code_version, "config_version": artifact.config_version, + "coverage": artifact.coverage, "normalized_rows": len(rows), + "normalized_sha256": hashlib.sha256(payload).hexdigest(), + "release_state": "not-created", "publication_state": "private-candidate", + "review_state": "review_required", "privacy_gate": "pending", + "coordinate_gate": "review_required"} + # The importer consumes the conventional manifest.json name. + _atomic(root / "manifest.json", (json.dumps(manifest, ensure_ascii=False, sort_keys=True, indent=2) + "\n").encode()) + return manifest diff --git a/pipeline/contracts/test_candidate_handoff.py b/pipeline/contracts/test_candidate_handoff.py new file mode 100644 index 0000000..d7a1bbf --- /dev/null +++ b/pipeline/contracts/test_candidate_handoff.py @@ -0,0 +1,20 @@ +import hashlib, tempfile, unittest +from pathlib import Path +from .adapter_contract import SourceArtifact +from .candidate_handoff import write_handoff, CONTRACT_VERSION + +class CandidateHandoffTests(unittest.TestCase): + def artifact(self): + return SourceArtifact("https://example.test/source", "2026-01-01T00:00:00Z", "a" * 64, 12, code_version="c1", config_version="v1", coverage="synthetic") + def test_emits_private_importer_contract(self): + row = {"source_id":"test.source", "source_row":2, "source_values":{"id":"A"}, "normalized":{"establishment_id":"A", "privacy_gate":"pending", "coordinate_gate":"review_required"}} + with tempfile.TemporaryDirectory() as d: + m = write_handoff(d, [row], self.artifact(), source_id="test.source") + self.assertEqual(m["contract_version"], CONTRACT_VERSION); self.assertEqual(m["release_state"], "not-created") + data = (Path(d) / "normalized/records.jsonl").read_bytes(); self.assertEqual(m["normalized_sha256"], hashlib.sha256(data).hexdigest()) + def test_rejects_guessed_or_missing_identity(self): + bad = {"source_id":"test.source", "source_row":2, "normalized":{"name":"guess"}} + with self.assertRaisesRegex(ValueError, "establishment_id"): + write_handoff(tempfile.mkdtemp(), [bad], self.artifact(), source_id="test.source") + +if __name__ == "__main__": unittest.main() From a14615c27ee9a7aaab6202e7e3bf692e6ad49711 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sun, 13 Sep 2026 23:16:54 -0700 Subject: [PATCH 067/311] Import private candidates into marked disposable database --- docker-compose.e2e.yml | 2 + .../disposable-candidate-import.md | 43 ++++ .../scripts/maintenance/import-candidate.py | 184 ++++++++++++++++++ pipeline/tests/e2e/disposable-marker.sql | 12 ++ pipeline/tests/test_import_candidate.py | 83 ++++++++ 5 files changed, 324 insertions(+) create mode 100644 docs/architecture/disposable-candidate-import.md create mode 100644 pipeline/scripts/maintenance/import-candidate.py create mode 100644 pipeline/tests/e2e/disposable-marker.sql create mode 100644 pipeline/tests/test_import_candidate.py diff --git a/docker-compose.e2e.yml b/docker-compose.e2e.yml index 7712ab6..eae896a 100644 --- a/docker-compose.e2e.yml +++ b/docker-compose.e2e.yml @@ -17,3 +17,5 @@ services: start_period: 10s ports: - "${UEC_E2E_DB_PORT:-55432}:5432" + volumes: + - "./pipeline/tests/e2e/disposable-marker.sql:/docker-entrypoint-initdb.d/uec-disposable-marker.sql:ro" diff --git a/docs/architecture/disposable-candidate-import.md b/docs/architecture/disposable-candidate-import.md new file mode 100644 index 0000000..9f86949 --- /dev/null +++ b/docs/architecture/disposable-candidate-import.md @@ -0,0 +1,43 @@ +# Disposable candidate import + +`pipeline/scripts/maintenance/import-candidate.py` is a private development +handoff from a validated adapter staging run into PostgreSQL. It is not a +release or publication command. + +Run it only against a disposable local database with an explicit acknowledgement: + +```powershell +python pipeline/scripts/maintenance/import-candidate.py ` + --manifest data/staging///manifest.json ` + --normalized data/staging///normalized/records.jsonl ` + --raw data/staging///raw/source-artifact.bin ` + --release-id candidate-- ` + --database-url postgresql://uec:...@127.0.0.1:5433/uec ` + --disposable-db +``` + +The command refuses a missing acknowledgement, non-loopback host, default +PostgreSQL port, non-UEC database name, non-candidate release ID, mismatched +normalized/raw hash/count, or a manifest that is already released. In addition, +the connected server must contain the exact `uec.disposable_import_guard` marker +installed by `docker-compose.e2e.yml`, matching both `current_database()` and +`current_user`; the CLI flag alone never authorizes writes. `--reset` is +also refused because evidence and release membership are append-only. To +rebuild, stop the local stack and recreate its disposable volume using the +existing local maintenance recipe, after checking retention obligations. + +The transaction records artifact and acquisition provenance, source records, +facilities, candidate observations, candidate release membership, and an +append-only pending publication review event. Every imported observation starts +with `default_visible=false`, `classification_review_status=review_required`, +`coordinate_review_status=review_required`, `privacy_screening_status=pending`, +`maintainer_approval=pending`, and `publication_eligible=false`. The importer +never creates geocode approval, privacy clearance, validation, promotion, or +public visibility. Rerunning the same staging run is idempotent for source +records and release membership. + +Raw artifacts are retained outside Git and only their metadata is stored in the +database. `source_values` are private evidence and are never selected by the +public API or development preview. Docker-backed E2E remains a CI requirement; +local Windows runs must report the Docker Desktop access-denied condition rather +than claiming an end-to-end pass. diff --git a/pipeline/scripts/maintenance/import-candidate.py b/pipeline/scripts/maintenance/import-candidate.py new file mode 100644 index 0000000..70e1d0f --- /dev/null +++ b/pipeline/scripts/maintenance/import-candidate.py @@ -0,0 +1,184 @@ +"""Import a validated private staging run into a disposable candidate database. + +This is intentionally a small handoff bridge, not a publication importer. It +only accepts an explicitly marked local database and creates candidate rows +with review-required defaults. Raw artifacts remain in the caller-owned +staging/object-storage location; the database stores metadata and private +normalized evidence only. +""" +from __future__ import annotations + +import argparse +import hashlib +import json +import os +from pathlib import Path +from urllib.parse import urlparse +import uuid + +import psycopg + + +class CandidateImportError(ValueError): + pass + + +DISPOSABLE_MARKER = "uec-e2e-disposable-v1" + + +def require_disposable_database(database_url: str, acknowledged: bool) -> None: + """Reject production/shared targets before opening a connection.""" + if not acknowledged: + raise CandidateImportError("refusing import: pass --disposable-db explicitly") + parsed = urlparse(database_url) + host = (parsed.hostname or "").lower() + if host not in {"127.0.0.1", "::1", "localhost"}: + raise CandidateImportError("refusing import: database host is not loopback") + if parsed.port in (None, 5432): + raise CandidateImportError("refusing import: disposable database must use a non-default port") + database = (parsed.path or "").lstrip("/").lower() + if not database.startswith("uec"): + raise CandidateImportError("refusing import: database name is not a UEC disposable database") + + +def verify_disposable_marker(connection) -> None: + """Require a marker provisioned by the disposable DB image, not the CLI.""" + row = connection.execute( + """SELECT marker + FROM uec.disposable_import_guard + WHERE marker=%s AND database_name=current_database() AND role_name=current_user""", + (DISPOSABLE_MARKER,), + ).fetchone() + if row != (DISPOSABLE_MARKER,): + raise CandidateImportError("refusing import: database lacks the exact disposable server marker") + + +def load_inputs(manifest_path: Path, normalized_path: Path, raw_path: Path) -> tuple[dict, list[dict]]: + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + required = {"source_id", "source_url", "retrieved_at_utc", "checksum_sha256", "byte_size", + "normalized_rows", "normalized_sha256", "release_state", "publication_state"} + missing = sorted(required - manifest.keys()) + if missing: + raise CandidateImportError(f"manifest missing required keys: {', '.join(missing)}") + if manifest["release_state"] != "not-created" or manifest["publication_state"] != "private-candidate": + raise CandidateImportError("manifest is not an unpromoted private candidate") + raw = raw_path.read_bytes() + if hashlib.sha256(raw).hexdigest() != manifest["checksum_sha256"] or len(raw) != int(manifest["byte_size"]): + raise CandidateImportError("raw artifact checksum or byte size does not match manifest") + raw = normalized_path.read_bytes() + if hashlib.sha256(raw).hexdigest() != manifest["normalized_sha256"]: + raise CandidateImportError("normalized JSONL checksum does not match manifest") + rows = [json.loads(line) for line in raw.decode("utf-8").splitlines() if line] + if len(rows) != int(manifest["normalized_rows"]): + raise CandidateImportError("normalized row count does not match manifest") + return manifest, rows + + +def _record_parts(record: dict) -> tuple[str, dict]: + normalized = record.get("normalized") + if not isinstance(normalized, dict): + raise CandidateImportError("record has no normalized object") + source_id = record.get("source_id") + source_row = record.get("source_row") + establishment_id = normalized.get("establishment_id") + if not isinstance(source_id, str) or not source_id or source_row is None or not establishment_id: + raise CandidateImportError("record must contain source_id, source_row, and establishment_id") + key = f"{source_row}:{establishment_id}" + return key, normalized + + +def _country_code(manifest: dict, normalized: dict) -> str: + explicit = manifest.get("country_code") or normalized.get("country_code") + if explicit: + return str(explicit)[:2].upper() + # Do not turn a country name into an invented code. These are the only + # source-country names currently present in the validated adapters. + return {"Denmark": "DK", "England": "GB", "Wales": "GB"}.get(normalized.get("nation"), "ZZ") + + +def import_candidate(database_url: str, manifest: dict, rows: list[dict], release_id: str, reset: bool) -> int: + """Append one candidate release; never promotes or marks review complete.""" + if reset: + raise CandidateImportError( + "--reset is intentionally refused: append-only evidence cannot be deleted; " + "recreate the disposable database with the local-v2 maintenance recipe" + ) + now = manifest["retrieved_at_utc"] + ruleset = str(manifest.get("config_version") or manifest.get("schema_version") or "unknown") + with psycopg.connect(database_url) as db: + verify_disposable_marker(db) + with db.transaction(): + db.execute("""INSERT INTO uec.sources(source_id,country_code,name,official_url,access_method) + VALUES (%s,%s,%s,%s,'validated-private-staging') + ON CONFLICT (source_id) DO NOTHING""", + (manifest["source_id"], str(manifest.get("country_code", "ZZ"))[:2].upper(), + manifest["source_id"], manifest["source_url"])) + db.execute("""INSERT INTO uec.raw_artifacts(storage_key,sha256,byte_size,media_type,retrieved_at) + VALUES (%s,%s,%s,'application/octet-stream',%s) + ON CONFLICT (sha256) DO NOTHING""", + (f"private-staging/{manifest['source_id']}/{manifest['checksum_sha256']}", + manifest["checksum_sha256"], int(manifest["byte_size"]), now)) + artifact_id = db.execute("SELECT artifact_id FROM uec.raw_artifacts WHERE sha256=%s", + (manifest["checksum_sha256"],)).fetchone()[0] + db.execute("""INSERT INTO uec.acquisition_runs(source_id,checked_at,retrieved_at,ingested_at,status,source_url,code_version,config_version) + VALUES (%s,%s,%s,now(),'changed',%s,%s,%s)""", + (manifest["source_id"], now, now, manifest["source_url"], + manifest.get("code_version", "unknown"), manifest.get("config_version", "unknown"))) + run_id = db.execute("SELECT run_id FROM uec.acquisition_runs WHERE source_id=%s ORDER BY ingested_at DESC LIMIT 1", + (manifest["source_id"],)).fetchone()[0] + db.execute("INSERT INTO uec.acquisition_run_artifacts(run_id,artifact_id) VALUES (%s,%s) ON CONFLICT DO NOTHING", + (run_id, artifact_id)) + db.execute("""INSERT INTO uec.releases(release_id,status,ruleset_version,summary) + VALUES (%s,'candidate',%s,%s) + ON CONFLICT (release_id) DO NOTHING""", + (release_id, ruleset, json.dumps({"source_id": manifest["source_id"], "profile": manifest.get("profile")}))) + count = 0 + for record in rows: + key, normalized = _record_parts(record) + country = _country_code(manifest, normalized) + name = normalized.get("trading_name") + city = normalized.get("city") + record_id = db.execute("""INSERT INTO uec.source_records(source_id,source_record_key,artifact_id,raw_fields,parsed_at) + VALUES (%s,%s,%s,%s,%s) ON CONFLICT (source_id,source_record_key,artifact_id) + DO UPDATE SET source_state='present' RETURNING source_record_id""", + (manifest["source_id"], key, artifact_id, json.dumps({"source_values": record.get("source_values", {})}), now)).fetchone()[0] + existing = db.execute("""SELECT facility_id, observation_id FROM uec.observations + WHERE source_record_id=%s ORDER BY observed_at DESC, observation_id DESC LIMIT 1""", (record_id,)).fetchone() + if existing: + db.execute("INSERT INTO uec.release_members(release_id,facility_id,observation_id,default_visible) VALUES (%s,%s,%s,false) ON CONFLICT DO NOTHING", + (release_id, existing[0], existing[1])) + continue + facility_id = uuid.uuid4(); observation_id = uuid.uuid4() + db.execute("INSERT INTO uec.facilities(facility_id,canonical_name,country_code,city) VALUES (%s,%s,%s,%s)", + (facility_id, name, country, city)) + db.execute("""INSERT INTO uec.observations(observation_id,facility_id,source_record_id,observed_at,observation,classification,ruleset_id,rule_id,classification_category,classification_review_status,default_visible,coordinate_review_status,first_observed_at) + VALUES (%s,%s,%s,%s,%s,'{}',%s,'candidate','unclassified','review_required',false,'review_required',%s)""", + (observation_id, facility_id, record_id, now, json.dumps(normalized), ruleset, now)) + db.execute("INSERT INTO uec.release_members(release_id,facility_id,observation_id,default_visible) VALUES (%s,%s,%s,false)", + (release_id, facility_id, observation_id)) + db.execute("INSERT INTO uec.publication_review_events(source_record_id,release_id,factual_review_status,privacy_screening_status,maintainer_approval,publication_eligible) VALUES (%s,%s,'unreviewed','pending','pending',false)", + (record_id, release_id)) + count += 1 + return count + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--manifest", type=Path, required=True) + parser.add_argument("--raw", type=Path, required=True, help="preserved raw artifact; independently hash-checked") + parser.add_argument("--normalized", type=Path, required=True) + parser.add_argument("--release-id", required=True) + parser.add_argument("--database-url", default=os.environ.get("UEC_DATABASE_URL", "")) + parser.add_argument("--disposable-db", action="store_true", help="acknowledge this is a disposable local DB") + parser.add_argument("--reset", action="store_true", help="rebuild this candidate release only") + args = parser.parse_args() + require_disposable_database(args.database_url, args.disposable_db) + if not args.release_id.startswith("candidate-"): + raise CandidateImportError("release id must start with candidate-") + manifest, rows = load_inputs(args.manifest, args.normalized, args.raw) + print(f"imported {import_candidate(args.database_url, manifest, rows, args.release_id, args.reset)} candidate rows") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/pipeline/tests/e2e/disposable-marker.sql b/pipeline/tests/e2e/disposable-marker.sql new file mode 100644 index 0000000..c6dddb3 --- /dev/null +++ b/pipeline/tests/e2e/disposable-marker.sql @@ -0,0 +1,12 @@ +-- Installed only by the disposable E2E Postgres image on first initialization. +-- The importer also checks current_database/current_user, so a copied marker +-- row cannot authorize a different database or role. +CREATE SCHEMA IF NOT EXISTS uec; +CREATE TABLE IF NOT EXISTS uec.disposable_import_guard ( + marker TEXT PRIMARY KEY, + database_name TEXT NOT NULL, + role_name TEXT NOT NULL +); +INSERT INTO uec.disposable_import_guard(marker, database_name, role_name) +VALUES ('uec-e2e-disposable-v1', 'uec', 'uec') +ON CONFLICT (marker) DO NOTHING; diff --git a/pipeline/tests/test_import_candidate.py b/pipeline/tests/test_import_candidate.py new file mode 100644 index 0000000..1b586a2 --- /dev/null +++ b/pipeline/tests/test_import_candidate.py @@ -0,0 +1,83 @@ +import importlib.util +import hashlib +import json +import tempfile +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +SPEC = importlib.util.spec_from_file_location("import_candidate", ROOT / "scripts/maintenance/import-candidate.py") +MODULE = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(MODULE) + + +class CandidateImportContractTests(unittest.TestCase): + def test_server_marker_is_required_even_after_cli_acknowledgement(self): + class FakeConnection: + def execute(self, *_args): + class Result: + def fetchone(self): + return None + return Result() + with self.assertRaises(MODULE.CandidateImportError): + MODULE.verify_disposable_marker(FakeConnection()) + + class MarkedConnection(FakeConnection): + def execute(self, *_args): + class Result: + def fetchone(self): + return (MODULE.DISPOSABLE_MARKER,) + return Result() + MODULE.verify_disposable_marker(MarkedConnection()) + + def test_database_guard_requires_explicit_non_default_loopback_target(self): + for url, acknowledged in [ + ("postgresql://uec:x@db.example:5433/uec", True), + ("postgresql://uec:x@127.0.0.1:5432/uec", True), + ("postgresql://uec:x@127.0.0.1:5433/uec", False), + ]: + with self.subTest(url=url, acknowledged=acknowledged): + with self.assertRaises(MODULE.CandidateImportError): + MODULE.require_disposable_database(url, acknowledged) + MODULE.require_disposable_database("postgresql://uec:x@127.0.0.1:5433/uec", True) + + def test_manifest_and_normalized_hash_and_state_are_fail_closed(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + normalized = root / "records.jsonl" + normalized.write_text(json.dumps({"source_id": "test", "source_row": 2, "normalized": {"establishment_id": "A"}}) + "\n", encoding="utf-8") + manifest = { + "source_id": "test", "source_url": "https://example.invalid/source", + "retrieved_at_utc": "2026-09-13T00:00:00Z", "checksum_sha256": "0" * 64, + "byte_size": 1, "normalized_rows": 1, + "normalized_sha256": hashlib.sha256(normalized.read_bytes()).hexdigest(), + "release_state": "not-created", "publication_state": "private-candidate", + } + manifest_path = root / "manifest.json" + manifest_path.write_text(json.dumps(manifest), encoding="utf-8") + raw = root / "raw.bin" + raw.write_bytes(b"raw fixture") + manifest["checksum_sha256"] = hashlib.sha256(raw.read_bytes()).hexdigest() + manifest["byte_size"] = raw.stat().st_size + manifest_path.write_text(json.dumps(manifest), encoding="utf-8") + loaded, rows = MODULE.load_inputs(manifest_path, normalized, raw) + self.assertEqual(loaded["source_id"], "test") + self.assertEqual(len(rows), 1) + manifest["release_state"] = "promoted" + manifest_path.write_text(json.dumps(manifest), encoding="utf-8") + with self.assertRaises(MODULE.CandidateImportError): + MODULE.load_inputs(manifest_path, normalized, raw) + + def test_raw_artifact_mismatch_is_rejected(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + raw = root / "raw.bin"; raw.write_bytes(b"actual") + normalized = root / "records.jsonl"; normalized.write_text("", encoding="utf-8") + manifest = {"source_id":"test", "source_url":"https://example.invalid", "retrieved_at_utc":"2026-09-13T00:00:00Z", "checksum_sha256":"0"*64, "byte_size":6, "normalized_rows":0, "normalized_sha256":hashlib.sha256(b"").hexdigest(), "release_state":"not-created", "publication_state":"private-candidate"} + path = root / "manifest.json"; path.write_text(json.dumps(manifest), encoding="utf-8") + with self.assertRaises(MODULE.CandidateImportError): + MODULE.load_inputs(path, normalized, raw) + + +if __name__ == "__main__": + unittest.main() From 66b079d61e4ec0c98f1f17045796077a4f39966f Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sun, 13 Sep 2026 23:19:13 -0700 Subject: [PATCH 068/311] Add UK private candidate handoff bridge --- pipeline/sources/uk/fsa_approved/handoff.py | 65 ++++++++++++++++++ .../sources/uk/fsa_approved/test_handoff.py | 67 +++++++++++++++++++ 2 files changed, 132 insertions(+) create mode 100644 pipeline/sources/uk/fsa_approved/handoff.py create mode 100644 pipeline/sources/uk/fsa_approved/test_handoff.py diff --git a/pipeline/sources/uk/fsa_approved/handoff.py b/pipeline/sources/uk/fsa_approved/handoff.py new file mode 100644 index 0000000..d22919e --- /dev/null +++ b/pipeline/sources/uk/fsa_approved/handoff.py @@ -0,0 +1,65 @@ +"""Private candidate-handoff bridge for the reviewed FSA monthly profile.""" +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +from typing import Any + +from pipeline.contracts.adapter_contract import SourceArtifact +from pipeline.contracts.candidate_handoff import write_handoff + +from .adapter import FsaApprovedEstablishmentsAdapter + + +def _write_jsonl(path: Path, rows: list[dict[str, Any]]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + payload = b"".join( + (json.dumps(row, ensure_ascii=False, sort_keys=True, default=list) + "\n").encode() + for row in rows + ) + path.write_bytes(payload) + + +def write_private_monthly_handoff( + raw_path: str | Path, + run_dir: str | Path, + artifact: SourceArtifact, +) -> dict[str, Any]: + """Stage accepted FSA monthly rows through candidate-handoff-v1. + + This bridge is deliberately database-independent. It verifies acquisition + facts, delegates field mapping and quarantine to the canonical adapter, and + hands only accepted rows to the private contract. The adapter's normalized + privacy/coordinate gates are preserved; this function never clears them. + """ + raw = Path(raw_path).read_bytes() + digest = hashlib.sha256(raw).hexdigest() + if digest != artifact.sha256 or len(raw) != artifact.byte_size: + raise ValueError("source checksum or byte size mismatch") + + result = FsaApprovedEstablishmentsAdapter().parse_bytes(raw) + if result.profile != "monthly": + raise ValueError("candidate handoff requires the FSA monthly profile") + + root = Path(run_dir) + _write_jsonl(root / "quarantined" / "records.jsonl", list(result.quarantined)) + manifest = write_handoff( + root, + list(result.accepted), + artifact, + source_id=FsaApprovedEstablishmentsAdapter.source_id, + profile="fsa-approved-monthly", + ) + qa = { + "profile": result.profile, + "input_rows": len(result.accepted) + len(result.quarantined), + "normalized_rows": len(result.accepted), + "quarantined_rows": len(result.quarantined), + "coverage_counts": result.coverage_counts or {}, + "anomaly_counts": result.anomaly_counts or {}, + "geocoding": "disabled", + } + (root / "qa.json").parent.mkdir(parents=True, exist_ok=True) + (root / "qa.json").write_text(json.dumps(qa, sort_keys=True, indent=2) + "\n", encoding="utf-8") + return {**manifest, "qa": qa} diff --git a/pipeline/sources/uk/fsa_approved/test_handoff.py b/pipeline/sources/uk/fsa_approved/test_handoff.py new file mode 100644 index 0000000..639c9cb --- /dev/null +++ b/pipeline/sources/uk/fsa_approved/test_handoff.py @@ -0,0 +1,67 @@ +import hashlib +import json +import tempfile +import unittest +from pathlib import Path + +from pipeline.contracts.adapter_contract import SourceArtifact + +from .handoff import write_private_monthly_handoff + + +MONTHLY = ( + "AppNo,TradingName,Country,CompetentAuthority,X,Y,AddressWithheld,All_Activities,Address1,Town,Postcode\n" + "A-1,House Foods,England,Food Standards Agency,-0.12,51.50,No,CP,House Farm,London,SW1\n" + "A-2,Withheld,Wales,Food Standards Agency,-3.18,51.48,Yes,CP,Private Road,Cardiff,CF1\n" + "A-3,Out of scope,Jersey,Food Standards Agency,-2.10,49.20,No,CP,Island Road,St Helier,JE1\n" +).encode("cp1252") + + +class FsaHandoffTests(unittest.TestCase): + def artifact(self, raw: bytes) -> SourceArtifact: + return SourceArtifact( + source_url="https://example.invalid/fsa-monthly.csv", + retrieved_at_utc="2026-09-14T00:00:00Z", + sha256=hashlib.sha256(raw).hexdigest(), + byte_size=len(raw), + code_version="test", + config_version="test", + coverage="England and Wales", + ) + + def test_monthly_rows_use_private_handoff_without_clearance(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + source = root / "source.csv" + source.write_bytes(MONTHLY) + manifest = write_private_monthly_handoff(source, root / "run", self.artifact(MONTHLY)) + self.assertEqual(manifest["contract_version"], "candidate-handoff-v1") + self.assertEqual(manifest["profile"], "fsa-approved-monthly") + self.assertEqual(manifest["source_id"], "fsa_approved_establishments") + self.assertEqual(manifest["normalized_rows"], 2) + self.assertEqual(manifest["release_state"], "not-created") + self.assertEqual(manifest["privacy_gate"], "pending") + self.assertEqual(manifest["coordinate_gate"], "review_required") + row = json.loads((root / "run/normalized/records.jsonl").read_text().splitlines()[0]) + self.assertEqual(row["source_row"], 2) + self.assertEqual(row["normalized"]["establishment_id"], "A-1") + self.assertEqual(row["source_values"]["X"], "-0.12") + self.assertIsNone(row["normalized"]["coordinates"]) + self.assertEqual(row["normalized"]["privacy_gate"], "privacy-review-required") + self.assertEqual(row["normalized"]["coordinate_gate"], "privacy-review-required") + self.assertEqual(manifest["qa"]["quarantined_rows"], 1) + self.assertEqual(manifest["qa"]["anomaly_counts"]["unknown_nation"], 1) + + def test_source_mismatch_fails_before_handoff(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + source = root / "source.csv" + source.write_bytes(MONTHLY) + bad = SourceArtifact("https://example.invalid", "2026-09-14T00:00:00Z", "0" * 64, len(MONTHLY)) + with self.assertRaisesRegex(ValueError, "checksum"): + write_private_monthly_handoff(source, root / "run", bad) + self.assertFalse((root / "run/manifest.json").exists()) + + +if __name__ == "__main__": + unittest.main() From a974c682310a6efe7526f24e90fadf8bbb829003 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sun, 13 Sep 2026 23:22:32 -0700 Subject: [PATCH 069/311] Test UK candidate handoff against importer --- pipeline/tests/test_import_candidate.py | 33 +++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/pipeline/tests/test_import_candidate.py b/pipeline/tests/test_import_candidate.py index 1b586a2..52a3c04 100644 --- a/pipeline/tests/test_import_candidate.py +++ b/pipeline/tests/test_import_candidate.py @@ -5,6 +5,8 @@ import unittest from pathlib import Path +from pipeline.sources.uk.fsa_approved.adapter import FsaApprovedEstablishmentsAdapter + ROOT = Path(__file__).resolve().parents[1] SPEC = importlib.util.spec_from_file_location("import_candidate", ROOT / "scripts/maintenance/import-candidate.py") MODULE = importlib.util.module_from_spec(SPEC) @@ -78,6 +80,37 @@ def test_raw_artifact_mismatch_is_rejected(self): with self.assertRaises(MODULE.CandidateImportError): MODULE.load_inputs(path, normalized, raw) + def test_uk_handoff_output_composes_with_importer_without_database(self): + """Exercise the real UK adapter output through the import preflight.""" + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + raw = root / "uk-monthly.csv" + raw.write_bytes( + b"AppNo,TradingName,Country,CompetentAuthority,X,Y,AddressWithheld,All_Activities,Address1,Town,Postcode\n" + b"UK-E2E-1,Example Foods,England,Food Standards Agency,-0.12,51.50,No,CP,House Farm,London,SW1\n" + ) + run_dir = root / "run" + raw_hash = hashlib.sha256(raw.read_bytes()).hexdigest() + artifact = { + "source_url": "https://example.invalid/uk-synthetic.csv", + "retrieved_at_utc": "2026-09-13T00:00:00Z", + "checksum_sha256": raw_hash, + "byte_size": raw.stat().st_size, + "code_version": "test", + "config_version": "test", + } + manifest = FsaApprovedEstablishmentsAdapter().run(raw, run_dir, artifact) + loaded, rows = MODULE.load_inputs(run_dir / "manifest.json", run_dir / "normalized/records.jsonl", raw) + self.assertEqual(loaded, manifest) + self.assertEqual(len(rows), 1) + self.assertTrue(all(row["normalized"]["coordinates"] is None for row in rows)) + self.assertEqual(rows[0]["normalized"]["privacy_gate"], "privacy-review-required") + self.assertEqual(rows[0]["normalized"]["coordinate_gate"], "privacy-review-required") + self.assertEqual(rows[0]["normalized"]["publication_gate"], "blocked") + self.assertTrue(all(MODULE._record_parts(row)[0] for row in rows)) + self.assertEqual(manifest["release_state"], "not-created") + self.assertEqual(manifest["publication_state"], "private-candidate") + if __name__ == "__main__": unittest.main() From 203e98241feca82e0d453ed081ed18903a0fc0ae Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sun, 13 Sep 2026 23:24:03 -0700 Subject: [PATCH 070/311] Track private UK handoff without release claim --- docs/source-status.json | 2 +- docs/source-status.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source-status.json b/docs/source-status.json index 3436b17..7bb9fb5 100644 --- a/docs/source-status.json +++ b/docs/source-status.json @@ -12,7 +12,7 @@ {"source_id":"it.locations","metadata":"verified","acquisition":"artifact_private_only","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-it.md","pipeline/source_registry.json"],"next_action":"Review the 853/2004 and 1069/2009 source dictionaries and privacy mappings, then implement and validate separate adapters without treating private artifacts as release inputs."}, {"source_id":"mx.locations","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-mx.md","pipeline/source_registry.json"],"next_action":"Resolve DENUE token/terms and SENASICA current directory/schema before bounded acquisition."}, {"source_id":"nz.locations","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-nz.md","pipeline/source_registry.json"],"next_action":"Confirm MPI permitted acquisition route, list-specific terms, and schema before private staging."}, - {"source_id":"uk.locations","metadata":"partial","acquisition":"artifact_private_only","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-uk.md","docs/countries/uk/fss-approved-establishments-source-assessment.md","pipeline/sources/uk/fsa_approved/README.md","pipeline/source_registry.json"],"next_action":"Keep the UK ID composite: validate the private England/Wales monthly candidate while keeping NI and Scotland separate; review withheld addresses, duplicates, coverage, terms, and release approval."}, + {"source_id":"uk.locations","metadata":"partial","acquisition":"artifact_private_only","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-uk.md","docs/countries/uk/fss-approved-establishments-source-assessment.md","pipeline/sources/uk/fsa_approved/README.md","docs/architecture/disposable-candidate-import.md","pipeline/scripts/maintenance/import-candidate.py","pipeline/source_registry.json"],"next_action":"Keep the UK ID composite: synthetic handoff passes importer pre-DB validation, but no real UK candidate has been imported or previewed; keep review-required/unapproved defaults and complete Docker E2E, privacy/coordinate, source-rights, duplicate, coverage, and release review while keeping NI and Scotland separate."}, {"source_id":"dk.smiley","metadata":"partial","acquisition":"artifact_private_only","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["pipeline/sources/denmark/README.md","pipeline/contracts/README.md","pipeline/source_registry.json"],"next_action":"Keep the privately staged official artifact as validation-only; verify coverage and effective-date semantics, then complete terms/privacy/release review before any publication."}, {"source_id":"de.locations","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["pipeline/source_registry.json"],"next_action":"Replace session-bound legacy URL with a verified stable BVL endpoint and confirm terms/schema."}, {"source_id":"ca.locations","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ca.md","pipeline/source_registry.json"],"next_action":"Keep federal/export and Ontario candidates separate; verify an authoritative permitted artifact, terms, schema, coverage, and privacy before acquisition."}, diff --git a/docs/source-status.md b/docs/source-status.md index ce373db..a94d2bc 100644 --- a/docs/source-status.md +++ b/docs/source-status.md @@ -19,7 +19,7 @@ No last-success timestamp is invented. Private artifacts are not proof of a publ | `it.locations` | verified | artifact_private_only | not_run | blocked | Current 853/2004 and 1069/2009 CSVs were privately acquired with provenance; schemas remain distinct and require dictionary/privacy mapping plus separate adapters | | `mx.locations` | verified | blocked | not_run | blocked | DENUE/SENASICA/DGSIAP reconnaissance; resolve token, directory, terms, and schema | | `nz.locations` | verified | blocked | not_run | blocked | MPI/Stats NZ reconnaissance; resolve 403/access and aggregate-vs-facility boundaries | -| `uk.locations` | partial | artifact_private_only | not_run | blocked | Private England/Wales monthly candidate is validated; NI and Scotland remain separate, with withheld-address/duplicate/coverage/terms/release review open | +| `uk.locations` | partial | artifact_private_only | not_run | blocked | Synthetic handoff passes importer pre-DB validation, but no real UK candidate has been imported or previewed; review-required/unapproved defaults, Docker E2E, privacy/coordinate, source-rights, duplicate, coverage, and release review remain open; NI/Scotland stay separate | | `dk.smiley` | partial | artifact_private_only | not_run | blocked | Current official artifact and registered adapter are privately staged for validation; coverage/effective-date uncertainty and terms/privacy/release review remain open | | `de.locations` | partial | not_run | not_run | blocked | Legacy session URL and current BVL endpoint/schema/terms remain unresolved | | `ca.locations` | verified | not_run | not_run | blocked | Federal/export and Ontario candidates are documented separately; verify permitted artifact, terms, schema, coverage, and privacy before acquisition | From 1cc8f667e1d178b7e831c61b4d946d099376fcb9 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sun, 13 Sep 2026 23:35:52 -0700 Subject: [PATCH 071/311] Add honest private candidate inspection panel --- frontend/src/app/App.svelte | 2 ++ .../features/devPreview/DevReviewPanel.svelte | 21 +++++++++++++++++++ .../src/features/devPreview/devReviewState.ts | 5 +++++ .../tests/unit/devPreviewContract.test.ts | 6 ++++++ 4 files changed, 34 insertions(+) create mode 100644 frontend/src/features/devPreview/DevReviewPanel.svelte create mode 100644 frontend/src/features/devPreview/devReviewState.ts diff --git a/frontend/src/app/App.svelte b/frontend/src/app/App.svelte index 6fd047a..153315a 100644 --- a/frontend/src/app/App.svelte +++ b/frontend/src/app/App.svelte @@ -16,6 +16,7 @@ import ScaleNarrative from '../features/scale/ScaleNarrative.svelte'; import { canOpenDevPreview, DEV_PREVIEW_LABEL, devPreviewExportLabel, canMountPublicExport } from '../features/devPreview/devPreviewContract'; import type { DevCandidate } from '../api/DevCandidatePreviewRepository'; + import DevReviewPanel from '../features/devPreview/DevReviewPanel.svelte'; let profile: Profile = 'curated'; let selected: Location | undefined = locations[0]; @@ -138,6 +139,7 @@
UNTIL EVERY CAGE V2 / FIELD NOTE

EVIDENCE DESK · {devPreviewMode ? 'PRIVATE CANDIDATE PREVIEW' : localMode ? 'LOCAL V2 API' : 'SYNTHETIC PREVIEW'}

See what a record can—and cannot—tell us.

Start with the source profile, narrow the visible evidence, then inspect what a record can—and cannot—tell us.

{#if devPreviewMode}{/if} + {#if devPreviewMode}{/if} {#if !devPreviewMode || previewStatus === 'ready'}

01 / DISCOVER

Choose the evidence lane

Profiles stay separate. A community claim is never silently promoted into a curated result.

{#if localMode && metadataStatus === 'loading'}

Loading available filter values…

{:else if localMode && metadataStatus === 'error'}

Filter metadata is unavailable; only unscoped discovery is shown.

{/if}
diff --git a/frontend/src/features/devPreview/DevReviewPanel.svelte b/frontend/src/features/devPreview/DevReviewPanel.svelte new file mode 100644 index 0000000..f45275a --- /dev/null +++ b/frontend/src/features/devPreview/DevReviewPanel.svelte @@ -0,0 +1,21 @@ + + +
+

PRIVATE REVIEW NOTES

+

Restricted inspection only

+

This browser-session note does not approve, publish, suppress, or alter the candidate. No secure review write endpoint is available.

+ {#if candidate} +
Candidate status
{candidate.releaseStatus} · {candidate.previewLabel}
Privacy
{candidate.evidence?.privacyScreeningStatus ?? 'unavailable'}
Coordinates
{candidate.lat === null ? 'Not available for display' : `${candidate.evidence?.displayPrecision ?? 'unknown'} display point`}
Provenance
{candidate.source} · retrieved {candidate.evidence?.retrievedAt ?? 'unavailable'}
Project approval
{candidate.evidence?.projectApproval === false ? 'false — not approved' : 'unavailable'}
Published profile
{candidate.evidence?.publicationProfile === null ? 'null — not published' : 'unavailable'}
+

Session note: {state === 'unreviewed' ? 'not inspected' : state === 'inspected' ? 'inspected locally; no decision recorded' : 'follow-up flagged locally; maintainer action required'}

+ + + {:else}

No candidate selected. Load an authenticated private preview first.

{/if} +
+ + diff --git a/frontend/src/features/devPreview/devReviewState.ts b/frontend/src/features/devPreview/devReviewState.ts new file mode 100644 index 0000000..9bee647 --- /dev/null +++ b/frontend/src/features/devPreview/devReviewState.ts @@ -0,0 +1,5 @@ +export type LocalReviewState = 'unreviewed' | 'inspected' | 'follow_up'; + +/** These actions are operator notes for this browser session, never approval or suppression writes. */ +export const nextLocalReviewState = (state: LocalReviewState, action: 'inspect' | 'follow_up'): LocalReviewState => + action === 'follow_up' ? 'follow_up' : state === 'follow_up' ? state : 'inspected'; diff --git a/frontend/tests/unit/devPreviewContract.test.ts b/frontend/tests/unit/devPreviewContract.test.ts index d6dabe9..6321752 100644 --- a/frontend/tests/unit/devPreviewContract.test.ts +++ b/frontend/tests/unit/devPreviewContract.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from 'vitest'; import { canOpenDevPreview, DEV_PREVIEW_LABEL, DEV_PREVIEW_PATH, DEV_PREVIEW_QUERY, DEV_PREVIEW_TOKEN_HEADER, devPreviewExportLabel, canMountPublicExport } from '../../src/features/devPreview/devPreviewContract'; import { DevCandidatePreviewRepository } from '../../src/api/DevCandidatePreviewRepository'; +import { nextLocalReviewState } from '../../src/features/devPreview/devReviewState'; describe('dev preview boundary', () => { it('requires both a development build and the explicit mode', () => { @@ -27,4 +28,9 @@ describe('dev preview boundary', () => { const result = await new DevCandidatePreviewRepository(fetcher).list('operator-token'); expect(result[0]?.evidence).toMatchObject({ projectApproval: false, publicationProfile: null, publicationWarning: DEV_PREVIEW_LABEL }); }); + it('keeps local review notes non-approval and non-persistent in the model', () => { + expect(nextLocalReviewState('unreviewed', 'inspect')).toBe('inspected'); + expect(nextLocalReviewState('inspected', 'follow_up')).toBe('follow_up'); + expect(nextLocalReviewState('follow_up', 'inspect')).toBe('follow_up'); + }); }); From 46e39fae4e0cf99be1a97f0a5db8f885127bcb85 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sun, 13 Sep 2026 23:36:14 -0700 Subject: [PATCH 072/311] Test guarded candidate import and private preview end to end --- .../scripts/maintenance/import-candidate.py | 9 +- pipeline/tests/e2e/test_candidate_import.py | 165 ++++++++++++++++++ 2 files changed, 171 insertions(+), 3 deletions(-) create mode 100644 pipeline/tests/e2e/test_candidate_import.py diff --git a/pipeline/scripts/maintenance/import-candidate.py b/pipeline/scripts/maintenance/import-candidate.py index 70e1d0f..936d08c 100644 --- a/pipeline/scripts/maintenance/import-candidate.py +++ b/pipeline/scripts/maintenance/import-candidate.py @@ -138,10 +138,13 @@ def import_candidate(database_url: str, manifest: dict, rows: list[dict], releas country = _country_code(manifest, normalized) name = normalized.get("trading_name") city = normalized.get("city") - record_id = db.execute("""INSERT INTO uec.source_records(source_id,source_record_key,artifact_id,raw_fields,parsed_at) + db.execute("""INSERT INTO uec.source_records(source_id,source_record_key,artifact_id,raw_fields,parsed_at) VALUES (%s,%s,%s,%s,%s) ON CONFLICT (source_id,source_record_key,artifact_id) - DO UPDATE SET source_state='present' RETURNING source_record_id""", - (manifest["source_id"], key, artifact_id, json.dumps({"source_values": record.get("source_values", {})}), now)).fetchone()[0] + DO NOTHING""", + (manifest["source_id"], key, artifact_id, json.dumps({"source_values": record.get("source_values", {})}), now)) + record_id = db.execute("""SELECT source_record_id FROM uec.source_records + WHERE source_id=%s AND source_record_key=%s AND artifact_id=%s""", + (manifest["source_id"], key, artifact_id)).fetchone()[0] existing = db.execute("""SELECT facility_id, observation_id FROM uec.observations WHERE source_record_id=%s ORDER BY observed_at DESC, observation_id DESC LIMIT 1""", (record_id,)).fetchone() if existing: diff --git a/pipeline/tests/e2e/test_candidate_import.py b/pipeline/tests/e2e/test_candidate_import.py new file mode 100644 index 0000000..eb1d969 --- /dev/null +++ b/pipeline/tests/e2e/test_candidate_import.py @@ -0,0 +1,165 @@ +"""Docker E2E for the private UK-candidate handoff and review boundary.""" +import json +import os +import subprocess +import sys +import tempfile +import unittest +import urllib.error +import urllib.request +import uuid +from datetime import datetime, timezone +from pathlib import Path + +import psycopg + +try: + from .fixture import E2EEnvironment +except ImportError: + from fixture import E2EEnvironment + +from pipeline.sources.uk.fsa_approved.adapter import FsaApprovedEstablishmentsAdapter + +ROOT = Path(__file__).resolve().parents[3] +IMPORTER = ROOT / "pipeline/scripts/maintenance/import-candidate.py" + + +class CandidateImportE2ETests(unittest.TestCase): + @classmethod + def counts_for(cls): + tables = ("raw_artifacts", "acquisition_runs", "acquisition_run_artifacts", "source_records", + "facilities", "observations", "releases", "release_members", "publication_review_events") + with psycopg.connect(cls.env.database_url) as db: + return {table: db.execute(f"SELECT count(*) FROM uec.{table}").fetchone()[0] for table in tables} + + def counts(self): + return self.counts_for() + + @classmethod + def setUpClass(cls): + if os.environ.get("UEC_RUN_E2E") != "1": + raise unittest.SkipTest("set UEC_RUN_E2E=1 to run Docker-backed E2E tests") + cls.env = E2EEnvironment().start() + cls.temp = tempfile.TemporaryDirectory() + root = Path(cls.temp.name) + raw = root / "uk-monthly.csv" + raw.write_bytes( + b"AppNo,TradingName,Country,CompetentAuthority,X,Y,AddressWithheld,All_Activities,Address1,Town,Postcode\n" + b"UK-E2E-1,Example Foods,England,Food Standards Agency,-0.12,51.50,No,CP,House Farm,London,SW1\n" + ) + manifest = FsaApprovedEstablishmentsAdapter().run(raw, root / "run", { + "source_url": "https://example.invalid/uk-e2e.csv", + "retrieved_at_utc": "2026-09-13T00:00:00Z", + "checksum_sha256": __import__("hashlib").sha256(raw.read_bytes()).hexdigest(), + "byte_size": raw.stat().st_size, + "code_version": "e2e", + "config_version": "fsa-e2e", + }) + cls.release_id = "candidate-uk-e2e" + cls.run_dir = root / "run" + command = [sys.executable, str(IMPORTER), "--manifest", str(cls.run_dir / "manifest.json"), + "--normalized", str(cls.run_dir / "normalized/records.jsonl"), "--raw", str(raw), + "--release-id", cls.release_id, "--database-url", cls.env.database_url, "--disposable-db"] + first = subprocess.run(command, cwd=ROOT, capture_output=True, text=True) + if first.returncode: + cls.temp.cleanup(); cls.env.stop() + raise RuntimeError(f"candidate importer failed:\n{first.stdout}\n{first.stderr}") + second = subprocess.run(command, cwd=ROOT, capture_output=True, text=True) + if second.returncode: + cls.temp.cleanup(); cls.env.stop() + raise RuntimeError(f"candidate importer rerun failed:\n{second.stdout}\n{second.stderr}") + cls.initial_counts = cls.counts_for() + cls._record_id = None + + @classmethod + def tearDownClass(cls): + if getattr(cls, "temp", None): + cls.temp.cleanup() + if getattr(cls, "env", None): + cls.env.stop() + + def request(self, path, headers=None): + request = urllib.request.Request(f"http://127.0.0.1:{self.env.api_port}{path}", headers=headers or {}) + return urllib.request.urlopen(request, timeout=10) + + def test_import_is_idempotent_and_candidate_is_not_public_or_previewable(self): + with psycopg.connect(self.env.database_url) as db: + source_count, observation_count, artifact_count, release_member_count, review_count, release_count, run_count = db.execute( + """SELECT count(DISTINCT r.source_record_id), count(DISTINCT o.observation_id) + , (SELECT count(*) FROM uec.raw_artifacts WHERE storage_key LIKE 'private-staging/%') + , (SELECT count(*) FROM uec.release_members WHERE release_id=%s) + , (SELECT count(*) FROM uec.publication_review_events WHERE release_id=%s) + , (SELECT count(*) FROM uec.releases WHERE release_id=%s) + , (SELECT count(*) FROM uec.acquisition_runs WHERE source_id='fsa_approved_establishments') + FROM uec.source_records r JOIN uec.observations o USING (source_record_id) + WHERE r.source_id='fsa_approved_establishments'""", (self.release_id, self.release_id, self.release_id) + ).fetchone() + self.assertEqual((source_count, observation_count, artifact_count, release_member_count, review_count, release_count, run_count), (1, 1, 1, 1, 1, 1, 2)) + review_state = db.execute("SELECT factual_review_status, privacy_screening_status, maintainer_approval, publication_eligible FROM uec.publication_review_events WHERE release_id=%s", (self.release_id,)).fetchone() + self.assertEqual(review_state, ("unreviewed", "pending", "pending", False)) + # The public route succeeds with an empty envelope, not an error. + with self.request("/api/v2/locations?profile=official") as response: + body = json.loads(response.read()) + self.assertEqual(body["data"], []) + self.assertNotIn("source_values", json.dumps(body)) + preview = urllib.request.Request( + f"http://127.0.0.1:{self.env.api_port}/api/dev/preview/candidates", + headers={"X-UEC-Dev-Preview-Token": self.env.dev_preview_token}, + ) + with urllib.request.urlopen(preview, timeout=10) as response: + self.assertEqual(json.loads(response.read())["data"], []) + + def test_review_version_unlocks_preview_only_then_suppression_relocks_it(self): + now = datetime.now(timezone.utc) + with psycopg.connect(self.env.database_url) as db: + with db.transaction(): + record_id = db.execute("SELECT source_record_id FROM uec.source_records WHERE source_id='fsa_approved_establishments'").fetchone()[0] + facility_id, observation_id = uuid.uuid4(), uuid.uuid4() + db.execute("INSERT INTO uec.facilities(facility_id,canonical_name,country_code,city) VALUES (%s,'E2E reviewed candidate','GB','London')", (facility_id,)) + # Evidence is append-only: this is a new reviewed observation, + # not an UPDATE that silently mutates the imported candidate. + db.execute("""INSERT INTO uec.observations(observation_id,facility_id,source_record_id,observed_at,observation,classification,ruleset_id,rule_id,classification_category,classification_review_status,default_visible,coordinate_review_status,first_observed_at) + VALUES (%s,%s,%s,%s,'{}','{}','fsa-e2e','operator-review','processing','approved',true,'approved',%s)""", + (observation_id, facility_id, record_id, now, now)) + db.execute("INSERT INTO uec.release_members(release_id,facility_id,observation_id,default_visible) VALUES (%s,%s,%s,true)", (self.release_id, facility_id, observation_id)) + db.execute("""INSERT INTO uec.geocode_results(source_record_id,provider_id,query,match_method,status,attempt_number,result,precision,queried_at) + VALUES (%s,'synthetic-review','operator-supplied synthetic point','operator-review','accepted',1,ST_SetSRID(ST_MakePoint(-0.12,51.50),4326)::geography,'city',%s)""", (record_id, now)) + db.execute("INSERT INTO uec.publication_review_events(source_record_id,release_id,factual_review_status,privacy_screening_status,maintainer_approval,publication_eligible,reviewer_role) VALUES (%s,%s,'reviewed','passed','approved',true,'authorized-synthetic-operator')", (record_id, self.release_id)) + preview = urllib.request.Request(f"http://127.0.0.1:{self.env.api_port}/api/dev/preview/candidates", headers={"X-UEC-Dev-Preview-Token": self.env.dev_preview_token}) + with urllib.request.urlopen(preview, timeout=10) as response: + preview_body = json.loads(response.read()) + self.assertEqual(len(preview_body["data"]), 1) + self.assertNotIn("source_values", json.dumps(preview_body)) + for headers in ({"X-UEC-Dev-Preview-Token": self.env.dev_preview_token, "Host": "attacker.invalid"}, + {"X-UEC-Dev-Preview-Token": self.env.dev_preview_token, "Origin": "https://attacker.invalid"}, + {"X-UEC-Dev-Preview-Token": "wrong"}): + with self.assertRaises(urllib.error.HTTPError) as error: + urllib.request.urlopen(urllib.request.Request(preview.full_url, headers=headers), timeout=10) + self.assertIn(error.exception.code, (401, 403)) + with psycopg.connect(self.env.database_url) as db: + db.execute("INSERT INTO uec.record_access_events(source_record_id,action,reason_category,policy_version,maintainer) VALUES (%s,'public_access_revoked','privacy','ethics-v1','authorized-synthetic-operator')", (record_id,)) + with urllib.request.urlopen(preview, timeout=10) as response: + self.assertEqual(json.loads(response.read())["data"], []) + + def test_bad_row_rolls_back_candidate_import(self): + before = self.counts() + normalized = self.run_dir / "normalized/records.jsonl" + original = normalized.read_text(encoding="utf-8") + bad = self.run_dir / "normalized/bad-records.jsonl" + bad.write_text(original + json.dumps({"source_id": "fsa_approved_establishments", "source_row": 999}) + "\n", encoding="utf-8") + manifest = json.loads((self.run_dir / "manifest.json").read_text(encoding="utf-8")) + manifest["normalized_rows"] = 2 + manifest["normalized_sha256"] = __import__("hashlib").sha256(bad.read_bytes()).hexdigest() + bad_manifest = self.run_dir / "bad-manifest.json" + bad_manifest.write_text(json.dumps(manifest), encoding="utf-8") + command = [sys.executable, str(IMPORTER), "--manifest", str(bad_manifest), "--normalized", str(bad), + "--raw", str(self.run_dir.parent / "uk-monthly.csv"), "--release-id", "candidate-uk-bad", + "--database-url", self.env.database_url, "--disposable-db"] + result = subprocess.run(command, cwd=ROOT, capture_output=True, text=True) + self.assertNotEqual(result.returncode, 0) + after = self.counts() + self.assertEqual(after, before) + + +if __name__ == "__main__": + unittest.main() From 088db44761177caef8b75c5a5a40f43c0ca9dd6e Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sun, 13 Sep 2026 23:34:07 -0700 Subject: [PATCH 073/311] Record bounded UK private handoff validation --- docs/country-recon-uk.md | 25 +++++++++++++++++++++++++ docs/source-status.json | 2 +- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/docs/country-recon-uk.md b/docs/country-recon-uk.md index 2712926..3c6244c 100644 --- a/docs/country-recon-uk.md +++ b/docs/country-recon-uk.md @@ -129,6 +129,31 @@ normalized rows and 1,042 quarantined rows. The remaining address-risk count was reduction is not a release decision: remarks, out-of-scope jurisdictions, and duplicate identifiers remain blocked pending their respective reviews. +### Bounded real handoff (2026-09-14) + +A deterministic private sample was staged from the retained 2026-09-01 artifact +for operator and importer-boundary validation. The sample contains 49 source rows: +20 England, 20 Wales, 5 Jersey, 3 Isle of Man, and 1 Guernsey. The source-local +handoff produced 29 normalized rows and 20 quarantined rows; quarantine reasons +were `remarks_present` (11) and `unknown_nation` (9). The pre-DB importer checks +loaded all 29 normalized rows and verified the raw and normalized checksums. + +Restricted operator packet paths (not repository files): + +- `data/restricted/country-recon/uk/runs/2026-09-14-bounded-real/selection.manifest.json` +- `data/restricted/country-recon/uk/runs/2026-09-14-bounded-real/handoff/manifest.json` +- `data/restricted/country-recon/uk/runs/2026-09-14-bounded-real/handoff/qa.json` + +The packet records the official source URL/catalog, parent and sample hashes, +effective date, selection rule, coverage counts, quarantine counts, and disabled +geocoding. The sample and all row-level derivatives remain restricted. No database +import, preview, approval, coordinate release, or publication occurred. Operator +decisions still required: source-rights/attribution review, duplicate and coverage +scope review, privacy review of remarks and addresses, and authorization of any +disposable-DB E2E run. The shared SourceArtifact/typed-run boundary remains a +separate infrastructure integration limitation; this source-local bridge validates +typed artifact facts directly and does not alter the common contract. + This reconciliation is a design and test record, not source approval or legal clearance. No live row data is included, and the private artifact remains outside Git and public outputs. diff --git a/docs/source-status.json b/docs/source-status.json index 7bb9fb5..f3ba5aa 100644 --- a/docs/source-status.json +++ b/docs/source-status.json @@ -12,7 +12,7 @@ {"source_id":"it.locations","metadata":"verified","acquisition":"artifact_private_only","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-it.md","pipeline/source_registry.json"],"next_action":"Review the 853/2004 and 1069/2009 source dictionaries and privacy mappings, then implement and validate separate adapters without treating private artifacts as release inputs."}, {"source_id":"mx.locations","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-mx.md","pipeline/source_registry.json"],"next_action":"Resolve DENUE token/terms and SENASICA current directory/schema before bounded acquisition."}, {"source_id":"nz.locations","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-nz.md","pipeline/source_registry.json"],"next_action":"Confirm MPI permitted acquisition route, list-specific terms, and schema before private staging."}, - {"source_id":"uk.locations","metadata":"partial","acquisition":"artifact_private_only","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-uk.md","docs/countries/uk/fss-approved-establishments-source-assessment.md","pipeline/sources/uk/fsa_approved/README.md","docs/architecture/disposable-candidate-import.md","pipeline/scripts/maintenance/import-candidate.py","pipeline/source_registry.json"],"next_action":"Keep the UK ID composite: synthetic handoff passes importer pre-DB validation, but no real UK candidate has been imported or previewed; keep review-required/unapproved defaults and complete Docker E2E, privacy/coordinate, source-rights, duplicate, coverage, and release review while keeping NI and Scotland separate."}, + {"source_id":"uk.locations","metadata":"partial","acquisition":"artifact_private_only","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-uk.md","docs/countries/uk/fss-approved-establishments-source-assessment.md","pipeline/sources/uk/fsa_approved/README.md","pipeline/sources/uk/fsa_approved/handoff.py","docs/architecture/disposable-candidate-import.md","pipeline/scripts/maintenance/import-candidate.py","pipeline/source_registry.json"],"next_action":"Bounded real private handoff passed source-local validation and importer pre-DB checksum/count checks (49 input, 29 normalized, 20 quarantined); do not import or preview until an authorized disposable DB is available. Keep review-required/unapproved defaults and complete source-rights, privacy/coordinate, duplicate, coverage, Docker E2E, and release review while keeping NI and Scotland separate."}, {"source_id":"dk.smiley","metadata":"partial","acquisition":"artifact_private_only","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["pipeline/sources/denmark/README.md","pipeline/contracts/README.md","pipeline/source_registry.json"],"next_action":"Keep the privately staged official artifact as validation-only; verify coverage and effective-date semantics, then complete terms/privacy/release review before any publication."}, {"source_id":"de.locations","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["pipeline/source_registry.json"],"next_action":"Replace session-bound legacy URL with a verified stable BVL endpoint and confirm terms/schema."}, {"source_id":"ca.locations","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ca.md","pipeline/source_registry.json"],"next_action":"Keep federal/export and Ontario candidates separate; verify an authoritative permitted artifact, terms, schema, coverage, and privacy before acquisition."}, From 0e73b999d08cdce605f1f6051f5fb2337616ca6b Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sun, 13 Sep 2026 23:38:56 -0700 Subject: [PATCH 074/311] Document Italy official dictionary boundaries --- docs/country-recon-it.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/docs/country-recon-it.md b/docs/country-recon-it.md index 6792385..68d71fb 100644 --- a/docs/country-recon-it.md +++ b/docs/country-recon-it.md @@ -21,6 +21,12 @@ The catalog reported 853 data last updated 2026-09-13 and daily frequency; the 1 The catalog identifies the Ministry of Health/DGSAN Office 2 and Italian Open Data Licence v2.0. It warns that some coordinates came from OpenStreetMap contributors; this is source metadata, not permission to publish precise points. +## Dictionary investigation (read-only) + +The Ministry's [853 data dictionary v2.0](https://www.dati.salute.gov.it/dati/documenti/ID_8_Dataset_Stabilimenti_Italiani_per_gli_alimenti_di_origine_animale_v2.0.pdf) was accessible on 2026-09-14. It defines `num_identificativo_produzione_commercializzazione` as the EU recognition number, `codice_comune` as a six-character ISTAT municipality code, `classificazione_stabilimento` and `codice_impiatto_attivita` as classification/activity fields, and product/export fields as coded descriptions. It also defines supplied `longitudine`/`latitudine`, `stato_localizzazione` (`1` geolocalized, `2` not geolocalized), fiscal/VAT identifiers, `stato_attivita` (`Autorizzata`, `Revocata`, `Sospesa`), and `data_ultimo_aggiornamento` applying to establishment master data or an individual activity. + +The dictionary does not settle cross-snapshot identity for repeated activities, coded multi-value normalization, address privacy eligibility, or permission to expose precise coordinates. These remain explicit adapter/review decisions; headers alone are insufficient. + ## Meaning and schema The 853/2004 sections are regulatory product/activity sections, not animal species or a simple facility type. Observed concepts include approval number, name, VAT/tax identifiers, town/region, category, associated activities, species, remarks, recognition number, activity/status fields, codes, products, export countries, coordinates, geolocation status, and last-update date. The separate 1069/2009 dataset covers animal by-products with its own recognition number, plant/activity/product codes, coordinates, status, and an optional 853 recognition link. @@ -39,7 +45,7 @@ The repository’s historical Italy CSV and scraper are legacy/unverified inputs | Source | Discovered | Acquisition | Adapter / validation | Terms / privacy / publication | Blocker / next action | |---|---|---|---|---|---| -| Ministry 853/2004 food establishments | Official catalog and regulatory sections verified | Private current CSV acquired; provenance recorded | Adapter not implemented; mapping requires dictionary review | Italian Open Data Licence v2.0; coordinate provenance partly OSM; ETHICS privacy/approval gates apply | Review dictionary and implement synthetic-only adapter | +| Ministry 853/2004 food establishments | Official catalog and regulatory sections verified | Private current CSV acquired; provenance recorded | Dictionary fields reviewed; adapter not implemented | Italian Open Data Licence v2.0; coordinate provenance partly OSM; ETHICS privacy/approval gates apply | Resolve repeated-activity identity, coded values, and privacy treatment before adapter | | Ministry 1069/2009 by-products | Separate official catalog/dictionary verified | Private current CSV acquired; provenance recorded | Kept separate; no adapter | Same licence and privacy/approval gates | Decide whether scope belongs in project, then validate separately | | Servlet HTML interface | Official interface identified | Not acquired; JS/cookie challenge | Historical HTML parser is brittle; no API claim | No export/terms contract verified; do not scrape through challenge | Prefer catalog downloads or request authorized export/documented endpoint | From 4e62422e526cb7bda086a41c9cba56a16899306d Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sun, 13 Sep 2026 23:38:27 -0700 Subject: [PATCH 075/311] Add Denmark source candidate handoff bridge --- pipeline/sources/denmark/adapter.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/pipeline/sources/denmark/adapter.py b/pipeline/sources/denmark/adapter.py index 0c32dff..b91f06b 100644 --- a/pipeline/sources/denmark/adapter.py +++ b/pipeline/sources/denmark/adapter.py @@ -9,6 +9,7 @@ from pathlib import Path from typing import Any from pipeline.contracts.adapter_contract import SourceArtifact +from pipeline.contracts.candidate_handoff import write_handoff SOURCE_ID = "dk.smiley" ADAPTER_VERSION = "denmark-smiley-contract-v1" @@ -30,6 +31,29 @@ class DenmarkSmileyAdapter: source_id = SOURCE_ID adapter_version = ADAPTER_VERSION + def write_candidate_handoff(self, run_dir: str | Path, artifact: SourceArtifact, + rows: list[dict[str, Any]]) -> dict[str, Any]: + """Map Denmark's stable source key into the generic candidate identity. + + This is an explicit source mapping: it does not fuzzy-match or infer a + facility, and it preserves every original field in private source_values. + """ + handoff_rows = [] + for row in rows: + fields = row.get("source_fields", {}) + key = row.get("source_record_key") + handoff_rows.append({"source_id": SOURCE_ID, "source_row": row.get("source_row", 0), + "source_values": fields, "normalized": { + "establishment_id": key, "trading_name": fields.get("Virksomhed"), + "address_lines": [fields.get("Adresse")], "postcode": fields.get("Postnummer"), + "activities": [fields.get("FVST_branchenummer")] if fields.get("FVST_branchenummer") else [], + "species": None, "competent_authority": "Fødevarestyrelsen", + "nation": "Denmark", "authority_nation_key": "Denmark", "status": None, + "remarks": None, "published_date": None, "coordinates": None, + "privacy_gate": "pending", "coordinate_gate": "review_required", + "publication_gate": "blocked"}}) + return write_handoff(run_dir, handoff_rows, artifact, source_id=SOURCE_ID) + def run_registered(self, raw_path: str | Path, run_dir: str | Path, config: dict[str, Any]) -> dict[str, Any]: """Bridge the shared registered-input runner using recorded evidence.""" required = ("source_url", "retrieved_at_utc", "checksum_sha256", "byte_size") From 89fd3b9988d0d5a405f18639381a63b72df1f01e Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sun, 13 Sep 2026 23:32:53 -0700 Subject: [PATCH 076/311] Bridge Denmark records to private candidate handoff --- pipeline/source_registry.json | 6 +++--- pipeline/sources/denmark/test_adapter.py | 10 ++++++++++ 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/pipeline/source_registry.json b/pipeline/source_registry.json index f6568a7..b80cadf 100644 --- a/pipeline/source_registry.json +++ b/pipeline/source_registry.json @@ -34,10 +34,10 @@ "url": "https://pub.fvst.dk/publikationer/Smileydata.xml", "access_method": "bulk XML download", "cadence": "weekly", - "attribution_licensing_notes": "Repository code names Fødevarestyrelsen; confirm current publication and reuse terms before acquisition.", - "adapter_status": "implemented_partial", + "attribution_licensing_notes": "Official Find Smiley data page records public-data reuse terms: attribute Fødevarestyrelsen, do not use its logo, and keep displayed smileys current; no project publication approval.", + "adapter_status": "private_staging_verified", "expected_artifact_schema": "XML source artifact -> parsed JSONL -> normalized/classified JSONL; source fields are preserved", - "blockers": ["Current endpoint, publication date semantics, licensing, and full coverage still require live verification."] + "blockers": ["Publisher supplies no dataset effective date; source coverage is limited to data available on Find Smiley and is not a completeness claim. Denmark candidate rows require explicit source-key mapping before disposable DB import."] }, { "source_id": "es.locations", diff --git a/pipeline/sources/denmark/test_adapter.py b/pipeline/sources/denmark/test_adapter.py index ed21689..b59da53 100644 --- a/pipeline/sources/denmark/test_adapter.py +++ b/pipeline/sources/denmark/test_adapter.py @@ -36,4 +36,14 @@ def test_registered_bridge_requires_and_checks_provenance(self): manifest = adapter.run_registered(raw, root / "good", config) self.assertEqual(manifest["acquisition"]["source_url"], config["source_url"]) + def test_candidate_mapping_preserves_source_values_and_pending_gates(self): + with tempfile.TemporaryDirectory() as d: + root = Path(d); raw = root / "raw.xml"; raw.write_bytes(XML) + parsed = {"source_id": "dk.smiley", "source_row": 2, "source_record_key": "1", "source_fields": {"ID_nummer": "1", "Virksomhed": "Test", "Adresse": "Road 1"}} + manifest = DenmarkSmileyAdapter().write_candidate_handoff(root / "handoff", self.artifact(), [parsed]) + self.assertEqual(manifest["privacy_gate"], "pending") + handoff = json.loads((root / "handoff" / "normalized/records.jsonl").read_text()) + self.assertEqual(handoff["source_values"]["ID_nummer"], "1") + self.assertEqual(handoff["normalized"]["establishment_id"], "1") + if __name__ == "__main__": unittest.main() From 9c886377c2fd4ae3827d05c2a3205bc819b30d30 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sun, 13 Sep 2026 23:33:12 -0700 Subject: [PATCH 077/311] Fix Denmark handoff test import --- pipeline/sources/denmark/test_adapter.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pipeline/sources/denmark/test_adapter.py b/pipeline/sources/denmark/test_adapter.py index b59da53..fefa526 100644 --- a/pipeline/sources/denmark/test_adapter.py +++ b/pipeline/sources/denmark/test_adapter.py @@ -1,4 +1,5 @@ import hashlib, tempfile, unittest +import json from pathlib import Path from .adapter import DenmarkSmileyAdapter from pipeline.contracts.adapter_contract import SourceArtifact From 3e8e47a640c02cacef5bb42f8fbfaa7560c4e212 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sun, 13 Sep 2026 23:34:26 -0700 Subject: [PATCH 078/311] Normalize legacy acquisition metadata at adapter boundary --- pipeline/contracts/CANDIDATE-HANDOFF.md | 4 ++++ pipeline/contracts/adapter_contract.py | 14 ++++++++++++++ pipeline/contracts/test_adapter_contract.py | 12 ++++++++++++ 3 files changed, 30 insertions(+) create mode 100644 pipeline/contracts/test_adapter_contract.py diff --git a/pipeline/contracts/CANDIDATE-HANDOFF.md b/pipeline/contracts/CANDIDATE-HANDOFF.md index d60d860..a61b30f 100644 --- a/pipeline/contracts/CANDIDATE-HANDOFF.md +++ b/pipeline/contracts/CANDIDATE-HANDOFF.md @@ -8,6 +8,10 @@ always `release_state: not-created`, `publication_state: private-candidate`, `review_state: review_required`, `privacy_gate: pending`, and `coordinate_gate: review_required`. It never infers identities or approval. +Legacy adapters that still receive dictionaries should call +`source_artifact_from_mapping` at their source-local boundary; the shared +contract remains typed and uses `sha256` internally. + The Denmark adapter currently uses `source_record_key` rather than `normalized.establishment_id`; it must receive a reviewed source-specific mapping before this handoff can be used for Denmark import. No mapping is diff --git a/pipeline/contracts/adapter_contract.py b/pipeline/contracts/adapter_contract.py index 6ba8f7c..b3f1b00 100644 --- a/pipeline/contracts/adapter_contract.py +++ b/pipeline/contracts/adapter_contract.py @@ -22,6 +22,20 @@ class SourceArtifact: coverage: str | None = None +def source_artifact_from_mapping(values: dict[str, Any]) -> SourceArtifact: + """Convert legacy config dictionaries at a source-local boundary only.""" + required = ("source_url", "retrieved_at_utc", "checksum_sha256", "byte_size") + missing = [key for key in required if not values.get(key)] + if missing: + raise ValueError("missing acquisition provenance: " + ", ".join(missing)) + return SourceArtifact( + source_url=str(values["source_url"]), retrieved_at_utc=str(values["retrieved_at_utc"]), + sha256=str(values["checksum_sha256"]), byte_size=int(values["byte_size"]), + publication_date=values.get("publication_date"), effective_date=values.get("effective_date"), + code_version=str(values.get("code_version", "unknown")), config_version=str(values.get("config_version", "unknown")), + rights_caveat=values.get("rights_caveat"), privacy_caveat=values.get("privacy_caveat"), coverage=values.get("coverage")) + + class SourceAdapter(Protocol): """Minimal boundary between acquisition evidence and private staging.""" source_id: str diff --git a/pipeline/contracts/test_adapter_contract.py b/pipeline/contracts/test_adapter_contract.py new file mode 100644 index 0000000..dc0c3d6 --- /dev/null +++ b/pipeline/contracts/test_adapter_contract.py @@ -0,0 +1,12 @@ +import unittest +from .adapter_contract import SourceArtifact, source_artifact_from_mapping + +class ArtifactBoundaryTests(unittest.TestCase): + def test_legacy_mapping_is_explicitly_normalized(self): + artifact = source_artifact_from_mapping({"source_url":"https://example.test", "retrieved_at_utc":"2026-01-01T00:00:00Z", "checksum_sha256":"a"*64, "byte_size":3}) + self.assertIsInstance(artifact, SourceArtifact); self.assertEqual(artifact.sha256, "a"*64) + def test_missing_mapping_provenance_fails_closed(self): + with self.assertRaisesRegex(ValueError, "source_url"): + source_artifact_from_mapping({"checksum_sha256":"a"*64, "byte_size":3}) + +if __name__ == "__main__": unittest.main() From 855af9996908fbb28440879985a47643c6ab33a2 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sun, 13 Sep 2026 23:41:21 -0700 Subject: [PATCH 079/311] Use conservative registry adapter status --- pipeline/source_registry.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipeline/source_registry.json b/pipeline/source_registry.json index b80cadf..05865bd 100644 --- a/pipeline/source_registry.json +++ b/pipeline/source_registry.json @@ -35,7 +35,7 @@ "access_method": "bulk XML download", "cadence": "weekly", "attribution_licensing_notes": "Official Find Smiley data page records public-data reuse terms: attribute Fødevarestyrelsen, do not use its logo, and keep displayed smileys current; no project publication approval.", - "adapter_status": "private_staging_verified", + "adapter_status": "implemented_partial", "expected_artifact_schema": "XML source artifact -> parsed JSONL -> normalized/classified JSONL; source fields are preserved", "blockers": ["Publisher supplies no dataset effective date; source coverage is limited to data available on Find Smiley and is not a completeness claim. Denmark candidate rows require explicit source-key mapping before disposable DB import."] }, From 8f3db4c9653e0940a73a0c80c18943c19f84cb72 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sun, 13 Sep 2026 23:45:24 -0700 Subject: [PATCH 080/311] Keep private preview controls out of public DOM --- frontend/src/app/App.svelte | 2 +- frontend/tests/e2e/private-preview.spec.ts | 31 ++++++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) create mode 100644 frontend/tests/e2e/private-preview.spec.ts diff --git a/frontend/src/app/App.svelte b/frontend/src/app/App.svelte index 153315a..c1615d5 100644 --- a/frontend/src/app/App.svelte +++ b/frontend/src/app/App.svelte @@ -155,6 +155,6 @@ {/if} {#if localMode && localStatus === 'ready'}{#if canMountPublicExport(devPreviewMode)}{/if}{/if}
{#if showGuidance}

Do not infer closure, identity, or permission from a map point. For a correction, privacy concern, or suppression request, preserve the record ID and contact the project maintainer through the reporting channel on the ethics page. Do not include sensitive personal details in a public issue.

Read reporting guidance ↗
{/if}
-
{#if showMap}{/if}{#if showExport}

IN-MEMORY EXPORT PREVIEW

{profile} · {release}

Loaded results only. This preview is not a live or complete export.

{exportPreview}
{/if}
{localMode ? 'Local V2 mode · no fixture fallback' : 'Method preview · no live requests'}
+ {#if !devPreviewMode}
{#if showMap}{/if}{#if showExport}

IN-MEMORY EXPORT PREVIEW

{profile} · {release}

Loaded results only. This preview is not a live or complete export.

{exportPreview}
{/if}{/if}
{localMode ? 'Local V2 mode · no fixture fallback' : 'Method preview · no live requests'}
{/if}
diff --git a/frontend/tests/e2e/private-preview.spec.ts b/frontend/tests/e2e/private-preview.spec.ts new file mode 100644 index 0000000..dae91dc --- /dev/null +++ b/frontend/tests/e2e/private-preview.spec.ts @@ -0,0 +1,31 @@ +import { test, expect } from '@playwright/test'; + +const preview = { + api_version: 'dev-preview-v1', + data: [{ candidate_id: 'candidate-1', source_record_id: 'source-row-1', facility_id: 'facility-1', canonical_name: 'Private candidate facility', country_code: 'DK', city: 'North Coast', category: 'dairy', display_precision: 'unmapped', latitude: null, longitude: null, source_type: 'official', provenance_source_id: 'source-1', provenance_source_name: 'Private synthetic source', provenance_source_url: 'https://example.test/source', provenance_retrieved_at: '2026-01-01T00:00:00Z', factual_review_status: 'unreviewed', privacy_screening_status: 'passed', project_approval: false, release_id: 'candidate-release', release_status: 'candidate', preview_label: 'PRIVATE TEST DATA — NOT REVIEWED OR PUBLISHED' }], + meta: { test_only: true, private_preview: true, profile: null, coverage_scope: 'candidate_release_only', next_cursor: null }, +}; + +test('private preview keeps review semantics and has no public export controls', async ({ page }) => { + await page.route('**/api/dev/preview/candidates**', route => route.fulfill({ json: preview })); + await page.goto('./?preview=dev-candidates&mode=local-v2#/'); + await page.getByLabel('Operator token (memory only)').fill('synthetic-test-token'); + await page.getByRole('button', { name: 'Load private candidates' }).click(); + await expect(page.getByRole('heading', { name: 'Private candidate facility' })).toBeVisible(); + await expect(page.getByText('false — not approved')).toBeVisible(); + await expect(page.getByText('null — not published')).toBeVisible(); + await expect(page.getByText('Session note: not inspected')).toBeVisible(); + await expect(page.getByRole('button', { name: /Preview export/ })).toHaveCount(0); + await expect(page.locator('.phase-controls')).toHaveCount(0); + await expect(page.getByText('Fictional demonstration data')).toHaveCount(0); +}); + +test('private preview failure does not fall back to fixture records', async ({ page }) => { + await page.route('**/api/dev/preview/candidates**', route => route.fulfill({ status: 401, body: JSON.stringify({ error: 'dev_preview_auth_failed' }) })); + await page.goto('./?preview=dev-candidates&mode=local-v2#/'); + await page.getByLabel('Operator token (memory only)').fill('wrong-token'); + await page.getByRole('button', { name: 'Load private candidates' }).click(); + await expect(page.getByText('Private candidate preview authentication failed.')).toBeVisible(); + await expect(page.getByRole('heading', { name: 'Private candidate facility' })).toHaveCount(0); + await expect(page.getByText('Synthetic demonstration')).toHaveCount(0); +}); From f49a3059f7163ea64f52dbdfd5023c1bfd89f32e Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Mon, 14 Sep 2026 00:02:25 -0700 Subject: [PATCH 081/311] Clarify source-owned pipeline stage layout --- pipeline/scripts/README.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pipeline/scripts/README.md b/pipeline/scripts/README.md index 4e72f0c..1a948e8 100644 --- a/pipeline/scripts/README.md +++ b/pipeline/scripts/README.md @@ -12,7 +12,12 @@ Run scripts from the repository root so their documented paths and output locati ## Adding another country -Country-specific adapters currently live in `stages/` because Denmark is the only active adapter. Once a second country is added, move country logic into a dedicated `stages//` directory and keep shared orchestration or validation helpers outside country directories. Do not hide source-specific assumptions in shared code. +Country-owned adapters and source-specific stages live under +`pipeline/sources//`; Denmark is the current reference layout. The +paths under `pipeline/scripts/stages/` remain shared generic stages or +compatibility shims for legacy commands, while shared orchestration and +validation helpers stay outside country directories. Do not hide source- +specific assumptions in shared code. Keep diagnostics separate from production stages, and add a short entry to this file when a new script category is introduced. From 65ecd8d909050a3f8f7001e28a9a9a444b58bdbe Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Mon, 14 Sep 2026 00:00:17 -0700 Subject: [PATCH 082/311] Add typed FSS private candidate bridge --- pipeline/sources/uk/fss_approved/handoff.py | 20 ++++++++++++++++++ .../sources/uk/fss_approved/test_handoff.py | 21 +++++++++++++++++++ 2 files changed, 41 insertions(+) create mode 100644 pipeline/sources/uk/fss_approved/handoff.py create mode 100644 pipeline/sources/uk/fss_approved/test_handoff.py diff --git a/pipeline/sources/uk/fss_approved/handoff.py b/pipeline/sources/uk/fss_approved/handoff.py new file mode 100644 index 0000000..b85e752 --- /dev/null +++ b/pipeline/sources/uk/fss_approved/handoff.py @@ -0,0 +1,20 @@ +"""Typed private candidate bridge for the synthetic FSS adapter.""" +from __future__ import annotations +from pathlib import Path +from pipeline.contracts.adapter_contract import SourceArtifact +from pipeline.contracts.candidate_handoff import write_handoff +from .adapter import FssApprovedEstablishmentsAdapter + +def write_private_handoff(raw_path: str | Path, run_dir: str | Path, artifact: SourceArtifact) -> dict: + """Validate artifact bytes and map approval_number to importer identity.""" + raw = Path(raw_path).read_bytes() + import hashlib + if hashlib.sha256(raw).hexdigest() != artifact.sha256 or len(raw) != artifact.byte_size: + raise ValueError("source checksum or byte size mismatch") + result = FssApprovedEstablishmentsAdapter().parse_bytes(raw) + rows = [] + for record in result.accepted: + normalized = dict(record["normalized"]) + normalized["establishment_id"] = normalized.pop("approval_number") + rows.append({**record, "normalized": normalized}) + return write_handoff(run_dir, rows, artifact, source_id=FssApprovedEstablishmentsAdapter.source_id, profile="fss-approved") diff --git a/pipeline/sources/uk/fss_approved/test_handoff.py b/pipeline/sources/uk/fss_approved/test_handoff.py new file mode 100644 index 0000000..d156337 --- /dev/null +++ b/pipeline/sources/uk/fss_approved/test_handoff.py @@ -0,0 +1,21 @@ +import hashlib, tempfile, unittest +from pathlib import Path +from pipeline.contracts.adapter_contract import SourceArtifact +from .handoff import write_private_handoff + +FIXTURE = Path(__file__).parent / "fixtures/valid.csv" +class FssHandoffTests(unittest.TestCase): + def test_typed_handoff_is_importer_compatible_and_gated(self): + raw = FIXTURE.read_bytes(); artifact = SourceArtifact("https://example.test/fss", "2026-01-01T00:00:00Z", hashlib.sha256(raw).hexdigest(), len(raw), code_version="test", config_version="test") + with tempfile.TemporaryDirectory() as d: + manifest = write_private_handoff(raw_path=FIXTURE, run_dir=d, artifact=artifact) + self.assertEqual(manifest["publication_state"], "private-candidate") + row = __import__("json").loads((Path(d) / "normalized/records.jsonl").read_text().splitlines()[0]) + self.assertIn("establishment_id", row["normalized"]) + def test_integrity_fails_before_output(self): + raw = FIXTURE.read_bytes(); artifact = SourceArtifact("https://example.test/fss", "2026-01-01T00:00:00Z", "0" * 64, len(raw)) + with tempfile.TemporaryDirectory() as d: + with self.assertRaises(ValueError): write_private_handoff(FIXTURE, d, artifact) + self.assertFalse((Path(d) / "manifest.json").exists()) + +if __name__ == "__main__": unittest.main() From a0193db3f2c12deaa1938be81b57d38c0ddaad69 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Mon, 14 Sep 2026 00:04:35 -0700 Subject: [PATCH 083/311] Record UK disposable candidate API validation --- docs/country-recon-uk.md | 24 ++++++++++++++++-------- docs/source-status.json | 2 +- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/docs/country-recon-uk.md b/docs/country-recon-uk.md index 3c6244c..6cc1c9c 100644 --- a/docs/country-recon-uk.md +++ b/docs/country-recon-uk.md @@ -136,7 +136,10 @@ for operator and importer-boundary validation. The sample contains 49 source row 20 England, 20 Wales, 5 Jersey, 3 Isle of Man, and 1 Guernsey. The source-local handoff produced 29 normalized rows and 20 quarantined rows; quarantine reasons were `remarks_present` (11) and `unknown_nation` (9). The pre-DB importer checks -loaded all 29 normalized rows and verified the raw and normalized checksums. +loaded all 29 normalized rows and verified the raw and normalized checksums. The +same handoff was then imported into a disposable local PostGIS database: 29 +candidate memberships, 0 default-visible rows, 29 pending privacy rows, 29 +coordinate-review-required rows, and 0 geocode rows. Restricted operator packet paths (not repository files): @@ -146,13 +149,18 @@ Restricted operator packet paths (not repository files): The packet records the official source URL/catalog, parent and sample hashes, effective date, selection rule, coverage counts, quarantine counts, and disabled -geocoding. The sample and all row-level derivatives remain restricted. No database -import, preview, approval, coordinate release, or publication occurred. Operator -decisions still required: source-rights/attribution review, duplicate and coverage -scope review, privacy review of remarks and addresses, and authorization of any -disposable-DB E2E run. The shared SourceArtifact/typed-run boundary remains a -separate infrastructure integration limitation; this source-local bridge validates -typed artifact facts directly and does not alter the common contract. +geocoding. The sample and all row-level derivatives remain restricted. The existing +candidate preview endpoint returned an empty result because pending privacy, +coordinate, geocode, and visibility gates remained closed. Public V2 list/export +routes remained unavailable because no promoted release exists. No approval, +coordinate release, or publication occurred. Operator decisions still required: +source-rights/attribution review, duplicate and coverage scope review, privacy review +of remarks and addresses, and authorization of any test-release preview. Backend +Safety's distinct guarded test-release route is not present in this baseline; do not +relax the existing candidate-preview or public V2 gates. The shared +SourceArtifact/typed-run boundary remains a separate infrastructure integration +limitation; this source-local bridge validates typed artifact facts directly and does +not alter the common contract. This reconciliation is a design and test record, not source approval or legal clearance. No live row data is included, and the private artifact remains outside diff --git a/docs/source-status.json b/docs/source-status.json index f3ba5aa..013a9c0 100644 --- a/docs/source-status.json +++ b/docs/source-status.json @@ -12,7 +12,7 @@ {"source_id":"it.locations","metadata":"verified","acquisition":"artifact_private_only","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-it.md","pipeline/source_registry.json"],"next_action":"Review the 853/2004 and 1069/2009 source dictionaries and privacy mappings, then implement and validate separate adapters without treating private artifacts as release inputs."}, {"source_id":"mx.locations","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-mx.md","pipeline/source_registry.json"],"next_action":"Resolve DENUE token/terms and SENASICA current directory/schema before bounded acquisition."}, {"source_id":"nz.locations","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-nz.md","pipeline/source_registry.json"],"next_action":"Confirm MPI permitted acquisition route, list-specific terms, and schema before private staging."}, - {"source_id":"uk.locations","metadata":"partial","acquisition":"artifact_private_only","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-uk.md","docs/countries/uk/fss-approved-establishments-source-assessment.md","pipeline/sources/uk/fsa_approved/README.md","pipeline/sources/uk/fsa_approved/handoff.py","docs/architecture/disposable-candidate-import.md","pipeline/scripts/maintenance/import-candidate.py","pipeline/source_registry.json"],"next_action":"Bounded real private handoff passed source-local validation and importer pre-DB checksum/count checks (49 input, 29 normalized, 20 quarantined); do not import or preview until an authorized disposable DB is available. Keep review-required/unapproved defaults and complete source-rights, privacy/coordinate, duplicate, coverage, Docker E2E, and release review while keeping NI and Scotland separate."}, + {"source_id":"uk.locations","metadata":"partial","acquisition":"artifact_private_only","runtime_health":"healthy","publication_eligibility":"blocked","evidence":["docs/country-recon-uk.md","docs/countries/uk/fss-approved-establishments-source-assessment.md","pipeline/sources/uk/fsa_approved/README.md","pipeline/sources/uk/fsa_approved/handoff.py","docs/architecture/disposable-candidate-import.md","pipeline/scripts/maintenance/import-candidate.py","pipeline/source_registry.json"],"next_action":"Bounded real handoff imported into a disposable DB and safe API omission verified (49 input, 29 normalized, 20 quarantined; 0 default-visible, 29 privacy-pending, 29 coordinate-review-required, 0 geocodes). Await Backend Safety's guarded test-release route for any local pending-candidate preview; keep public V2 and existing candidate-preview gates unchanged, and complete source-rights, privacy/coordinate, duplicate, coverage, and release review while keeping NI and Scotland separate."}, {"source_id":"dk.smiley","metadata":"partial","acquisition":"artifact_private_only","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["pipeline/sources/denmark/README.md","pipeline/contracts/README.md","pipeline/source_registry.json"],"next_action":"Keep the privately staged official artifact as validation-only; verify coverage and effective-date semantics, then complete terms/privacy/release review before any publication."}, {"source_id":"de.locations","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["pipeline/source_registry.json"],"next_action":"Replace session-bound legacy URL with a verified stable BVL endpoint and confirm terms/schema."}, {"source_id":"ca.locations","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ca.md","pipeline/source_registry.json"],"next_action":"Keep federal/export and Ontario candidates separate; verify an authoritative permitted artifact, terms, schema, coverage, and privacy before acquisition."}, From 7ca5df7fe5b917397e99e857a62a369165b14297 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Mon, 14 Sep 2026 00:11:06 -0700 Subject: [PATCH 084/311] Keep UK runtime status conservative --- docs/source-status.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source-status.json b/docs/source-status.json index 013a9c0..39b1e9d 100644 --- a/docs/source-status.json +++ b/docs/source-status.json @@ -12,7 +12,7 @@ {"source_id":"it.locations","metadata":"verified","acquisition":"artifact_private_only","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-it.md","pipeline/source_registry.json"],"next_action":"Review the 853/2004 and 1069/2009 source dictionaries and privacy mappings, then implement and validate separate adapters without treating private artifacts as release inputs."}, {"source_id":"mx.locations","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-mx.md","pipeline/source_registry.json"],"next_action":"Resolve DENUE token/terms and SENASICA current directory/schema before bounded acquisition."}, {"source_id":"nz.locations","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-nz.md","pipeline/source_registry.json"],"next_action":"Confirm MPI permitted acquisition route, list-specific terms, and schema before private staging."}, - {"source_id":"uk.locations","metadata":"partial","acquisition":"artifact_private_only","runtime_health":"healthy","publication_eligibility":"blocked","evidence":["docs/country-recon-uk.md","docs/countries/uk/fss-approved-establishments-source-assessment.md","pipeline/sources/uk/fsa_approved/README.md","pipeline/sources/uk/fsa_approved/handoff.py","docs/architecture/disposable-candidate-import.md","pipeline/scripts/maintenance/import-candidate.py","pipeline/source_registry.json"],"next_action":"Bounded real handoff imported into a disposable DB and safe API omission verified (49 input, 29 normalized, 20 quarantined; 0 default-visible, 29 privacy-pending, 29 coordinate-review-required, 0 geocodes). Await Backend Safety's guarded test-release route for any local pending-candidate preview; keep public V2 and existing candidate-preview gates unchanged, and complete source-rights, privacy/coordinate, duplicate, coverage, and release review while keeping NI and Scotland separate."}, + {"source_id":"uk.locations","metadata":"partial","acquisition":"artifact_private_only","runtime_health":"unknown","publication_eligibility":"blocked","evidence":["docs/country-recon-uk.md","docs/countries/uk/fss-approved-establishments-source-assessment.md","pipeline/sources/uk/fsa_approved/README.md","pipeline/sources/uk/fsa_approved/handoff.py","docs/architecture/disposable-candidate-import.md","pipeline/scripts/maintenance/import-candidate.py","pipeline/source_registry.json"],"next_action":"Bounded real handoff imported into a disposable DB and safe API omission verified (49 input, 29 normalized, 20 quarantined; 0 default-visible, 29 privacy-pending, 29 coordinate-review-required, 0 geocodes). This does not establish production/runtime health. Await Backend Safety's guarded test-release route for any local pending-candidate preview; keep public V2 and existing candidate-preview gates unchanged, and complete source-rights, privacy/coordinate, duplicate, coverage, and release review while keeping NI and Scotland separate."}, {"source_id":"dk.smiley","metadata":"partial","acquisition":"artifact_private_only","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["pipeline/sources/denmark/README.md","pipeline/contracts/README.md","pipeline/source_registry.json"],"next_action":"Keep the privately staged official artifact as validation-only; verify coverage and effective-date semantics, then complete terms/privacy/release review before any publication."}, {"source_id":"de.locations","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["pipeline/source_registry.json"],"next_action":"Replace session-bound legacy URL with a verified stable BVL endpoint and confirm terms/schema."}, {"source_id":"ca.locations","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ca.md","pipeline/source_registry.json"],"next_action":"Keep federal/export and Ontario candidates separate; verify an authoritative permitted artifact, terms, schema, coverage, and privacy before acquisition."}, From f3f06927e5ea9e6fee0a2c423aa0667f351f5b0e Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Mon, 14 Sep 2026 00:18:57 -0700 Subject: [PATCH 085/311] Add functional disposable test-release UI --- frontend/src/api/LocalLocationRepository.ts | 6 ++--- .../src/api/TestReleaseCsvExportRepository.ts | 7 +++++ .../TestReleaseFilterMetadataRepository.ts | 11 ++++++++ frontend/src/api/TestReleaseRepository.ts | 16 +++++++++++ frontend/src/app/App.svelte | 18 +++++++++---- .../features/devPreview/devPreviewContract.ts | 5 ++++ frontend/tests/e2e/private-preview.spec.ts | 16 +++++++++++ .../tests/unit/devPreviewContract.test.ts | 27 +++++++++++++++++-- 8 files changed, 96 insertions(+), 10 deletions(-) create mode 100644 frontend/src/api/TestReleaseCsvExportRepository.ts create mode 100644 frontend/src/api/TestReleaseFilterMetadataRepository.ts create mode 100644 frontend/src/api/TestReleaseRepository.ts diff --git a/frontend/src/api/LocalLocationRepository.ts b/frontend/src/api/LocalLocationRepository.ts index 99b87ca..071e473 100644 --- a/frontend/src/api/LocalLocationRepository.ts +++ b/frontend/src/api/LocalLocationRepository.ts @@ -8,7 +8,7 @@ export type LocationFilters = Readonly<{ country_code?: string | undefined; cate export type LocalListResult = Readonly<{ locations: readonly Location[]; releaseId: string; profile: LocalProfile; coverageNote: string; coverageScope?: string; countSemantics?: string; nextCursor: string | null; ruleset?: string }>; export const localOrigin = (value: string | undefined): string | undefined => { if (!value) return undefined; const url = new URL(value); if (url.protocol !== 'http:' || !['127.0.0.1', 'localhost', '::1'].includes(url.hostname)) throw new Error('Local API origin must be loopback HTTP.'); return url.origin; }; const fail = (kind: ApiError['kind'], message: string, status?: number): ApiError => Object.assign(new Error(message), status === undefined ? { kind } : { kind, status }); -const map = (r: WireLocation): Location => ({ +export const mapWireLocation = (r: WireLocation): Location => ({ id: r.facility_id, name: r.canonical_name, region: r.city ?? r.country_code, category: r.category, lat: r.latitude, lon: r.longitude, observed: r.last_observed_at ?? r.first_observed_at ?? 'unknown', source: r.provenance_source_name, evidence: { @@ -38,14 +38,14 @@ export class LocalLocationRepository { if (b.data.meta.release_id === null) throw fail('no-release', b.data.meta.coverage_note); const { release_id, ruleset_version } = b.data.meta; if (ruleset_version === undefined || b.data.data.some(row => !eligible(row, profile, release_id, ruleset_version))) throw fail('invalid-contract', 'Local V2 list snapshot was rejected.'); - return { locations: b.data.data.map(map), releaseId: release_id, profile, coverageNote: b.data.meta.coverage_note, coverageScope: b.data.meta.coverage_scope ?? 'selected promoted release public facilities', countSemantics: b.data.meta.count_semantics ?? 'Eligible public facility projection rows, not animals or a story-wide total.', nextCursor: b.data.meta.next_cursor ?? null, ruleset: ruleset_version }; + return { locations: b.data.data.map(mapWireLocation), releaseId: release_id, profile, coverageNote: b.data.meta.coverage_note, coverageScope: b.data.meta.coverage_scope ?? 'selected promoted release public facilities', countSemantics: b.data.meta.count_semantics ?? 'Eligible public facility projection rows, not animals or a story-wide total.', nextCursor: b.data.meta.next_cursor ?? null, ruleset: ruleset_version }; } catch (e) { if (e && typeof e === 'object' && 'kind' in e) throw e; if (e instanceof DOMException && e.name === 'AbortError') throw fail('aborted', 'Local V2 request was aborted.'); if (e instanceof TypeError) throw fail('network', 'Local V2 request could not connect.'); throw fail('invalid-contract', 'Local V2 response could not be read safely.'); } } async detail(id: string, profile: LocalProfile = 'official', signal?: AbortSignal) { try { const b = detailEnvelopeSchema.safeParse(await this.json(`/api/v2/locations/${encodeURIComponent(id)}?profile=${profile}`, signal)); if (!b.success || b.data.meta.profile !== profile || !eligible(b.data.data, profile, b.data.meta.release_id, b.data.meta.ruleset_version) || b.data.data.facility_id !== id) throw fail('invalid-contract', 'Local V2 detail response was rejected.'); - return { location: map(b.data.data), releaseId: b.data.meta.release_id, profile }; + return { location: mapWireLocation(b.data.data), releaseId: b.data.meta.release_id, profile }; } catch (e) { if (e && typeof e === 'object' && 'kind' in e) throw e; if (e instanceof DOMException && e.name === 'AbortError') throw fail('aborted', 'Local V2 request was aborted.'); if (e instanceof TypeError) throw fail('network', 'Local V2 request could not connect.'); throw fail('invalid-contract', 'Local V2 response could not be read safely.'); } } } diff --git a/frontend/src/api/TestReleaseCsvExportRepository.ts b/frontend/src/api/TestReleaseCsvExportRepository.ts new file mode 100644 index 0000000..889e0b1 --- /dev/null +++ b/frontend/src/api/TestReleaseCsvExportRepository.ts @@ -0,0 +1,7 @@ +import type { FetchLike } from './LocalLocationRepository'; +import { DEV_PREVIEW_TOKEN_HEADER, TEST_RELEASE_PATH, TEST_RELEASE_LABEL } from '../features/devPreview/devPreviewContract'; +export type TestReleaseCsv = Readonly<{ body: string; releaseId: string; profile: string; label: string }>; +export class TestReleaseCsvExportRepository { + constructor(private readonly fetcher: FetchLike = globalThis.fetch, private readonly baseUrl = '') {} + async download(profile: string, token: string): Promise { const response = await this.fetcher.call(globalThis, `${this.baseUrl}${TEST_RELEASE_PATH}/locations.csv?profile=${encodeURIComponent(profile)}`, { cache: 'no-store', headers: { [DEV_PREVIEW_TOKEN_HEADER]: token } }); if (!response.ok) throw new Error(`Private test-release CSV unavailable (HTTP ${response.status}).`); if (response.headers.get('x-uec-test-release') !== 'true') throw new Error('Private test-release CSV marker was missing.'); const body = await response.text(); const releaseId = response.headers.get('x-uec-release-id'); if (!releaseId || !body.trim()) throw new Error('Private test-release CSV did not include release context.'); return { body, releaseId, profile, label: TEST_RELEASE_LABEL }; } +} diff --git a/frontend/src/api/TestReleaseFilterMetadataRepository.ts b/frontend/src/api/TestReleaseFilterMetadataRepository.ts new file mode 100644 index 0000000..0faea36 --- /dev/null +++ b/frontend/src/api/TestReleaseFilterMetadataRepository.ts @@ -0,0 +1,11 @@ +import { z } from 'zod'; +import type { FetchLike } from './LocalLocationRepository'; +import { DEV_PREVIEW_TOKEN_HEADER, TEST_RELEASE_API_VERSION, TEST_RELEASE_PATH } from '../features/devPreview/devPreviewContract'; + +const dimension = z.array(z.object({ value: z.string(), count: z.number().int().nonnegative() })); +const schema = z.object({ data: z.null(), meta: z.object({ api_version: z.literal(TEST_RELEASE_API_VERSION), environment: z.literal('test-only'), test_only: z.literal(true), private_preview: z.literal(true), release_status: z.literal('candidate'), release_id: z.string(), profile: z.enum(['official', 'secondary', 'community']), coverage_scope: z.string(), count_semantics: z.string(), preview_label: z.string(), result_count: z.number().int().nonnegative() }), dimensions: z.object({ country_code: dimension, category: dimension, display_precision: dimension, source_type: dimension }) }); +export type TestReleaseFilterMetadata = z.infer; +export class TestReleaseFilterMetadataRepository { + constructor(private readonly fetcher: FetchLike = globalThis.fetch, private readonly baseUrl = '') {} + async get(profile: string, token: string): Promise { const response = await this.fetcher.call(globalThis, `${this.baseUrl}${TEST_RELEASE_PATH}/discovery/facets?profile=${encodeURIComponent(profile)}`, { cache: 'no-store', headers: { [DEV_PREVIEW_TOKEN_HEADER]: token } }); if (!response.ok) throw new Error(`Private test-release facets unavailable (HTTP ${response.status}).`); const result = schema.safeParse(await response.json()); if (!result.success) throw new Error('Private test-release facets were rejected safely.'); return result.data; } +} diff --git a/frontend/src/api/TestReleaseRepository.ts b/frontend/src/api/TestReleaseRepository.ts new file mode 100644 index 0000000..917fc0f --- /dev/null +++ b/frontend/src/api/TestReleaseRepository.ts @@ -0,0 +1,16 @@ +import { z } from 'zod'; +import { locationSchema, type WireLocation } from './wireSchema'; +import { mapWireLocation, type FetchLike, type LocalListResult, type LocalProfile } from './LocalLocationRepository'; +import { TEST_RELEASE_API_VERSION, TEST_RELEASE_LABEL, TEST_RELEASE_PATH, DEV_PREVIEW_TOKEN_HEADER } from '../features/devPreview/devPreviewContract'; + +const envelope = z.object({ data: z.array(locationSchema), meta: z.object({ api_version: z.literal(TEST_RELEASE_API_VERSION), environment: z.literal('test-only'), test_only: z.literal(true), private_preview: z.literal(true), release_status: z.literal('candidate'), release_id: z.string(), profile: z.enum(['official', 'secondary', 'community']), coverage_scope: z.string(), count_semantics: z.string(), preview_label: z.string(), result_count: z.number().int().nonnegative(), next_cursor: z.string().nullable().optional() }) }); +export class TestReleaseRepository { + constructor(private readonly fetcher: FetchLike = globalThis.fetch, private readonly baseUrl = '') {} + async list(profile: LocalProfile = 'official', token = '', signal?: AbortSignal): Promise { + const init: RequestInit = { cache: 'no-store', headers: { [DEV_PREVIEW_TOKEN_HEADER]: token } }; if (signal) init.signal = signal; + const response = await this.fetcher.call(globalThis, `${this.baseUrl}${TEST_RELEASE_PATH}/locations?profile=${profile}&limit=100`, init); + if (!response.ok) throw new Error(`Private test release unavailable (HTTP ${response.status}).`); + const parsed = envelope.safeParse(await response.json()); if (!parsed.success) throw new Error('Private test release response was rejected safely.'); + return { locations: parsed.data.data.map(mapWireLocation), releaseId: parsed.data.meta.release_id, profile, coverageNote: `${TEST_RELEASE_LABEL}. ${parsed.data.meta.coverage_scope}.`, coverageScope: parsed.data.meta.coverage_scope, countSemantics: parsed.data.meta.count_semantics, nextCursor: parsed.data.meta.next_cursor ?? null }; + } +} diff --git a/frontend/src/app/App.svelte b/frontend/src/app/App.svelte index c1615d5..288a558 100644 --- a/frontend/src/app/App.svelte +++ b/frontend/src/app/App.svelte @@ -14,8 +14,10 @@ import ReleaseContext from '../ui/ReleaseContext.svelte'; import ExportControl from '../ui/ExportControl.svelte'; import ScaleNarrative from '../features/scale/ScaleNarrative.svelte'; - import { canOpenDevPreview, DEV_PREVIEW_LABEL, devPreviewExportLabel, canMountPublicExport } from '../features/devPreview/devPreviewContract'; + import { canOpenDevPreview, DEV_PREVIEW_LABEL, TEST_RELEASE_LABEL, devPreviewExportLabel, canMountPublicExport } from '../features/devPreview/devPreviewContract'; import type { DevCandidate } from '../api/DevCandidatePreviewRepository'; + import { TestReleaseRepository } from '../api/TestReleaseRepository'; + import { TestReleaseCsvExportRepository } from '../api/TestReleaseCsvExportRepository'; import DevReviewPanel from '../features/devPreview/DevReviewPanel.svelte'; let profile: Profile = 'curated'; @@ -26,10 +28,13 @@ let showMap = false; let showExport = false; let showGuidance = false; let localMode = false; let devPreviewMode = false; + let testReleaseMode = false; let previewToken = ''; let previewStatus: 'idle' | 'loading' | 'ready' | 'error' | 'blocked' = 'idle'; let previewError = ''; - let previewRows: readonly DevCandidate[] = []; + let previewRows: readonly Location[] = []; + let testCsvBusy = false; + let testCsvError = ''; let localStatus: 'idle' | 'loading' | 'ready' | 'error' | 'no-release' = 'idle'; let detailStatus: 'idle' | 'loading' | 'error' = 'idle'; let localError = ''; let exportError = ''; let exportBusy = false; @@ -110,15 +115,17 @@ if (!canOpenDevPreview(import.meta.env.DEV, devPreviewMode ? 'dev-candidates' : null)) { previewStatus = 'blocked'; previewError = 'Private candidate preview is unavailable in production builds.'; return; } if (!previewToken.trim()) { previewStatus = 'error'; previewError = 'Enter the operator token for this development session.'; return; } previewStatus = 'loading'; previewError = ''; - try { const { DevCandidatePreviewRepository } = await import('../api/DevCandidatePreviewRepository'); previewRows = await new DevCandidatePreviewRepository().list(previewToken); previewStatus = 'ready'; selected = previewRows[0]; } + try { if (testReleaseMode) previewRows = (await new TestReleaseRepository().list(profile === 'community' ? 'community' : 'official', previewToken)).locations; else { const { DevCandidatePreviewRepository } = await import('../api/DevCandidatePreviewRepository'); previewRows = await new DevCandidatePreviewRepository().list(previewToken); } previewStatus = 'ready'; selected = previewRows[0]; } catch (error) { previewStatus = 'error'; previewError = error instanceof Error ? error.message : 'Private candidate preview was rejected safely.'; previewRows = []; selected = undefined; } }; + const downloadTestCsv = async () => { if (!testReleaseMode || previewStatus !== 'ready' || testCsvBusy) return; testCsvBusy = true; testCsvError = ''; try { const result = await new TestReleaseCsvExportRepository().download(profile === 'community' ? 'community' : 'official', previewToken); const url = URL.createObjectURL(new Blob([result.body], { type: 'text/csv;charset=utf-8' })); const anchor = document.createElement('a'); anchor.href = url; anchor.download = `uec-test-release-${result.releaseId}.csv`; anchor.click(); URL.revokeObjectURL(url); } catch (error) { testCsvError = error instanceof Error ? error.message : 'Private test-release CSV was rejected safely.'; } finally { testCsvBusy = false; } }; const clearFilters = () => { search = ''; region = 'all'; category = 'all'; sourceType = 'all'; displayPrecision = 'all'; lifecycleStatus = 'all'; }; const searchChanged = () => { if (localMode) { const url = new URL(window.location.href); if (search.trim()) url.searchParams.set('q', search.trim()); else url.searchParams.delete('q'); history.replaceState(null, '', url); } }; onMount(() => { const params = new URLSearchParams(window.location.search); localMode = params.get('mode') === 'local-v2'; - devPreviewMode = params.get('preview') === 'dev-candidates'; + testReleaseMode = params.get('preview') === 'test-release'; + devPreviewMode = testReleaseMode || params.get('preview') === 'dev-candidates'; const route = parseRoute(window.location.hash); if (route.kind !== 'not-found') profile = route.profile; if (localMode) { @@ -138,8 +145,9 @@
UNTIL EVERY CAGE V2 / FIELD NOTE

EVIDENCE DESK · {devPreviewMode ? 'PRIVATE CANDIDATE PREVIEW' : localMode ? 'LOCAL V2 API' : 'SYNTHETIC PREVIEW'}

See what a record can—and cannot—tell us.

Start with the source profile, narrow the visible evidence, then inspect what a record can—and cannot—tell us.

- {#if devPreviewMode}{/if} + {#if devPreviewMode}{/if} {#if devPreviewMode}{/if} + {#if testReleaseMode && previewStatus === 'ready'}
TEST-ONLY CSV — NOT PROJECT-APPROVED OR PUBLISHEDComplete bounded test-release rows only; this action never uses the public export route.{#if testCsvError}

{testCsvError}

{/if}
{/if} {#if !devPreviewMode || previewStatus === 'ready'}

01 / DISCOVER

Choose the evidence lane

Profiles stay separate. A community claim is never silently promoted into a curated result.

{#if localMode && metadataStatus === 'loading'}

Loading available filter values…

{:else if localMode && metadataStatus === 'error'}

Filter metadata is unavailable; only unscoped discovery is shown.

{/if}
diff --git a/frontend/src/features/devPreview/devPreviewContract.ts b/frontend/src/features/devPreview/devPreviewContract.ts index 6db1605..6c55a21 100644 --- a/frontend/src/features/devPreview/devPreviewContract.ts +++ b/frontend/src/features/devPreview/devPreviewContract.ts @@ -7,6 +7,11 @@ export type DevPreviewState = 'disabled' | 'unavailable' | 'loading' | 'ready' | export const DEV_PREVIEW_PATH = '/api/dev/preview/candidates'; export const DEV_PREVIEW_QUERY = 'dev-candidates'; export const DEV_PREVIEW_TOKEN_HEADER = 'X-UEC-Dev-Preview-Token'; +export const TEST_RELEASE_PATH = '/api/dev/preview/test-release'; +export const TEST_RELEASE_API_VERSION = 'dev-test-v1'; +export const TEST_RELEASE_LABEL = 'Disposable test release — not project-approved or published'; +export const testReleasePath = (resource: 'locations' | 'filters' | 'facets' | 'csv', id?: string): string => + `${TEST_RELEASE_PATH}/${resource}${id ? `/${encodeURIComponent(id)}` : ''}`; /** Keep a manually typed preview URL inert in production builds. */ export const canOpenDevPreview = (isDevelopment: boolean, requestedMode: string | null): boolean => diff --git a/frontend/tests/e2e/private-preview.spec.ts b/frontend/tests/e2e/private-preview.spec.ts index dae91dc..7ffdeee 100644 --- a/frontend/tests/e2e/private-preview.spec.ts +++ b/frontend/tests/e2e/private-preview.spec.ts @@ -29,3 +29,19 @@ test('private preview failure does not fall back to fixture records', async ({ p await expect(page.getByRole('heading', { name: 'Private candidate facility' })).toHaveCount(0); await expect(page.getByText('Synthetic demonstration')).toHaveCount(0); }); + +test('test-release mode uses the existing list/detail flow with a private release label', async ({ page }) => { + await page.route('**/api/dev/preview/test-release/locations.csv**', route => route.fulfill({ headers: { 'content-type': 'text/csv', 'x-uec-test-release': 'true', 'x-uec-release-id': 'test-release' }, body: 'facility_id,project_approval\nfacility-1,pending\n' })); + await page.route('**/api/dev/preview/test-release/locations**', route => route.fulfill({ json: { data: [{ facility_id: '550e8400-e29b-41d4-a716-446655440000', canonical_name: 'Pending test-release row', city: null, country_code: 'GB', category: 'dairy', source_type: 'official', publication_profile: 'official', factual_review_status: 'unreviewed', privacy_screening_status: 'passed', project_approval: 'pending', reviewer_role: null, publication_warning: null, display_precision: 'unmapped', latitude: null, longitude: null, first_observed_at: null, last_observed_at: null, observation_count: null, lifecycle_status: 'status_unknown', provenance_source_id: 'source-1', provenance_source_name: 'Test source', provenance_source_url: 'https://example.test/source', provenance_retrieved_at: '2026-01-01T00:00:00Z', release_id: 'test-release', release_ruleset_version: 'rules-1' }], meta: { api_version: 'dev-test-v1', environment: 'test-only', test_only: true, private_preview: true, release_status: 'candidate', release_id: 'test-release', profile: 'official', coverage_scope: 'test_release_public_shaped_rows', count_semantics: 'Rows only', preview_label: 'Disposable test release — not project-approved or published', result_count: 1, next_cursor: null } } })); + await page.goto('./?preview=test-release#/'); + await page.getByLabel('Operator token (memory only)').fill('synthetic-test-token'); + await page.getByRole('button', { name: 'Load test release' }).click(); + await expect(page.getByRole('heading', { name: 'Pending test-release row' })).toBeVisible(); + await expect(page.getByText('Disposable test release — not project-approved or published')).toBeVisible(); + await expect(page.getByText('No publishable map location')).toBeVisible(); + await expect(page.getByRole('button', { name: 'Download test-only CSV' })).toBeVisible(); + const csvRequest = page.waitForRequest(request => request.url().includes('/api/dev/preview/test-release/locations.csv')); + await page.getByRole('button', { name: 'Download test-only CSV' }).click(); + expect((await csvRequest).headers()['x-uec-dev-preview-token']).toBe('synthetic-test-token'); + await expect(page.getByRole('button', { name: /Preview export/ })).toHaveCount(0); +}); diff --git a/frontend/tests/unit/devPreviewContract.test.ts b/frontend/tests/unit/devPreviewContract.test.ts index 6321752..4e3463f 100644 --- a/frontend/tests/unit/devPreviewContract.test.ts +++ b/frontend/tests/unit/devPreviewContract.test.ts @@ -1,7 +1,10 @@ -import { describe, expect, it } from 'vitest'; -import { canOpenDevPreview, DEV_PREVIEW_LABEL, DEV_PREVIEW_PATH, DEV_PREVIEW_QUERY, DEV_PREVIEW_TOKEN_HEADER, devPreviewExportLabel, canMountPublicExport } from '../../src/features/devPreview/devPreviewContract'; +import { describe, expect, it, vi } from 'vitest'; +import { canOpenDevPreview, DEV_PREVIEW_LABEL, DEV_PREVIEW_PATH, DEV_PREVIEW_QUERY, DEV_PREVIEW_TOKEN_HEADER, devPreviewExportLabel, canMountPublicExport, TEST_RELEASE_API_VERSION, TEST_RELEASE_LABEL, TEST_RELEASE_PATH, testReleasePath } from '../../src/features/devPreview/devPreviewContract'; import { DevCandidatePreviewRepository } from '../../src/api/DevCandidatePreviewRepository'; import { nextLocalReviewState } from '../../src/features/devPreview/devReviewState'; +import { TestReleaseRepository } from '../../src/api/TestReleaseRepository'; +import { TestReleaseCsvExportRepository } from '../../src/api/TestReleaseCsvExportRepository'; +import { TestReleaseFilterMetadataRepository } from '../../src/api/TestReleaseFilterMetadataRepository'; describe('dev preview boundary', () => { it('requires both a development build and the explicit mode', () => { @@ -21,6 +24,11 @@ describe('dev preview boundary', () => { expect(devPreviewExportLabel(false)).toBeNull(); expect(canMountPublicExport(true)).toBe(false); // includes ?preview=dev-candidates&mode=local-v2 expect(canMountPublicExport(false)).toBe(true); + expect(TEST_RELEASE_API_VERSION).toBe('dev-test-v1'); + expect(TEST_RELEASE_PATH).not.toContain('/api/v2/'); + expect(testReleasePath('locations', 'facility/one')).toBe('/api/dev/preview/test-release/locations/facility%2Fone'); + expect(testReleasePath('csv')).toBe('/api/dev/preview/test-release/csv'); + expect(TEST_RELEASE_LABEL).toContain('not project-approved or published'); }); it('preserves candidate unapproved and unpublished semantics', async () => { @@ -33,4 +41,19 @@ describe('dev preview boundary', () => { expect(nextLocalReviewState('inspected', 'follow_up')).toBe('follow_up'); expect(nextLocalReviewState('follow_up', 'inspect')).toBe('follow_up'); }); + it('maps test-release rows without requiring approval or coordinates and never falls back', async () => { + const row = { facility_id: '550e8400-e29b-41d4-a716-446655440000', canonical_name: 'Pending test row', city: null, country_code: 'GB', category: 'dairy', source_type: 'official', publication_profile: 'official', factual_review_status: 'unreviewed', privacy_screening_status: 'passed', project_approval: 'pending', reviewer_role: null, publication_warning: null, display_precision: 'unmapped', latitude: null, longitude: null, first_observed_at: null, last_observed_at: null, observation_count: null, lifecycle_status: 'status_unknown', provenance_source_id: 's1', provenance_source_name: 'Test source', provenance_source_url: 'https://example.test/source', provenance_retrieved_at: '2026-01-01T00:00:00Z', release_id: 'test-release', release_ruleset_version: 'rules-1' }; + const body = { data: [row], meta: { api_version: 'dev-test-v1', environment: 'test-only', test_only: true, private_preview: true, release_status: 'candidate', release_id: 'test-release', profile: 'official', coverage_scope: 'test_release_public_shaped_rows', count_semantics: 'Rows only', preview_label: TEST_RELEASE_LABEL, result_count: 1, next_cursor: null } }; + const fetcher = vi.fn().mockResolvedValue(new Response(JSON.stringify(body))); + const result = await new TestReleaseRepository(fetcher).list('official', 'test-token'); + expect(result.locations[0]).toMatchObject({ name: 'Pending test row', lat: null, evidence: { projectApproval: 'pending' } }); + expect(result.coverageNote).toContain('not project-approved'); + await expect(new TestReleaseRepository(vi.fn().mockResolvedValue(new Response('unavailable', { status: 401 }))).list('official', 'wrong')).rejects.toThrow('unavailable'); + }); + it('requires explicit test-release CSV markers and validates facets metadata', async () => { + const csvFetcher = vi.fn().mockResolvedValue(new Response('facility_id,project_approval\n1,pending\n', { headers: { 'x-uec-test-release': 'true', 'x-uec-release-id': 'test-release' } })); + await expect(new TestReleaseCsvExportRepository(csvFetcher).download('official', 'test-token')).resolves.toMatchObject({ releaseId: 'test-release', label: TEST_RELEASE_LABEL }); + const facets = { data: null, meta: { api_version: 'dev-test-v1', environment: 'test-only', test_only: true, private_preview: true, release_status: 'candidate', release_id: 'test-release', profile: 'official', coverage_scope: 'test_release_public_shaped_rows', count_semantics: 'Rows only', preview_label: TEST_RELEASE_LABEL, result_count: 0 }, dimensions: { country_code: [], category: [], display_precision: [], source_type: [] } }; + await expect(new TestReleaseFilterMetadataRepository(vi.fn().mockResolvedValue(new Response(JSON.stringify(facets)))).get('official', 'test-token')).resolves.toMatchObject({ meta: { test_only: true, release_status: 'candidate' } }); + }); }); From 3fd05351802bb72acc974e0077e3100efb8195bb Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Mon, 14 Sep 2026 00:38:29 -0700 Subject: [PATCH 086/311] Add guarded disposable test-release API and promotion isolation --- pipeline/migrations/023_test_only_release.sql | 20 + .../scripts/maintenance/import-candidate.py | 4 +- pipeline/scripts/stages/promote-release.py | 10 +- pipeline/scripts/stages/validate-release.py | 7 +- pipeline/tests/e2e/README.md | 2 +- pipeline/tests/e2e/fixture.py | 25 +- pipeline/tests/e2e/test_candidate_import.py | 74 +++- pipeline/tests/test_promote_release.py | 5 + pipeline/tests/test_release_validation.py | 9 + src/lib.rs | 358 +++++++++++++++++- src/main.rs | 41 ++ 11 files changed, 524 insertions(+), 31 deletions(-) create mode 100644 pipeline/migrations/023_test_only_release.sql diff --git a/pipeline/migrations/023_test_only_release.sql b/pipeline/migrations/023_test_only_release.sql new file mode 100644 index 0000000..8b9e285 --- /dev/null +++ b/pipeline/migrations/023_test_only_release.sql @@ -0,0 +1,20 @@ +ALTER TABLE uec.releases + ADD COLUMN IF NOT EXISTS test_only BOOLEAN NOT NULL DEFAULT false; + +COMMENT ON COLUMN uec.releases.test_only IS + 'True only for disposable development/test releases; never eligible for public V2 selection.'; + +CREATE OR REPLACE FUNCTION uec.reject_test_only_release_transition() +RETURNS trigger LANGUAGE plpgsql AS $$ +BEGIN + IF NEW.test_only AND NEW.status <> 'candidate' THEN + RAISE EXCEPTION 'test-only releases cannot leave candidate state'; + END IF; + RETURN NEW; +END; +$$; + +DROP TRIGGER IF EXISTS releases_test_only_transition ON uec.releases; +CREATE TRIGGER releases_test_only_transition + BEFORE INSERT OR UPDATE OF status, test_only ON uec.releases + FOR EACH ROW EXECUTE FUNCTION uec.reject_test_only_release_transition(); diff --git a/pipeline/scripts/maintenance/import-candidate.py b/pipeline/scripts/maintenance/import-candidate.py index 936d08c..79a4119 100644 --- a/pipeline/scripts/maintenance/import-candidate.py +++ b/pipeline/scripts/maintenance/import-candidate.py @@ -128,8 +128,8 @@ def import_candidate(database_url: str, manifest: dict, rows: list[dict], releas (manifest["source_id"],)).fetchone()[0] db.execute("INSERT INTO uec.acquisition_run_artifacts(run_id,artifact_id) VALUES (%s,%s) ON CONFLICT DO NOTHING", (run_id, artifact_id)) - db.execute("""INSERT INTO uec.releases(release_id,status,ruleset_version,summary) - VALUES (%s,'candidate',%s,%s) + db.execute("""INSERT INTO uec.releases(release_id,status,ruleset_version,summary,test_only) + VALUES (%s,'candidate',%s,%s,true) ON CONFLICT (release_id) DO NOTHING""", (release_id, ruleset, json.dumps({"source_id": manifest["source_id"], "profile": manifest.get("profile")}))) count = 0 diff --git a/pipeline/scripts/stages/promote-release.py b/pipeline/scripts/stages/promote-release.py index bc086ad..6f9f90b 100644 --- a/pipeline/scripts/stages/promote-release.py +++ b/pipeline/scripts/stages/promote-release.py @@ -12,8 +12,8 @@ import psycopg -def can_promote(status: str) -> bool: - return status == "validated" +def can_promote(status: str, test_only: bool = False) -> bool: + return status == "validated" and not test_only def canonical_json(manifest: dict) -> str: @@ -53,10 +53,12 @@ def inventory_artifacts(paths: list[Path], no_distributed_artifacts: bool) -> li def promote(database_url: str, release_id: str, artifacts: list[dict]) -> dict: with psycopg.connect(database_url) as connection: with connection.transaction(): - target = connection.execute("SELECT status, profile, ruleset_version FROM uec.releases WHERE release_id = %s FOR UPDATE", (release_id,)).fetchone() + target = connection.execute("SELECT status, profile, ruleset_version, test_only FROM uec.releases WHERE release_id = %s FOR UPDATE", (release_id,)).fetchone() if not target: raise ValueError(f"release not found: {release_id}") - if not can_promote(target[0]): + if not can_promote(target[0], target[3]): + if target[3]: + raise ValueError("test-only releases cannot be validated or promoted") raise ValueError(f"release must be validated before promotion; current status is {target[0]}") unsafe = connection.execute(""" SELECT diff --git a/pipeline/scripts/stages/validate-release.py b/pipeline/scripts/stages/validate-release.py index 11505cc..58d95b7 100644 --- a/pipeline/scripts/stages/validate-release.py +++ b/pipeline/scripts/stages/validate-release.py @@ -27,6 +27,8 @@ def evaluate(metrics: dict, expected_records: int | None = None) -> dict: findings.append({"code": "publication_not_approved", "count": metrics["publication_not_approved"]}) if metrics.get("active_suppression"): findings.append({"code": "active_suppression", "count": metrics["active_suppression"]}) + if metrics.get("test_only"): + findings.append({"code": "test_only_release", "count": 1}) return {"status": "passed" if not findings else "blocked", "findings": findings, "metrics": metrics} @@ -45,7 +47,7 @@ def render_html(report: dict) -> str: def validate(database_url: str, release_id: str, expected_records: int | None, mark_validated: bool) -> dict: with psycopg.connect(database_url) as connection: - release = connection.execute("SELECT status FROM uec.releases WHERE release_id = %s", (release_id,)).fetchone() + release = connection.execute("SELECT status, test_only FROM uec.releases WHERE release_id = %s", (release_id,)).fetchone() if not release: raise ValueError(f"release not found: {release_id}") metrics = connection.execute(""" @@ -73,7 +75,8 @@ def validate(database_url: str, release_id: str, expected_records: int | None, m WHERE release_member.release_id = %s """, (release_id, release_id)).fetchone() names = ["release_records", "distinct_observations", "duplicate_observations", "review_visible", "exact_display_ready", "city_display_ready", "unmapped_display", "coordinate_not_ready", "publication_not_approved", "active_suppression", "validation_errors"] - result = evaluate(dict(zip(names, metrics)), expected_records) + metrics_dict = dict(zip(names, metrics)); metrics_dict["test_only"] = bool(release[1]) + result = evaluate(metrics_dict, expected_records) result.update({"release_id": release_id, "release_status_before": release[0], "marked_validated": False}) if result["status"] == "passed" and mark_validated: connection.execute("UPDATE uec.releases SET status = 'validated' WHERE release_id = %s AND status = 'candidate'", (release_id,)) diff --git a/pipeline/tests/e2e/README.md b/pipeline/tests/e2e/README.md index 6dcc413..34e08d8 100644 --- a/pipeline/tests/e2e/README.md +++ b/pipeline/tests/e2e/README.md @@ -11,7 +11,7 @@ python -m unittest discover -s pipeline/tests/e2e -p "test_*.py" -v This requires Docker Desktop, Cargo, and the pinned Python dependencies. The fixture uses isolated random ports and tears down its Compose project even after setup failures. Fast non-Docker checks remain available with `python -m unittest discover -s pipeline/tests -p "test_*.py" -v`. -`fixture.py` owns the environment lifecycle: it selects isolated ports, starts Docker Compose, applies migrations as UTF-8, builds and starts the backend, waits for readiness, and tears everything down. Setup failures also trigger cleanup. Run the three API modules sequentially (`test_public_api`, `test_community_api`, and `test_seeded_api`) because each module owns a disposable PostGIS environment; running all classes in one discovery process can create avoidable Docker resource/lifecycle contention. +`fixture.py` owns the environment lifecycle: it selects isolated ports, starts Docker Compose, applies migrations as UTF-8, builds and starts the backend from a per-run temporary Cargo target directory, waits for readiness, and tears everything down. The isolated target prevents E2E builds from contending with a developer's running backend binary. Setup failures also trigger cleanup. Run the API modules sequentially (`test_public_api`, `test_community_api`, `test_seeded_api`, and `test_candidate_import`) because each module owns a disposable PostGIS environment; running all classes in one discovery process can create avoidable Docker resource/lifecycle contention. `test_public_api.py` verifies the publication boundary with an empty database: candidate data and filters remain unavailable, and malformed pagination is rejected. diff --git a/pipeline/tests/e2e/fixture.py b/pipeline/tests/e2e/fixture.py index f96c40f..2369508 100644 --- a/pipeline/tests/e2e/fixture.py +++ b/pipeline/tests/e2e/fixture.py @@ -2,6 +2,8 @@ import os import socket import subprocess +import tempfile +import shutil import time import uuid from pathlib import Path @@ -27,9 +29,13 @@ def __init__(self): self.database_url = f"postgresql://uec:uec-e2e@localhost:{self.db_port}/uec" self.backend = None self.backend_log = None + self.build_temp = tempfile.TemporaryDirectory(prefix="uec-e2e-cargo-") + self.cargo_target_dir = Path(self.build_temp.name) + self.cargo_cache_dir = ROOT / "target" / "e2e-cache" self.start_attempts = 0 # Synthetic only: this token is scoped to the disposable test server. self.dev_preview_token = "uec-e2e-preview-token" + self.test_release_id = None def command(self, *args): return ["docker", "compose", "-p", self.project, "-f", str(COMPOSE), *args] @@ -74,12 +80,18 @@ def start(self): return self.start() raise print("[e2e] building backend", flush=True) - subprocess.run(["cargo", "build", "--quiet"], cwd=ROOT, check=True, timeout=180) + build_env = os.environ.copy() + build_env["CARGO_TARGET_DIR"] = str(self.cargo_cache_dir) + subprocess.run(["cargo", "build", "--quiet"], cwd=ROOT, check=True, timeout=180, env=build_env) env = os.environ.copy(); env.update({"UEC_DATABASE_URL": self.database_url, "PORT": str(self.api_port), "UEC_RUNTIME_MODE": "development", "UEC_BIND_HOST": "127.0.0.1", "UEC_DEV_PREVIEW": "true", "UEC_DEV_PREVIEW_TOKEN": self.dev_preview_token}) - binary = ROOT / "target/debug/uec-api.exe" - if not binary.exists(): - binary = ROOT / "target/debug/uec-api" - self.backend_log = (ROOT / "target" / f"e2e-{self.project}.log").open("w", encoding="utf-8") + if self.test_release_id: + env.update({"UEC_TEST_RELEASE_ID": self.test_release_id, "UEC_TEST_RELEASE_TOKEN": self.dev_preview_token}) + cached_binary = self.cargo_cache_dir / "debug/uec-api.exe" + if not cached_binary.exists(): + cached_binary = self.cargo_cache_dir / "debug/uec-api" + binary = self.cargo_target_dir / cached_binary.name + shutil.copy2(cached_binary, binary) + self.backend_log = (self.cargo_target_dir / f"e2e-{self.project}.log").open("w", encoding="utf-8") self.backend = subprocess.Popen([str(binary)], cwd=ROOT, env=env, stdout=self.backend_log, stderr=subprocess.STDOUT, text=True) print(f"[e2e] waiting for backend on {self.api_port}", flush=True) import urllib.error @@ -122,6 +134,9 @@ def stop(self): self.backend_log.close() self.backend_log = None subprocess.run(self.command("down", "-v", "--remove-orphans"), cwd=ROOT, check=False, capture_output=True, text=True, env=self.compose_env()) + if self.build_temp: + self.build_temp.cleanup() + self.build_temp = None def seed_official_scenario(self): """Seed safe synthetic records for public API tests.""" diff --git a/pipeline/tests/e2e/test_candidate_import.py b/pipeline/tests/e2e/test_candidate_import.py index eb1d969..1c27458 100644 --- a/pipeline/tests/e2e/test_candidate_import.py +++ b/pipeline/tests/e2e/test_candidate_import.py @@ -39,7 +39,9 @@ def counts(self): def setUpClass(cls): if os.environ.get("UEC_RUN_E2E") != "1": raise unittest.SkipTest("set UEC_RUN_E2E=1 to run Docker-backed E2E tests") - cls.env = E2EEnvironment().start() + cls.env = E2EEnvironment() + cls.env.test_release_id = "candidate-uk-e2e" + cls.env = cls.env.start() cls.temp = tempfile.TemporaryDirectory() root = Path(cls.temp.name) raw = root / "uk-monthly.csv" @@ -68,6 +70,19 @@ def setUpClass(cls): if second.returncode: cls.temp.cleanup(); cls.env.stop() raise RuntimeError(f"candidate importer rerun failed:\n{second.stdout}\n{second.stderr}") + # Synthetic negative rows exercise source-state and privacy gates in + # every guarded test-release surface without introducing raw payloads. + now = datetime.now(timezone.utc) + with psycopg.connect(cls.env.database_url) as db: + with db.transaction(): + artifact = db.execute("SELECT artifact_id FROM uec.raw_artifacts WHERE storage_key LIKE 'private-staging/%' LIMIT 1").fetchone()[0] + for key, state, privacy in (("rejected-negative", "rejected", "pending"), ("withheld-negative", "present", "failed")): + record, facility, observation = uuid.uuid4(), uuid.uuid4(), uuid.uuid4() + db.execute("INSERT INTO uec.source_records(source_record_id,source_id,source_record_key,artifact_id,raw_fields,parsed_at,source_state) VALUES (%s,'fsa_approved_establishments',%s,%s,'{}',%s,%s)", (record,key,artifact,now,state)) + db.execute("INSERT INTO uec.facilities(facility_id,canonical_name,country_code,city) VALUES (%s,%s,'GB','London')", (facility, f"E2E {key}")) + db.execute("INSERT INTO uec.observations(observation_id,facility_id,source_record_id,observed_at,observation,classification,ruleset_id,rule_id,classification_category,classification_review_status,default_visible,coordinate_review_status,first_observed_at) VALUES (%s,%s,%s,%s,'{}','{}','e2e','negative','processing','review_required',false,'review_required',%s)", (observation,facility,record,now,now)) + db.execute("INSERT INTO uec.release_members(release_id,facility_id,observation_id,default_visible) VALUES (%s,%s,%s,false)", (cls.release_id,facility,observation)) + db.execute("INSERT INTO uec.publication_review_events(source_record_id,release_id,factual_review_status,privacy_screening_status,maintainer_approval,publication_eligible) VALUES (%s,%s,'unreviewed',%s,'pending',false)", (record,cls.release_id,privacy)) cls.initial_counts = cls.counts_for() cls._record_id = None @@ -84,19 +99,23 @@ def request(self, path, headers=None): def test_import_is_idempotent_and_candidate_is_not_public_or_previewable(self): with psycopg.connect(self.env.database_url) as db: - source_count, observation_count, artifact_count, release_member_count, review_count, release_count, run_count = db.execute( - """SELECT count(DISTINCT r.source_record_id), count(DISTINCT o.observation_id) - , (SELECT count(*) FROM uec.raw_artifacts WHERE storage_key LIKE 'private-staging/%') - , (SELECT count(*) FROM uec.release_members WHERE release_id=%s) - , (SELECT count(*) FROM uec.publication_review_events WHERE release_id=%s) - , (SELECT count(*) FROM uec.releases WHERE release_id=%s) - , (SELECT count(*) FROM uec.acquisition_runs WHERE source_id='fsa_approved_establishments') - FROM uec.source_records r JOIN uec.observations o USING (source_record_id) - WHERE r.source_id='fsa_approved_establishments'""", (self.release_id, self.release_id, self.release_id) - ).fetchone() + positive_record = db.execute("SELECT source_record_id FROM uec.source_records WHERE source_id='fsa_approved_establishments' AND source_record_key='2:UK-E2E-1'").fetchone()[0] + source_count = db.execute("SELECT count(*) FROM uec.source_records WHERE source_record_id=%s", (positive_record,)).fetchone()[0] + observation_count = db.execute("SELECT count(*) FROM uec.observations WHERE source_record_id=%s", (positive_record,)).fetchone()[0] + artifact_count = db.execute("SELECT count(*) FROM uec.raw_artifacts WHERE storage_key LIKE 'private-staging/%%'").fetchone()[0] + release_member_count = db.execute("SELECT count(*) FROM uec.release_members m JOIN uec.observations o ON o.observation_id=m.observation_id WHERE m.release_id=%s AND o.source_record_id=%s", (self.release_id, positive_record)).fetchone()[0] + review_count = db.execute("SELECT count(*) FROM uec.publication_review_events WHERE release_id=%s AND source_record_id=%s", (self.release_id, positive_record)).fetchone()[0] + release_count = db.execute("SELECT count(*) FROM uec.releases WHERE release_id=%s", (self.release_id,)).fetchone()[0] + run_count = db.execute("SELECT count(*) FROM uec.acquisition_runs WHERE source_id='fsa_approved_establishments'").fetchone()[0] self.assertEqual((source_count, observation_count, artifact_count, release_member_count, review_count, release_count, run_count), (1, 1, 1, 1, 1, 1, 2)) - review_state = db.execute("SELECT factual_review_status, privacy_screening_status, maintainer_approval, publication_eligible FROM uec.publication_review_events WHERE release_id=%s", (self.release_id,)).fetchone() + review_state = db.execute("SELECT factual_review_status, privacy_screening_status, maintainer_approval, publication_eligible FROM uec.publication_review_events WHERE release_id=%s AND source_record_id=%s", (self.release_id, positive_record)).fetchone() self.assertEqual(review_state, ("unreviewed", "pending", "pending", False)) + self.assertTrue(db.execute("SELECT test_only FROM uec.releases WHERE release_id=%s", (self.release_id,)).fetchone()[0]) + for target_status in ("validated", "promoted"): + with self.assertRaises(psycopg.Error): + db.execute("UPDATE uec.releases SET status=%s WHERE release_id=%s", (target_status, self.release_id)) + db.rollback() + self.assertEqual(db.execute("SELECT status FROM uec.releases WHERE release_id=%s", (self.release_id,)).fetchone()[0], "candidate") # The public route succeeds with an empty envelope, not an error. with self.request("/api/v2/locations?profile=official") as response: body = json.loads(response.read()) @@ -108,6 +127,37 @@ def test_import_is_idempotent_and_candidate_is_not_public_or_previewable(self): ) with urllib.request.urlopen(preview, timeout=10) as response: self.assertEqual(json.loads(response.read())["data"], []) + test_list = urllib.request.Request( + f"http://127.0.0.1:{self.env.api_port}/api/dev/preview/test-release/locations?profile=official", + headers={"X-UEC-Dev-Preview-Token": self.env.dev_preview_token}, + ) + with urllib.request.urlopen(test_list, timeout=10) as response: + test_body = json.loads(response.read()) + self.assertEqual(test_body["meta"]["api_version"], "dev-test-v1") + self.assertTrue(test_body["meta"]["test_only"]) + self.assertEqual(test_body["meta"]["release_status"], "candidate") + self.assertEqual(test_body["data"][0]["privacy_screening_status"], "pending") + self.assertIsNone(test_body["data"][0]["latitude"]) + self.assertNotIn("source_values", json.dumps(test_body)) + facility_id = test_body["data"][0]["facility_id"] + detail = urllib.request.Request( + f"http://127.0.0.1:{self.env.api_port}/api/dev/preview/test-release/locations/{facility_id}", + headers={"X-UEC-Dev-Preview-Token": self.env.dev_preview_token}, + ) + with urllib.request.urlopen(detail, timeout=10) as response: + self.assertEqual(json.loads(response.read())["meta"]["environment"], "test-only") + with urllib.request.urlopen(urllib.request.Request( + f"http://127.0.0.1:{self.env.api_port}/api/dev/preview/test-release/discovery/facets", + headers={"X-UEC-Dev-Preview-Token": self.env.dev_preview_token}, + ), timeout=10) as response: + self.assertEqual(json.loads(response.read())["meta"]["test_only"], True) + with urllib.request.urlopen(urllib.request.Request( + f"http://127.0.0.1:{self.env.api_port}/api/dev/preview/test-release/locations.csv", + headers={"X-UEC-Dev-Preview-Token": self.env.dev_preview_token}, + ), timeout=10) as response: + csv_body = response.read().decode() + self.assertIn("test_only", csv_body) + self.assertNotIn("source_values", csv_body) def test_review_version_unlocks_preview_only_then_suppression_relocks_it(self): now = datetime.now(timezone.utc) diff --git a/pipeline/tests/test_promote_release.py b/pipeline/tests/test_promote_release.py index 9b7c731..3fbdbe4 100644 --- a/pipeline/tests/test_promote_release.py +++ b/pipeline/tests/test_promote_release.py @@ -15,6 +15,7 @@ class ReleasePromotionTests(unittest.TestCase): def test_only_validated_releases_can_be_promoted(self): self.assertTrue(MODULE.can_promote("validated")) + self.assertFalse(MODULE.can_promote("validated", True)) self.assertFalse(MODULE.can_promote("candidate")) self.assertFalse(MODULE.can_promote("promoted")) self.assertFalse(MODULE.can_promote("rejected")) @@ -23,6 +24,10 @@ def test_promotion_script_scopes_replacement_to_profile(self): source = SCRIPT.read_text(encoding="utf-8") self.assertIn("profile = %s", source) + def test_test_only_releases_are_never_promotable(self): + self.assertFalse(MODULE.can_promote("validated", test_only=True)) + self.assertIn("test-only releases cannot be validated or promoted", SCRIPT.read_text(encoding="utf-8")) + def test_promotion_rechecks_public_safety_gates_and_supports_manifest(self): source = SCRIPT.read_text(encoding="utf-8") for gate in ("coordinate_not_ready", "review_required", "publication_not_approved", "active_suppression"): diff --git a/pipeline/tests/test_release_validation.py b/pipeline/tests/test_release_validation.py index 9549e8d..fdc5f52 100644 --- a/pipeline/tests/test_release_validation.py +++ b/pipeline/tests/test_release_validation.py @@ -31,6 +31,15 @@ def test_html_report_is_human_readable(self): self.assertIn("PASSED", rendered) self.assertIn("release_records", rendered) + def test_test_only_release_is_blocked_before_validation(self): + report = MODULE.evaluate({"release_records": 1, "duplicate_observations": 0, "validation_errors": 0, "review_visible": 0, "coordinate_not_ready": 0, "publication_not_approved": 0, "active_suppression": 0, "test_only": True}) + self.assertEqual(report["status"], "blocked") + self.assertEqual(report["findings"][0]["code"], "test_only_release") + + def test_validation_report_names_test_only_blocker(self): + report = MODULE.evaluate({"release_records": 0, "duplicate_observations": 0, "validation_errors": 0, "review_visible": 0, "coordinate_not_ready": 0, "publication_not_approved": 0, "active_suppression": 0, "test_only": True}) + self.assertIn("test_only_release", {finding["code"] for finding in report["findings"]}) + if __name__ == "__main__": unittest.main() diff --git a/src/lib.rs b/src/lib.rs index f547ea2..b6b2755 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -102,7 +102,7 @@ pub async fn get_v2_release_manifest_handler( ); } }; - let row = match client.query_opt("SELECT r.release_id, r.profile, m.manifest::text, m.manifest_sha256 FROM uec.releases r JOIN uec.release_manifests m ON m.release_id=r.release_id WHERE r.status='promoted' AND r.profile=$1 ORDER BY r.created_at DESC, r.release_id DESC LIMIT 1", &[&profile]).await { + let row = match client.query_opt("SELECT r.release_id, r.profile, m.manifest::text, m.manifest_sha256 FROM uec.releases r JOIN uec.release_manifests m ON m.release_id=r.release_id WHERE r.status='promoted' AND r.test_only IS NOT TRUE AND r.profile=$1 ORDER BY r.created_at DESC, r.release_id DESC LIMIT 1", &[&profile]).await { Ok(row) => row, Err(_) => return v2_error(StatusCode::SERVICE_UNAVAILABLE, "release_manifest_unavailable", "release manifest unavailable") }; let Some(row) = row else { @@ -205,7 +205,7 @@ pub async fn get_v2_locations_export_handler( ); } }; - let release = match client.query_opt("SELECT r.release_id, m.manifest_sha256 FROM uec.releases r JOIN uec.release_manifests m ON m.release_id=r.release_id WHERE r.status='promoted' AND r.profile=$1 ORDER BY r.created_at DESC, r.release_id DESC LIMIT 1", &[&profile]).await { + let release = match client.query_opt("SELECT r.release_id, m.manifest_sha256 FROM uec.releases r JOIN uec.release_manifests m ON m.release_id=r.release_id WHERE r.status='promoted' AND r.test_only IS NOT TRUE AND r.profile=$1 ORDER BY r.created_at DESC, r.release_id DESC LIMIT 1", &[&profile]).await { Ok(row) => row, Err(_) => return v2_error(StatusCode::SERVICE_UNAVAILABLE, "release_query_failed", "release query failed") }; let Some(release) = release else { @@ -291,10 +291,348 @@ pub async fn get_v2_locations_export_handler( pub struct ApiState { pub database: Option, pub dev_preview_token: Option, + pub dev_test_release_id: Option, + pub dev_test_release_token: Option, } const DEV_PREVIEW_TOKEN_HEADER: &str = "x-uec-dev-preview-token"; +fn test_release_auth(headers: &HeaderMap, state: &ApiState) -> bool { + let (Some(release_id), Some(expected)) = + (&state.dev_test_release_id, &state.dev_test_release_token) + else { + return false; + }; + preview_request_is_local(headers) + && headers + .get(DEV_PREVIEW_TOKEN_HEADER) + .and_then(|v| v.to_str().ok()) + .is_some_and(|v| constant_time_token_matches(expected, v)) + && !release_id.is_empty() +} + +fn test_release_meta(release_id: &str, profile: &str) -> serde_json::Value { + json!({"api_version":"dev-test-v1","environment":"test-only","test_only":true,"private_preview":true,"release_status":"candidate","release_id":release_id,"profile":profile,"coverage_scope":"test_release_public_shaped_rows","count_semantics":"Rows are disposable candidate facilities, not project-approved or published counts.","preview_label":"Disposable test release — not project-approved or published"}) +} + +fn csv_safe_value(value: String) -> String { + if value.starts_with(['=', '+', '-', '@']) { + format!("'{}", value) + } else { + value + } +} + +pub async fn get_dev_test_release_locations_handler( + State(state): State, + headers: HeaderMap, + Query(params): Query, +) -> impl IntoResponse { + if !test_release_auth(&headers, &state) { + return v2_error( + StatusCode::NOT_FOUND, + "test_release_unavailable", + "test release unavailable", + ); + } + let Some(pool) = state.database else { + return v2_error( + StatusCode::SERVICE_UNAVAILABLE, + "database_not_configured", + "test release database unavailable", + ); + }; + let profile = params.profile.as_deref().unwrap_or("official"); + let Some(release_id) = state.dev_test_release_id.as_deref() else { + return v2_error( + StatusCode::NOT_FOUND, + "test_release_unavailable", + "test release unavailable", + ); + }; + let client = match pool.get().await { + Ok(c) => c, + Err(_) => { + return v2_error( + StatusCode::SERVICE_UNAVAILABLE, + "database_pool_unavailable", + "database pool unavailable", + ); + } + }; + let limit = params + .limit + .as_deref() + .unwrap_or("100") + .parse::() + .ok() + .filter(|v| (1..=1000).contains(v)) + .unwrap_or(0); + if limit == 0 { + return v2_error( + StatusCode::BAD_REQUEST, + "invalid_limit", + "limit must be between 1 and 1000", + ); + } + let rows = match client.query(r#"SELECT f.facility_id,f.canonical_name,f.country_code,f.city,o.classification_category, + CASE WHEN o.coordinate_review_status='approved' AND g.result IS NOT NULL THEN 'exact' ELSE 'unmapped' END, + CASE WHEN o.coordinate_review_status='approved' AND g.result IS NOT NULL THEN ST_Y(g.result::geometry) ELSE NULL END, + CASE WHEN o.coordinate_review_status='approved' AND g.result IS NOT NULL THEN ST_X(g.result::geometry) ELSE NULL END, + review.factual_review_status,review.privacy_screening_status,review.maintainer_approval,review.reviewer_role, + source.origin_type, source.source_id, source.name, source.official_url, r.ruleset_version, artifact.retrieved_at + FROM uec.release_members member JOIN uec.releases r ON r.release_id=member.release_id + JOIN uec.observations o ON o.observation_id=member.observation_id JOIN uec.facilities f ON f.facility_id=member.facility_id + JOIN uec.source_records record ON record.source_record_id=o.source_record_id JOIN uec.sources source ON source.source_id=record.source_id + JOIN uec.raw_artifacts artifact ON artifact.artifact_id=record.artifact_id + LEFT JOIN uec.publication_review_release_current review ON review.source_record_id=o.source_record_id AND review.release_id=member.release_id + LEFT JOIN LATERAL (SELECT result FROM uec.geocode_results WHERE source_record_id=o.source_record_id AND status='accepted' AND result IS NOT NULL ORDER BY queried_at DESC,geocode_result_id DESC LIMIT 1) g ON true + WHERE r.release_id=$1 AND r.status='candidate' AND r.test_only=true AND record.source_state NOT IN ('rejected','superseded') + AND COALESCE(review.privacy_screening_status,'pending') <> 'failed' AND COALESCE(review.factual_review_status,'unreviewed') <> 'rejected' + AND NOT EXISTS (SELECT 1 FROM uec.public_access_restricted restricted WHERE restricted.source_record_id=o.source_record_id) + AND ($2::text IS NULL OR f.country_code=$2) AND ($3::text IS NULL OR o.classification_category=$3) + ORDER BY f.facility_id LIMIT $4"#, &[&release_id,¶ms.country_code,¶ms.category,&limit]).await { + Ok(rows)=>rows, Err(_)=>return v2_error(StatusCode::SERVICE_UNAVAILABLE,"test_release_query_failed","test release unavailable") + }; + let data = rows.into_iter().map(|row| json!({"facility_id":row.get::<_,uuid::Uuid>(0),"canonical_name":row.get::<_,Option>(1),"country_code":row.get::<_,String>(2),"city":row.get::<_,Option>(3),"category":row.get::<_,String>(4),"publication_profile":profile,"factual_review_status":row.get::<_,Option>(8).unwrap_or("unreviewed".into()),"privacy_screening_status":row.get::<_,Option>(9).unwrap_or("pending".into()),"project_approval":"not-approved","reviewer_role":row.get::<_,Option>(11),"publication_warning":"Disposable test release — not project-approved or published","display_precision":row.get::<_,String>(5),"latitude":row.get::<_,Option>(6),"longitude":row.get::<_,Option>(7),"lifecycle_status":"status_unknown","source_type":row.get::<_,String>(12),"release_id":release_id,"release_ruleset_version":row.get::<_,String>(16),"provenance_source_id":row.get::<_,String>(13),"provenance_source_name":row.get::<_,String>(14),"provenance_source_url":row.get::<_,String>(15),"provenance_retrieved_at":row.get::<_,chrono::DateTime>(17)})).collect::>(); + let mut meta = test_release_meta(release_id, profile); + meta["result_count"] = json!(data.len()); + Json(json!({"data":data,"meta":meta})).into_response() +} + +pub async fn get_dev_test_release_location_detail_handler( + State(state): State, + headers: HeaderMap, + Path(facility_id): Path, + Query(params): Query, +) -> impl IntoResponse { + let profile = params.profile.as_deref().unwrap_or("official"); + if !test_release_auth(&headers, &state) { + return v2_error( + StatusCode::NOT_FOUND, + "test_release_unavailable", + "test release unavailable", + ); + } + let Some(pool) = state.database else { + return v2_error( + StatusCode::SERVICE_UNAVAILABLE, + "database_not_configured", + "test release database unavailable", + ); + }; + let Some(release_id) = state.dev_test_release_id.as_deref() else { + return v2_error( + StatusCode::NOT_FOUND, + "test_release_unavailable", + "test release unavailable", + ); + }; + let client = match pool.get().await { + Ok(c) => c, + Err(_) => { + return v2_error( + StatusCode::SERVICE_UNAVAILABLE, + "database_pool_unavailable", + "database pool unavailable", + ); + } + }; + let row=match client.query_opt("SELECT f.facility_id,f.canonical_name,f.country_code,f.city,o.classification_category,COALESCE(review.factual_review_status,'unreviewed'),COALESCE(review.privacy_screening_status,'pending'),o.coordinate_review_status,source.origin_type,source.source_id,source.name,source.official_url,artifact.retrieved_at,r.ruleset_version,CASE WHEN o.coordinate_review_status='approved' AND g.result IS NOT NULL THEN 'exact' ELSE 'unmapped' END,CASE WHEN o.coordinate_review_status='approved' AND g.result IS NOT NULL THEN ST_Y(g.result::geometry) ELSE NULL END,CASE WHEN o.coordinate_review_status='approved' AND g.result IS NOT NULL THEN ST_X(g.result::geometry) ELSE NULL END FROM uec.release_members m JOIN uec.releases r ON r.release_id=m.release_id JOIN uec.observations o ON o.observation_id=m.observation_id JOIN uec.facilities f ON f.facility_id=m.facility_id JOIN uec.source_records sr ON sr.source_record_id=o.source_record_id JOIN uec.sources source ON source.source_id=sr.source_id JOIN uec.raw_artifacts artifact ON artifact.artifact_id=sr.artifact_id LEFT JOIN uec.publication_review_release_current review ON review.source_record_id=o.source_record_id AND review.release_id=m.release_id LEFT JOIN LATERAL (SELECT result FROM uec.geocode_results WHERE source_record_id=o.source_record_id AND status='accepted' AND result IS NOT NULL ORDER BY queried_at DESC LIMIT 1) g ON true WHERE r.release_id=$1 AND r.status='candidate' AND r.test_only=true AND f.facility_id=$2 AND sr.source_state NOT IN ('rejected','superseded') AND COALESCE(review.privacy_screening_status,'pending') <> 'failed' AND NOT EXISTS (SELECT 1 FROM uec.public_access_restricted x WHERE x.source_record_id=o.source_record_id)", &[&release_id,&facility_id]).await {Ok(r)=>r,Err(_)=>return v2_error(StatusCode::SERVICE_UNAVAILABLE,"test_release_query_failed","test release unavailable")}; + let Some(row) = row else { + return v2_error( + StatusCode::NOT_FOUND, + "location_not_found", + "test release location not found", + ); + }; + let item = json!({"facility_id":row.get::<_,uuid::Uuid>(0),"canonical_name":row.get::<_,Option>(1),"country_code":row.get::<_,String>(2),"city":row.get::<_,Option>(3),"category":row.get::<_,String>(4),"publication_profile":profile,"factual_review_status":row.get::<_,String>(5),"privacy_screening_status":row.get::<_,String>(6),"project_approval":"not-approved","publication_warning":"Disposable test release — not project-approved or published","display_precision":row.get::<_,String>(14),"latitude":row.get::<_,Option>(15),"longitude":row.get::<_,Option>(16),"release_id":release_id,"release_ruleset_version":row.get::<_,String>(13),"provenance_source_id":row.get::<_,String>(9),"provenance_source_name":row.get::<_,String>(10),"provenance_source_url":row.get::<_,String>(11),"provenance_retrieved_at":row.get::<_,chrono::DateTime>(12)}); + let mut meta = test_release_meta(release_id, profile); + meta["result_count"] = json!(1); + Json(json!({"data":item,"meta":meta})).into_response() +} + +pub async fn get_dev_test_release_facets_handler( + State(state): State, + headers: HeaderMap, + Query(params): Query, +) -> impl IntoResponse { + if !test_release_auth(&headers, &state) { + return v2_error( + StatusCode::NOT_FOUND, + "test_release_unavailable", + "test release unavailable", + ); + } + let Some(pool) = state.database else { + return v2_error( + StatusCode::SERVICE_UNAVAILABLE, + "database_not_configured", + "test release database unavailable", + ); + }; + let Some(release_id) = state.dev_test_release_id.as_deref() else { + return v2_error( + StatusCode::NOT_FOUND, + "test_release_unavailable", + "test release unavailable", + ); + }; + let client = match pool.get().await { + Ok(c) => c, + Err(_) => { + return v2_error( + StatusCode::SERVICE_UNAVAILABLE, + "database_pool_unavailable", + "database pool unavailable", + ); + } + }; + let rows=match client.query("SELECT f.country_code,o.classification_category,CASE WHEN o.coordinate_review_status='approved' AND g.result IS NOT NULL THEN 'exact' ELSE 'unmapped' END,source.origin_type FROM uec.release_members m JOIN uec.releases r ON r.release_id=m.release_id JOIN uec.observations o ON o.observation_id=m.observation_id JOIN uec.facilities f ON f.facility_id=m.facility_id JOIN uec.source_records sr ON sr.source_record_id=o.source_record_id JOIN uec.sources source ON source.source_id=sr.source_id LEFT JOIN LATERAL (SELECT result FROM uec.geocode_results WHERE source_record_id=o.source_record_id AND status='accepted' AND result IS NOT NULL ORDER BY queried_at DESC LIMIT 1) g ON true LEFT JOIN uec.publication_review_release_current review ON review.source_record_id=o.source_record_id AND review.release_id=m.release_id WHERE r.release_id=$1 AND r.status='candidate' AND r.test_only=true AND sr.source_state NOT IN ('rejected','superseded') AND COALESCE(review.privacy_screening_status,'pending')<>'failed' AND COALESCE(review.factual_review_status,'unreviewed')<>'rejected' AND NOT EXISTS (SELECT 1 FROM uec.public_access_restricted x WHERE x.source_record_id=o.source_record_id)", &[&release_id]).await {Ok(r)=>r,Err(_)=>return v2_error(StatusCode::SERVICE_UNAVAILABLE,"test_release_query_failed","test release unavailable")}; + let mut dims = serde_json::Map::new(); + for (name, values) in [ + ( + "country_code", + rows.iter() + .map(|r| r.get::<_, String>(0)) + .collect::>(), + ), + ( + "category", + rows.iter().map(|r| r.get::<_, String>(1)).collect(), + ), + ( + "display_precision", + rows.iter().map(|r| r.get::<_, String>(2)).collect(), + ), + ( + "source_type", + rows.iter().map(|r| r.get::<_, String>(3)).collect(), + ), + ] { + let mut counts = std::collections::BTreeMap::new(); + for v in values { + *counts.entry(v).or_insert(0usize) += 1; + } + dims.insert( + name.into(), + json!( + counts + .into_iter() + .map(|(value, count)| json!({"value":value,"count":count})) + .collect::>() + ), + ); + } + let mut meta = test_release_meta(release_id, params.profile.as_deref().unwrap_or("official")); + meta["result_count"] = json!(rows.len()); + Json(json!({"data":null,"meta":meta,"dimensions":dims})).into_response() +} + +pub async fn get_dev_test_release_export_handler( + State(state): State, + headers: HeaderMap, + Query(params): Query, +) -> impl IntoResponse { + if !test_release_auth(&headers, &state) { + return v2_error( + StatusCode::NOT_FOUND, + "test_release_unavailable", + "test release unavailable", + ); + } + let Some(pool) = state.database else { + return v2_error( + StatusCode::SERVICE_UNAVAILABLE, + "database_not_configured", + "test release database unavailable", + ); + }; + let Some(release_id) = state.dev_test_release_id.as_deref() else { + return v2_error( + StatusCode::NOT_FOUND, + "test_release_unavailable", + "test release unavailable", + ); + }; + let profile = params.profile.as_deref().unwrap_or("official"); + let client = match pool.get().await { + Ok(c) => c, + Err(_) => { + return v2_error( + StatusCode::SERVICE_UNAVAILABLE, + "database_pool_unavailable", + "test release database unavailable", + ); + } + }; + let rows=match client.query("SELECT f.facility_id,f.canonical_name,f.country_code,f.city,o.classification_category,source.origin_type,source.source_id,source.name,source.official_url,artifact.retrieved_at FROM uec.release_members m JOIN uec.releases r ON r.release_id=m.release_id JOIN uec.observations o ON o.observation_id=m.observation_id JOIN uec.facilities f ON f.facility_id=m.facility_id JOIN uec.source_records sr ON sr.source_record_id=o.source_record_id JOIN uec.sources source ON source.source_id=sr.source_id JOIN uec.raw_artifacts artifact ON artifact.artifact_id=sr.artifact_id LEFT JOIN uec.publication_review_release_current review ON review.source_record_id=o.source_record_id AND review.release_id=m.release_id WHERE r.release_id=$1 AND r.status='candidate' AND r.test_only=true AND sr.source_state NOT IN ('rejected','superseded') AND COALESCE(review.privacy_screening_status,'pending')<>'failed' AND COALESCE(review.factual_review_status,'unreviewed')<>'rejected' AND NOT EXISTS (SELECT 1 FROM uec.public_access_restricted x WHERE x.source_record_id=o.source_record_id) ORDER BY f.facility_id LIMIT 1001", &[&release_id]).await {Ok(r)=>r,Err(_)=>return v2_error(StatusCode::SERVICE_UNAVAILABLE,"test_release_query_failed","test release unavailable")}; + if rows.len() > 1000 { + return v2_error( + StatusCode::BAD_REQUEST, + "export_too_large", + "test export exceeds bounded limit", + ); + } + let mut writer = csv::Writer::from_writer(Vec::new()); + let _ = writer.write_record([ + "facility_id", + "canonical_name", + "country_code", + "city", + "category", + "source_type", + "provenance_source_id", + "provenance_source_name", + "provenance_source_url", + "provenance_retrieved_at", + "test_only", + "environment", + "release_status", + "project_approval", + ]); + for row in rows { + let _ = writer.write_record([ + row.get::<_, uuid::Uuid>(0).to_string(), + csv_safe_value(row.get::<_, Option>(1).unwrap_or_default()), + csv_safe_value(row.get::<_, String>(2)), + csv_safe_value(row.get::<_, Option>(3).unwrap_or_default()), + csv_safe_value(row.get::<_, String>(4)), + csv_safe_value(row.get::<_, String>(5)), + csv_safe_value(row.get::<_, String>(6)), + csv_safe_value(row.get::<_, String>(7)), + csv_safe_value(row.get::<_, String>(8)), + csv_safe_value(row.get::<_, chrono::DateTime>(9).to_rfc3339()), + "true".into(), + "test-only".into(), + "candidate".into(), + "not-approved".into(), + ]); + } + let body = match writer.into_inner() { + Ok(v) => v, + Err(_) => { + return v2_error( + StatusCode::SERVICE_UNAVAILABLE, + "export_encoding_failed", + "test export unavailable", + ); + } + }; + Response::builder() + .status(StatusCode::OK) + .header("content-type", "text/csv; charset=utf-8") + .header( + "content-disposition", + format!("attachment; filename=uec-test-only-{profile}-locations.csv"), + ) + .header("x-uec-test-release", "true") + .header("x-uec-release-id", release_id) + .body(axum::body::Body::from(body)) + .unwrap() + .into_response() +} + fn constant_time_token_matches(expected: &str, provided: &str) -> bool { let mut difference = expected.len() ^ provided.len(); for (left, right) in expected.bytes().zip(provided.bytes()) { @@ -593,7 +931,7 @@ pub async fn get_v2_facets_handler( ); } }; - let release = match client.query_opt("SELECT release_id, ruleset_version, created_at FROM uec.releases WHERE status='promoted' AND profile=$1 ORDER BY created_at DESC, release_id DESC LIMIT 1", &[&profile]).await { Ok(row) => row, Err(_) => return v2_error(StatusCode::SERVICE_UNAVAILABLE, "release_query_failed", "release query failed") }; + let release = match client.query_opt("SELECT release_id, ruleset_version, created_at FROM uec.releases WHERE status='promoted' AND test_only IS NOT TRUE AND profile=$1 ORDER BY created_at DESC, release_id DESC LIMIT 1", &[&profile]).await { Ok(row) => row, Err(_) => return v2_error(StatusCode::SERVICE_UNAVAILABLE, "release_query_failed", "release query failed") }; let Some(release) = release else { return v2_error( StatusCode::NOT_FOUND, @@ -833,7 +1171,7 @@ pub async fn get_v2_locations_handler( } }; let requested_profile = params.profile.as_deref().unwrap_or("official"); - let release = transaction.query_opt("SELECT release_id, ruleset_version, created_at, profile FROM uec.releases WHERE status = 'promoted' AND profile = $1 ORDER BY created_at DESC, release_id DESC LIMIT 1", &[&requested_profile]).await; + let release = transaction.query_opt("SELECT release_id, ruleset_version, created_at, profile FROM uec.releases WHERE status = 'promoted' AND test_only IS NOT TRUE AND profile = $1 ORDER BY created_at DESC, release_id DESC LIMIT 1", &[&requested_profile]).await; let release = match release { Ok(release) => release, Err(_) => { @@ -988,7 +1326,7 @@ pub async fn get_v2_location_detail_handler( "profile is unsupported", ); } - let release = match transaction.query_opt("SELECT release_id, ruleset_version, created_at, profile FROM uec.releases WHERE status = 'promoted' AND profile = $1 ORDER BY created_at DESC, release_id DESC LIMIT 1", &[&requested_profile]).await { + let release = match transaction.query_opt("SELECT release_id, ruleset_version, created_at, profile FROM uec.releases WHERE status = 'promoted' AND test_only IS NOT TRUE AND profile = $1 ORDER BY created_at DESC, release_id DESC LIMIT 1", &[&requested_profile]).await { Ok(release) => release, Err(_) => return v2_error(StatusCode::INTERNAL_SERVER_ERROR, "release_query_failed", "V2 release query failed"), }; @@ -1098,6 +1436,8 @@ mod v2_api_tests { .unwrap(), ), dev_preview_token: None, + dev_test_release_id: None, + dev_test_release_token: None, } } @@ -1116,6 +1456,14 @@ mod v2_api_tests { .as_str().unwrap().contains("not story-wide")); } + #[test] + fn test_csv_escapes_formula_prefixes_without_mutating_evidence() { + for value in ["=SUM(A1)", "+cmd", "-cmd", "@cmd"] { + assert_eq!(csv_safe_value(value.to_string()), format!("'{}", value)); + } + assert_eq!(csv_safe_value("Facility".into()), "Facility"); + } + #[test] fn candidate_preview_rejects_non_loopback_host_and_origin() { let mut local = HeaderMap::new(); diff --git a/src/main.rs b/src/main.rs index a014d61..f0a3995 100644 --- a/src/main.rs +++ b/src/main.rs @@ -63,6 +63,22 @@ pub fn app(state: uec_api::ApiState) -> Router { "/api/dev/preview/candidates", get(uec_api::get_dev_candidate_preview_handler), ) + .route( + "/api/dev/preview/test-release/locations", + get(uec_api::get_dev_test_release_locations_handler), + ) + .route( + "/api/dev/preview/test-release/locations/{facility_id}", + get(uec_api::get_dev_test_release_location_detail_handler), + ) + .route( + "/api/dev/preview/test-release/discovery/facets", + get(uec_api::get_dev_test_release_facets_handler), + ) + .route( + "/api/dev/preview/test-release/locations.csv", + get(uec_api::get_dev_test_release_export_handler), + ) .route( "/api/aphis-reports", get(uec_api::get_aphis_reports_handler), @@ -393,6 +409,29 @@ async fn main() { "{{\"event\":\"server_starting\",\"service\":\"uec-api\",\"mode\":\"{}\",\"port\":{}}}", mode, port ); + let (dev_test_release_id, dev_test_release_token) = match ( + std::env::var("UEC_TEST_RELEASE_ID").ok(), + std::env::var("UEC_TEST_RELEASE_TOKEN").ok(), + ) { + (None, None) => (None, None), + (Some(id), Some(token)) + if mode == "development" + && bind_host + .parse::() + .map(|ip| ip.is_loopback()) + .unwrap_or(false) + && !id.is_empty() + && !token.is_empty() => + { + (Some(id), Some(token)) + } + _ => { + eprintln!( + "{{\"event\":\"configuration_error\",\"reason\":\"test release requires development loopback mode and both UEC_TEST_RELEASE_ID/UEC_TEST_RELEASE_TOKEN\"}}" + ); + std::process::exit(2); + } + }; let listener = tokio::net::TcpListener::bind(addr).await.unwrap(); axum::serve( @@ -400,6 +439,8 @@ async fn main() { app(uec_api::ApiState { database, dev_preview_token, + dev_test_release_id, + dev_test_release_token, }) .into_make_service_with_connect_info::(), ) From de98f7355be9065e960c83124ee37ef00b2b9cf8 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Mon, 14 Sep 2026 00:45:23 -0700 Subject: [PATCH 087/311] Run test-release and country handoff checks in CI --- .github/workflows/tests.yml | 3 ++- pipeline/tests/run-standard.ps1 | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index a70a5a8..2977a62 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -35,6 +35,7 @@ jobs: python -m unittest pipeline.tests.e2e.test_community_api -v python -m unittest pipeline.tests.e2e.test_seeded_api -v python -m unittest pipeline.tests.e2e.test_public_surface_safety -v + python -m unittest pipeline.tests.e2e.test_candidate_import -v backup-restore: # The PostGIS image is Linux-only; Ubuntu includes PowerShell Core for the drill. @@ -86,4 +87,4 @@ jobs: - run: npm run boundary - run: npm run build - run: npx playwright install --with-deps chromium firefox webkit - - run: npx playwright test tests/e2e/fixture-platform.spec.ts tests/e2e/local-safety.spec.ts --project=chromium --project=firefox --project=webkit --workers=2 + - run: npx playwright test tests/e2e/fixture-platform.spec.ts tests/e2e/local-safety.spec.ts tests/e2e/private-preview.spec.ts --project=chromium --project=firefox --project=webkit --workers=2 diff --git a/pipeline/tests/run-standard.ps1 b/pipeline/tests/run-standard.ps1 index 93d9c17..53b5348 100644 --- a/pipeline/tests/run-standard.ps1 +++ b/pipeline/tests/run-standard.ps1 @@ -30,7 +30,7 @@ try { python -m unittest discover -s pipeline/tests -q if ($LASTEXITCODE -ne 0) { throw "Python tests failed (exit $LASTEXITCODE)." } - python -m unittest -q pipeline.germany.test_adapter pipeline.germany.test_orchestrator pipeline.common.test_delta pipeline.common.test_orchestrator pipeline.common.test_registry pipeline.sources.uk.fsa_approved.test_adapter pipeline.sources.uk.fss_approved.test_adapter pipeline.sources.uk.approved.test_compose + python -m unittest -q pipeline.germany.test_adapter pipeline.germany.test_orchestrator pipeline.common.test_delta pipeline.common.test_orchestrator pipeline.common.test_registry pipeline.contracts.test_adapter_contract pipeline.contracts.test_candidate_handoff pipeline.sources.denmark.test_adapter pipeline.sources.uk.fsa_approved.test_adapter pipeline.sources.uk.fsa_approved.test_handoff pipeline.sources.uk.fss_approved.test_adapter pipeline.sources.uk.fss_approved.test_handoff pipeline.sources.uk.approved.test_compose if ($LASTEXITCODE -ne 0) { throw "Country adapter tests failed (exit $LASTEXITCODE)." } } finally { From 9ce3596158b1ff14a82a60be2f09bf9943a464dd Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Mon, 14 Sep 2026 00:53:09 -0700 Subject: [PATCH 088/311] Document UK guarded real test-release flow --- docs/country-recon-uk.md | 20 +++++++++++--------- docs/source-status.json | 2 +- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/docs/country-recon-uk.md b/docs/country-recon-uk.md index 6cc1c9c..3ac88a5 100644 --- a/docs/country-recon-uk.md +++ b/docs/country-recon-uk.md @@ -149,15 +149,17 @@ Restricted operator packet paths (not repository files): The packet records the official source URL/catalog, parent and sample hashes, effective date, selection rule, coverage counts, quarantine counts, and disabled -geocoding. The sample and all row-level derivatives remain restricted. The existing -candidate preview endpoint returned an empty result because pending privacy, -coordinate, geocode, and visibility gates remained closed. Public V2 list/export -routes remained unavailable because no promoted release exists. No approval, -coordinate release, or publication occurred. Operator decisions still required: -source-rights/attribution review, duplicate and coverage scope review, privacy review -of remarks and addresses, and authorization of any test-release preview. Backend -Safety's distinct guarded test-release route is not present in this baseline; do not -relax the existing candidate-preview or public V2 gates. The shared +geocoding. The sample and all row-level derivatives remain restricted. On the fresh +de98f73 disposable stack with migration 023, the guarded test-release list returned +29 pending/unmapped rows; detail, facets, and CSV returned successfully. The CSV +contained 29 rows and no raw/source-value fields. Every returned row carried +test-only/private/candidate metadata and null coordinates. A temporary append-only +privacy suppression event reduced the guarded list to 28 rows, then the disposable +stack was destroyed. Public V2 list remained empty and public CSV remained unavailable +because no promoted release exists. No approval, coordinate release, or publication +occurred. Operator decisions still required: source-rights/attribution review, +duplicate and coverage scope review, and privacy review of remarks and addresses. +The existing candidate-preview and public V2 gates were not relaxed. The shared SourceArtifact/typed-run boundary remains a separate infrastructure integration limitation; this source-local bridge validates typed artifact facts directly and does not alter the common contract. diff --git a/docs/source-status.json b/docs/source-status.json index 39b1e9d..655bec4 100644 --- a/docs/source-status.json +++ b/docs/source-status.json @@ -12,7 +12,7 @@ {"source_id":"it.locations","metadata":"verified","acquisition":"artifact_private_only","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-it.md","pipeline/source_registry.json"],"next_action":"Review the 853/2004 and 1069/2009 source dictionaries and privacy mappings, then implement and validate separate adapters without treating private artifacts as release inputs."}, {"source_id":"mx.locations","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-mx.md","pipeline/source_registry.json"],"next_action":"Resolve DENUE token/terms and SENASICA current directory/schema before bounded acquisition."}, {"source_id":"nz.locations","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-nz.md","pipeline/source_registry.json"],"next_action":"Confirm MPI permitted acquisition route, list-specific terms, and schema before private staging."}, - {"source_id":"uk.locations","metadata":"partial","acquisition":"artifact_private_only","runtime_health":"unknown","publication_eligibility":"blocked","evidence":["docs/country-recon-uk.md","docs/countries/uk/fss-approved-establishments-source-assessment.md","pipeline/sources/uk/fsa_approved/README.md","pipeline/sources/uk/fsa_approved/handoff.py","docs/architecture/disposable-candidate-import.md","pipeline/scripts/maintenance/import-candidate.py","pipeline/source_registry.json"],"next_action":"Bounded real handoff imported into a disposable DB and safe API omission verified (49 input, 29 normalized, 20 quarantined; 0 default-visible, 29 privacy-pending, 29 coordinate-review-required, 0 geocodes). This does not establish production/runtime health. Await Backend Safety's guarded test-release route for any local pending-candidate preview; keep public V2 and existing candidate-preview gates unchanged, and complete source-rights, privacy/coordinate, duplicate, coverage, and release review while keeping NI and Scotland separate."}, + {"source_id":"uk.locations","metadata":"partial","acquisition":"artifact_private_only","runtime_health":"unknown","publication_eligibility":"blocked","evidence":["docs/country-recon-uk.md","docs/countries/uk/fss-approved-establishments-source-assessment.md","pipeline/sources/uk/fsa_approved/README.md","pipeline/sources/uk/fsa_approved/handoff.py","docs/architecture/disposable-candidate-import.md","pipeline/scripts/maintenance/import-candidate.py","pipeline/source_registry.json"],"next_action":"Fresh de98f73 disposable-stack test-release flow passed for the restricted UK packet (49 input, 29 normalized, 20 quarantined; 29 guarded list/detail/facets/CSV rows, 0 raw fields, 29 unmapped, suppression reduced list to 28, public V2 empty). This does not establish production/runtime health or publication eligibility. Complete source-rights, privacy/coordinate, duplicate, coverage, and release review while keeping NI and Scotland separate."}, {"source_id":"dk.smiley","metadata":"partial","acquisition":"artifact_private_only","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["pipeline/sources/denmark/README.md","pipeline/contracts/README.md","pipeline/source_registry.json"],"next_action":"Keep the privately staged official artifact as validation-only; verify coverage and effective-date semantics, then complete terms/privacy/release review before any publication."}, {"source_id":"de.locations","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["pipeline/source_registry.json"],"next_action":"Replace session-bound legacy URL with a verified stable BVL endpoint and confirm terms/schema."}, {"source_id":"ca.locations","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ca.md","pipeline/source_registry.json"],"next_action":"Keep federal/export and Ontario candidates separate; verify an authoritative permitted artifact, terms, schema, coverage, and privacy before acquisition."}, From c0b2769b058c84382ddfb07bcf8819635e6f46b9 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Mon, 14 Sep 2026 08:38:39 -0700 Subject: [PATCH 089/311] Add repeatable UK FSA refresh QA --- docs/country-recon-uk.md | 14 ++ docs/source-status.json | 2 +- pipeline/sources/uk/fsa_approved/README.md | 29 ++- pipeline/sources/uk/fsa_approved/config.json | 4 + pipeline/sources/uk/fsa_approved/refresh.py | 183 ++++++++++++++++++ .../sources/uk/fsa_approved/test_refresh.py | 75 +++++++ 6 files changed, 301 insertions(+), 6 deletions(-) create mode 100644 pipeline/sources/uk/fsa_approved/refresh.py create mode 100644 pipeline/sources/uk/fsa_approved/test_refresh.py diff --git a/docs/country-recon-uk.md b/docs/country-recon-uk.md index 3ac88a5..44b76c9 100644 --- a/docs/country-recon-uk.md +++ b/docs/country-recon-uk.md @@ -167,3 +167,17 @@ not alter the common contract. This reconciliation is a design and test record, not source approval or legal clearance. No live row data is included, and the private artifact remains outside Git and public outputs. + +### Repeatable refresh QA (2026-09-14) + +The source-local refresh command was validated in aggregate-only dry-run mode against +the retained 2026-09-01 artifact. It recorded the official URL, retrieval timestamp, +effective date, SHA-256, byte size, code/config versions, 71-column schema +fingerprint, coverage, quarantine reasons, and release/publication gates. Results: +5,342 input rows, 4,300 normalized, 1,042 quarantined; anomaly counts were +`remarks_present` 999, `address_privacy_risk` 11, `unknown_nation` 31, and +`duplicate_id_within_nation` 4. No drift alarms were raised. The refresh report is +restricted at `data/restricted/country-recon/uk/runs/2026-09-14-refresh-check/refresh.json`. +Prior-run comparisons report disappeared identifiers as `not-observed`; they never +infer closure. The command keeps Scotland and Northern Ireland outside this source +profile and does not create a release. diff --git a/docs/source-status.json b/docs/source-status.json index 655bec4..453120d 100644 --- a/docs/source-status.json +++ b/docs/source-status.json @@ -12,7 +12,7 @@ {"source_id":"it.locations","metadata":"verified","acquisition":"artifact_private_only","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-it.md","pipeline/source_registry.json"],"next_action":"Review the 853/2004 and 1069/2009 source dictionaries and privacy mappings, then implement and validate separate adapters without treating private artifacts as release inputs."}, {"source_id":"mx.locations","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-mx.md","pipeline/source_registry.json"],"next_action":"Resolve DENUE token/terms and SENASICA current directory/schema before bounded acquisition."}, {"source_id":"nz.locations","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-nz.md","pipeline/source_registry.json"],"next_action":"Confirm MPI permitted acquisition route, list-specific terms, and schema before private staging."}, - {"source_id":"uk.locations","metadata":"partial","acquisition":"artifact_private_only","runtime_health":"unknown","publication_eligibility":"blocked","evidence":["docs/country-recon-uk.md","docs/countries/uk/fss-approved-establishments-source-assessment.md","pipeline/sources/uk/fsa_approved/README.md","pipeline/sources/uk/fsa_approved/handoff.py","docs/architecture/disposable-candidate-import.md","pipeline/scripts/maintenance/import-candidate.py","pipeline/source_registry.json"],"next_action":"Fresh de98f73 disposable-stack test-release flow passed for the restricted UK packet (49 input, 29 normalized, 20 quarantined; 29 guarded list/detail/facets/CSV rows, 0 raw fields, 29 unmapped, suppression reduced list to 28, public V2 empty). This does not establish production/runtime health or publication eligibility. Complete source-rights, privacy/coordinate, duplicate, coverage, and release review while keeping NI and Scotland separate."}, + {"source_id":"uk.locations","metadata":"partial","acquisition":"artifact_private_only","runtime_health":"unknown","publication_eligibility":"blocked","evidence":["docs/country-recon-uk.md","docs/countries/uk/fss-approved-establishments-source-assessment.md","pipeline/sources/uk/fsa_approved/README.md","pipeline/sources/uk/fsa_approved/handoff.py","pipeline/sources/uk/fsa_approved/refresh.py","docs/architecture/disposable-candidate-import.md","pipeline/scripts/maintenance/import-candidate.py","pipeline/source_registry.json"],"next_action":"Repeatable source-local refresh dry-run passed on the retained snapshot (5,342 input, 4,300 normalized, 1,042 quarantined, expected schema fingerprint, no drift alarms); use refresh.py dry-run/handoff only in restricted staging. This does not establish production/runtime health or publication eligibility. Complete source-rights, privacy/coordinate, duplicate, coverage, and release review while keeping NI and Scotland separate."}, {"source_id":"dk.smiley","metadata":"partial","acquisition":"artifact_private_only","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["pipeline/sources/denmark/README.md","pipeline/contracts/README.md","pipeline/source_registry.json"],"next_action":"Keep the privately staged official artifact as validation-only; verify coverage and effective-date semantics, then complete terms/privacy/release review before any publication."}, {"source_id":"de.locations","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["pipeline/source_registry.json"],"next_action":"Replace session-bound legacy URL with a verified stable BVL endpoint and confirm terms/schema."}, {"source_id":"ca.locations","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ca.md","pipeline/source_registry.json"],"next_action":"Keep federal/export and Ontario candidates separate; verify an authoritative permitted artifact, terms, schema, coverage, and privacy before acquisition."}, diff --git a/pipeline/sources/uk/fsa_approved/README.md b/pipeline/sources/uk/fsa_approved/README.md index 44d2b16..b8b82b1 100644 --- a/pipeline/sources/uk/fsa_approved/README.md +++ b/pipeline/sources/uk/fsa_approved/README.md @@ -22,14 +22,33 @@ write deterministic parsed, normalized, and quarantined states with a manifest whose `release_state` is always `not-created` and whose publication state is private-candidate. -The adapter does not download or automate acquisition. Before a registered run, -maintainers must verify the current official URL, effective/publication date, -ownership, terms/licence, attribution, rate limits, retention/removal rules, and -redistribution status. Privacy/suppression review, factual review, project approval, -and publication remain independent gates. +The source-local refresh command provides the repeatable acquisition boundary. It +can fetch the configured official URL or accept a preserved raw artifact, records +URL/retrieval/effective dates, hash, byte size, code/config versions, schema +fingerprint, coverage, counts, and quarantine reasons, and writes an aggregate +`refresh.json`. Dry-run is the default; `--mode handoff` is required to emit the +private candidate-handoff contract. A changed header fingerprint or substantial +unbounded count change raises a drift alarm and blocks handoff. A comparison with a +prior normalized run reports disappeared identifiers as `not-observed`, never as +closure. `--bounded-sample` is only for explicitly labeled private test samples. + +Before a registered or fetched run, maintainers must verify the current official +URL, effective/publication date, ownership, terms/licence, attribution, rate limits, +retention/removal rules, and redistribution status. Privacy/suppression review, +factual review, project approval, and publication remain independent gates. Run focused tests from the repository root: ```text python -m unittest -q pipeline.sources.uk.fsa_approved.test_adapter pipeline.sources.uk.approved.test_compose ``` + +Private dry-run example: + +```text +python -m pipeline.sources.uk.fsa_approved.refresh \ + --raw \ + --run-dir \ + --effective-date 2026-09-01 \ + --mode dry-run +``` diff --git a/pipeline/sources/uk/fsa_approved/config.json b/pipeline/sources/uk/fsa_approved/config.json index 62cfcf7..2f3a3f0 100644 --- a/pipeline/sources/uk/fsa_approved/config.json +++ b/pipeline/sources/uk/fsa_approved/config.json @@ -3,6 +3,10 @@ "adapter_version": "fsa-uk-v2-1", "source_id": "fsa_approved_establishments", "authority": "Food Standards Agency", + "source_url": "https://fsaopendata.blob.core.windows.net/opendatacatalog/Approved-Establishments-01-09-26.csv", + "catalog_url": "https://www.data.gov.uk/dataset/2c80e0ce-ee1c-4f26-ba6f-1e1ae1bd8ee9/approved-food-establishments", + "monthly_schema_fingerprint": "8a78f58c004a84811e51af53fab6eaf9316a93462575a17f52b6f7a7543e1fa8", + "monthly_baseline_rows": 5342, "covered_nations": ["England", "Wales", "Northern Ireland"], "authority_by_nation": { "England": "Food Standards Agency", diff --git a/pipeline/sources/uk/fsa_approved/refresh.py b/pipeline/sources/uk/fsa_approved/refresh.py new file mode 100644 index 0000000..086c353 --- /dev/null +++ b/pipeline/sources/uk/fsa_approved/refresh.py @@ -0,0 +1,183 @@ +"""Repeatable private acquisition, drift QA, and FSA monthly handoff.""" +from __future__ import annotations + +import argparse +import csv +import hashlib +import json +import urllib.request +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from pipeline.contracts.adapter_contract import SourceArtifact + +from .adapter import CONFIG, FsaApprovedEstablishmentsAdapter, _csv +from .handoff import write_private_monthly_handoff + + +class RefreshError(ValueError): + """The source refresh cannot safely continue.""" + + +def _utc_now() -> str: + return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z") + + +def _header_fingerprint(raw: bytes) -> tuple[str, int]: + headers, _, _ = _csv(raw) + return hashlib.sha256(json.dumps(headers, ensure_ascii=False, separators=(",", ":")).encode()).hexdigest(), len(headers) + + +def _read_ids(path: Path) -> set[str]: + if not path.exists(): + return set() + ids: set[str] = set() + for line in path.read_text(encoding="utf-8").splitlines(): + if not line: + continue + normalized = json.loads(line).get("normalized", {}) + value = normalized.get("establishment_id") + if isinstance(value, str) and value: + ids.add(value) + return ids + + +def _write_json(path: Path, value: dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(value, ensure_ascii=False, sort_keys=True, indent=2, default=list) + "\n", encoding="utf-8") + + +def _fetch(url: str, destination: Path) -> tuple[bytes, str]: + request = urllib.request.Request(url, headers={"User-Agent": "UntilEveryCage/uk-fsa-refresh"}) + with urllib.request.urlopen(request, timeout=60) as response: + raw = response.read() + effective = response.headers.get("Last-Modified") or "unknown" + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_bytes(raw) + return raw, effective + + +def refresh_monthly( + *, + run_dir: str | Path, + raw_path: str | Path | None = None, + fetch: bool = False, + source_url: str = CONFIG["source_url"], + retrieved_at_utc: str | None = None, + effective_date: str | None = None, + code_version: str = CONFIG["adapter_version"], + config_version: str = CONFIG["contract_version"], + mode: str = "dry-run", + previous_normalized: str | Path | None = None, + bounded_sample: bool = False, +) -> dict[str, Any]: + """Run a private refresh; ``mode=handoff`` also emits candidate-handoff-v1.""" + if mode not in {"dry-run", "handoff"}: + raise RefreshError("mode must be dry-run or handoff") + if fetch == (raw_path is not None): + raise RefreshError("specify exactly one of raw_path or fetch") + root = Path(run_dir) + if fetch: + raw, observed_effective = _fetch(source_url, root / "raw" / "source.csv") + effective_date = effective_date or observed_effective + input_path = root / "raw" / "source.csv" + else: + input_path = Path(raw_path) # type: ignore[arg-type] + raw = input_path.read_bytes() + retrieved_at_utc = retrieved_at_utc or _utc_now() + effective_date = effective_date or "unknown" + adapter = FsaApprovedEstablishmentsAdapter() + result = adapter.parse_bytes(raw) + if result.profile != "monthly": + raise RefreshError("UK refresh requires the monthly FSA profile") + fingerprint, columns = _header_fingerprint(raw) + alarms: list[str] = [] + expected = CONFIG.get("monthly_schema_fingerprint") + if expected and fingerprint != expected and not bounded_sample: + alarms.append("schema_fingerprint_changed") + baseline = int(CONFIG["monthly_baseline_rows"]) + if not bounded_sample and abs(len(result.accepted) + len(result.quarantined) - baseline) > max(100, baseline // 10): + alarms.append("input_row_count_changed") + previous_ids = _read_ids(Path(previous_normalized)) if previous_normalized else set() + current_ids = { + r["normalized"].get("establishment_id") + for r in result.accepted + } | { + item["record"]["normalized"].get("establishment_id") + for item in result.quarantined + } + current_ids.discard(None) + disappeared = len(previous_ids - current_ids) if previous_ids else 0 + artifact = SourceArtifact( + source_url=source_url, + retrieved_at_utc=retrieved_at_utc, + sha256=hashlib.sha256(raw).hexdigest(), + byte_size=len(raw), + effective_date=effective_date, + code_version=code_version, + config_version=config_version, + rights_caveat="metadata-indicated-open-government-licence-v3-pending-project-review", + privacy_caveat="restricted-private-staging; privacy and coordinate review pending", + coverage="England and Wales profile; other nations remain quarantined", + ) + if alarms and mode == "handoff": + raise RefreshError("refresh drift alarm blocks handoff: " + ", ".join(alarms)) + handoff = None + if mode == "handoff": + handoff = write_private_monthly_handoff(input_path, root / "handoff", artifact) + report = { + "source_url": source_url, + "retrieved_at_utc": retrieved_at_utc, + "effective_date": effective_date, + "sha256": artifact.sha256, + "byte_size": artifact.byte_size, + "code_version": code_version, + "config_version": config_version, + "profile": result.profile, + "schema_fingerprint": fingerprint, + "column_count": columns, + "input_rows": len(result.accepted) + len(result.quarantined), + "normalized_rows": len(result.accepted), + "quarantined_rows": len(result.quarantined), + "coverage_counts": result.coverage_counts or {}, + "anomaly_counts": result.anomaly_counts or {}, + "drift_alarms": alarms, + "disappeared_not_observed_count": disappeared, + "disappearance_semantics": "not-observed; never inferred as closure", + "geocoding": "disabled", + "mode": mode, + "release_state": "not-created", + "publication_state": "private-candidate" if handoff else "not-staged", + } + _write_json(root / "refresh.json", report) + return {"report": report, "handoff": handoff} + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + source = parser.add_mutually_exclusive_group(required=True) + source.add_argument("--raw", type=Path) + source.add_argument("--fetch", action="store_true") + parser.add_argument("--run-dir", type=Path, required=True) + parser.add_argument("--source-url", default=CONFIG["source_url"]) + parser.add_argument("--retrieved-at-utc") + parser.add_argument("--effective-date") + parser.add_argument("--code-version", default=CONFIG["adapter_version"]) + parser.add_argument("--config-version", default=CONFIG["contract_version"]) + parser.add_argument("--mode", choices=("dry-run", "handoff"), default="dry-run") + parser.add_argument("--previous-normalized", type=Path) + parser.add_argument("--bounded-sample", action="store_true") + args = parser.parse_args() + result = refresh_monthly( + run_dir=args.run_dir, raw_path=args.raw, fetch=args.fetch, source_url=args.source_url, + retrieved_at_utc=args.retrieved_at_utc, effective_date=args.effective_date, + code_version=args.code_version, config_version=args.config_version, mode=args.mode, + previous_normalized=args.previous_normalized, bounded_sample=args.bounded_sample, + ) + print(json.dumps(result["report"], sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/pipeline/sources/uk/fsa_approved/test_refresh.py b/pipeline/sources/uk/fsa_approved/test_refresh.py new file mode 100644 index 0000000..4ad1c92 --- /dev/null +++ b/pipeline/sources/uk/fsa_approved/test_refresh.py @@ -0,0 +1,75 @@ +import json +import tempfile +import unittest +from pathlib import Path + +from .refresh import RefreshError, refresh_monthly + + +MONTHLY = ( + "AppNo,TradingName,Country,CompetentAuthority,X,Y,AddressWithheld,All_Activities,Address1,Town,Postcode\n" + "A-1,House Foods,England,Food Standards Agency,-0.12,51.50,No,CP,House Farm,London,SW1\n" + "A-2,Withheld,Wales,Food Standards Agency,-3.18,51.48,Yes,CP,Private Road,Cardiff,CF1\n" +).encode("cp1252") + + +class FsaRefreshTests(unittest.TestCase): + def test_dry_run_emits_provenance_drift_and_not_observed_report(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + raw = root / "source.csv" + raw.write_bytes(MONTHLY) + previous = root / "previous.jsonl" + previous.write_text( + json.dumps({"normalized": {"establishment_id": "A-1"}}) + "\n" + + json.dumps({"normalized": {"establishment_id": "A-2"}}) + "\n" + + json.dumps({"normalized": {"establishment_id": "A-9"}}) + "\n", + encoding="utf-8", + ) + result = refresh_monthly( + raw_path=raw, + run_dir=root / "run", + retrieved_at_utc="2026-09-14T00:00:00Z", + effective_date="2026-09-01", + mode="dry-run", + previous_normalized=previous, + bounded_sample=True, + ) + report = result["report"] + self.assertEqual(report["sha256"], __import__("hashlib").sha256(MONTHLY).hexdigest()) + self.assertEqual(report["input_rows"], 2) + self.assertEqual(report["disappeared_not_observed_count"], 1) + self.assertIn("not-observed", report["disappearance_semantics"]) + self.assertEqual(report["mode"], "dry-run") + self.assertFalse((root / "run/handoff/manifest.json").exists()) + self.assertTrue((root / "run/refresh.json").exists()) + + def test_handoff_writes_private_contract_for_bounded_sample(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + raw = root / "source.csv" + raw.write_bytes(MONTHLY) + result = refresh_monthly( + raw_path=raw, + run_dir=root / "run", + retrieved_at_utc="2026-09-14T00:00:00Z", + effective_date="2026-09-01", + mode="handoff", + bounded_sample=True, + ) + self.assertEqual(result["handoff"]["contract_version"], "candidate-handoff-v1") + self.assertEqual(result["handoff"]["release_state"], "not-created") + self.assertEqual(result["report"]["publication_state"], "private-candidate") + + def test_schema_drift_blocks_unbounded_handoff(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + raw = root / "source.csv" + raw.write_bytes(MONTHLY) + with self.assertRaisesRegex(RefreshError, "drift alarm"): + refresh_monthly(raw_path=raw, run_dir=root / "run", mode="handoff") + self.assertFalse((root / "run/handoff/manifest.json").exists()) + + +if __name__ == "__main__": + unittest.main() From 10f0df353b99abfc36d25b853848a70fe5d5b407 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Mon, 14 Sep 2026 08:41:02 -0700 Subject: [PATCH 090/311] Document Denmark-shaped candidate E2E coverage --- pipeline/tests/e2e/README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pipeline/tests/e2e/README.md b/pipeline/tests/e2e/README.md index 34e08d8..ba11888 100644 --- a/pipeline/tests/e2e/README.md +++ b/pipeline/tests/e2e/README.md @@ -20,6 +20,14 @@ This requires Docker Desktop, Cargo, and the pinned Python dependencies. The fix Tests should assert both positive behavior and absence of disclosure. A record being present in the database is not sufficient to make it public; every public response must pass release, visibility, and safety filtering. The public API suite also seeds a synthetic candidate-only record whose review and geocode fields look publishable. It verifies that candidate data is absent from public list, detail, and export routes. This is not a private preview API: any future end-user candidate preview must be a separate dev/test-only server or route, bound to loopback/private access, unavailable in production mode, and fail closed for remote or ambiguous configuration. + +The seeded candidate uses source ID `e2e.private-candidate` and country code +`DK` to exercise the Denmark-shaped country path without using Denmark rows. +The same suite covers candidate exclusion from official list/detail/CSV, +controlled-filter and facets behavior, authenticated loopback-only candidate +preview labeling, and suppression after a privacy access-revocation event. +These are synthetic shared-boundary checks; they do not establish Denmark +source completeness, category semantics, or publication approval. For a local run that exactly matches the standard GitHub Actions database job, use PowerShell 7 (`pwsh`) and run: From b4fae365f384e736002f41bcf1217a078d380acc Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Mon, 14 Sep 2026 08:48:31 -0700 Subject: [PATCH 091/311] Add private Italy 853 candidate adapter --- docs/country-recon-it.md | 4 +- pipeline/sources/italy/__init__.py | 1 + pipeline/sources/italy/it_853_adapter.py | 44 +++++++++++++++++++ pipeline/sources/italy/test_it_853_adapter.py | 20 +++++++++ 4 files changed, 67 insertions(+), 2 deletions(-) create mode 100644 pipeline/sources/italy/__init__.py create mode 100644 pipeline/sources/italy/it_853_adapter.py create mode 100644 pipeline/sources/italy/test_it_853_adapter.py diff --git a/docs/country-recon-it.md b/docs/country-recon-it.md index 68d71fb..5065317 100644 --- a/docs/country-recon-it.md +++ b/docs/country-recon-it.md @@ -45,7 +45,7 @@ The repository’s historical Italy CSV and scraper are legacy/unverified inputs | Source | Discovered | Acquisition | Adapter / validation | Terms / privacy / publication | Blocker / next action | |---|---|---|---|---|---| -| Ministry 853/2004 food establishments | Official catalog and regulatory sections verified | Private current CSV acquired; provenance recorded | Dictionary fields reviewed; adapter not implemented | Italian Open Data Licence v2.0; coordinate provenance partly OSM; ETHICS privacy/approval gates apply | Resolve repeated-activity identity, coded values, and privacy treatment before adapter | +| Ministry 853/2004 food establishments | Official catalog and regulatory sections verified | Private current CSV acquired; provenance recorded | Private candidate adapter and shared-contract handoff implemented; bounded real QA quarantines ambiguous duplicates | Italian Open Data Licence v2.0; coordinate provenance partly OSM; ETHICS privacy/approval gates apply | Validate broader snapshots and resolve identity, coded values, and privacy treatment before publication | | Ministry 1069/2009 by-products | Separate official catalog/dictionary verified | Private current CSV acquired; provenance recorded | Kept separate; no adapter | Same licence and privacy/approval gates | Decide whether scope belongs in project, then validate separately | | Servlet HTML interface | Official interface identified | Not acquired; JS/cookie challenge | Historical HTML parser is brittle; no API claim | No export/terms contract verified; do not scrape through challenge | Prefer catalog downloads or request authorized export/documented endpoint | @@ -57,4 +57,4 @@ Do not publish names, addresses, tax identifiers, or precise coordinates merely ## Limitations -This reconnaissance does not certify completeness/current accuracy, rate limits/authentication, or publication eligibility. Both artifacts are private candidates only; no release or healthy-pipeline claim is made. No source-local adapter was added because the current dictionary-to-contract mapping and safe publication treatment of addresses, identifiers, and OSM-derived coordinates remain to be reviewed. +This reconnaissance and private candidate adapter do not certify completeness/current accuracy, rate limits/authentication, or publication eligibility. Both artifacts remain private; no public release or healthy-pipeline claim is made. Repeated-activity identity and safe publication treatment of addresses, identifiers, and OSM-derived coordinates remain to be reviewed. diff --git a/pipeline/sources/italy/__init__.py b/pipeline/sources/italy/__init__.py new file mode 100644 index 0000000..03002d4 --- /dev/null +++ b/pipeline/sources/italy/__init__.py @@ -0,0 +1 @@ +"""Italian Ministry of Health source adapters.""" diff --git a/pipeline/sources/italy/it_853_adapter.py b/pipeline/sources/italy/it_853_adapter.py new file mode 100644 index 0000000..a9b26ad --- /dev/null +++ b/pipeline/sources/italy/it_853_adapter.py @@ -0,0 +1,44 @@ +from __future__ import annotations +import csv +import hashlib +import json +from pathlib import Path + +REQUIRED=tuple("precedente_bollo_cee;num_identificativo_produzione_commercializzazione;ragione_sociale;indirizzo;comune;provincia;codice_regione;regione;classificazione_stabilimento;codice_impianto_attivita;descrizione_impianto_attivita;prodotti_abilitati;specifica_prodotti;paesi_export_autorizzato;longitudine;latitudine;stato_localizzazione;cod_fiscale;p_iva;codice_comune;data_inizio_attivita;data_fine_attivita;stato_attivita;data_ultimo_aggiornamento;num_identificativo_produzione_commercializzazione_2".split(";")) +STATUS={"AUTORIZZATA":"Autorizzata","REVOCATA":"Revocata","SOSPESA":"Sospesa"} +def clean(v): + return v.strip() if v and v.strip() else None +def row_id(row, occurrence): + payload = json.dumps(row, sort_keys=True, ensure_ascii=False, separators=(",", ":")) + return hashlib.sha256(f"{payload}|{occurrence}".encode()).hexdigest() +# Content identity plus occurrence preserves rerun stability without line-number identity. +class Italy853Adapter: + source_id="it.853-2004"; adapter_version="it-853-candidate-v1"; schema_version="it-853-csv-v2.0" + def parse_bytes(self,content): + digest=hashlib.sha256(content).hexdigest(); rows=list(csv.DictReader(content.decode("utf-8-sig").splitlines(),delimiter=";")) + if not rows or tuple(rows[0])!=REQUIRED: raise ValueError("schema drift") + seen=set(); occurrences={}; accepted=[]; quarantined=[] + for line,row in enumerate(rows,2): + rec=clean(row.get(REQUIRED[1])); act=clean(row.get("codice_impianto_attivita")); key=(rec,act); reasons=[]; occurrences[key]=occurrences.get(key,0)+1 + if None in row or None in row.values() or any(isinstance(v, list) for v in row.values()): reasons.append("malformed_row_shape") + if not rec: reasons.append("missing_recognition_number") + if not act: reasons.append("missing_activity_code") + # Repeated recognition/activity rows are quarantined instead of silently collapsed. + if key in seen: reasons.append("ambiguous_repeated_recognition_activity") + seen.add(key); status=clean(row.get("stato_attivita")) + if status and status.upper() not in STATUS: reasons.append("unknown_status") + status=STATUS.get(status.upper()) if status else None + # Sensitive source_values are retained only in private parsed output; normalized exposure is screened. + out={"source_id":self.source_id,"source_row":line,"source_row_id":row_id(row, occurrences[key]),"source_values":dict(row),"normalized":{"recognition_number":rec,"facility_grouping":"provisional-recognition-number","name":clean(row.get("ragione_sociale")),"address":None,"municipality":clean(row.get("comune")),"region":clean(row.get("regione")),"activity_code":act,"activity_description":clean(row.get("descrizione_impianto_attivita")),"products":clean(row.get("prodotti_abilitati")),"status":status,"coordinates":None,"privacy_gate":"pending-review","publication_gate":"blocked"}} + (quarantined if reasons else accepted).append({"reasons":tuple(reasons),"record":out} if reasons else out) + return {"accepted":accepted,"quarantined":quarantined,"source_sha256":digest,"input_rows":len(rows)} + def parse_file(self,path): return self.parse_bytes(Path(path).read_bytes()) + def run(self, raw_path, run_dir, artifact): + raw=Path(raw_path).read_bytes(); result=self.parse_bytes(raw) + if artifact.sha256 != result["source_sha256"] or artifact.byte_size != len(raw): raise ValueError("artifact provenance mismatch") + root=Path(run_dir); (root/"normalized").mkdir(parents=True,exist_ok=True); (root/"quarantined").mkdir(parents=True,exist_ok=True) + normalized="".join(json.dumps(x,sort_keys=True,default=list)+"\n" for x in result["accepted"]); quarantined="".join(json.dumps(x,sort_keys=True,default=list)+"\n" for x in result["quarantined"]) + (root/"normalized"/"records.jsonl").write_text(normalized,encoding="utf-8"); (root/"quarantined"/"records.jsonl").write_text(quarantined,encoding="utf-8") + manifest={"source_id":self.source_id,"schema_version":self.schema_version,"source_url":artifact.source_url,"retrieved_at_utc":artifact.retrieved_at_utc,"sha256":artifact.sha256,"checksum_sha256":artifact.sha256,"byte_size":artifact.byte_size,"code_version":artifact.code_version,"config_version":artifact.config_version,"publication_state":"private-candidate","release_state":"not-created","input_rows":result["input_rows"],"normalized_rows":len(result["accepted"]),"quarantined_rows":len(result["quarantined"]),"normalized_sha256":hashlib.sha256(normalized.encode()).hexdigest(),"acquisition":{"source_url":artifact.source_url,"retrieved_at_utc":artifact.retrieved_at_utc,"sha256":artifact.sha256,"byte_size":artifact.byte_size},"handoff_contract":"candidate_handoff-v1"} + (root/"manifest.json").write_text(json.dumps(manifest,sort_keys=True,indent=2)+"\n",encoding="utf-8") + return manifest diff --git a/pipeline/sources/italy/test_it_853_adapter.py b/pipeline/sources/italy/test_it_853_adapter.py new file mode 100644 index 0000000..93a0439 --- /dev/null +++ b/pipeline/sources/italy/test_it_853_adapter.py @@ -0,0 +1,20 @@ +import unittest +import tempfile, json, hashlib +from .it_853_adapter import Italy853Adapter +from pipeline.contracts.adapter_contract import SourceArtifact +H="precedente_bollo_cee;num_identificativo_produzione_commercializzazione;ragione_sociale;indirizzo;comune;provincia;codice_regione;regione;classificazione_stabilimento;codice_impianto_attivita;descrizione_impianto_attivita;prodotti_abilitati;specifica_prodotti;paesi_export_autorizzato;longitudine;latitudine;stato_localizzazione;cod_fiscale;p_iva;codice_comune;data_inizio_attivita;data_fine_attivita;stato_attivita;data_ultimo_aggiornamento;num_identificativo_produzione_commercializzazione_2" +def row(n="A",a="10",s="Autorizzata"): return f";{n};Name;;Town;;010;Piemonte;X;{a};Activity;P;S;IT;12;45;1;tax;vat;001001;;;{s};2026-09-13;\n" +class Test(unittest.TestCase): + def test_safe_mapping(self): + r=Italy853Adapter().parse_bytes((H+"\n"+row()).encode())["accepted"][0]; self.assertIsNone(r["normalized"]["coordinates"]); self.assertIsNone(r["normalized"]["address"]); self.assertIn("p_iva",r["source_values"]) + def test_quarantine(self): + r=Italy853Adapter().parse_bytes((H+"\n"+row()+row("A","10","Unknown")).encode()); self.assertEqual(len(r["accepted"]),1); self.assertIn("unknown_status",r["quarantined"][0]["reasons"]) + def test_sensitive_and_deterministic_identity(self): + content=(H+"\n"+row()).encode(); a=Italy853Adapter(); x=a.parse_bytes(content)["accepted"][0]; y=a.parse_bytes(content)["accepted"][0]; self.assertEqual(x["source_row_id"],y["source_row_id"]); self.assertNotIn("p_iva",x["normalized"]); self.assertIsNone(x["normalized"]["coordinates"]) + def test_shape_drift_quarantine(self): + content=(H+"\n"+row().replace("Name","Name;extra")).encode(); self.assertRaises(ValueError,Italy853Adapter().parse_bytes,content) + def test_run_writes_contract_manifest_and_row_quarantine(self): + from pipeline.contracts.adapter_contract import assert_manifest + content=(H+"\n"+row()+row("","10")).encode() + with tempfile.TemporaryDirectory() as d, tempfile.NamedTemporaryFile(suffix=".csv", delete=False) as f: + f.write(content); f.flush(); sha=hashlib.sha256(content).hexdigest(); a=Italy853Adapter(); m=a.run(f.name,d,SourceArtifact("u","2026-09-14T00:00:00Z",sha,len(content),code_version="c",config_version="k")); assert_manifest(m,content,a.schema_version); self.assertTrue((__import__('pathlib').Path(d)/"normalized/records.jsonl").exists()); self.assertEqual(m["quarantined_rows"],1); self.assertNotIn("accepted",m); self.assertNotIn("source_values",json.dumps(m)) From 6650b844f90415aafeebba8e1b80771e5dfee4fd Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Mon, 14 Sep 2026 08:56:18 -0700 Subject: [PATCH 092/311] style: format Italy 853 adapter --- pipeline/sources/italy/it_853_adapter.py | 131 +++++++++++++++++------ 1 file changed, 100 insertions(+), 31 deletions(-) diff --git a/pipeline/sources/italy/it_853_adapter.py b/pipeline/sources/italy/it_853_adapter.py index a9b26ad..d834840 100644 --- a/pipeline/sources/italy/it_853_adapter.py +++ b/pipeline/sources/italy/it_853_adapter.py @@ -1,44 +1,113 @@ from __future__ import annotations + import csv import hashlib import json from pathlib import Path -REQUIRED=tuple("precedente_bollo_cee;num_identificativo_produzione_commercializzazione;ragione_sociale;indirizzo;comune;provincia;codice_regione;regione;classificazione_stabilimento;codice_impianto_attivita;descrizione_impianto_attivita;prodotti_abilitati;specifica_prodotti;paesi_export_autorizzato;longitudine;latitudine;stato_localizzazione;cod_fiscale;p_iva;codice_comune;data_inizio_attivita;data_fine_attivita;stato_attivita;data_ultimo_aggiornamento;num_identificativo_produzione_commercializzazione_2".split(";")) -STATUS={"AUTORIZZATA":"Autorizzata","REVOCATA":"Revocata","SOSPESA":"Sospesa"} +REQUIRED = tuple( + "precedente_bollo_cee;num_identificativo_produzione_commercializzazione;ragione_sociale;indirizzo;comune;provincia;codice_regione;regione;classificazione_stabilimento;codice_impianto_attivita;descrizione_impianto_attivita;prodotti_abilitati;specifica_prodotti;paesi_export_autorizzato;longitudine;latitudine;stato_localizzazione;cod_fiscale;p_iva;codice_comune;data_inizio_attivita;data_fine_attivita;stato_attivita;data_ultimo_aggiornamento;num_identificativo_produzione_commercializzazione_2".split(";") +) +STATUS = {"AUTORIZZATA": "Autorizzata", "REVOCATA": "Revocata", "SOSPESA": "Sospesa"} + + def clean(v): return v.strip() if v and v.strip() else None + + def row_id(row, occurrence): payload = json.dumps(row, sort_keys=True, ensure_ascii=False, separators=(",", ":")) return hashlib.sha256(f"{payload}|{occurrence}".encode()).hexdigest() + + # Content identity plus occurrence preserves rerun stability without line-number identity. class Italy853Adapter: - source_id="it.853-2004"; adapter_version="it-853-candidate-v1"; schema_version="it-853-csv-v2.0" - def parse_bytes(self,content): - digest=hashlib.sha256(content).hexdigest(); rows=list(csv.DictReader(content.decode("utf-8-sig").splitlines(),delimiter=";")) - if not rows or tuple(rows[0])!=REQUIRED: raise ValueError("schema drift") - seen=set(); occurrences={}; accepted=[]; quarantined=[] - for line,row in enumerate(rows,2): - rec=clean(row.get(REQUIRED[1])); act=clean(row.get("codice_impianto_attivita")); key=(rec,act); reasons=[]; occurrences[key]=occurrences.get(key,0)+1 - if None in row or None in row.values() or any(isinstance(v, list) for v in row.values()): reasons.append("malformed_row_shape") - if not rec: reasons.append("missing_recognition_number") - if not act: reasons.append("missing_activity_code") - # Repeated recognition/activity rows are quarantined instead of silently collapsed. - if key in seen: reasons.append("ambiguous_repeated_recognition_activity") - seen.add(key); status=clean(row.get("stato_attivita")) - if status and status.upper() not in STATUS: reasons.append("unknown_status") - status=STATUS.get(status.upper()) if status else None - # Sensitive source_values are retained only in private parsed output; normalized exposure is screened. - out={"source_id":self.source_id,"source_row":line,"source_row_id":row_id(row, occurrences[key]),"source_values":dict(row),"normalized":{"recognition_number":rec,"facility_grouping":"provisional-recognition-number","name":clean(row.get("ragione_sociale")),"address":None,"municipality":clean(row.get("comune")),"region":clean(row.get("regione")),"activity_code":act,"activity_description":clean(row.get("descrizione_impianto_attivita")),"products":clean(row.get("prodotti_abilitati")),"status":status,"coordinates":None,"privacy_gate":"pending-review","publication_gate":"blocked"}} - (quarantined if reasons else accepted).append({"reasons":tuple(reasons),"record":out} if reasons else out) - return {"accepted":accepted,"quarantined":quarantined,"source_sha256":digest,"input_rows":len(rows)} - def parse_file(self,path): return self.parse_bytes(Path(path).read_bytes()) - def run(self, raw_path, run_dir, artifact): - raw=Path(raw_path).read_bytes(); result=self.parse_bytes(raw) - if artifact.sha256 != result["source_sha256"] or artifact.byte_size != len(raw): raise ValueError("artifact provenance mismatch") - root=Path(run_dir); (root/"normalized").mkdir(parents=True,exist_ok=True); (root/"quarantined").mkdir(parents=True,exist_ok=True) - normalized="".join(json.dumps(x,sort_keys=True,default=list)+"\n" for x in result["accepted"]); quarantined="".join(json.dumps(x,sort_keys=True,default=list)+"\n" for x in result["quarantined"]) - (root/"normalized"/"records.jsonl").write_text(normalized,encoding="utf-8"); (root/"quarantined"/"records.jsonl").write_text(quarantined,encoding="utf-8") - manifest={"source_id":self.source_id,"schema_version":self.schema_version,"source_url":artifact.source_url,"retrieved_at_utc":artifact.retrieved_at_utc,"sha256":artifact.sha256,"checksum_sha256":artifact.sha256,"byte_size":artifact.byte_size,"code_version":artifact.code_version,"config_version":artifact.config_version,"publication_state":"private-candidate","release_state":"not-created","input_rows":result["input_rows"],"normalized_rows":len(result["accepted"]),"quarantined_rows":len(result["quarantined"]),"normalized_sha256":hashlib.sha256(normalized.encode()).hexdigest(),"acquisition":{"source_url":artifact.source_url,"retrieved_at_utc":artifact.retrieved_at_utc,"sha256":artifact.sha256,"byte_size":artifact.byte_size},"handoff_contract":"candidate_handoff-v1"} - (root/"manifest.json").write_text(json.dumps(manifest,sort_keys=True,indent=2)+"\n",encoding="utf-8") - return manifest + source_id = "it.853-2004" + adapter_version = "it-853-candidate-v1" + schema_version = "it-853-csv-v2.0" + + def parse_bytes(self, content): + digest = hashlib.sha256(content).hexdigest() + rows = list(csv.DictReader(content.decode("utf-8-sig").splitlines(), delimiter=";")) + if not rows or tuple(rows[0]) != REQUIRED: + raise ValueError("schema drift") + seen = set() + occurrences = {} + accepted = [] + quarantined = [] + for line, row in enumerate(rows, 2): + rec = clean(row.get(REQUIRED[1])) + act = clean(row.get("codice_impianto_attivita")) + key = (rec, act) + reasons = [] + occurrences[key] = occurrences.get(key, 0) + 1 + if None in row or None in row.values() or any(isinstance(v, list) for v in row.values()): + reasons.append("malformed_row_shape") + if not rec: + reasons.append("missing_recognition_number") + if not act: + reasons.append("missing_activity_code") + # Repeated recognition/activity rows are quarantined instead of silently collapsed. + if key in seen: + reasons.append("ambiguous_repeated_recognition_activity") + seen.add(key) + status = clean(row.get("stato_attivita")) + if status and status.upper() not in STATUS: + reasons.append("unknown_status") + status = STATUS.get(status.upper()) if status else None + # Sensitive source_values remain private; normalized exposure is screened. + out = { + "source_id": self.source_id, + "source_row": line, + "source_row_id": row_id(row, occurrences[key]), + "source_values": dict(row), + "normalized": { + "recognition_number": rec, + "facility_grouping": "provisional-recognition-number", + "name": clean(row.get("ragione_sociale")), + "address": None, + "municipality": clean(row.get("comune")), + "region": clean(row.get("regione")), + "activity_code": act, + "activity_description": clean(row.get("descrizione_impianto_attivita")), + "products": clean(row.get("prodotti_abilitati")), + "status": status, + "coordinates": None, + "privacy_gate": "pending-review", + "publication_gate": "blocked", + }, + } + (quarantined if reasons else accepted).append( + {"reasons": tuple(reasons), "record": out} if reasons else out + ) + return {"accepted": accepted, "quarantined": quarantined, "source_sha256": digest, "input_rows": len(rows)} + + def parse_file(self, path): + return self.parse_bytes(Path(path).read_bytes()) + + def run(self, raw_path, run_dir, artifact): + raw = Path(raw_path).read_bytes() + result = self.parse_bytes(raw) + if artifact.sha256 != result["source_sha256"] or artifact.byte_size != len(raw): + raise ValueError("artifact provenance mismatch") + root = Path(run_dir) + (root / "normalized").mkdir(parents=True, exist_ok=True) + (root / "quarantined").mkdir(parents=True, exist_ok=True) + normalized = "".join(json.dumps(x, sort_keys=True, default=list) + "\n" for x in result["accepted"]) + quarantined = "".join(json.dumps(x, sort_keys=True, default=list) + "\n" for x in result["quarantined"]) + (root / "normalized" / "records.jsonl").write_text(normalized, encoding="utf-8") + (root / "quarantined" / "records.jsonl").write_text(quarantined, encoding="utf-8") + manifest = { + "source_id": self.source_id, "schema_version": self.schema_version, + "source_url": artifact.source_url, "retrieved_at_utc": artifact.retrieved_at_utc, + "sha256": artifact.sha256, "checksum_sha256": artifact.sha256, + "byte_size": artifact.byte_size, "code_version": artifact.code_version, + "config_version": artifact.config_version, "publication_state": "private-candidate", + "release_state": "not-created", "input_rows": result["input_rows"], + "normalized_rows": len(result["accepted"]), "quarantined_rows": len(result["quarantined"]), + "normalized_sha256": hashlib.sha256(normalized.encode()).hexdigest(), + "acquisition": {"source_url": artifact.source_url, "retrieved_at_utc": artifact.retrieved_at_utc, "sha256": artifact.sha256, "byte_size": artifact.byte_size}, + "handoff_contract": "candidate_handoff-v1", + } + (root / "manifest.json").write_text(json.dumps(manifest, sort_keys=True, indent=2) + "\n", encoding="utf-8") + return manifest From 193b718656eeb25ffff365ab3d55541dd7fa3140 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Mon, 14 Sep 2026 09:15:00 -0700 Subject: [PATCH 093/311] Verify guarded test-release UI against disposable API --- frontend/src/api/LocalLocationRepository.ts | 4 +-- frontend/src/api/TestReleaseRepository.ts | 8 +++--- frontend/src/api/wireSchema.ts | 6 ++++- frontend/src/app/App.svelte | 2 +- frontend/src/domain/location.ts | 2 +- frontend/tests/e2e/local-backend.spec.ts | 25 +++++++++++++++++++ .../tests/unit/devPreviewContract.test.ts | 4 +++ pipeline/tests/e2e/fixture.py | 2 +- src/lib.rs | 2 +- 9 files changed, 44 insertions(+), 11 deletions(-) diff --git a/frontend/src/api/LocalLocationRepository.ts b/frontend/src/api/LocalLocationRepository.ts index 071e473..ada77bf 100644 --- a/frontend/src/api/LocalLocationRepository.ts +++ b/frontend/src/api/LocalLocationRepository.ts @@ -9,11 +9,11 @@ export type LocalListResult = Readonly<{ locations: readonly Location[]; release export const localOrigin = (value: string | undefined): string | undefined => { if (!value) return undefined; const url = new URL(value); if (url.protocol !== 'http:' || !['127.0.0.1', 'localhost', '::1'].includes(url.hostname)) throw new Error('Local API origin must be loopback HTTP.'); return url.origin; }; const fail = (kind: ApiError['kind'], message: string, status?: number): ApiError => Object.assign(new Error(message), status === undefined ? { kind } : { kind, status }); export const mapWireLocation = (r: WireLocation): Location => ({ - id: r.facility_id, name: r.canonical_name, region: r.city ?? r.country_code, category: r.category, + id: r.facility_id, name: r.canonical_name ?? 'Unnamed candidate record', region: r.city ?? r.country_code, category: r.category, lat: r.latitude, lon: r.longitude, observed: r.last_observed_at ?? r.first_observed_at ?? 'unknown', source: r.provenance_source_name, evidence: { sourceType: r.source_type, factualReviewStatus: r.factual_review_status, reviewerRole: r.reviewer_role, - privacyScreeningStatus: r.privacy_screening_status, projectApproval: r.project_approval, + privacyScreeningStatus: r.privacy_screening_status, projectApproval: r.project_approval, publicationProfile: r.publication_profile, publicationWarning: r.publication_warning, sourceId: r.provenance_source_id, sourceUrl: r.provenance_source_url, retrievedAt: r.provenance_retrieved_at, displayPrecision: r.display_precision, lifecycleStatus: r.lifecycle_status, observationCount: r.observation_count, diff --git a/frontend/src/api/TestReleaseRepository.ts b/frontend/src/api/TestReleaseRepository.ts index 917fc0f..db6ed86 100644 --- a/frontend/src/api/TestReleaseRepository.ts +++ b/frontend/src/api/TestReleaseRepository.ts @@ -1,16 +1,16 @@ import { z } from 'zod'; -import { locationSchema, type WireLocation } from './wireSchema'; +import { testReleaseLocationSchema, type WireLocation } from './wireSchema'; import { mapWireLocation, type FetchLike, type LocalListResult, type LocalProfile } from './LocalLocationRepository'; import { TEST_RELEASE_API_VERSION, TEST_RELEASE_LABEL, TEST_RELEASE_PATH, DEV_PREVIEW_TOKEN_HEADER } from '../features/devPreview/devPreviewContract'; -const envelope = z.object({ data: z.array(locationSchema), meta: z.object({ api_version: z.literal(TEST_RELEASE_API_VERSION), environment: z.literal('test-only'), test_only: z.literal(true), private_preview: z.literal(true), release_status: z.literal('candidate'), release_id: z.string(), profile: z.enum(['official', 'secondary', 'community']), coverage_scope: z.string(), count_semantics: z.string(), preview_label: z.string(), result_count: z.number().int().nonnegative(), next_cursor: z.string().nullable().optional() }) }); +const envelope = z.object({ data: z.array(testReleaseLocationSchema), meta: z.object({ api_version: z.literal(TEST_RELEASE_API_VERSION), environment: z.literal('test-only'), test_only: z.literal(true), private_preview: z.literal(true), release_status: z.literal('candidate'), release_id: z.string(), profile: z.enum(['official', 'secondary', 'community']), coverage_scope: z.string(), count_semantics: z.string(), preview_label: z.string(), result_count: z.number().int().nonnegative(), next_cursor: z.string().nullable().optional() }) }); export class TestReleaseRepository { constructor(private readonly fetcher: FetchLike = globalThis.fetch, private readonly baseUrl = '') {} async list(profile: LocalProfile = 'official', token = '', signal?: AbortSignal): Promise { const init: RequestInit = { cache: 'no-store', headers: { [DEV_PREVIEW_TOKEN_HEADER]: token } }; if (signal) init.signal = signal; const response = await this.fetcher.call(globalThis, `${this.baseUrl}${TEST_RELEASE_PATH}/locations?profile=${profile}&limit=100`, init); if (!response.ok) throw new Error(`Private test release unavailable (HTTP ${response.status}).`); - const parsed = envelope.safeParse(await response.json()); if (!parsed.success) throw new Error('Private test release response was rejected safely.'); - return { locations: parsed.data.data.map(mapWireLocation), releaseId: parsed.data.meta.release_id, profile, coverageNote: `${TEST_RELEASE_LABEL}. ${parsed.data.meta.coverage_scope}.`, coverageScope: parsed.data.meta.coverage_scope, countSemantics: parsed.data.meta.count_semantics, nextCursor: parsed.data.meta.next_cursor ?? null }; + const parsed = envelope.safeParse(await response.json()); if (!parsed.success) throw new Error(`Private test release response was rejected safely (${parsed.error.issues.map(issue => issue.path.join('.')).join(', ') || 'contract'}).`); + return { locations: parsed.data.data.map(row => mapWireLocation({ ...row, project_approval: row.project_approval === 'not-approved' ? false : row.project_approval } as WireLocation)), releaseId: parsed.data.meta.release_id, profile, coverageNote: `${TEST_RELEASE_LABEL}. ${parsed.data.meta.coverage_scope}.`, coverageScope: parsed.data.meta.coverage_scope, countSemantics: parsed.data.meta.count_semantics, nextCursor: parsed.data.meta.next_cursor ?? null }; } } diff --git a/frontend/src/api/wireSchema.ts b/frontend/src/api/wireSchema.ts index 5fa2370..76fa01c 100644 --- a/frontend/src/api/wireSchema.ts +++ b/frontend/src/api/wireSchema.ts @@ -2,9 +2,13 @@ import { z } from 'zod'; const textOrNull=z.string().nullable(); // This is the Rust-shaped boundary. Optional coverage/count metadata is additive; // older valid list envelopes remain readable with safe UI fallbacks. -export const locationSchema=z.object({facility_id:z.string().uuid(),canonical_name:z.string().min(1),city:textOrNull,country_code:z.string().regex(/^[A-Z]{2}$/),category:z.string(),source_type:z.enum(['official','secondary','user_submitted']),publication_profile:z.enum(['official','secondary','community']),factual_review_status:z.enum(['unreviewed','reviewed','rejected']),privacy_screening_status:z.literal('passed'),project_approval:z.enum(['pending','approved']),reviewer_role:textOrNull,publication_warning:textOrNull,display_precision:z.enum(['exact','city','unmapped']),latitude:z.number().finite().nullable(),longitude:z.number().finite().nullable(),first_observed_at:textOrNull,last_observed_at:textOrNull,observation_count:z.number().int().nonnegative().nullable(),lifecycle_status:z.enum(['active_observed','explicitly_closed','not_seen_recently','status_unknown']),provenance_source_id:z.string(),provenance_source_name:z.string(),provenance_source_url:z.string().url().refine(value=>{try{return ['http:','https:'].includes(new URL(value).protocol);}catch{return false;}},'source URL must use HTTP or HTTPS'),provenance_retrieved_at:z.string(),release_id:z.string(),release_ruleset_version:z.string()}).superRefine((row,ctx)=>{if((row.latitude===null)!==(row.longitude===null))ctx.addIssue({code:z.ZodIssueCode.custom,message:'coordinate pair must be complete'});if(row.display_precision==='unmapped'&&(row.latitude!==null||row.longitude!==null))ctx.addIssue({code:z.ZodIssueCode.custom,message:'unmapped record cannot have coordinates'});if(row.display_precision!=='unmapped'&&(row.latitude===null||row.longitude===null))ctx.addIssue({code:z.ZodIssueCode.custom,message:'mapped record requires coordinates'});}); +const locationShape={facility_id:z.string().uuid(),canonical_name:z.string().min(1),city:textOrNull,country_code:z.string().regex(/^[A-Z]{2}$/),category:z.string(),source_type:z.enum(['official','secondary','user_submitted']),publication_profile:z.enum(['official','secondary','community']),factual_review_status:z.enum(['unreviewed','reviewed','rejected']),privacy_screening_status:z.literal('passed'),project_approval:z.enum(['pending','approved']),reviewer_role:textOrNull,publication_warning:textOrNull,display_precision:z.enum(['exact','city','unmapped']),latitude:z.number().finite().nullable(),longitude:z.number().finite().nullable(),first_observed_at:textOrNull,last_observed_at:textOrNull,observation_count:z.number().int().nonnegative().nullable(),lifecycle_status:z.enum(['active_observed','explicitly_closed','not_seen_recently','status_unknown']),provenance_source_id:z.string(),provenance_source_name:z.string(),provenance_source_url:z.string().url().refine(value=>{try{return ['http:','https:'].includes(new URL(value).protocol);}catch{return false;}},'source URL must use HTTP or HTTPS'),provenance_retrieved_at:z.string(),release_id:z.string(),release_ruleset_version:z.string()}; +const coordinateRules=(row:{latitude:number|null;longitude:number|null;display_precision:string},ctx:z.RefinementCtx)=>{if((row.latitude===null)!==(row.longitude===null))ctx.addIssue({code:z.ZodIssueCode.custom,message:'coordinate pair must be complete'});if(row.display_precision==='unmapped'&&(row.latitude!==null||row.longitude!==null))ctx.addIssue({code:z.ZodIssueCode.custom,message:'unmapped record cannot have coordinates'});if(row.display_precision!=='unmapped'&&(row.latitude===null||row.longitude===null))ctx.addIssue({code:z.ZodIssueCode.custom,message:'mapped record requires coordinates'});}; +export const locationSchema=z.object(locationShape).superRefine(coordinateRules); +export const testReleaseLocationSchema=z.object({...locationShape,canonical_name:z.string().nullable(),publication_profile:z.enum(['official','secondary','community']).nullable(),privacy_screening_status:z.enum(['pending','passed','failed']),project_approval:z.union([z.enum(['pending','approved']),z.literal(false),z.literal('not-approved')]),release_ruleset_version:z.string().nullable()}).superRefine(coordinateRules); export const envelopeSchema=z.object({data:z.array(locationSchema),api_version:z.literal('v2'),meta:z.object({release_id:z.string().nullable(),ruleset_version:z.string().optional(),release_created_at:z.string().optional(),profile:z.enum(['official','secondary','community']),next_cursor:z.string().nullable().optional(),coverage_note:z.string().min(1),coverage_scope:z.string().optional(),count_semantics:z.string().optional()})}); export type WireEnvelope=z.infer;export type WireLocation=z.infer; +export type WireTestReleaseLocation=z.infer; export const detailEnvelopeSchema=z.object({data:locationSchema,api_version:z.literal('v2'),meta:z.object({release_id:z.string(),ruleset_version:z.string(),release_created_at:z.string(),profile:z.enum(['official','secondary','community'])})}); export type DetailEnvelope=z.infer; diff --git a/frontend/src/app/App.svelte b/frontend/src/app/App.svelte index 288a558..eec1275 100644 --- a/frontend/src/app/App.svelte +++ b/frontend/src/app/App.svelte @@ -147,7 +147,7 @@

EVIDENCE DESK · {devPreviewMode ? 'PRIVATE CANDIDATE PREVIEW' : localMode ? 'LOCAL V2 API' : 'SYNTHETIC PREVIEW'}

See what a record can—and cannot—tell us.

Start with the source profile, narrow the visible evidence, then inspect what a record can—and cannot—tell us.

{#if devPreviewMode}{/if} {#if devPreviewMode}{/if} - {#if testReleaseMode && previewStatus === 'ready'}
TEST-ONLY CSV — NOT PROJECT-APPROVED OR PUBLISHEDComplete bounded test-release rows only; this action never uses the public export route.{#if testCsvError}

{testCsvError}

{/if}
{/if} + {#if testReleaseMode}
TEST-ONLY CSV — NOT PROJECT-APPROVED OR PUBLISHEDComplete bounded test-release rows only; this action never uses the public export route.{#if testCsvError}

{testCsvError}

{/if}
{/if} {#if !devPreviewMode || previewStatus === 'ready'}

01 / DISCOVER

Choose the evidence lane

Profiles stay separate. A community claim is never silently promoted into a curated result.

{#if localMode && metadataStatus === 'loading'}

Loading available filter values…

{:else if localMode && metadataStatus === 'error'}

Filter metadata is unavailable; only unscoped discovery is shown.

{/if}
diff --git a/frontend/src/domain/location.ts b/frontend/src/domain/location.ts index a73822f..29e3967 100644 --- a/frontend/src/domain/location.ts +++ b/frontend/src/domain/location.ts @@ -5,7 +5,7 @@ export type LocationEvidence = Readonly<{ sourceType: 'official' | 'secondary' | 'user_submitted'; factualReviewStatus: 'unreviewed' | 'reviewed' | 'rejected'; reviewerRole: string | null; - privacyScreeningStatus: 'passed'; + privacyScreeningStatus: 'pending' | 'passed' | 'failed'; projectApproval: 'pending' | 'approved' | false; publicationProfile: 'official' | 'secondary' | 'community' | null; publicationWarning: string | null; diff --git a/frontend/tests/e2e/local-backend.spec.ts b/frontend/tests/e2e/local-backend.spec.ts index b9d8bc2..dfe6d16 100644 --- a/frontend/tests/e2e/local-backend.spec.ts +++ b/frontend/tests/e2e/local-backend.spec.ts @@ -9,6 +9,8 @@ test('renders the real seeded local V2 record and opens its detail route', async expect(response.ok).toBeTruthy(); list = await response.json() as typeof list; const record = list.data?.[0]; + // The disposable candidate fixture must remain absent from public V2. + if (!record) { expect(list.data).toEqual([]); return; } expect(record?.facility_id).toBeTruthy(); expect(record?.canonical_name).toBeTruthy(); await page.route('**/api/v2/**', async route => { @@ -27,3 +29,26 @@ test('renders the real seeded local V2 record and opens its detail route', async await expect(page.getByRole('heading', { name: record?.canonical_name ?? '' })).toBeVisible(); await expect(page.locator('article')).toContainText('Project approval'); }); + +test.skip(process.env.TEST_RELEASE_E2E !== '1', 'Set TEST_RELEASE_E2E=1 with a disposable guarded test-release backend'); + +test('renders the guarded disposable test release without public fallback', async ({ page }) => { + const apiUrl = process.env.UEC_E2E_API_URL ?? 'http://127.0.0.1:8000'; + const token = process.env.UEC_TEST_RELEASE_TOKEN ?? process.env.UEC_DEV_PREVIEW_TOKEN; + expect(token, 'UEC_TEST_RELEASE_TOKEN must be supplied in memory by the test runner').toBeTruthy(); + const probe = await fetch(`${apiUrl}/api/dev/preview/test-release/locations?profile=official&limit=100`, { headers: { 'X-UEC-Dev-Preview-Token': token ?? '' } }); + expect(probe.ok).toBeTruthy(); + const body = await probe.json() as { data?: Array<{ canonical_name?: string }> }; + expect(body.data?.length).toBeGreaterThan(0); + await page.route('**/api/dev/preview/test-release/**', async route => { + const requestUrl = new URL(route.request().url()); + const upstream = await fetch(`${apiUrl}${requestUrl.pathname}${requestUrl.search}`, { headers: { 'X-UEC-Dev-Preview-Token': token ?? '' } }); + await route.fulfill({ status: upstream.status, headers: Object.fromEntries(upstream.headers.entries()), body: await upstream.text() }); + }); + await page.goto('./?preview=test-release#/'); + await page.getByLabel('Operator token (memory only)').fill(token ?? ''); + await page.getByRole('button', { name: 'Load test release' }).click(); + await expect(page.getByText('Disposable test release — not project-approved or published')).toBeVisible(); + await expect(page.getByRole('button', { name: /Download test-only CSV/ })).toBeVisible(); + await expect(page.getByRole('button', { name: /Preview export/ })).toHaveCount(0); +}); diff --git a/frontend/tests/unit/devPreviewContract.test.ts b/frontend/tests/unit/devPreviewContract.test.ts index 4e3463f..c01ab8b 100644 --- a/frontend/tests/unit/devPreviewContract.test.ts +++ b/frontend/tests/unit/devPreviewContract.test.ts @@ -5,6 +5,7 @@ import { nextLocalReviewState } from '../../src/features/devPreview/devReviewSta import { TestReleaseRepository } from '../../src/api/TestReleaseRepository'; import { TestReleaseCsvExportRepository } from '../../src/api/TestReleaseCsvExportRepository'; import { TestReleaseFilterMetadataRepository } from '../../src/api/TestReleaseFilterMetadataRepository'; +import { locationSchema, testReleaseLocationSchema } from '../../src/api/wireSchema'; describe('dev preview boundary', () => { it('requires both a development build and the explicit mode', () => { @@ -44,6 +45,9 @@ describe('dev preview boundary', () => { it('maps test-release rows without requiring approval or coordinates and never falls back', async () => { const row = { facility_id: '550e8400-e29b-41d4-a716-446655440000', canonical_name: 'Pending test row', city: null, country_code: 'GB', category: 'dairy', source_type: 'official', publication_profile: 'official', factual_review_status: 'unreviewed', privacy_screening_status: 'passed', project_approval: 'pending', reviewer_role: null, publication_warning: null, display_precision: 'unmapped', latitude: null, longitude: null, first_observed_at: null, last_observed_at: null, observation_count: null, lifecycle_status: 'status_unknown', provenance_source_id: 's1', provenance_source_name: 'Test source', provenance_source_url: 'https://example.test/source', provenance_retrieved_at: '2026-01-01T00:00:00Z', release_id: 'test-release', release_ruleset_version: 'rules-1' }; const body = { data: [row], meta: { api_version: 'dev-test-v1', environment: 'test-only', test_only: true, private_preview: true, release_status: 'candidate', release_id: 'test-release', profile: 'official', coverage_scope: 'test_release_public_shaped_rows', count_semantics: 'Rows only', preview_label: TEST_RELEASE_LABEL, result_count: 1, next_cursor: null } }; + const candidateVariant = { ...row, canonical_name: null, privacy_screening_status: 'pending', project_approval: 'not-approved', release_ruleset_version: null }; + expect(testReleaseLocationSchema.safeParse(candidateVariant).success).toBe(true); + expect(locationSchema.safeParse(candidateVariant).success).toBe(false); const fetcher = vi.fn().mockResolvedValue(new Response(JSON.stringify(body))); const result = await new TestReleaseRepository(fetcher).list('official', 'test-token'); expect(result.locations[0]).toMatchObject({ name: 'Pending test row', lat: null, evidence: { projectApproval: 'pending' } }); diff --git a/pipeline/tests/e2e/fixture.py b/pipeline/tests/e2e/fixture.py index 2369508..5a9184a 100644 --- a/pipeline/tests/e2e/fixture.py +++ b/pipeline/tests/e2e/fixture.py @@ -187,7 +187,7 @@ def seed_private_candidate_scenario(self): with psycopg.connect(self.database_url) as db: with db.transaction(): db.execute("INSERT INTO uec.sources (source_id,country_code,name,official_url,access_method) VALUES ('e2e.private-candidate','DK','Synthetic private candidate source','https://example.invalid/private-candidate','fixture')") - db.execute("INSERT INTO uec.releases (release_id,status,ruleset_version,profile,summary) VALUES ('e2e-private-candidate','candidate','e2e-private-v1','official','{}')") + db.execute("INSERT INTO uec.releases (release_id,status,ruleset_version,profile,test_only,summary) VALUES ('e2e-private-candidate','candidate','e2e-private-v1','official',true,'{}')") record, facility, observation, artifact = uuid.uuid4(), uuid.uuid4(), uuid.uuid4(), uuid.uuid4() db.execute("INSERT INTO uec.raw_artifacts (artifact_id,storage_key,sha256,byte_size,retrieved_at) VALUES (%s,'e2e/private-candidate',%s,1,%s)", (artifact, uuid.uuid4().hex * 2, now)) db.execute("INSERT INTO uec.source_records (source_record_id,source_id,source_record_key,artifact_id,raw_fields,parsed_at) VALUES (%s,'e2e.private-candidate','candidate-only',%s,'{}',%s)", (record, artifact, now)) diff --git a/src/lib.rs b/src/lib.rs index b6b2755..f8f1568 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -394,7 +394,7 @@ pub async fn get_dev_test_release_locations_handler( ORDER BY f.facility_id LIMIT $4"#, &[&release_id,¶ms.country_code,¶ms.category,&limit]).await { Ok(rows)=>rows, Err(_)=>return v2_error(StatusCode::SERVICE_UNAVAILABLE,"test_release_query_failed","test release unavailable") }; - let data = rows.into_iter().map(|row| json!({"facility_id":row.get::<_,uuid::Uuid>(0),"canonical_name":row.get::<_,Option>(1),"country_code":row.get::<_,String>(2),"city":row.get::<_,Option>(3),"category":row.get::<_,String>(4),"publication_profile":profile,"factual_review_status":row.get::<_,Option>(8).unwrap_or("unreviewed".into()),"privacy_screening_status":row.get::<_,Option>(9).unwrap_or("pending".into()),"project_approval":"not-approved","reviewer_role":row.get::<_,Option>(11),"publication_warning":"Disposable test release — not project-approved or published","display_precision":row.get::<_,String>(5),"latitude":row.get::<_,Option>(6),"longitude":row.get::<_,Option>(7),"lifecycle_status":"status_unknown","source_type":row.get::<_,String>(12),"release_id":release_id,"release_ruleset_version":row.get::<_,String>(16),"provenance_source_id":row.get::<_,String>(13),"provenance_source_name":row.get::<_,String>(14),"provenance_source_url":row.get::<_,String>(15),"provenance_retrieved_at":row.get::<_,chrono::DateTime>(17)})).collect::>(); + let data = rows.into_iter().map(|row| json!({"facility_id":row.get::<_,uuid::Uuid>(0),"canonical_name":row.get::<_,Option>(1),"country_code":row.get::<_,String>(2),"city":row.get::<_,Option>(3),"category":row.get::<_,String>(4),"publication_profile":profile,"factual_review_status":row.get::<_,Option>(8).unwrap_or("unreviewed".into()),"privacy_screening_status":row.get::<_,Option>(9).unwrap_or("pending".into()),"project_approval":"not-approved","reviewer_role":row.get::<_,Option>(11),"publication_warning":"Disposable test release — not project-approved or published","display_precision":row.get::<_,String>(5),"latitude":row.get::<_,Option>(6),"longitude":row.get::<_,Option>(7),"first_observed_at":null,"last_observed_at":null,"observation_count":null,"lifecycle_status":"status_unknown","source_type":row.get::<_,String>(12),"release_id":release_id,"release_ruleset_version":row.get::<_,String>(16),"provenance_source_id":row.get::<_,String>(13),"provenance_source_name":row.get::<_,String>(14),"provenance_source_url":row.get::<_,String>(15),"provenance_retrieved_at":row.get::<_,chrono::DateTime>(17)})).collect::>(); let mut meta = test_release_meta(release_id, profile); meta["result_count"] = json!(data.len()); Json(json!({"data":data,"meta":meta})).into_response() From 7701f37022897077fdaf72d7726b8252cc8ca9b3 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Mon, 14 Sep 2026 09:17:36 -0700 Subject: [PATCH 094/311] Correct Italy 853 source row count --- docs/country-recon-it.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/country-recon-it.md b/docs/country-recon-it.md index 5065317..c254c29 100644 --- a/docs/country-recon-it.md +++ b/docs/country-recon-it.md @@ -17,7 +17,7 @@ The strongest candidate is the Italian Ministry of Health open-data catalog rath - 853 schema dictionary: - 1069 schema dictionary: -The catalog reported 853 data last updated 2026-09-13 and daily frequency; the 1069 catalog reported last updated 2026-09-11 and daily frequency. Private retrieval was 2026-09-14T05:44:54.7132759Z UTC. The 853 artifact is 49,927,230 bytes (SHA-256 `af1ec6eb7b530fef8dd420cdd08215b36d1b29cb202cf355b87a95fc938d6fea`) with 47,369 data rows; the separate 1069 artifact is 7,708,707 bytes (SHA-256 `4071c10f00f59070f75435988c7b22858bbd24abdc456613d448ff873bd9a2ce`) with 9,935 data rows. Raw files and sanitized metadata remain under ignored `data/raw/italy/` and are not release inputs. +The catalog reported 853 data last updated 2026-09-13 and daily frequency; the 1069 catalog reported last updated 2026-09-11 and daily frequency. Private retrieval was 2026-09-14T05:44:54.7132759Z UTC. The 853 artifact is 49,927,230 bytes (SHA-256 `af1ec6eb7b530fef8dd420cdd08215b36d1b29cb202cf355b87a95fc938d6fea`) with 47,370 data rows; the separate 1069 artifact is 7,708,707 bytes (SHA-256 `4071c10f00f59070f75435988c7b22858bbd24abdc456613d448ff873bd9a2ce`) with 9,935 data rows. Raw files and sanitized metadata remain under ignored `data/raw/italy/` and are not release inputs. The catalog identifies the Ministry of Health/DGSAN Office 2 and Italian Open Data Licence v2.0. It warns that some coordinates came from OpenStreetMap contributors; this is source metadata, not permission to publish precise points. From c3c05aaf15d868e9f50d7f1593f9d88c591db5b0 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Mon, 14 Sep 2026 09:27:26 -0700 Subject: [PATCH 095/311] Harden production readiness checks --- pipeline/tests/e2e/fixture.py | 4 +++- src/main.rs | 21 ++++++++++++++++++--- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/pipeline/tests/e2e/fixture.py b/pipeline/tests/e2e/fixture.py index 5a9184a..02807f0 100644 --- a/pipeline/tests/e2e/fixture.py +++ b/pipeline/tests/e2e/fixture.py @@ -1,5 +1,6 @@ """Disposable PostGIS and backend fixture used by API E2E tests.""" import os +import json import socket import subprocess import tempfile @@ -100,7 +101,8 @@ def start(self): for _ in range(80): try: with urllib.request.urlopen(f"http://127.0.0.1:{self.api_port}/health/ready", timeout=1) as response: - if response.status == 200: + payload = json.load(response) + if response.status == 200 and payload.get("schema") == "migrated": return self except (urllib.error.HTTPError, urllib.error.URLError, TimeoutError, OSError) as exc: last_error = repr(exc) diff --git a/src/main.rs b/src/main.rs index f0a3995..33bbecf 100644 --- a/src/main.rs +++ b/src/main.rs @@ -294,8 +294,15 @@ async fn readiness( .into_response(); }; match pool.get().await { - Ok(client) => match client.query_one("SELECT 1", &[]).await { - Ok(_) => Json(serde_json::json!({"status": "ready", "database": "ok"})).into_response(), + Ok(client) => match client.query_one("SELECT to_regclass('uec.releases')::text", &[]).await { + Ok(row) if row.get::<_, Option>(0).is_some() => { + Json(serde_json::json!({"status": "ready", "database": "ok", "schema": "migrated"})).into_response() + } + Ok(_) => ( + StatusCode::SERVICE_UNAVAILABLE, + Json(serde_json::json!({"status": "not_ready", "reason": "database_schema_not_migrated"})), + ) + .into_response(), Err(_) => ( StatusCode::SERVICE_UNAVAILABLE, Json(serde_json::json!({"status": "not_ready", "reason": "database_query_failed"})), @@ -315,7 +322,11 @@ fn validate_runtime( database_url: Option<&str>, port: &str, ) -> Result { - if mode == "production" && database_url.is_none() { + if mode == "production" + && database_url + .map(|url| url.trim().is_empty()) + .unwrap_or(true) + { return Err("UEC_DATABASE_URL is required in production"); } if !matches!(mode, "development" | "production") { @@ -461,6 +472,10 @@ mod config_tests { validate_runtime("production", None, "8000"), Err("UEC_DATABASE_URL is required in production") ); + assert_eq!( + validate_runtime("production", Some(" "), "8000"), + Err("UEC_DATABASE_URL is required in production") + ); } #[test] fn invalid_mode_and_port_are_rejected() { From 3ca989b6ac99842bf63412140e37b0692491f76c Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Mon, 14 Sep 2026 09:32:47 -0700 Subject: [PATCH 096/311] Validate complete database schema before readiness --- pipeline/tests/e2e/fixture.py | 7 +++-- pipeline/tests/e2e/test_readiness.py | 40 ++++++++++++++++++++++++++++ src/main.rs | 40 ++++++++++++++++++++++++++-- 3 files changed, 83 insertions(+), 4 deletions(-) create mode 100644 pipeline/tests/e2e/test_readiness.py diff --git a/pipeline/tests/e2e/fixture.py b/pipeline/tests/e2e/fixture.py index 02807f0..3378fe7 100644 --- a/pipeline/tests/e2e/fixture.py +++ b/pipeline/tests/e2e/fixture.py @@ -44,13 +44,14 @@ def command(self, *args): def compose_env(self): env = os.environ.copy(); env["UEC_E2E_DB_PORT"] = str(self.db_port); return env - def start(self): + def start(self, migration_files=None, wait_for_ready=True): try: print(f"[e2e] starting {self.project}", flush=True) startup = subprocess.run(self.command("up", "-d", "--wait"), cwd=ROOT, capture_output=True, text=True, env=self.compose_env()) if startup.returncode: raise RuntimeError(f"Docker Compose startup failed (exit {startup.returncode})\n{startup.stdout}\n{startup.stderr}") - migrations = "\n".join(p.read_text(encoding="utf-8") for p in sorted((ROOT / "pipeline/migrations").glob("*.sql"))) + files = migration_files if migration_files is not None else sorted((ROOT / "pipeline/migrations").glob("*.sql")) + migrations = "\n".join(p.read_text(encoding="utf-8") for p in files) print("[e2e] applying migrations", flush=True) for _ in range(60): # pg_isready only confirms that Postgres accepts connections; @@ -95,6 +96,8 @@ def start(self): self.backend_log = (self.cargo_target_dir / f"e2e-{self.project}.log").open("w", encoding="utf-8") self.backend = subprocess.Popen([str(binary)], cwd=ROOT, env=env, stdout=self.backend_log, stderr=subprocess.STDOUT, text=True) print(f"[e2e] waiting for backend on {self.api_port}", flush=True) + if not wait_for_ready: + return self import urllib.error import urllib.request last_error = None diff --git a/pipeline/tests/e2e/test_readiness.py b/pipeline/tests/e2e/test_readiness.py new file mode 100644 index 0000000..eaee7e3 --- /dev/null +++ b/pipeline/tests/e2e/test_readiness.py @@ -0,0 +1,40 @@ +"""Disposable readiness checks for incomplete and fully migrated databases.""" +import json +import os +import unittest +import urllib.error +import urllib.request +from .fixture import E2EEnvironment, ROOT + + +@unittest.skipUnless(os.environ.get("UEC_RUN_E2E") == "1", "set UEC_RUN_E2E=1 to run Docker-backed E2E tests") +class ReadinessE2ETests(unittest.TestCase): + def test_partial_schema_is_not_ready(self): + env = E2EEnvironment() + try: + migrations = sorted((ROOT / "pipeline/migrations").glob("*.sql")) + env.start(migration_files=migrations[:1], wait_for_ready=False) + try: + urllib.request.urlopen(f"http://127.0.0.1:{env.api_port}/health/ready", timeout=2) + except urllib.error.HTTPError as response: + self.assertEqual(response.code, 503) + body = json.load(response) + self.assertEqual(body["reason"], "database_schema_not_migrated") + else: + self.fail("partial schema was reported ready") + finally: + env.stop() + + def test_full_schema_is_ready(self): + env = E2EEnvironment() + try: + env.start() + with urllib.request.urlopen(f"http://127.0.0.1:{env.api_port}/health/ready", timeout=2) as response: + self.assertEqual(response.status, 200) + self.assertEqual(json.load(response)["schema"], "migrated") + finally: + env.stop() + + +if __name__ == "__main__": + unittest.main() diff --git a/src/main.rs b/src/main.rs index 33bbecf..c142abd 100644 --- a/src/main.rs +++ b/src/main.rs @@ -294,8 +294,44 @@ async fn readiness( .into_response(); }; match pool.get().await { - Ok(client) => match client.query_one("SELECT to_regclass('uec.releases')::text", &[]).await { - Ok(row) if row.get::<_, Option>(0).is_some() => { + Ok(client) => match client.query_one(r#" + WITH required_relations(name) AS ( + VALUES ('uec.releases'), ('uec.release_manifests'), + ('uec.map_facilities_display_history'), + ('uec.publication_review_release_current'), + ('uec.public_access_restricted') + ), required_columns(schema_name, table_name, column_name) AS ( + VALUES ('uec','releases','release_id'), ('uec','releases','status'), + ('uec','releases','test_only'), ('uec','releases','profile'), + ('uec','releases','ruleset_version'), ('uec','releases','created_at'), + ('uec','release_manifests','release_id'), ('uec','release_manifests','manifest'), + ('uec','release_manifests','manifest_sha256'), + ('uec','map_facilities_display_history','facility_id'), + ('uec','map_facilities_display_history','release_id'), + ('uec','map_facilities_display_history','release_ruleset_version'), + ('uec','map_facilities_display_history','provenance_origin_type'), + ('uec','publication_review_release_current','source_record_id'), + ('uec','publication_review_release_current','release_id'), + ('uec','publication_review_release_current','factual_review_status'), + ('uec','publication_review_release_current','privacy_screening_status'), + ('uec','publication_review_release_current','maintainer_approval'), + ('uec','publication_review_release_current','publication_eligible'), + ('uec','public_access_restricted','source_record_id') + ) + SELECT NOT EXISTS ( + SELECT 1 FROM required_relations + WHERE to_regclass(name) IS NULL + ) AND NOT EXISTS ( + SELECT 1 FROM required_columns required + WHERE NOT EXISTS ( + SELECT 1 FROM information_schema.columns found + WHERE found.table_schema = required.schema_name + AND found.table_name = required.table_name + AND found.column_name = required.column_name + ) + ) + "#, &[]).await { + Ok(row) if row.get::<_, bool>(0) => { Json(serde_json::json!({"status": "ready", "database": "ok", "schema": "migrated"})).into_response() } Ok(_) => ( From c5977297f4b1afc7301cf6ce9a9d89a9307fb707 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Mon, 14 Sep 2026 09:25:24 -0700 Subject: [PATCH 097/311] Add shared private run QA seam --- pipeline/contracts/README.md | 16 +++ pipeline/contracts/private_run.py | 138 +++++++++++++++++++++++++ pipeline/contracts/test_private_run.py | 65 ++++++++++++ 3 files changed, 219 insertions(+) create mode 100644 pipeline/contracts/private_run.py create mode 100644 pipeline/contracts/test_private_run.py diff --git a/pipeline/contracts/README.md b/pipeline/contracts/README.md index 477fab5..ce9935a 100644 --- a/pipeline/contracts/README.md +++ b/pipeline/contracts/README.md @@ -1,5 +1,21 @@ # Source-adapter contract +## Shared private-run QA seam + +`private_run.run_typed_adapter` provides the common runner for typed +`SourceAdapter` implementations. It writes a deterministic, row-free +`qa.json` beside the adapter manifest, validates count and release-state +invariants, and accepts provenance recorded either at the manifest root or in +the nested `acquisition` object. The report includes schema/count/anomaly and +drift summaries without copying source values, coordinates, or other row data. + +When a prior normalized artifact is supplied, disappeared identifiers are +reported only as `not-observed`; they are never converted into closure or +deauthorization claims. Acquisition, ingestion, release approval, and +publication remain separate stages so the scrape-to-ingestion pipeline can be +fully automated without silently turning a failed or incomplete run into a +public result. + Adapters receive a preserved raw artifact and `SourceArtifact` facts: source URL, UTC retrieval time, SHA-256, byte size, supplied publication/effective dates, code/config versions, rights/privacy caveats, and coverage. They must retain diff --git a/pipeline/contracts/private_run.py b/pipeline/contracts/private_run.py new file mode 100644 index 0000000..78f56d5 --- /dev/null +++ b/pipeline/contracts/private_run.py @@ -0,0 +1,138 @@ +"""Shared private-run validation and aggregate QA reporting.""" +from __future__ import annotations + +import json +import os +from pathlib import Path +from typing import Any, Iterable + +from .adapter_contract import SourceAdapter, SourceArtifact + + +class PrivateRunError(ValueError): + """A private run manifest or report violates the shared contract.""" + + +def _atomic_json(path: Path, value: dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + payload = (json.dumps(value, ensure_ascii=False, sort_keys=True, indent=2, default=list) + "\n").encode() + temporary = path.with_name(path.name + ".tmp") + temporary.write_bytes(payload) + os.replace(temporary, path) + + +def _manifest_value(manifest: dict[str, Any], key: str) -> Any: + if key in manifest: + return manifest[key] + acquisition = manifest.get("acquisition") + if isinstance(acquisition, dict): + return acquisition.get(key) + return None + + +def _record_ids(path: Path) -> set[str]: + if not path.exists(): + return set() + ids: set[str] = set() + for line in path.read_text(encoding="utf-8").splitlines(): + if not line: + continue + row = json.loads(line) + normalized = row.get("normalized", {}) + candidates = ( + normalized.get("establishment_id"), + normalized.get("recognition_number"), + row.get("source_record_key"), + row.get("source_row_id"), + ) + value = next((candidate for candidate in candidates if isinstance(candidate, str) and candidate), None) + if value: + ids.add(value) + return ids + + +def validate_manifest(manifest: dict[str, Any]) -> None: + required = {"source_id", "input_rows", "normalized_rows", "quarantined_rows", "release_state"} + missing = sorted(required - manifest.keys()) + if missing: + raise PrivateRunError("manifest missing required keys: " + ", ".join(missing)) + if manifest["input_rows"] != manifest["normalized_rows"] + manifest["quarantined_rows"]: + raise PrivateRunError("manifest row counts do not reconcile") + if manifest["release_state"] != "not-created": + raise PrivateRunError("private run cannot have a release") + if manifest.get("publication_state") not in {None, "private-candidate", "not-staged"}: + raise PrivateRunError("private run publication state is not restricted") + + +def summarize_private_run( + manifest: dict[str, Any], + *, + normalized_path: str | Path | None = None, + previous_normalized_path: str | Path | None = None, + drift_alarms: Iterable[str] = (), +) -> dict[str, Any]: + """Return a row-free deterministic QA summary for a private run.""" + validate_manifest(manifest) + disappeared = 0 + if normalized_path is not None and previous_normalized_path is not None: + disappeared = len(_record_ids(Path(previous_normalized_path)) - _record_ids(Path(normalized_path))) + provenance = { + key: _manifest_value(manifest, key) + for key in ("source_url", "retrieved_at_utc", "publication_date", "effective_date", "sha256", "checksum_sha256", "byte_size", "code_version", "config_version") + if _manifest_value(manifest, key) is not None + } + report = { + "source_id": manifest["source_id"], + "adapter_version": manifest.get("adapter_version"), + "schema_version": manifest.get("schema_version"), + "provenance": provenance, + "input_rows": manifest["input_rows"], + "normalized_rows": manifest["normalized_rows"], + "quarantined_rows": manifest["quarantined_rows"], + "coverage_counts": manifest.get("coverage_counts", {}), + "anomaly_counts": manifest.get("anomaly_counts", {}), + "drift_alarms": sorted(set(drift_alarms)), + "disappeared_not_observed_count": disappeared, + "disappearance_semantics": "not-observed; never inferred as closure", + "geocoding": manifest.get("geocoding", "unavailable"), + "release_state": manifest["release_state"], + "publication_state": manifest.get("publication_state", "private-candidate"), + } + return report + + +def write_private_run_report( + run_dir: str | Path, + manifest: dict[str, Any], + *, + normalized_path: str | Path | None = None, + previous_normalized_path: str | Path | None = None, + drift_alarms: Iterable[str] = (), +) -> dict[str, Any]: + report = summarize_private_run( + manifest, + normalized_path=normalized_path, + previous_normalized_path=previous_normalized_path, + drift_alarms=drift_alarms, + ) + _atomic_json(Path(run_dir) / "qa.json", report) + return report + + +def run_typed_adapter( + adapter: SourceAdapter, + raw_path: str | Path, + run_dir: str | Path, + artifact: SourceArtifact, + *, + previous_normalized_path: str | Path | None = None, +) -> tuple[dict[str, Any], dict[str, Any]]: + """Run a typed adapter and emit the shared aggregate QA report.""" + manifest = adapter.run(raw_path, run_dir, artifact) + report = write_private_run_report( + run_dir, + manifest, + normalized_path=Path(run_dir) / "normalized" / "records.jsonl", + previous_normalized_path=previous_normalized_path, + ) + return manifest, report diff --git a/pipeline/contracts/test_private_run.py b/pipeline/contracts/test_private_run.py new file mode 100644 index 0000000..5401f19 --- /dev/null +++ b/pipeline/contracts/test_private_run.py @@ -0,0 +1,65 @@ +import hashlib +import json +import tempfile +import unittest +from pathlib import Path + +from .adapter_contract import SourceArtifact +from .private_run import PrivateRunError, run_typed_adapter, summarize_private_run + + +class StubAdapter: + source_id = "test.source" + adapter_version = "test-v1" + + def run(self, raw_path, run_dir, artifact): + raw = Path(raw_path).read_bytes() + if hashlib.sha256(raw).hexdigest() != artifact.sha256: + raise ValueError("hash mismatch") + root = Path(run_dir) + root.joinpath("normalized").mkdir(parents=True) + rows = [{"source_id": self.source_id, "normalized": {"establishment_id": "A"}}] + root.joinpath("normalized/records.jsonl").write_text(json.dumps(rows[0]) + "\n", encoding="utf-8") + return { + "source_id": self.source_id, + "adapter_version": self.adapter_version, + "schema_version": "test-schema", + "source_url": artifact.source_url, + "retrieved_at_utc": artifact.retrieved_at_utc, + "checksum_sha256": artifact.sha256, + "byte_size": len(raw), + "input_rows": 2, + "normalized_rows": 1, + "quarantined_rows": 1, + "publication_state": "private-candidate", + "release_state": "not-created", + "geocoding": "disabled", + "anomaly_counts": {"synthetic_quarantine": 1}, + } + + +class PrivateRunTests(unittest.TestCase): + def artifact(self, raw): + return SourceArtifact("https://example.test", "2026-09-14T00:00:00Z", hashlib.sha256(raw).hexdigest(), len(raw), code_version="c", config_version="k") + + def test_typed_runner_writes_row_free_report_and_not_observed_delta(self): + raw = b"synthetic" + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + raw_path = root / "raw.bin" + raw_path.write_bytes(raw) + previous = root / "previous.jsonl" + previous.write_text(json.dumps({"normalized": {"establishment_id": "A"}}) + "\n" + json.dumps({"normalized": {"establishment_id": "B"}}) + "\n", encoding="utf-8") + manifest, report = run_typed_adapter(StubAdapter(), raw_path, root / "run", self.artifact(raw), previous_normalized_path=previous) + self.assertEqual(report["disappeared_not_observed_count"], 1) + self.assertIn("not-observed", report["disappearance_semantics"]) + self.assertEqual(report["anomaly_counts"], {"synthetic_quarantine": 1}) + self.assertNotIn("source_values", (root / "run/qa.json").read_text()) + + def test_manifest_invariants_fail_closed(self): + with self.assertRaisesRegex(PrivateRunError, "row counts"): + summarize_private_run({"source_id": "x", "input_rows": 2, "normalized_rows": 2, "quarantined_rows": 2, "release_state": "not-created"}) + + +if __name__ == "__main__": + unittest.main() From 19710f317ccbbe28a1d266b044a1b945c8939d2b Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Mon, 14 Sep 2026 09:30:05 -0700 Subject: [PATCH 098/311] Add typed runner and resumable candidate batches --- pipeline/common/orchestrator.py | 22 +++++ pipeline/common/test_orchestrator.py | 18 +++- .../scripts/maintenance/import-candidate.py | 90 ++++++++++++------- 3 files changed, 96 insertions(+), 34 deletions(-) diff --git a/pipeline/common/orchestrator.py b/pipeline/common/orchestrator.py index f89ce53..ecb552a 100644 --- a/pipeline/common/orchestrator.py +++ b/pipeline/common/orchestrator.py @@ -9,6 +9,7 @@ from pathlib import Path from typing import Any, Callable +from pipeline.contracts.adapter_contract import SourceAdapter, source_artifact_from_mapping from .identity import record_key ORCHESTRATOR_VERSION = "v2-orchestrator-3" @@ -86,3 +87,24 @@ def run_registered_input(raw_path: str | Path, runs_dir: str | Path, config: dic status["run_dir"] = str(run_dir) _atomic(run_dir / "run-status.json", (json.dumps(status, ensure_ascii=False, sort_keys=True, indent=2) + "\n").encode()) return status + + +def run_registered_typed_input(raw_path: str | Path, runs_dir: str | Path, + config: dict[str, Any], adapter: SourceAdapter, + suppressed_ids: set[str | tuple[str, str, str]] | None = None, + prior_eligible_release: dict[str, Any] | None = None) -> dict[str, Any]: + """Run a typed adapter from registered acquisition metadata. + + This is the compatibility seam for adapters whose ``run`` method accepts + ``SourceArtifact`` rather than the legacy config mapping. The mapping is + converted once at the shared boundary; the adapter still validates the + raw bytes and writes its own private manifest. + """ + artifact = source_artifact_from_mapping(config) + + def invoke(raw: str | Path, run_dir: str | Path, _config: dict[str, Any]) -> dict[str, Any]: + return adapter.run(raw, run_dir, artifact) + + return run_registered_input(raw_path, runs_dir, config, invoke, + suppressed_ids=suppressed_ids, + prior_eligible_release=prior_eligible_release) diff --git a/pipeline/common/test_orchestrator.py b/pipeline/common/test_orchestrator.py index e6fffff..7825a97 100644 --- a/pipeline/common/test_orchestrator.py +++ b/pipeline/common/test_orchestrator.py @@ -1,14 +1,30 @@ import json +import hashlib import tempfile import unittest from pathlib import Path from .adapter_registry import load -from .orchestrator import register_input, run_registered_input +from .orchestrator import register_input, run_registered_input, run_registered_typed_input from pipeline.sources.uk.fss_approved.adapter import FssApprovedEstablishmentsAdapter +from pipeline.sources.italy.it_853_adapter import Italy853Adapter class SharedPipelineTests(unittest.TestCase): + def test_registered_typed_adapter_compatibility(self): + header = "precedente_bollo_cee;num_identificativo_produzione_commercializzazione;ragione_sociale;indirizzo;comune;provincia;codice_regione;regione;classificazione_stabilimento;codice_impianto_attivita;descrizione_impianto_attivita;prodotti_abilitati;specifica_prodotti;paesi_export_autorizzato;longitudine;latitudine;stato_localizzazione;cod_fiscale;p_iva;codice_comune;data_inizio_attivita;data_fine_attivita;stato_attivita;data_ultimo_aggiornamento;num_identificativo_produzione_commercializzazione_2" + row = ";A;Name;;Town;;010;Piemonte;X;10;Activity;P;S;IT;12;45;1;tax;vat;001001;;;Autorizzata;2026-09-13;\n" + raw = (header + "\n" + row).encode() + with tempfile.TemporaryDirectory() as directory: + raw_path = Path(directory) / "italy.csv" + raw_path.write_bytes(raw) + config = {"source_url": "https://example.test/italy", "retrieved_at_utc": "2026-09-14T00:00:00Z", + "checksum_sha256": hashlib.sha256(raw).hexdigest(), "byte_size": len(raw), + "code_version": "test", "config_version": "test"} + status = run_registered_typed_input(raw_path, Path(directory) / "runs", config, Italy853Adapter()) + self.assertEqual(status["status"], "candidate-ready") + self.assertEqual(status["manifest"]["source_id"], "it.853-2004") + def test_registry_and_suppression_are_shared(self): root = Path(__file__).parents[1] registry = load(root / "adapter-capabilities.json") diff --git a/pipeline/scripts/maintenance/import-candidate.py b/pipeline/scripts/maintenance/import-candidate.py index 79a4119..7ccfaa7 100644 --- a/pipeline/scripts/maintenance/import-candidate.py +++ b/pipeline/scripts/maintenance/import-candidate.py @@ -24,6 +24,7 @@ class CandidateImportError(ValueError): DISPOSABLE_MARKER = "uec-e2e-disposable-v1" +DEFAULT_BATCH_SIZE = 500 def require_disposable_database(database_url: str, acknowledged: bool) -> None: @@ -96,13 +97,25 @@ def _country_code(manifest: dict, normalized: dict) -> str: return {"Denmark": "DK", "England": "GB", "Wales": "GB"}.get(normalized.get("nation"), "ZZ") -def import_candidate(database_url: str, manifest: dict, rows: list[dict], release_id: str, reset: bool) -> int: - """Append one candidate release; never promotes or marks review complete.""" +def _stable_uuid(*parts: object) -> uuid.UUID: + return uuid.uuid5(uuid.NAMESPACE_URL, "uec-candidate:" + "|".join(str(part) for part in parts)) + + +def import_candidate(database_url: str, manifest: dict, rows: list[dict], release_id: str, + reset: bool, batch_size: int = DEFAULT_BATCH_SIZE) -> int: + """Append one candidate release; never promotes or marks review complete. + + Batches commit independently so a bounded failure can resume with the same + release ID. Deterministic IDs and conflict-safe inserts make retries + idempotent; every committed row remains private and review-required. + """ if reset: raise CandidateImportError( "--reset is intentionally refused: append-only evidence cannot be deleted; " "recreate the disposable database with the local-v2 maintenance recipe" ) + if batch_size <= 0: + raise CandidateImportError("batch size must be positive") now = manifest["retrieved_at_utc"] ruleset = str(manifest.get("config_version") or manifest.get("schema_version") or "unknown") with psycopg.connect(database_url) as db: @@ -132,37 +145,47 @@ def import_candidate(database_url: str, manifest: dict, rows: list[dict], releas VALUES (%s,'candidate',%s,%s,true) ON CONFLICT (release_id) DO NOTHING""", (release_id, ruleset, json.dumps({"source_id": manifest["source_id"], "profile": manifest.get("profile")}))) - count = 0 - for record in rows: - key, normalized = _record_parts(record) - country = _country_code(manifest, normalized) - name = normalized.get("trading_name") - city = normalized.get("city") - db.execute("""INSERT INTO uec.source_records(source_id,source_record_key,artifact_id,raw_fields,parsed_at) - VALUES (%s,%s,%s,%s,%s) ON CONFLICT (source_id,source_record_key,artifact_id) - DO NOTHING""", - (manifest["source_id"], key, artifact_id, json.dumps({"source_values": record.get("source_values", {})}), now)) - record_id = db.execute("""SELECT source_record_id FROM uec.source_records - WHERE source_id=%s AND source_record_key=%s AND artifact_id=%s""", - (manifest["source_id"], key, artifact_id)).fetchone()[0] - existing = db.execute("""SELECT facility_id, observation_id FROM uec.observations - WHERE source_record_id=%s ORDER BY observed_at DESC, observation_id DESC LIMIT 1""", (record_id,)).fetchone() - if existing: + count = 0 + for offset in range(0, len(rows), batch_size): + batch_count = 0 + with db.transaction(): + for record in rows[offset:offset + batch_size]: + key, normalized = _record_parts(record) + country = _country_code(manifest, normalized) + name = normalized.get("trading_name") + city = normalized.get("city") + record_id = _stable_uuid(manifest["source_id"], key, artifact_id) + source_inserted = db.execute("""INSERT INTO uec.source_records(source_record_id,source_id,source_record_key,artifact_id,raw_fields,parsed_at) + VALUES (%s,%s,%s,%s,%s,%s) ON CONFLICT (source_id,source_record_key,artifact_id) + DO NOTHING RETURNING source_record_id""", + (record_id, manifest["source_id"], key, artifact_id, + json.dumps({"source_values": record.get("source_values", {})}), now)).fetchone() + if source_inserted: + record_id = source_inserted[0] + else: + record_id = db.execute("""SELECT source_record_id FROM uec.source_records + WHERE source_id=%s AND source_record_key=%s AND artifact_id=%s""", + (manifest["source_id"], key, artifact_id)).fetchone()[0] + facility_id = _stable_uuid("facility", manifest["source_id"], key, artifact_id) + observation_id = _stable_uuid("observation", manifest["source_id"], key, artifact_id) + db.execute("""INSERT INTO uec.facilities(facility_id,canonical_name,country_code,city) + VALUES (%s,%s,%s,%s) ON CONFLICT (facility_id) DO NOTHING""", + (facility_id, name, country, city)) + created = db.execute("""INSERT INTO uec.observations(observation_id,facility_id,source_record_id,observed_at,observation,classification,ruleset_id,rule_id,classification_category,classification_review_status,default_visible,coordinate_review_status,first_observed_at) + VALUES (%s,%s,%s,%s,%s,'{}',%s,'candidate','unclassified','review_required',false,'review_required',%s) + ON CONFLICT (facility_id,source_record_id,observed_at) DO NOTHING RETURNING observation_id""", + (observation_id, facility_id, record_id, now, json.dumps(normalized), ruleset, now)).fetchone() + observation_ref = created[0] if created else db.execute("""SELECT observation_id FROM uec.observations + WHERE facility_id=%s AND source_record_id=%s AND observed_at=%s""", + (facility_id, record_id, now)).fetchone()[0] db.execute("INSERT INTO uec.release_members(release_id,facility_id,observation_id,default_visible) VALUES (%s,%s,%s,false) ON CONFLICT DO NOTHING", - (release_id, existing[0], existing[1])) - continue - facility_id = uuid.uuid4(); observation_id = uuid.uuid4() - db.execute("INSERT INTO uec.facilities(facility_id,canonical_name,country_code,city) VALUES (%s,%s,%s,%s)", - (facility_id, name, country, city)) - db.execute("""INSERT INTO uec.observations(observation_id,facility_id,source_record_id,observed_at,observation,classification,ruleset_id,rule_id,classification_category,classification_review_status,default_visible,coordinate_review_status,first_observed_at) - VALUES (%s,%s,%s,%s,%s,'{}',%s,'candidate','unclassified','review_required',false,'review_required',%s)""", - (observation_id, facility_id, record_id, now, json.dumps(normalized), ruleset, now)) - db.execute("INSERT INTO uec.release_members(release_id,facility_id,observation_id,default_visible) VALUES (%s,%s,%s,false)", - (release_id, facility_id, observation_id)) - db.execute("INSERT INTO uec.publication_review_events(source_record_id,release_id,factual_review_status,privacy_screening_status,maintainer_approval,publication_eligible) VALUES (%s,%s,'unreviewed','pending','pending',false)", - (record_id, release_id)) - count += 1 - return count + (release_id, facility_id, observation_ref)) + if created: + db.execute("INSERT INTO uec.publication_review_events(source_record_id,release_id,factual_review_status,privacy_screening_status,maintainer_approval,publication_eligible) VALUES (%s,%s,'unreviewed','pending','pending',false)", + (record_id, release_id)) + batch_count += 1 + count += batch_count + return count def main() -> int: @@ -174,12 +197,13 @@ def main() -> int: parser.add_argument("--database-url", default=os.environ.get("UEC_DATABASE_URL", "")) parser.add_argument("--disposable-db", action="store_true", help="acknowledge this is a disposable local DB") parser.add_argument("--reset", action="store_true", help="rebuild this candidate release only") + parser.add_argument("--batch-size", type=int, default=DEFAULT_BATCH_SIZE) args = parser.parse_args() require_disposable_database(args.database_url, args.disposable_db) if not args.release_id.startswith("candidate-"): raise CandidateImportError("release id must start with candidate-") manifest, rows = load_inputs(args.manifest, args.normalized, args.raw) - print(f"imported {import_candidate(args.database_url, manifest, rows, args.release_id, args.reset)} candidate rows") + print(f"imported {import_candidate(args.database_url, manifest, rows, args.release_id, args.reset, args.batch_size)} candidate rows") return 0 From 83a494081303e1cd81461a50e1cd81cb4258ad02 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Mon, 14 Sep 2026 09:34:22 -0700 Subject: [PATCH 099/311] Commit guard transaction before import batches --- pipeline/scripts/maintenance/import-candidate.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pipeline/scripts/maintenance/import-candidate.py b/pipeline/scripts/maintenance/import-candidate.py index 7ccfaa7..d693762 100644 --- a/pipeline/scripts/maintenance/import-candidate.py +++ b/pipeline/scripts/maintenance/import-candidate.py @@ -120,6 +120,10 @@ def import_candidate(database_url: str, manifest: dict, rows: list[dict], releas ruleset = str(manifest.get("config_version") or manifest.get("schema_version") or "unknown") with psycopg.connect(database_url) as db: verify_disposable_marker(db) + # The guard query starts an implicit read transaction in psycopg. + # End it before opening the independently resumable write batches; + # otherwise psycopg nests them as savepoints under one outer rollback. + db.commit() with db.transaction(): db.execute("""INSERT INTO uec.sources(source_id,country_code,name,official_url,access_method) VALUES (%s,%s,%s,%s,'validated-private-staging') From 5d114ff22d79e6eacb5c613e719d818713b1b0a8 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Mon, 14 Sep 2026 09:46:01 -0700 Subject: [PATCH 100/311] Add disposable batch resume E2E --- pipeline/tests/e2e/test_candidate_import.py | 49 +++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/pipeline/tests/e2e/test_candidate_import.py b/pipeline/tests/e2e/test_candidate_import.py index 1c27458..98aead5 100644 --- a/pipeline/tests/e2e/test_candidate_import.py +++ b/pipeline/tests/e2e/test_candidate_import.py @@ -210,6 +210,55 @@ def test_bad_row_rolls_back_candidate_import(self): after = self.counts() self.assertEqual(after, before) + def test_partial_batch_failure_resumes_idempotently_and_stays_private(self): + root = Path(self.temp.name) / "batch-resume" + root.mkdir() + raw = root / "batch.raw" + raw.write_bytes(b"synthetic-batch-resume") + source_id = "e2e.batch-resume" + release_id = "candidate-batch-resume" + digest = __import__("hashlib").sha256(raw.read_bytes()).hexdigest() + rows = [ + {"source_id": source_id, "source_row": index, "source_values": {"name": f"private-{index}"}, + "normalized": {"establishment_id": f"BATCH-{index}", "trading_name": f"Batch {index}", + "city": "Testville", "country_code": "GB", "nation": "England"}} + for index in (1, 2, 3) + ] + manifest = {"source_id": source_id, "source_url": "https://example.invalid/batch", + "retrieved_at_utc": "2026-09-14T00:00:00Z", "checksum_sha256": digest, + "byte_size": raw.stat().st_size, "normalized_rows": 4, + "release_state": "not-created", "publication_state": "private-candidate", + "normalized_sha256": "placeholder", "config_version": "e2e"} + normalized = root / "normalized.jsonl" + bad = rows + [{"source_id": source_id, "source_row": 99, "normalized": {"trading_name": "bad"}}] + normalized.write_text("".join(json.dumps(row) + "\n" for row in bad), encoding="utf-8") + manifest["normalized_sha256"] = __import__("hashlib").sha256(normalized.read_bytes()).hexdigest() + manifest_path = root / "manifest.json" + manifest_path.write_text(json.dumps(manifest), encoding="utf-8") + command = [sys.executable, str(IMPORTER), "--manifest", str(manifest_path), "--normalized", str(normalized), + "--raw", str(raw), "--release-id", release_id, "--database-url", self.env.database_url, + "--disposable-db", "--batch-size", "2"] + failed = subprocess.run(command, cwd=ROOT, capture_output=True, text=True) + self.assertNotEqual(failed.returncode, 0) + with psycopg.connect(self.env.database_url) as db: + self.assertEqual(db.execute("SELECT count(*) FROM uec.source_records WHERE source_id=%s", (source_id,)).fetchone()[0], 2) + + normalized.write_text("".join(json.dumps(row) + "\n" for row in rows), encoding="utf-8") + manifest["normalized_rows"] = len(rows) + manifest["normalized_sha256"] = __import__("hashlib").sha256(normalized.read_bytes()).hexdigest() + manifest_path.write_text(json.dumps(manifest), encoding="utf-8") + resumed = subprocess.run(command, cwd=ROOT, capture_output=True, text=True) + self.assertEqual(resumed.returncode, 0, resumed.stderr) + repeated = subprocess.run(command, cwd=ROOT, capture_output=True, text=True) + self.assertEqual(repeated.returncode, 0, repeated.stderr) + self.assertIn("imported 1 candidate rows", resumed.stdout) + self.assertIn("imported 0 candidate rows", repeated.stdout) + with psycopg.connect(self.env.database_url) as db: + self.assertEqual(db.execute("SELECT count(*) FROM uec.source_records WHERE source_id=%s", (source_id,)).fetchone()[0], 3) + self.assertEqual(db.execute("SELECT count(*) FROM uec.release_members WHERE release_id=%s", (release_id,)).fetchone()[0], 3) + self.assertEqual(db.execute("SELECT bool_or(default_visible) FROM uec.release_members WHERE release_id=%s", (release_id,)).fetchone()[0], False) + self.assertEqual(db.execute("SELECT bool_or(publication_eligible) FROM uec.publication_review_events WHERE release_id=%s", (release_id,)).fetchone()[0], False) + if __name__ == "__main__": unittest.main() From 007342ba998f2b173de3205c333b4466adee4e56 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Mon, 14 Sep 2026 09:52:01 -0700 Subject: [PATCH 101/311] Exercise private Italy 853 handoff end to end --- docs/country-recon-it.md | 2 + pipeline/sources/italy/it_853_adapter.py | 13 ++++ pipeline/sources/italy/test_it_853_adapter.py | 4 ++ .../tests/e2e/test_italy_candidate_import.py | 59 +++++++++++++++++++ 4 files changed, 78 insertions(+) create mode 100644 pipeline/tests/e2e/test_italy_candidate_import.py diff --git a/docs/country-recon-it.md b/docs/country-recon-it.md index c254c29..3ed5f72 100644 --- a/docs/country-recon-it.md +++ b/docs/country-recon-it.md @@ -53,6 +53,8 @@ The repository’s historical Italy CSV and scraper are legacy/unverified inputs Build a deterministic catalog-download adapter with an explicit dataset variant and format. Validate encoding, delimiter/header, recognition identifiers, status vocabulary, category/activity codes, dates, coordinate ranges, duplicate identifiers, and count changes. Quarantine schema drift and malformed rows. Geocoding, if approved later, must be a separate derived event with provider/query/time/precision/review fields. +Private aggregate QA of the 853 snapshot found 41,844 distinct recognition/activity pairs; 4,529 pairs repeat, covering 10,055 rows. Multiplicity was 3,666 pairs occurring twice, 739 three times, 114 four times, and 10 five times. The adapter therefore retains source-row identity and quarantines repeated pair collisions rather than merging them; this is not evidence of duplicate facilities or an operating-status conclusion. + Do not publish names, addresses, tax identifiers, or precise coordinates merely because the Ministry publishes them. Apply residential/private-location screening, source-origin labels, project approval, and publication profile independently. Government-sourced does not mean current, complete, project-approved, or safe to expose. ## Limitations diff --git a/pipeline/sources/italy/it_853_adapter.py b/pipeline/sources/italy/it_853_adapter.py index d834840..5c602c9 100644 --- a/pipeline/sources/italy/it_853_adapter.py +++ b/pipeline/sources/italy/it_853_adapter.py @@ -4,6 +4,8 @@ import hashlib import json from pathlib import Path +from pipeline.contracts.adapter_contract import SourceArtifact +from pipeline.contracts.candidate_handoff import write_handoff REQUIRED = tuple( "precedente_bollo_cee;num_identificativo_produzione_commercializzazione;ragione_sociale;indirizzo;comune;provincia;codice_regione;regione;classificazione_stabilimento;codice_impianto_attivita;descrizione_impianto_attivita;prodotti_abilitati;specifica_prodotti;paesi_export_autorizzato;longitudine;latitudine;stato_localizzazione;cod_fiscale;p_iva;codice_comune;data_inizio_attivita;data_fine_attivita;stato_attivita;data_ultimo_aggiornamento;num_identificativo_produzione_commercializzazione_2".split(";") @@ -26,6 +28,12 @@ class Italy853Adapter: adapter_version = "it-853-candidate-v1" schema_version = "it-853-csv-v2.0" + def write_candidate_handoff(self, run_dir, artifact: SourceArtifact, parsed): + rows = [item["record"] if "record" in item else item for item in parsed["accepted"]] + for row in rows: + row["normalized"]["establishment_id"] = row["normalized"]["recognition_number"] + return write_handoff(run_dir, rows, artifact, source_id=self.source_id) + def parse_bytes(self, content): digest = hashlib.sha256(content).hexdigest() rows = list(csv.DictReader(content.decode("utf-8-sig").splitlines(), delimiter=";")) @@ -62,11 +70,16 @@ def parse_bytes(self, content): "source_row_id": row_id(row, occurrences[key]), "source_values": dict(row), "normalized": { + "establishment_id": rec, "recognition_number": rec, "facility_grouping": "provisional-recognition-number", "name": clean(row.get("ragione_sociale")), + "trading_name": clean(row.get("ragione_sociale")), "address": None, "municipality": clean(row.get("comune")), + "city": clean(row.get("comune")), + "country_code": "IT", + "nation": "Italy", "region": clean(row.get("regione")), "activity_code": act, "activity_description": clean(row.get("descrizione_impianto_attivita")), diff --git a/pipeline/sources/italy/test_it_853_adapter.py b/pipeline/sources/italy/test_it_853_adapter.py index 93a0439..b6d58b8 100644 --- a/pipeline/sources/italy/test_it_853_adapter.py +++ b/pipeline/sources/italy/test_it_853_adapter.py @@ -18,3 +18,7 @@ def test_run_writes_contract_manifest_and_row_quarantine(self): content=(H+"\n"+row()+row("","10")).encode() with tempfile.TemporaryDirectory() as d, tempfile.NamedTemporaryFile(suffix=".csv", delete=False) as f: f.write(content); f.flush(); sha=hashlib.sha256(content).hexdigest(); a=Italy853Adapter(); m=a.run(f.name,d,SourceArtifact("u","2026-09-14T00:00:00Z",sha,len(content),code_version="c",config_version="k")); assert_manifest(m,content,a.schema_version); self.assertTrue((__import__('pathlib').Path(d)/"normalized/records.jsonl").exists()); self.assertEqual(m["quarantined_rows"],1); self.assertNotIn("accepted",m); self.assertNotIn("source_values",json.dumps(m)) + def test_candidate_handoff_uses_shared_writer(self): + content=(H+"\n"+row()).encode(); sha=hashlib.sha256(content).hexdigest(); a=Italy853Adapter() + with tempfile.TemporaryDirectory() as d, tempfile.NamedTemporaryFile(suffix=".csv", delete=False) as f: + f.write(content); f.flush(); m=a.write_candidate_handoff(d,SourceArtifact("u","2026-09-14T00:00:00Z",sha,len(content)),a.parse_bytes(content)); self.assertEqual(m["contract_version"],"candidate-handoff-v1"); self.assertEqual(m["normalized_rows"],1) diff --git a/pipeline/tests/e2e/test_italy_candidate_import.py b/pipeline/tests/e2e/test_italy_candidate_import.py new file mode 100644 index 0000000..aca966b --- /dev/null +++ b/pipeline/tests/e2e/test_italy_candidate_import.py @@ -0,0 +1,59 @@ +import hashlib, json, os, subprocess, sys, tempfile, unittest, urllib.request, urllib.error +from pathlib import Path +import psycopg +from .fixture import E2EEnvironment +from pipeline.contracts.adapter_contract import SourceArtifact +from pipeline.sources.italy.it_853_adapter import Italy853Adapter + +ROOT=Path(__file__).resolve().parents[3] +HEADER="precedente_bollo_cee;num_identificativo_produzione_commercializzazione;ragione_sociale;indirizzo;comune;provincia;codice_regione;regione;classificazione_stabilimento;codice_impianto_attivita;descrizione_impianto_attivita;prodotti_abilitati;specifica_prodotti;paesi_export_autorizzato;longitudine;latitudine;stato_localizzazione;cod_fiscale;p_iva;codice_comune;data_inizio_attivita;data_fine_attivita;stato_attivita;data_ultimo_aggiornamento;num_identificativo_produzione_commercializzazione_2" +ROW=";IT-E2E-1;Synthetic Italy Facility;Private Address;Synthetic Town;XX;010;Piemonte;CODE=desc;A1;Synthetic activity;P;S;IT;12;45;1;restricted-tax;restricted-vat;001001;;;AUTORIZZATA;2026-09-14;" + +class ItalyCandidateImportE2E(unittest.TestCase): + @classmethod + def setUpClass(cls): + if os.environ.get("UEC_RUN_E2E")!="1": raise unittest.SkipTest("set UEC_RUN_E2E=1") + cls.env=E2EEnvironment(); cls.env.test_release_id="e2e-private-candidate"; cls.env=cls.env.start(); cls.temp=tempfile.TemporaryDirectory(); root=Path(cls.temp.name); cls.raw=root/"it.csv"; cls.raw.write_text(HEADER+"\n"+ROW+"\n",encoding="utf-8") + a=Italy853Adapter(); raw=cls.raw.read_bytes(); artifact=SourceArtifact("https://example.invalid/it-853.csv","2026-09-14T00:00:00Z",hashlib.sha256(raw).hexdigest(),len(raw),code_version=a.adapter_version,config_version=a.schema_version) + parsed=a.parse_bytes(raw); cls.run_dir=root/"handoff"; a.write_candidate_handoff(cls.run_dir,artifact,parsed) + cls.release_id="candidate-italy-e2e"; cmd=[sys.executable,str(ROOT/"pipeline/scripts/maintenance/import-candidate.py"),"--manifest",str(cls.run_dir/"manifest.json"),"--normalized",str(cls.run_dir/"normalized/records.jsonl"),"--raw",str(cls.raw),"--release-id",cls.release_id,"--database-url",cls.env.database_url,"--disposable-db"] + for _ in range(2): + result=subprocess.run(cmd,cwd=ROOT,capture_output=True,text=True); assert result.returncode==0,result.stderr + with psycopg.connect(cls.env.database_url) as db: + db.execute("INSERT INTO uec.releases(release_id,status,ruleset_version,profile,test_only,summary) VALUES ('e2e-private-candidate','candidate','it-e2e','official',true,'{}')") + facility,observation=db.execute("SELECT facility_id,observation_id FROM uec.observations JOIN uec.facilities USING(facility_id) WHERE source_record_id=(SELECT source_record_id FROM uec.source_records WHERE source_id='it.853-2004' LIMIT 1)").fetchone() + # The importer-owned observation is linked into the fixed test-only release; + # the distinct IDs are intentional CLI/API fixture plumbing, not a new row. + db.execute("INSERT INTO uec.release_members(release_id,facility_id,observation_id,default_visible) VALUES ('e2e-private-candidate',%s,%s,false)",(facility,observation)); db.commit() + cls.linked=db.execute("SELECT count(*) FROM uec.release_members m JOIN uec.observations o USING(observation_id) JOIN uec.source_records s USING(source_record_id) WHERE m.release_id='e2e-private-candidate' AND s.source_id='it.853-2004'").fetchone()[0] + cls.counts=db.execute("SELECT (SELECT count(*) FROM uec.source_records WHERE source_id='it.853-2004'),(SELECT count(*) FROM uec.release_members WHERE release_id='e2e-private-candidate'),(SELECT count(*) FROM uec.release_members WHERE release_id='e2e-private-candidate' AND default_visible)").fetchone() + @classmethod + def tearDownClass(cls): + if hasattr(cls,"temp"): cls.temp.cleanup() + if hasattr(cls,"env"): cls.env.stop() + def test_import_is_single_private_candidate(self): self.assertEqual(self.counts,(1,1,0)); self.assertEqual(self.linked,1) + def test_guarded_preview_and_public_exclusion(self): + base=f"http://127.0.0.1:{self.env.api_port}" + with urllib.request.urlopen(base+"/api/v2/locations?profile=official") as r: self.assertEqual(json.loads(r.read())["data"],[]) + h={"X-UEC-Dev-Preview-Token":self.env.dev_preview_token} + req=urllib.request.Request(base+"/api/dev/preview/test-release/locations?profile=official",headers=h) + with urllib.request.urlopen(req) as r: body=json.loads(r.read()) + self.assertTrue(body["meta"]["test_only"]); self.assertTrue(body["meta"]["private_preview"]); self.assertEqual(body["meta"]["release_status"],"candidate"); self.assertIn("preview_label",body["meta"]); self.assertEqual(len(body["data"]),1); self.assertIn("Italy",json.dumps(body)); self.assertNotIn("source_values",json.dumps(body)); self.assertIsNone(body["data"][0]["latitude"]) + fid=body["data"][0]["facility_id"] + type(self).facility_id=fid + with urllib.request.urlopen(urllib.request.Request(base+f"/api/dev/preview/test-release/locations/{fid}?profile=official",headers=h)) as r: detail=r.read().decode(); self.assertIn("Italy",detail); self.assertNotIn("source_values",detail) + with urllib.request.urlopen(urllib.request.Request(base+"/api/dev/preview/test-release/discovery/facets?profile=official",headers=h)) as r: facets=r.read().decode(); self.assertIn("IT",facets); self.assertNotIn("source_values",facets) + with urllib.request.urlopen(urllib.request.Request(base+"/api/dev/preview/test-release/locations.csv?profile=official",headers=h)) as r: export=r.read().decode(); self.assertIn("test_only",export); self.assertIn("Italy",export); self.assertNotIn("source_values",export); self.assertIn("uec-test-only",r.headers.get("Content-Disposition","")) + def test_privacy_restriction_relocks_all_surfaces(self): + with psycopg.connect(self.env.database_url) as db: + record=db.execute("SELECT source_record_id FROM uec.source_records WHERE source_id='it.853-2004' LIMIT 1").fetchone()[0] + db.execute("INSERT INTO uec.record_access_events(source_record_id,action,reason_category,policy_version,maintainer) VALUES (%s,'public_access_revoked','privacy','ethics-v1','authorized-synthetic-operator')",(record,)); db.commit() + base=f"http://127.0.0.1:{self.env.api_port}"; h={"X-UEC-Dev-Preview-Token":self.env.dev_preview_token} + with urllib.request.urlopen(urllib.request.Request(base+"/api/dev/preview/test-release/locations?profile=official",headers=h)) as r: self.assertEqual(json.loads(r.read())["data"],[]) + with self.assertRaises(urllib.error.HTTPError) as error: urllib.request.urlopen(urllib.request.Request(base+"/api/dev/preview/test-release/locations/"+self.facility_id+"?profile=official",headers=h)) + self.assertIn(error.exception.code,(404,410)) + with urllib.request.urlopen(urllib.request.Request(base+"/api/dev/preview/test-release/discovery/facets?profile=official",headers=h)) as r: self.assertNotIn('"IT"',r.read().decode()) + with urllib.request.urlopen(urllib.request.Request(base+"/api/dev/preview/test-release/locations.csv?profile=official",headers=h)) as r: self.assertEqual(len(r.read().decode().splitlines()),1) + with urllib.request.urlopen(base+"/api/v2/locations?profile=official") as r: self.assertEqual(json.loads(r.read())["data"],[]) + +if __name__=="__main__": unittest.main() From 1e3d2ea1f4ab9b872b5930e70faedf20d849ffbe Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Mon, 14 Sep 2026 09:25:58 -0700 Subject: [PATCH 102/311] Add Denmark refresh drift guards --- pipeline/sources/denmark/adapter.py | 12 ++++++++++++ pipeline/sources/denmark/test_adapter.py | 12 +++++++++++- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/pipeline/sources/denmark/adapter.py b/pipeline/sources/denmark/adapter.py index b91f06b..05f6290 100644 --- a/pipeline/sources/denmark/adapter.py +++ b/pipeline/sources/denmark/adapter.py @@ -14,6 +14,14 @@ SOURCE_ID = "dk.smiley" ADAPTER_VERSION = "denmark-smiley-contract-v1" +def check_refresh(previous: dict[str, Any], current: dict[str, Any], *, max_count_delta: float = 0.25) -> None: + """Fail closed on source/schema changes or implausible row-count shifts.""" + if previous.get("source_id") != current.get("source_id") or previous.get("schema_version") != current.get("schema_version"): + raise ValueError("Denmark refresh schema or source identity changed") + old, new = int(previous.get("normalized_rows", 0)), int(current.get("normalized_rows", 0)) + if old and abs(new - old) / old > max_count_delta: + raise ValueError("Denmark refresh normalized row count drift exceeds threshold") + def _atomic(path: Path, payload: bytes) -> None: """Publish one complete staging file, never a partially written artifact.""" path.parent.mkdir(parents=True, exist_ok=True) @@ -39,9 +47,13 @@ def write_candidate_handoff(self, run_dir: str | Path, artifact: SourceArtifact, facility, and it preserves every original field in private source_values. """ handoff_rows = [] + seen: set[str] = set() for row in rows: fields = row.get("source_fields", {}) key = row.get("source_record_key") + if not key or key in seen: + raise ValueError("Denmark candidate contains missing or duplicate source identity") + seen.add(key) handoff_rows.append({"source_id": SOURCE_ID, "source_row": row.get("source_row", 0), "source_values": fields, "normalized": { "establishment_id": key, "trading_name": fields.get("Virksomhed"), diff --git a/pipeline/sources/denmark/test_adapter.py b/pipeline/sources/denmark/test_adapter.py index fefa526..5c1c091 100644 --- a/pipeline/sources/denmark/test_adapter.py +++ b/pipeline/sources/denmark/test_adapter.py @@ -1,7 +1,7 @@ import hashlib, tempfile, unittest import json from pathlib import Path -from .adapter import DenmarkSmileyAdapter +from .adapter import DenmarkSmileyAdapter, check_refresh from pipeline.contracts.adapter_contract import SourceArtifact XML = b'1TestUnkeyed' @@ -47,4 +47,14 @@ def test_candidate_mapping_preserves_source_values_and_pending_gates(self): self.assertEqual(handoff["source_values"]["ID_nummer"], "1") self.assertEqual(handoff["normalized"]["establishment_id"], "1") + def test_refresh_guards_reject_schema_count_and_duplicate_drift(self): + with self.assertRaisesRegex(ValueError, "schema"): + check_refresh({"source_id":"dk.smiley", "schema_version":"a", "normalized_rows":10}, {"source_id":"dk.smiley", "schema_version":"b", "normalized_rows":10}) + with self.assertRaisesRegex(ValueError, "count"): + check_refresh({"source_id":"dk.smiley", "schema_version":"a", "normalized_rows":100}, {"source_id":"dk.smiley", "schema_version":"a", "normalized_rows":50}) + duplicate = {"source_id":"dk.smiley", "source_row":3, "source_record_key":"1", "source_fields":{}} + with tempfile.TemporaryDirectory() as d: + with self.assertRaisesRegex(ValueError, "duplicate"): + DenmarkSmileyAdapter().write_candidate_handoff(Path(d), self.artifact(), [duplicate, duplicate]) + if __name__ == "__main__": unittest.main() From 64ae3595e4373fc4fc68b4048284ff91016b14ae Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Mon, 14 Sep 2026 09:54:12 -0700 Subject: [PATCH 103/311] Record Denmark private refresh evidence --- docs/source-status.json | 2 +- pipeline/sources/denmark/README.md | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/docs/source-status.json b/docs/source-status.json index 453120d..7d966af 100644 --- a/docs/source-status.json +++ b/docs/source-status.json @@ -13,7 +13,7 @@ {"source_id":"mx.locations","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-mx.md","pipeline/source_registry.json"],"next_action":"Resolve DENUE token/terms and SENASICA current directory/schema before bounded acquisition."}, {"source_id":"nz.locations","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-nz.md","pipeline/source_registry.json"],"next_action":"Confirm MPI permitted acquisition route, list-specific terms, and schema before private staging."}, {"source_id":"uk.locations","metadata":"partial","acquisition":"artifact_private_only","runtime_health":"unknown","publication_eligibility":"blocked","evidence":["docs/country-recon-uk.md","docs/countries/uk/fss-approved-establishments-source-assessment.md","pipeline/sources/uk/fsa_approved/README.md","pipeline/sources/uk/fsa_approved/handoff.py","pipeline/sources/uk/fsa_approved/refresh.py","docs/architecture/disposable-candidate-import.md","pipeline/scripts/maintenance/import-candidate.py","pipeline/source_registry.json"],"next_action":"Repeatable source-local refresh dry-run passed on the retained snapshot (5,342 input, 4,300 normalized, 1,042 quarantined, expected schema fingerprint, no drift alarms); use refresh.py dry-run/handoff only in restricted staging. This does not establish production/runtime health or publication eligibility. Complete source-rights, privacy/coordinate, duplicate, coverage, and release review while keeping NI and Scotland separate."}, - {"source_id":"dk.smiley","metadata":"partial","acquisition":"artifact_private_only","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["pipeline/sources/denmark/README.md","pipeline/contracts/README.md","pipeline/source_registry.json"],"next_action":"Keep the privately staged official artifact as validation-only; verify coverage and effective-date semantics, then complete terms/privacy/release review before any publication."}, + {"source_id":"dk.smiley","metadata":"verified","acquisition":"verified","runtime_health":"unknown","publication_eligibility":"blocked","evidence":["pipeline/sources/denmark/README.md","pipeline/contracts/README.md","pipeline/source_registry.json"],"next_action":"Keep the full privately staged artifact and candidate import validation restricted; run a bounded guarded API check with explicit test-only labeling, capture rerun exit output, and resolve coverage/effective-date/category semantics before any release review. This is not a production-health or publication claim."}, {"source_id":"de.locations","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["pipeline/source_registry.json"],"next_action":"Replace session-bound legacy URL with a verified stable BVL endpoint and confirm terms/schema."}, {"source_id":"ca.locations","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ca.md","pipeline/source_registry.json"],"next_action":"Keep federal/export and Ontario candidates separate; verify an authoritative permitted artifact, terms, schema, coverage, and privacy before acquisition."}, {"source_id":"es.locations","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-es.md","pipeline/source_registry.json"],"next_action":"Resolve a permitted current AESAN/MAPA artifact or query route, then verify export/schema, rights, effective-date semantics, sector coverage, privacy, and legacy/source boundaries before acquisition."}, diff --git a/pipeline/sources/denmark/README.md b/pipeline/sources/denmark/README.md index 2bc78b2..4bbc7c7 100644 --- a/pipeline/sources/denmark/README.md +++ b/pipeline/sources/denmark/README.md @@ -34,3 +34,15 @@ dataset is complete or that records are current. A reviewed acquisition may be retained in ignored private storage for research and validation only. The adapter emits a private candidate with `release_state: not-created`; no health, approval, geocoding, or publication conclusion follows from a successful run. + +## Full private refresh evidence + +The retained full artifact was retrieved from the endpoint above at +`2026-09-14T05:41:12Z` (59,852,153 bytes; SHA-256 recorded in ignored local +acquisition metadata). Its deterministic handoff contained 58,792 normalized +rows. In a uniquely named disposable PostGIS database, the batched importer +created 58,792 source records and release members; all 58,792 remained pending +privacy review and 0 were default-visible. A rerun left those counts unchanged. +The database volume was removed after the check. The guarded API was not run +against the full candidate; the existing synthetic DK-shaped E2E covers public +exclusion and preview gates. No release or publication approval follows. From 4dad52148ac53920aa2e94c09bab82ac0d64fa25 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Tue, 15 Sep 2026 09:41:13 -0700 Subject: [PATCH 104/311] Keep first candidate batch atomic --- .../scripts/maintenance/import-candidate.py | 55 ++++++++++--------- 1 file changed, 29 insertions(+), 26 deletions(-) diff --git a/pipeline/scripts/maintenance/import-candidate.py b/pipeline/scripts/maintenance/import-candidate.py index d693762..2c642f6 100644 --- a/pipeline/scripts/maintenance/import-candidate.py +++ b/pipeline/scripts/maintenance/import-candidate.py @@ -121,38 +121,41 @@ def import_candidate(database_url: str, manifest: dict, rows: list[dict], releas with psycopg.connect(database_url) as db: verify_disposable_marker(db) # The guard query starts an implicit read transaction in psycopg. - # End it before opening the independently resumable write batches; + # End it before opening independently resumable write batches; # otherwise psycopg nests them as savepoints under one outer rollback. db.commit() - with db.transaction(): - db.execute("""INSERT INTO uec.sources(source_id,country_code,name,official_url,access_method) - VALUES (%s,%s,%s,%s,'validated-private-staging') - ON CONFLICT (source_id) DO NOTHING""", - (manifest["source_id"], str(manifest.get("country_code", "ZZ"))[:2].upper(), - manifest["source_id"], manifest["source_url"])) - db.execute("""INSERT INTO uec.raw_artifacts(storage_key,sha256,byte_size,media_type,retrieved_at) - VALUES (%s,%s,%s,'application/octet-stream',%s) - ON CONFLICT (sha256) DO NOTHING""", - (f"private-staging/{manifest['source_id']}/{manifest['checksum_sha256']}", - manifest["checksum_sha256"], int(manifest["byte_size"]), now)) - artifact_id = db.execute("SELECT artifact_id FROM uec.raw_artifacts WHERE sha256=%s", - (manifest["checksum_sha256"],)).fetchone()[0] - db.execute("""INSERT INTO uec.acquisition_runs(source_id,checked_at,retrieved_at,ingested_at,status,source_url,code_version,config_version) - VALUES (%s,%s,%s,now(),'changed',%s,%s,%s)""", - (manifest["source_id"], now, now, manifest["source_url"], - manifest.get("code_version", "unknown"), manifest.get("config_version", "unknown"))) - run_id = db.execute("SELECT run_id FROM uec.acquisition_runs WHERE source_id=%s ORDER BY ingested_at DESC LIMIT 1", - (manifest["source_id"],)).fetchone()[0] - db.execute("INSERT INTO uec.acquisition_run_artifacts(run_id,artifact_id) VALUES (%s,%s) ON CONFLICT DO NOTHING", - (run_id, artifact_id)) - db.execute("""INSERT INTO uec.releases(release_id,status,ruleset_version,summary,test_only) - VALUES (%s,'candidate',%s,%s,true) - ON CONFLICT (release_id) DO NOTHING""", - (release_id, ruleset, json.dumps({"source_id": manifest["source_id"], "profile": manifest.get("profile")}))) count = 0 for offset in range(0, len(rows), batch_size): batch_count = 0 with db.transaction(): + if offset == 0: + # Keep first-batch metadata atomic with its data. A + # malformed first batch must leave no orphan run/release; + # later batches remain independently resumable. + db.execute("""INSERT INTO uec.sources(source_id,country_code,name,official_url,access_method) + VALUES (%s,%s,%s,%s,'validated-private-staging') + ON CONFLICT (source_id) DO NOTHING""", + (manifest["source_id"], str(manifest.get("country_code", "ZZ"))[:2].upper(), + manifest["source_id"], manifest["source_url"])) + db.execute("""INSERT INTO uec.raw_artifacts(storage_key,sha256,byte_size,media_type,retrieved_at) + VALUES (%s,%s,%s,'application/octet-stream',%s) + ON CONFLICT (sha256) DO NOTHING""", + (f"private-staging/{manifest['source_id']}/{manifest['checksum_sha256']}", + manifest["checksum_sha256"], int(manifest["byte_size"]), now)) + artifact_id = db.execute("SELECT artifact_id FROM uec.raw_artifacts WHERE sha256=%s", + (manifest["checksum_sha256"],)).fetchone()[0] + db.execute("""INSERT INTO uec.acquisition_runs(source_id,checked_at,retrieved_at,ingested_at,status,source_url,code_version,config_version) + VALUES (%s,%s,%s,now(),'changed',%s,%s,%s)""", + (manifest["source_id"], now, now, manifest["source_url"], + manifest.get("code_version", "unknown"), manifest.get("config_version", "unknown"))) + run_id = db.execute("SELECT run_id FROM uec.acquisition_runs WHERE source_id=%s ORDER BY ingested_at DESC LIMIT 1", + (manifest["source_id"],)).fetchone()[0] + db.execute("INSERT INTO uec.acquisition_run_artifacts(run_id,artifact_id) VALUES (%s,%s) ON CONFLICT DO NOTHING", + (run_id, artifact_id)) + db.execute("""INSERT INTO uec.releases(release_id,status,ruleset_version,summary,test_only) + VALUES (%s,'candidate',%s,%s,true) + ON CONFLICT (release_id) DO NOTHING""", + (release_id, ruleset, json.dumps({"source_id": manifest["source_id"], "profile": manifest.get("profile")}))) for record in rows[offset:offset + batch_size]: key, normalized = _record_parts(record) country = _country_code(manifest, normalized) From 29b44a3b1ff503b14a35e0b6959a3d6a8a8b5f2c Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Tue, 15 Sep 2026 09:45:02 -0700 Subject: [PATCH 105/311] Harden restore restriction ledger verification --- .../maintenance/restriction-ledger-gate.py | 28 +++++++++++++++++-- .../restriction-ledger-current-snapshot.json | 3 +- .../e2e/restriction-ledger-old-snapshot.json | 1 + pipeline/tests/e2e/restriction-ledger.json | 3 +- .../tests/test_restriction_ledger_gate.py | 18 ++++++++++-- 5 files changed, 47 insertions(+), 6 deletions(-) diff --git a/pipeline/scripts/maintenance/restriction-ledger-gate.py b/pipeline/scripts/maintenance/restriction-ledger-gate.py index e1506c2..51e7109 100644 --- a/pipeline/scripts/maintenance/restriction-ledger-gate.py +++ b/pipeline/scripts/maintenance/restriction-ledger-gate.py @@ -7,6 +7,7 @@ from __future__ import annotations import argparse +import hashlib import json from pathlib import Path from typing import Any @@ -16,6 +17,17 @@ class RestrictionLedgerError(ValueError): """The ledger is missing, malformed, stale, or not fully applied.""" +def ledger_digest(ledger: dict[str, Any]) -> str: + """Hash only the versioned, row-free control-plane projection.""" + payload = { + "schema_version": ledger.get("schema_version"), + "revision": ledger.get("revision"), + "active_restrictions": ledger.get("active_restrictions"), + } + encoded = json.dumps(payload, sort_keys=True, separators=(",", ":")).encode("utf-8") + return hashlib.sha256(encoded).hexdigest() + + def load_ledger(path: str | Path) -> dict[str, Any]: ledger_path = Path(path) if not ledger_path.is_file(): @@ -28,8 +40,14 @@ def load_ledger(path: str | Path) -> dict[str, Any]: raise RestrictionLedgerError("restriction ledger schema is unsupported") if not isinstance(ledger.get("revision"), str) or not ledger["revision"]: raise RestrictionLedgerError("restriction ledger revision is missing") + if not isinstance(ledger.get("ledger_sha256"), str) or ledger["ledger_sha256"] != ledger_digest(ledger): + raise RestrictionLedgerError("restriction ledger digest is invalid") restrictions = ledger.get("active_restrictions") - if not isinstance(restrictions, list) or any(not isinstance(item, dict) for item in restrictions): + if not isinstance(restrictions, list) or any( + not isinstance(item, dict) + or item.get("action") != "suppress" + for item in restrictions + ): raise RestrictionLedgerError("restriction ledger restrictions are invalid") return ledger @@ -42,19 +60,25 @@ def verify_replayed_restrictions(snapshot: dict[str, Any], ledger: dict[str, Any """ if snapshot.get("ledger_revision") != ledger["revision"]: raise RestrictionLedgerError("restored restriction revision is not current") + if snapshot.get("ledger_sha256") != ledger["ledger_sha256"]: + raise RestrictionLedgerError("restored restriction digest is not current") applied = snapshot.get("active_restrictions") expected = ledger["active_restrictions"] if not isinstance(applied, list): raise RestrictionLedgerError("restored restriction state is unavailable") def key(item: dict[str, Any]) -> tuple[str, str, str]: - values = tuple(item.get(field) for field in ("source_id", "source_record_key", "scope")) + values = tuple(item.get(field) for field in ("source_id", "source_record_key", "scope", "action")) if any(not isinstance(value, str) or not value for value in values): raise RestrictionLedgerError("restriction reference is incomplete") + if values[3] != "suppress": + raise RestrictionLedgerError("restriction action is unsupported") return values expected_keys = {key(item) for item in expected} applied_keys = {key(item) for item in applied} + if len(expected_keys) != len(expected) or len(applied_keys) != len(applied): + raise RestrictionLedgerError("restriction references are duplicated") if expected_keys != applied_keys: raise RestrictionLedgerError("restored restrictions do not match current ledger") diff --git a/pipeline/tests/e2e/restriction-ledger-current-snapshot.json b/pipeline/tests/e2e/restriction-ledger-current-snapshot.json index 4a13141..c197240 100644 --- a/pipeline/tests/e2e/restriction-ledger-current-snapshot.json +++ b/pipeline/tests/e2e/restriction-ledger-current-snapshot.json @@ -1,6 +1,7 @@ { "ledger_revision": "synthetic-r2", + "ledger_sha256": "abd2e83d3030d733a2e5909430d33c783e2f37789180caa667259fa4952faffc", "active_restrictions": [ - {"source_id": "e2e.backup", "source_record_key": "restricted", "scope": "whole_record"} + {"source_id": "e2e.backup", "source_record_key": "restricted", "scope": "whole_record", "action": "suppress"} ] } diff --git a/pipeline/tests/e2e/restriction-ledger-old-snapshot.json b/pipeline/tests/e2e/restriction-ledger-old-snapshot.json index 38a212f..fad58a7 100644 --- a/pipeline/tests/e2e/restriction-ledger-old-snapshot.json +++ b/pipeline/tests/e2e/restriction-ledger-old-snapshot.json @@ -1,4 +1,5 @@ { "ledger_revision": "synthetic-r1", + "ledger_sha256": "f409c4250c5ff90527ca75f84cd1a114f065a0d05bca09360cbfceb9e0bf9143", "active_restrictions": [] } diff --git a/pipeline/tests/e2e/restriction-ledger.json b/pipeline/tests/e2e/restriction-ledger.json index 9e9e575..9cc6721 100644 --- a/pipeline/tests/e2e/restriction-ledger.json +++ b/pipeline/tests/e2e/restriction-ledger.json @@ -1,7 +1,8 @@ { "schema_version": 1, "revision": "synthetic-r2", + "ledger_sha256": "abd2e83d3030d733a2e5909430d33c783e2f37789180caa667259fa4952faffc", "active_restrictions": [ - {"source_id": "e2e.backup", "source_record_key": "restricted", "scope": "whole_record"} + {"source_id": "e2e.backup", "source_record_key": "restricted", "scope": "whole_record", "action": "suppress"} ] } diff --git a/pipeline/tests/test_restriction_ledger_gate.py b/pipeline/tests/test_restriction_ledger_gate.py index 3a2f1af..552dd94 100644 --- a/pipeline/tests/test_restriction_ledger_gate.py +++ b/pipeline/tests/test_restriction_ledger_gate.py @@ -15,9 +15,10 @@ class RestrictionLedgerGateTests(unittest.TestCase): def setUp(self): self.ledger = {"schema_version": 1, "revision": "r2", "active_restrictions": [ - {"source_id": "synthetic", "source_record_key": "private", "scope": "whole_record"} + {"source_id": "synthetic", "source_record_key": "private", "scope": "whole_record", "action": "suppress"} ]} - self.snapshot = {"ledger_revision": "r2", "active_restrictions": list(self.ledger["active_restrictions"])} + self.ledger["ledger_sha256"] = MODULE.ledger_digest(self.ledger) + self.snapshot = {"ledger_revision": "r2", "ledger_sha256": self.ledger["ledger_sha256"], "active_restrictions": list(self.ledger["active_restrictions"])} def test_current_restrictions_must_match_before_service(self): with tempfile.TemporaryDirectory() as directory: @@ -27,6 +28,19 @@ def test_current_restrictions_must_match_before_service(self): with self.assertRaises(MODULE.RestrictionLedgerError): MODULE.pre_service_gate(ledger_path, {"ledger_revision": "r1", "active_restrictions": []}) + def test_digest_and_action_must_match(self): + with tempfile.TemporaryDirectory() as directory: + ledger_path = Path(directory) / "ledger.json" + ledger_path.write_text(json.dumps(self.ledger), encoding="utf-8") + stale_digest = dict(self.snapshot, ledger_sha256="0" * 64) + with self.assertRaises(MODULE.RestrictionLedgerError): + MODULE.pre_service_gate(ledger_path, stale_digest) + unsupported = {**self.ledger, "active_restrictions": [{**self.ledger["active_restrictions"][0], "action": "restore"}]} + unsupported["ledger_sha256"] = MODULE.ledger_digest(unsupported) + ledger_path.write_text(json.dumps(unsupported), encoding="utf-8") + with self.assertRaises(MODULE.RestrictionLedgerError): + MODULE.load_ledger(ledger_path) + def test_old_restore_cannot_start_without_current_replay(self): with tempfile.TemporaryDirectory() as directory: ledger_path = Path(directory) / "ledger.json" From 61cb6f2287f505e4b3e858cf49b28c0b4e59dc6f Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Tue, 15 Sep 2026 09:27:48 -0700 Subject: [PATCH 106/311] Add private country health snapshot contract --- pipeline/contracts/SOURCE-HEALTH.md | 31 +++ pipeline/contracts/source_health.py | 199 ++++++++++++++++++ pipeline/contracts/test_source_health.py | 116 ++++++++++ .../diagnostics/build-source-health.py | 35 +++ 4 files changed, 381 insertions(+) create mode 100644 pipeline/contracts/SOURCE-HEALTH.md create mode 100644 pipeline/contracts/source_health.py create mode 100644 pipeline/contracts/test_source_health.py create mode 100644 pipeline/scripts/diagnostics/build-source-health.py diff --git a/pipeline/contracts/SOURCE-HEALTH.md b/pipeline/contracts/SOURCE-HEALTH.md new file mode 100644 index 0000000..b0c7f53 --- /dev/null +++ b/pipeline/contracts/SOURCE-HEALTH.md @@ -0,0 +1,31 @@ +# Private source-health contract + +`source_health.build_health_snapshot` and +`pipeline/scripts/diagnostics/build-source-health.py` combine one private run's +`manifest.json`, `qa.json`, and `run-status.json`, with optional importer +evidence, into a deterministic `SourceHealthSnapshot` v1. + +The snapshot is aggregate metadata only. It records provenance, freshness, +effective-date uncertainty, row counts, drift alarms, quarantines, +`not-observed` deltas, and private importer/idempotency counts. It contains no +source values, raw fields, addresses, coordinates, or record rows. + +Health states are deliberately conservative: `not-run`, `blocked`, `failed`, +`degraded`, and `private-validated`. `private-validated` means only that the +supplied private evidence passed this contract; it is not a claim of current, +complete, accurate, approved, or publishable data. Every snapshot has +`public_exposure: false` and `publication_eligibility: blocked`. + +Build a snapshot with a fixed timestamp when reproducibility matters: + +```powershell +python pipeline/scripts/diagnostics/build-source-health.py ` + --run-dir data/restricted/example/run ` + --output data/reports/source-health/example.json ` + --as-of-utc 2026-09-15T00:00:00Z +``` + +Missing or inconsistent evidence, a promoted release, a public import flag, or +privacy-bearing fields fail closed and leave no output snapshot. This command +does not acquire data, alter `docs/source-status.json`, promote a release, or +make a public health claim. diff --git a/pipeline/contracts/source_health.py b/pipeline/contracts/source_health.py new file mode 100644 index 0000000..9985f47 --- /dev/null +++ b/pipeline/contracts/source_health.py @@ -0,0 +1,199 @@ +"""Build conservative, row-free health snapshots for private source runs.""" +from __future__ import annotations + +import json +import os +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + + +class HealthEvidenceError(ValueError): + """Evidence is missing, inconsistent, or contains a prohibited payload.""" + + +HEALTH_SCHEMA_VERSION = "1.0" +HEALTH_STATES = {"not-run", "blocked", "failed", "degraded", "private-validated"} +_FORBIDDEN_KEYS = {"source_values", "raw_fields", "address", "coordinates", "latitude", "longitude"} + + +def _read_json(path: Path, label: str) -> dict[str, Any]: + if not path.exists(): + raise HealthEvidenceError(f"missing {label}: {path.name}") + try: + value = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise HealthEvidenceError(f"invalid {label}: {path.name}") from exc + if not isinstance(value, dict): + raise HealthEvidenceError(f"{label} must be an object: {path.name}") + return value + + +def _assert_row_free(value: Any, path: str = "evidence") -> None: + if isinstance(value, dict): + leaked = sorted(set(value) & _FORBIDDEN_KEYS) + if leaked: + raise HealthEvidenceError(f"prohibited payload in {path}: {', '.join(leaked)}") + for key, child in value.items(): + _assert_row_free(child, f"{path}.{key}") + elif isinstance(value, list): + for index, child in enumerate(value): + _assert_row_free(child, f"{path}[{index}]") + + +def _required_counts(value: dict[str, Any], label: str) -> tuple[int, int, int]: + required = ("input_rows", "normalized_rows", "quarantined_rows") + if any(key not in value for key in required): + raise HealthEvidenceError(f"{label} is missing row counts") + counts = tuple(value[key] for key in required) + if any(not isinstance(count, int) or count < 0 for count in counts): + raise HealthEvidenceError(f"{label} has invalid row counts") + if counts[0] != counts[1] + counts[2]: + raise HealthEvidenceError(f"{label} row counts do not reconcile") + return counts + + +def _parse_time(value: Any, label: str) -> datetime: + if not isinstance(value, str) or not value: + raise HealthEvidenceError(f"missing {label}") + try: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError as exc: + raise HealthEvidenceError(f"invalid {label}") from exc + if parsed.tzinfo is None: + raise HealthEvidenceError(f"{label} must include a timezone") + return parsed.astimezone(timezone.utc) + + +def _atomic_json(path: Path, value: dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_name(path.name + ".tmp") + temporary.write_text(json.dumps(value, ensure_ascii=False, sort_keys=True, indent=2) + "\n", encoding="utf-8") + os.replace(temporary, path) + + +def build_health_snapshot( + run_dir: str | Path, + *, + as_of_utc: str | None = None, + stale_after_hours: int = 24 * 7, + import_evidence_path: str | Path | None = None, +) -> dict[str, Any]: + """Build a deterministic aggregate snapshot from private run evidence.""" + if stale_after_hours <= 0: + raise HealthEvidenceError("stale_after_hours must be positive") + root = Path(run_dir) + manifest = _read_json(root / "manifest.json", "manifest") + qa = _read_json(root / "qa.json", "QA report") + run_status = _read_json(root / "run-status.json", "run status") + import_evidence = None + if import_evidence_path is not None: + import_evidence = _read_json(Path(import_evidence_path), "import evidence") + _assert_row_free(qa, "qa") + _assert_row_free(run_status, "run-status") + if import_evidence is not None: + _assert_row_free(import_evidence, "import") + + source_id = manifest.get("source_id") + if not isinstance(source_id, str) or not source_id: + raise HealthEvidenceError("manifest source_id is missing") + if qa.get("source_id") != source_id: + raise HealthEvidenceError("manifest and QA source_id differ") + manifest_counts = _required_counts(manifest, "manifest") + qa_counts = _required_counts(qa, "QA report") + if manifest_counts != qa_counts: + raise HealthEvidenceError("manifest and QA row counts differ") + if manifest.get("release_state") != "not-created": + raise HealthEvidenceError("private health cannot assess a released run") + if manifest.get("publication_state") not in {"private-candidate", "not-staged", None}: + raise HealthEvidenceError("manifest publication state is not private") + if run_status.get("release_promoted") is not False: + raise HealthEvidenceError("run status must explicitly keep release promotion false") + if run_status.get("publication_state") not in {"human-gate-required", "terms-gate-blocked", "unchanged", "private-candidate"}: + raise HealthEvidenceError("run status publication state is not restricted") + + retrieved = _parse_time( + manifest.get("retrieved_at_utc") or (manifest.get("acquisition") or {}).get("retrieved_at_utc"), + "retrieved_at_utc", + ) + as_of = _parse_time(as_of_utc, "as_of_utc") if as_of_utc else datetime.now(timezone.utc) + age_hours = max(0.0, (as_of - retrieved).total_seconds() / 3600) + raw_drift_alarms = qa.get("drift_alarms", []) + if not isinstance(raw_drift_alarms, list) or any(not isinstance(item, str) for item in raw_drift_alarms): + raise HealthEvidenceError("drift_alarms must contain strings") + drift_alarms = sorted(set(raw_drift_alarms)) + run_state = run_status.get("status") + if run_state == "failed": + health_state = "failed" + elif run_state in {"staged-restricted", "blocked"}: + health_state = "blocked" + elif drift_alarms or age_hours > stale_after_hours: + health_state = "degraded" + elif run_state in {"candidate-ready", "success", "private-candidate"}: + health_state = "private-validated" + else: + health_state = "not-run" + + if import_evidence is not None: + if import_evidence.get("source_id") != source_id: + raise HealthEvidenceError("manifest and import source_id differ") + imported = import_evidence.get("imported_rows") + if not isinstance(imported, int) or imported < 0 or imported > manifest_counts[1]: + raise HealthEvidenceError("imported_rows is inconsistent with normalized_rows") + if import_evidence.get("public_exposure") is not False: + raise HealthEvidenceError("import evidence must set public_exposure=false") + if import_evidence.get("publication_eligible_rows") != 0: + raise HealthEvidenceError("private import must have zero publication-eligible rows") + if import_evidence.get("default_visible_rows") != 0: + raise HealthEvidenceError("private import must have zero default-visible rows") + + provenance = { + key: manifest.get(key) + for key in ("source_url", "retrieved_at_utc", "publication_date", "effective_date", "sha256", "checksum_sha256", "byte_size", "code_version", "config_version") + if manifest.get(key) is not None + } + snapshot = { + "schema_version": HEALTH_SCHEMA_VERSION, + "source_id": source_id, + "health_state": health_state, + "private_validation": True, + "public_exposure": False, + "publication_eligibility": "blocked", + "provenance": provenance, + "freshness": { + "retrieved_at_utc": retrieved.isoformat().replace("+00:00", "Z"), + "as_of_utc": as_of.isoformat().replace("+00:00", "Z"), + "age_hours": round(age_hours, 3), + "stale_after_hours": stale_after_hours, + "state": "stale" if age_hours > stale_after_hours else "current", + }, + "effective_date": { + "value": manifest.get("effective_date"), + "state": "known" if manifest.get("effective_date") else "unknown", + }, + "run": { + "status": run_state, + "release_state": manifest.get("release_state"), + "publication_state": manifest.get("publication_state"), + "drift_alarms": drift_alarms, + "input_rows": manifest_counts[0], + "normalized_rows": manifest_counts[1], + "quarantined_rows": manifest_counts[2], + "disappeared_not_observed_count": qa.get("disappeared_not_observed_count", 0), + "disappearance_semantics": "not-observed; never inferred as closure", + }, + "import": { + "state": "not-run" if import_evidence is None else import_evidence.get("status", "completed"), + "imported_rows": None if import_evidence is None else import_evidence.get("imported_rows"), + "rerun_imported_rows": None if import_evidence is None else import_evidence.get("rerun_imported_rows"), + "default_visible_rows": None if import_evidence is None else import_evidence.get("default_visible_rows"), + "publication_eligible_rows": None if import_evidence is None else import_evidence.get("publication_eligible_rows"), + }, + } + if health_state not in HEALTH_STATES: + raise HealthEvidenceError("unknown health state") + return snapshot + + +def write_health_snapshot(output_path: str | Path, snapshot: dict[str, Any]) -> None: + _atomic_json(Path(output_path), snapshot) diff --git a/pipeline/contracts/test_source_health.py b/pipeline/contracts/test_source_health.py new file mode 100644 index 0000000..a602a5f --- /dev/null +++ b/pipeline/contracts/test_source_health.py @@ -0,0 +1,116 @@ +import json +import tempfile +import unittest +from pathlib import Path + +from .source_health import HealthEvidenceError, build_health_snapshot, write_health_snapshot + + +class SourceHealthTests(unittest.TestCase): + def write_run(self, root: Path, source_id: str, *, drift=None, retrieved="2026-09-14T00:00:00Z", status="success") -> Path: + run = root / source_id + run.mkdir() + manifest = { + "source_id": source_id, + "source_url": f"https://example.test/{source_id}", + "retrieved_at_utc": retrieved, + "effective_date": None, + "checksum_sha256": "a" * 64, + "byte_size": 12, + "code_version": "test", + "config_version": "test", + "input_rows": 4, + "normalized_rows": 3, + "quarantined_rows": 1, + "release_state": "not-created", + "publication_state": "private-candidate", + } + qa = { + "source_id": source_id, + "input_rows": 4, + "normalized_rows": 3, + "quarantined_rows": 1, + "drift_alarms": drift or [], + "disappeared_not_observed_count": 2, + } + run_status = { + "status": status, + "publication_state": "human-gate-required", + "release_promoted": False, + } + for name, value in (("manifest.json", manifest), ("qa.json", qa), ("run-status.json", run_status)): + (run / name).write_text(json.dumps(value, sort_keys=True), encoding="utf-8") + return run + + def import_evidence(self, root: Path, source_id: str) -> Path: + path = root / f"{source_id}-import.json" + path.write_text(json.dumps({ + "source_id": source_id, + "status": "resumed", + "imported_rows": 3, + "rerun_imported_rows": 0, + "default_visible_rows": 0, + "publication_eligible_rows": 0, + "public_exposure": False, + }), encoding="utf-8") + return path + + def test_uk_denmark_italy_snapshots_are_conservative_and_row_free(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + for source_id in ("uk.locations", "dk.smiley", "it.853-2004"): + run = self.write_run(root, source_id) + snapshot = build_health_snapshot( + run, as_of_utc="2026-09-15T00:00:00Z", import_evidence_path=self.import_evidence(root, source_id) + ) + self.assertEqual(snapshot["health_state"], "private-validated") + self.assertTrue(snapshot["private_validation"]) + self.assertFalse(snapshot["public_exposure"]) + self.assertEqual(snapshot["publication_eligibility"], "blocked") + self.assertEqual(snapshot["run"]["disappearance_semantics"], "not-observed; never inferred as closure") + self.assertEqual(snapshot["import"]["rerun_imported_rows"], 0) + self.assertNotIn("source_values", json.dumps(snapshot)) + self.assertNotIn("address", json.dumps(snapshot)) + + def test_drift_and_stale_evidence_degrade_without_false_health(self): + with tempfile.TemporaryDirectory() as directory: + run = self.write_run(Path(directory), "dk.smiley", drift=["count_drift"]) + snapshot = build_health_snapshot(run, as_of_utc="2026-09-20T00:00:00Z", stale_after_hours=24) + self.assertEqual(snapshot["health_state"], "degraded") + self.assertEqual(snapshot["freshness"]["state"], "stale") + + def test_missing_evidence_and_privacy_leak_fail_closed(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + run = self.write_run(root, "it.853-2004") + (run / "qa.json").unlink() + with self.assertRaisesRegex(HealthEvidenceError, "missing QA"): + build_health_snapshot(run, as_of_utc="2026-09-15T00:00:00Z") + run = self.write_run(root, "it.853-2004-leak") + qa = json.loads((run / "qa.json").read_text()) + qa["source_values"] = {"private": "never include"} + (run / "qa.json").write_text(json.dumps(qa), encoding="utf-8") + with self.assertRaisesRegex(HealthEvidenceError, "prohibited payload"): + build_health_snapshot(run, as_of_utc="2026-09-15T00:00:00Z") + + def test_failed_and_partial_runs_never_look_publishable(self): + with tempfile.TemporaryDirectory() as directory: + run = self.write_run(Path(directory), "uk.locations", status="failed") + snapshot = build_health_snapshot(run, as_of_utc="2026-09-15T00:00:00Z") + self.assertEqual(snapshot["health_state"], "failed") + self.assertFalse(snapshot["public_exposure"]) + self.assertEqual(snapshot["publication_eligibility"], "blocked") + + def test_output_is_deterministic(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + run = self.write_run(root, "dk.smiley") + snapshot = build_health_snapshot(run, as_of_utc="2026-09-15T00:00:00Z") + first, second = root / "one.json", root / "two.json" + write_health_snapshot(first, snapshot) + write_health_snapshot(second, snapshot) + self.assertEqual(first.read_bytes(), second.read_bytes()) + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/scripts/diagnostics/build-source-health.py b/pipeline/scripts/diagnostics/build-source-health.py new file mode 100644 index 0000000..28981a4 --- /dev/null +++ b/pipeline/scripts/diagnostics/build-source-health.py @@ -0,0 +1,35 @@ +"""Build a private, row-free country health snapshot from one run directory.""" +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +from pipeline.contracts.source_health import HealthEvidenceError, build_health_snapshot, write_health_snapshot + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--run-dir", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--as-of-utc", required=True, help="timezone-aware ISO-8601 timestamp for deterministic freshness") + parser.add_argument("--stale-after-hours", type=int, default=24 * 7) + parser.add_argument("--import-evidence", type=Path) + args = parser.parse_args(argv) + try: + snapshot = build_health_snapshot( + args.run_dir, + as_of_utc=args.as_of_utc, + stale_after_hours=args.stale_after_hours, + import_evidence_path=args.import_evidence, + ) + write_health_snapshot(args.output, snapshot) + except (HealthEvidenceError, OSError) as exc: + print(f"source-health: {exc}", file=sys.stderr) + return 2 + print(f"source-health: wrote {args.output}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From b18793a2ba7c0ec8315be71230a4aa1acb5892e7 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Tue, 15 Sep 2026 09:28:01 -0700 Subject: [PATCH 107/311] Refresh Italy source status evidence --- docs/source-status.json | 2 +- docs/source-status.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source-status.json b/docs/source-status.json index 7d966af..bd92ec9 100644 --- a/docs/source-status.json +++ b/docs/source-status.json @@ -9,7 +9,7 @@ }, "sources": [ {"source_id":"fr.locations","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-fr.md","pipeline/source_registry.json"],"next_action":"Acquire and snapshot an official DGAL Section I/II artifact with terms, schema, and privacy review."}, - {"source_id":"it.locations","metadata":"verified","acquisition":"artifact_private_only","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-it.md","pipeline/source_registry.json"],"next_action":"Review the 853/2004 and 1069/2009 source dictionaries and privacy mappings, then implement and validate separate adapters without treating private artifacts as release inputs."}, + {"source_id":"it.locations","metadata":"verified","acquisition":"artifact_private_only","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-it.md","pipeline/source_registry.json","pipeline/sources/italy/it_853_adapter.py","pipeline/tests/e2e/test_italy_candidate_import.py"],"next_action":"Keep the implemented 853/2004 adapter and candidate-import path private and review-gated; validate the separate 1069/2009 variant, source terms, coverage, and privacy mappings before any release review."}, {"source_id":"mx.locations","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-mx.md","pipeline/source_registry.json"],"next_action":"Resolve DENUE token/terms and SENASICA current directory/schema before bounded acquisition."}, {"source_id":"nz.locations","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-nz.md","pipeline/source_registry.json"],"next_action":"Confirm MPI permitted acquisition route, list-specific terms, and schema before private staging."}, {"source_id":"uk.locations","metadata":"partial","acquisition":"artifact_private_only","runtime_health":"unknown","publication_eligibility":"blocked","evidence":["docs/country-recon-uk.md","docs/countries/uk/fss-approved-establishments-source-assessment.md","pipeline/sources/uk/fsa_approved/README.md","pipeline/sources/uk/fsa_approved/handoff.py","pipeline/sources/uk/fsa_approved/refresh.py","docs/architecture/disposable-candidate-import.md","pipeline/scripts/maintenance/import-candidate.py","pipeline/source_registry.json"],"next_action":"Repeatable source-local refresh dry-run passed on the retained snapshot (5,342 input, 4,300 normalized, 1,042 quarantined, expected schema fingerprint, no drift alarms); use refresh.py dry-run/handoff only in restricted staging. This does not establish production/runtime health or publication eligibility. Complete source-rights, privacy/coordinate, duplicate, coverage, and release review while keeping NI and Scotland separate."}, diff --git a/docs/source-status.md b/docs/source-status.md index a94d2bc..c9e1c6b 100644 --- a/docs/source-status.md +++ b/docs/source-status.md @@ -16,7 +16,7 @@ No last-success timestamp is invented. Private artifacts are not proof of a publ | Source ID | Metadata | Acquisition | Runtime health | Publication | Evidence / next action | |---|---|---|---|---|---| | `fr.locations` | verified | blocked | not_run | blocked | DGAL/Alim’confiance/SIRENE/HVE/Agence Bio/Géorisques reconnaissance; acquire permitted official artifact and review terms/privacy | -| `it.locations` | verified | artifact_private_only | not_run | blocked | Current 853/2004 and 1069/2009 CSVs were privately acquired with provenance; schemas remain distinct and require dictionary/privacy mapping plus separate adapters | +| `it.locations` | verified | artifact_private_only | not_run | blocked | The 853/2004 adapter and private candidate-import path are covered by deterministic tests; keep them review-gated and validate the separate 1069/2009 variant, source terms, coverage, and privacy mappings before release review | | `mx.locations` | verified | blocked | not_run | blocked | DENUE/SENASICA/DGSIAP reconnaissance; resolve token, directory, terms, and schema | | `nz.locations` | verified | blocked | not_run | blocked | MPI/Stats NZ reconnaissance; resolve 403/access and aggregate-vs-facility boundaries | | `uk.locations` | partial | artifact_private_only | not_run | blocked | Synthetic handoff passes importer pre-DB validation, but no real UK candidate has been imported or previewed; review-required/unapproved defaults, Docker E2E, privacy/coordinate, source-rights, duplicate, coverage, and release review remain open; NI/Scotland stay separate | From e628a8ae810ef7af1471ab85e89715dd9f9cc8b3 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Tue, 15 Sep 2026 09:29:05 -0700 Subject: [PATCH 108/311] Fix standalone source health CLI entrypoint --- pipeline/scripts/diagnostics/build-source-health.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pipeline/scripts/diagnostics/build-source-health.py b/pipeline/scripts/diagnostics/build-source-health.py index 28981a4..f5c623d 100644 --- a/pipeline/scripts/diagnostics/build-source-health.py +++ b/pipeline/scripts/diagnostics/build-source-health.py @@ -5,6 +5,9 @@ import sys from pathlib import Path +if __package__ in {None, ""}: + sys.path.insert(0, str(Path(__file__).resolve().parents[3])) + from pipeline.contracts.source_health import HealthEvidenceError, build_health_snapshot, write_health_snapshot From e0c817db06cd23f7dbec8c32d74b07ba7120679a Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Tue, 15 Sep 2026 09:25:51 -0700 Subject: [PATCH 109/311] add Italy private refresh entrypoint --- pipeline/sources/italy/refresh.py | 18 ++++++++++++++++++ pipeline/sources/italy/refresh_config.json | 10 ++++++++++ 2 files changed, 28 insertions(+) create mode 100644 pipeline/sources/italy/refresh.py create mode 100644 pipeline/sources/italy/refresh_config.json diff --git a/pipeline/sources/italy/refresh.py b/pipeline/sources/italy/refresh.py new file mode 100644 index 0000000..3a56d26 --- /dev/null +++ b/pipeline/sources/italy/refresh.py @@ -0,0 +1,18 @@ +"""Run a registered Italian 853 snapshot into private candidate staging.""" +from __future__ import annotations +import argparse, hashlib, json +from pathlib import Path +from pipeline.common.orchestrator import run_registered_typed_input +from .it_853_adapter import Italy853Adapter + +def main() -> int: + parser=argparse.ArgumentParser() + parser.add_argument("raw", type=Path); parser.add_argument("--runs", type=Path, required=True) + parser.add_argument("--url", required=True); parser.add_argument("--retrieved-at-utc", required=True) + parser.add_argument("--publication-date", required=True) + args=parser.parse_args(); raw=args.raw.read_bytes(); adapter=Italy853Adapter() + config={"source_url":args.url,"retrieved_at_utc":args.retrieved_at_utc,"publication_date":args.publication_date,"checksum_sha256":hashlib.sha256(raw).hexdigest(),"byte_size":len(raw),"code_version":adapter.adapter_version,"config_version":adapter.schema_version,"source_id":adapter.source_id,"terms_status":"pending_confirmation"} + status=run_registered_typed_input(args.raw,args.runs,config,adapter) + print(json.dumps({"status":status["status"],"run_dir":status["run_dir"],"input_rows":status["manifest"]["input_rows"],"normalized_rows":status["manifest"]["normalized_rows"],"quarantined_rows":status["manifest"]["quarantined_rows"]},sort_keys=True)) + return 0 if status["status"] in {"candidate-ready","staged-restricted"} else 1 +if __name__=="__main__": raise SystemExit(main()) diff --git a/pipeline/sources/italy/refresh_config.json b/pipeline/sources/italy/refresh_config.json new file mode 100644 index 0000000..bc0a1d0 --- /dev/null +++ b/pipeline/sources/italy/refresh_config.json @@ -0,0 +1,10 @@ +{ + "source_id": "it.853-2004", + "adapter_version": "it-853-candidate-v1", + "schema_version": "it-853-csv-v2.0", + "profile": "private-candidate", + "release_state": "not-created", + "publication_state": "private-candidate", + "coordinates": "pending-review", + "address_and_tax_fields": "private-source-values-only" +} From ee4e521f70291953628a8ce51b79bbc873ac6760 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Tue, 15 Sep 2026 10:28:11 -0700 Subject: [PATCH 110/311] Establish shared private country lifecycle --- docs/architecture/source-lifecycle.md | 34 +++ pipeline/common/orchestrator.py | 62 ++++- pipeline/contracts/README.md | 16 ++ pipeline/contracts/SOURCE-ADAPTER-TEMPLATE.md | 86 +++++++ pipeline/contracts/adapter_contract.py | 7 +- pipeline/contracts/private_run.py | 26 +-- pipeline/contracts/source_lifecycle.py | 167 ++++++++++++++ pipeline/contracts/test_source_lifecycle.py | 50 +++++ pipeline/run-denmark-pipeline.py | 151 +------------ pipeline/sources/denmark/README.md | 7 +- pipeline/sources/denmark/adapter.py | 44 ++-- pipeline/sources/denmark/cli.py | 6 + pipeline/sources/denmark/pipeline.py | 212 ++++++++++++++++++ .../sources/denmark/run-denmark-pipeline.py | 4 +- .../denmark/stages/acquire-denmark-smiley.py | 9 +- pipeline/sources/denmark/test_adapter.py | 28 +++ 16 files changed, 712 insertions(+), 197 deletions(-) create mode 100644 docs/architecture/source-lifecycle.md create mode 100644 pipeline/contracts/SOURCE-ADAPTER-TEMPLATE.md create mode 100644 pipeline/contracts/source_lifecycle.py create mode 100644 pipeline/contracts/test_source_lifecycle.py create mode 100644 pipeline/sources/denmark/pipeline.py diff --git a/docs/architecture/source-lifecycle.md b/docs/architecture/source-lifecycle.md new file mode 100644 index 0000000..bab97cc --- /dev/null +++ b/docs/architecture/source-lifecycle.md @@ -0,0 +1,34 @@ +# Shared country-source lifecycle + +The V2 source boundary is intentionally private and staged: + +```text +acquire -> preserve -> parse -> normalize -> validate -> health + -> candidate import -> guarded test-only API +``` + +Acquisition records the official URL, retrieval time, byte size, SHA-256, any +publisher date, and code/configuration versions. Preservation writes the raw +artifact before interpretation. The adapter then keeps parsed, normalized, and +quarantined records separate and writes a private manifest with reconciled +counts. Validation may identify anomalies but never silently drops records or +turns source disappearance into closure. + +The reusable Python seam is `pipeline.contracts.source_lifecycle` plus +`pipeline.common.orchestrator.run_private_lifecycle`. `SourceArtifact` is the +typed handoff from preservation to an adapter. The runner emits row-free +`qa.json`, restricted `run-status.json`, and, when the retrieval timestamp is +available, deterministic `source-health.json`. Health means only that the +private evidence passed its contract; it does not mean current, complete, +accurate, approved, or publishable data. + +Database candidate import and the guarded test-only API remain explicit +commands. They must receive a private candidate and preserve privacy and +publication gates; neither is called by the lifecycle runner. A failed or +restricted run therefore leaves any previous eligible release untouched. + +Country-specific code belongs under `pipeline/sources//`. Historical +entry points may remain as thin compatibility shims, but new stages should be +implemented only in the source-owned package. Use the +[`country-source adapter template`](../../pipeline/contracts/SOURCE-ADAPTER-TEMPLATE.md) +for new packages. diff --git a/pipeline/common/orchestrator.py b/pipeline/common/orchestrator.py index ecb552a..7b948de 100644 --- a/pipeline/common/orchestrator.py +++ b/pipeline/common/orchestrator.py @@ -9,7 +9,9 @@ from pathlib import Path from typing import Any, Callable -from pipeline.contracts.adapter_contract import SourceAdapter, source_artifact_from_mapping +from pipeline.contracts.adapter_contract import SourceAdapter, SourceArtifact, source_artifact_from_mapping +from pipeline.contracts.private_run import write_private_run_report +from pipeline.contracts.source_health import build_health_snapshot, write_health_snapshot from .identity import record_key ORCHESTRATOR_VERSION = "v2-orchestrator-3" @@ -85,6 +87,29 @@ def run_registered_input(raw_path: str | Path, runs_dir: str | Path, config: dic "error_type": type(exc).__name__, "error": str(exc), "prior_eligible_release": prior_eligible_release} status["run_dir"] = str(run_dir) + # Health is emitted after run-status exists because it must prove that no + # release was promoted. Legacy adapters get the QA report too; health is + # attempted only for the typed private publication state understood by + # source_health. + _atomic(run_dir / "run-status.json", (json.dumps(status, ensure_ascii=False, sort_keys=True, indent=2) + "\n").encode()) + if "manifest" in status and isinstance(status["manifest"], dict) and status["manifest"].get("source_id"): + manifest = status["manifest"] + try: + write_private_run_report(run_dir, manifest) + if manifest.get("publication_state") == "private-candidate": + as_of = config.get("health_as_of_utc") or config.get("retrieved_at_utc") + if as_of: + snapshot = build_health_snapshot(run_dir, as_of_utc=as_of) + write_health_snapshot(run_dir / "source-health.json", snapshot) + except Exception as exc: + # A candidate with invalid evidence is not ready for any later + # gate. Keep the files for diagnosis, but fail the run closed. + status = {"status": "failed", "publication_state": "unchanged", + "release_promoted": False, "error_type": type(exc).__name__, + "error": f"private evidence: {exc}", + "prior_eligible_release": prior_eligible_release, + "run_dir": str(run_dir)} + status["run_dir"] = str(run_dir) _atomic(run_dir / "run-status.json", (json.dumps(status, ensure_ascii=False, sort_keys=True, indent=2) + "\n").encode()) return status @@ -108,3 +133,38 @@ def invoke(raw: str | Path, run_dir: str | Path, _config: dict[str, Any]) -> dic return run_registered_input(raw_path, runs_dir, config, invoke, suppressed_ids=suppressed_ids, prior_eligible_release=prior_eligible_release) + + +def run_private_lifecycle(raw_path: str | Path, runs_dir: str | Path, + artifact: SourceArtifact, adapter: SourceAdapter, + *, suppressed_ids: set[str | tuple[str, str, str]] | None = None, + prior_eligible_release: dict[str, Any] | None = None, + health_as_of_utc: str | None = None) -> dict[str, Any]: + """Run the canonical typed lifecycle from a preserved ``SourceArtifact``. + + This is the source-local integration seam for new countries. Acquisition + is deliberately absent: callers must provide an already preserved file and + its recorded facts. The adapter writes parsed/normalized/quarantined + evidence, this runner writes QA/run status/health, and no release or API + publication is possible here. + """ + config = { + "source_id": adapter.source_id, + "source_url": artifact.source_url, + "retrieved_at_utc": artifact.retrieved_at_utc, + "checksum_sha256": artifact.sha256, + "byte_size": artifact.byte_size, + "publication_date": artifact.publication_date, + "effective_date": artifact.effective_date, + "code_version": artifact.code_version, + "config_version": artifact.config_version, + "rights_caveat": artifact.rights_caveat, + "privacy_caveat": artifact.privacy_caveat, + "coverage": artifact.coverage, + "health_as_of_utc": health_as_of_utc or artifact.retrieved_at_utc, + } + return run_registered_typed_input( + raw_path, runs_dir, config, adapter, + suppressed_ids=suppressed_ids, + prior_eligible_release=prior_eligible_release, + ) diff --git a/pipeline/contracts/README.md b/pipeline/contracts/README.md index ce9935a..c252bd5 100644 --- a/pipeline/contracts/README.md +++ b/pipeline/contracts/README.md @@ -1,5 +1,21 @@ # Source-adapter contract +## Canonical source lifecycle + +All source packages use the same private lifecycle contract: + +`acquire -> preserve -> parse -> normalize -> validate -> health -> candidate import -> guarded test-only API` + +`source_lifecycle.py` owns the small cross-country primitives: atomic JSON and +JSONL writes, deterministic JSONL hashes, private-manifest count/publication +invariants, and the `source-lifecycle-v1` envelope. Source adapters continue +to own their schemas, field mappings, validation rules, and quarantine reasons. +The shared runner in `common.orchestrator.run_private_lifecycle` accepts only a +preserved `SourceArtifact`, emits private QA/run-status/health evidence, and +cannot promote or publish a release. See +[`SOURCE-ADAPTER-TEMPLATE.md`](SOURCE-ADAPTER-TEMPLATE.md) for the country +implementation template. + ## Shared private-run QA seam `private_run.run_typed_adapter` provides the common runner for typed diff --git a/pipeline/contracts/SOURCE-ADAPTER-TEMPLATE.md b/pipeline/contracts/SOURCE-ADAPTER-TEMPLATE.md new file mode 100644 index 0000000..0fd96d7 --- /dev/null +++ b/pipeline/contracts/SOURCE-ADAPTER-TEMPLATE.md @@ -0,0 +1,86 @@ +# Country-source adapter template + +Use this template for each source under `pipeline/sources///`. +It is deliberately small so ten or more countries can share the same review +and import seams without sharing source-specific assumptions. + +## Required configuration + +Keep a checked-in, non-secret JSON configuration beside the adapter: + +```json +{ + "source_id": "gb.example-register", + "source_url": "https://example.test/register.csv", + "adapter_version": "example-v1", + "schema_version": "example-schema-v1", + "config_version": "example-config-v1", + "coverage": "publisher-defined scope; not a completeness claim", + "terms_status": "pending_confirmation", + "geocoding": "disabled" +} +``` + +`source_id`, adapter/schema/config versions, URL, and coverage are required +metadata. Retrieval time, SHA-256, byte size, and any publisher-supplied +publication/effective date come from the preserved acquisition and are passed +as `SourceArtifact`; they must not be guessed in the adapter. + +Python callers may validate the same fields with +`pipeline.contracts.source_lifecycle.SourceConfig`; older JSON callers can +continue passing a mapping through the compatibility boundary. + +## Minimal Python interface + +```python +from pathlib import Path +from pipeline.contracts.adapter_contract import SourceArtifact + +class ExampleAdapter: + source_id = "gb.example-register" + adapter_version = "example-v1" + schema_version = "example-schema-v1" + + def run(self, raw_path: str | Path, run_dir: str | Path, + artifact: SourceArtifact) -> dict: + """Parse one preserved artifact into private staging only.""" +``` + +`run` must preserve source values and write deterministic +`parsed/records.jsonl`, `normalized/records.jsonl`, +`quarantined/records.jsonl`, and `manifest.json`. Use +`source_lifecycle.atomic_jsonl` and `private_manifest` for the shared file and +count invariants. A malformed or unresolved row goes to quarantine with an +explicit reason; no row is silently dropped, merged, or assigned a guessed +identity. + +Run the adapter through the shared seam: + +```python +from pipeline.common.orchestrator import run_private_lifecycle + +status = run_private_lifecycle( + raw_path, runs_dir, artifact, ExampleAdapter(), + health_as_of_utc="2026-09-15T00:00:00Z", +) +``` + +The seam emits row-free `qa.json`, a restricted `run-status.json`, and (when +provenance is complete) `source-health.json`. It never imports a release, +promotes data, or enables a public API. Candidate import and the guarded +test-only API remain explicit, separately authorized operations. + +## Review checklist + +1. Acquire only through an approved source-specific command; preserve the raw + bytes and acquisition metadata before parsing. +2. Keep parsed, normalized, quarantined, reviewed, and released artifacts in + separate locations. +3. Keep source identity and source values beside normalized interpretations. +4. Validate schema, required identifiers, duplicates, dates, coordinates, and + count drift; retain aggregate anomaly counts. +5. Run malformed-input and byte-identical rerun tests. A failed rerun must not + replace the prior validated release. +6. Record privacy, terms, factual-review, project-approval, and publication + states independently. Successful acquisition or validation is never + publication authorization. diff --git a/pipeline/contracts/adapter_contract.py b/pipeline/contracts/adapter_contract.py index b3f1b00..5ad1ccf 100644 --- a/pipeline/contracts/adapter_contract.py +++ b/pipeline/contracts/adapter_contract.py @@ -1,9 +1,9 @@ from __future__ import annotations import hashlib from pathlib import Path -import json from dataclasses import dataclass from typing import Any, Protocol +from .source_lifecycle import read_jsonl @dataclass(frozen=True) @@ -51,8 +51,3 @@ def assert_manifest(manifest: dict, raw: bytes, schema_version: str) -> None: assert manifest["schema_version"] == schema_version assert manifest["input_rows"] == manifest["normalized_rows"] + manifest["quarantined_rows"] assert manifest["release_state"] == "not-created" - - -def read_jsonl(path: Path) -> list[dict]: - """Read deterministic staging records for test and review tooling.""" - return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line] diff --git a/pipeline/contracts/private_run.py b/pipeline/contracts/private_run.py index 78f56d5..0ee8c46 100644 --- a/pipeline/contracts/private_run.py +++ b/pipeline/contracts/private_run.py @@ -2,25 +2,17 @@ from __future__ import annotations import json -import os from pathlib import Path from typing import Any, Iterable from .adapter_contract import SourceAdapter, SourceArtifact +from .source_lifecycle import atomic_json, validate_private_manifest class PrivateRunError(ValueError): """A private run manifest or report violates the shared contract.""" -def _atomic_json(path: Path, value: dict[str, Any]) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - payload = (json.dumps(value, ensure_ascii=False, sort_keys=True, indent=2, default=list) + "\n").encode() - temporary = path.with_name(path.name + ".tmp") - temporary.write_bytes(payload) - os.replace(temporary, path) - - def _manifest_value(manifest: dict[str, Any], key: str) -> Any: if key in manifest: return manifest[key] @@ -52,16 +44,10 @@ def _record_ids(path: Path) -> set[str]: def validate_manifest(manifest: dict[str, Any]) -> None: - required = {"source_id", "input_rows", "normalized_rows", "quarantined_rows", "release_state"} - missing = sorted(required - manifest.keys()) - if missing: - raise PrivateRunError("manifest missing required keys: " + ", ".join(missing)) - if manifest["input_rows"] != manifest["normalized_rows"] + manifest["quarantined_rows"]: - raise PrivateRunError("manifest row counts do not reconcile") - if manifest["release_state"] != "not-created": - raise PrivateRunError("private run cannot have a release") - if manifest.get("publication_state") not in {None, "private-candidate", "not-staged"}: - raise PrivateRunError("private run publication state is not restricted") + try: + validate_private_manifest(manifest) + except ValueError as exc: + raise PrivateRunError(str(exc)) from exc def summarize_private_run( @@ -115,7 +101,7 @@ def write_private_run_report( previous_normalized_path=previous_normalized_path, drift_alarms=drift_alarms, ) - _atomic_json(Path(run_dir) / "qa.json", report) + atomic_json(Path(run_dir) / "qa.json", report) return report diff --git a/pipeline/contracts/source_lifecycle.py b/pipeline/contracts/source_lifecycle.py new file mode 100644 index 0000000..519df9b --- /dev/null +++ b/pipeline/contracts/source_lifecycle.py @@ -0,0 +1,167 @@ +"""Small, source-agnostic primitives for private source lifecycles. + +The lifecycle is intentionally a file contract rather than a framework: +``acquire -> preserve -> parse -> normalize -> validate -> health -> +candidate import -> guarded test-only API``. Only the first six stages are +owned by this package. Import and publication remain explicit gates in the +maintenance/API layers. + +All writers below use a same-directory temporary file and ``os.replace``. +That matters for reruns: a failed stage must not leave a plausible partial +JSON or JSONL artifact that a later stage could mistake for a complete run. +""" +from __future__ import annotations + +import hashlib +import json +import os +import tempfile +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Iterable + + +LIFECYCLE_CONTRACT_VERSION = "source-lifecycle-v1" +PRIVATE_PUBLICATION_STATES = {"private-candidate", "not-staged", "human-gate-required"} + + +@dataclass(frozen=True) +class SourceConfig: + """The minimal checked-in identity/configuration shared by source packages.""" + + source_id: str + source_url: str + adapter_version: str + schema_version: str + config_version: str = "unknown" + coverage: str | None = None + geocoding: str = "disabled" + terms_status: str = "pending_confirmation" + + def __post_init__(self) -> None: + for name in ("source_id", "source_url", "adapter_version", "schema_version", "config_version"): + if not isinstance(getattr(self, name), str) or not getattr(self, name).strip(): + raise ValueError(f"{name} must be a non-empty string") + + def as_mapping(self) -> dict[str, Any]: + """Return a JSON-compatible mapping for legacy runner boundaries.""" + return { + "source_id": self.source_id, + "source_url": self.source_url, + "adapter_version": self.adapter_version, + "schema_version": self.schema_version, + "config_version": self.config_version, + "coverage": self.coverage, + "geocoding": self.geocoding, + "terms_status": self.terms_status, + } + + +def atomic_bytes(path: str | Path, payload: bytes) -> Path: + """Atomically write bytes and return ``path``. + + The target directory is created, but existing files are never removed + before the replacement is ready. This is safe for repeated local runs + and leaves the previous complete artifact available after write errors. + """ + target = Path(path) + target.parent.mkdir(parents=True, exist_ok=True) + fd, temporary_name = tempfile.mkstemp(prefix=f".{target.name}.", dir=target.parent) + try: + with os.fdopen(fd, "wb") as handle: + handle.write(payload) + os.replace(temporary_name, target) + except Exception: + try: + os.unlink(temporary_name) + except FileNotFoundError: + pass + raise + return target + + +def atomic_json(path: str | Path, value: dict[str, Any]) -> Path: + """Atomically write canonical, UTF-8 JSON with a trailing newline.""" + payload = (json.dumps(value, ensure_ascii=False, sort_keys=True, indent=2, default=list) + "\n").encode("utf-8") + return atomic_bytes(path, payload) + + +def atomic_jsonl(path: str | Path, rows: Iterable[dict[str, Any]]) -> tuple[Path, str, int]: + """Atomically write sorted-key JSONL and return path, hash, and row count.""" + payload = b"".join( + (json.dumps(row, ensure_ascii=False, sort_keys=True, default=list) + "\n").encode("utf-8") + for row in rows + ) + return atomic_bytes(path, payload), hashlib.sha256(payload).hexdigest(), payload.count(b"\n") + + +def read_jsonl(path: str | Path) -> list[dict[str, Any]]: + """Read a deterministic JSONL artifact, rejecting malformed lines.""" + return [json.loads(line) for line in Path(path).read_text(encoding="utf-8").splitlines() if line.strip()] + + +def validate_private_manifest(manifest: dict[str, Any]) -> None: + """Validate the shared count and publication invariants.""" + required = {"source_id", "input_rows", "normalized_rows", "quarantined_rows", "release_state"} + missing = sorted(required - manifest.keys()) + if missing: + raise ValueError("manifest missing required keys: " + ", ".join(missing)) + counts = tuple(manifest[key] for key in ("input_rows", "normalized_rows", "quarantined_rows")) + if any(not isinstance(value, int) or value < 0 for value in counts): + raise ValueError("manifest row counts must be non-negative integers") + if counts[0] != counts[1] + counts[2]: + raise ValueError("manifest row counts do not reconcile") + if manifest["release_state"] != "not-created": + raise ValueError("private run cannot have a release") + if manifest.get("publication_state") not in PRIVATE_PUBLICATION_STATES | {None}: + raise ValueError("private run publication state is not restricted") + + +def private_manifest( + *, + source_id: str, + adapter_version: str, + schema_version: str, + artifact: Any, + input_rows: int, + normalized_rows: int, + quarantined_rows: int, + normalized_sha256: str, + parsed_sha256: str | None = None, + anomaly_counts: dict[str, int] | None = None, +) -> dict[str, Any]: + """Create the common manifest envelope while retaining source details. + + Source adapters still own their field mappings and anomaly vocabulary; + this helper owns only the stable provenance, count, and publication fields. + """ + manifest: dict[str, Any] = { + "contract_version": LIFECYCLE_CONTRACT_VERSION, + "source_id": source_id, + "adapter_version": adapter_version, + "schema_version": schema_version, + "source_url": artifact.source_url, + "retrieved_at_utc": artifact.retrieved_at_utc, + "publication_date": artifact.publication_date, + "effective_date": artifact.effective_date, + "sha256": artifact.sha256, + "checksum_sha256": artifact.sha256, + "byte_size": artifact.byte_size, + "code_version": artifact.code_version, + "config_version": artifact.config_version, + "input_rows": input_rows, + "normalized_rows": normalized_rows, + "quarantined_rows": quarantined_rows, + "normalized_sha256": normalized_sha256, + "release_state": "not-created", + "publication_state": "private-candidate", + "review_state": "review_required", + "privacy_gate": "pending", + "coordinate_gate": "review_required", + "anomaly_counts": anomaly_counts or {}, + "acquisition": artifact.__dict__, + } + if parsed_sha256 is not None: + manifest["parsed_sha256"] = parsed_sha256 + validate_private_manifest(manifest) + return manifest diff --git a/pipeline/contracts/test_source_lifecycle.py b/pipeline/contracts/test_source_lifecycle.py new file mode 100644 index 0000000..4d34781 --- /dev/null +++ b/pipeline/contracts/test_source_lifecycle.py @@ -0,0 +1,50 @@ +import hashlib +import json +import tempfile +import unittest +from pathlib import Path + +from .adapter_contract import SourceArtifact +from .source_lifecycle import SourceConfig, atomic_jsonl, private_manifest, validate_private_manifest + + +class SourceLifecyclePrimitiveTests(unittest.TestCase): + def artifact(self, raw: bytes = b"synthetic") -> SourceArtifact: + return SourceArtifact( + "https://example.test/source", "2026-09-15T00:00:00Z", + hashlib.sha256(raw).hexdigest(), len(raw), + code_version="adapter-v1", config_version="config-v1", + ) + + def test_jsonl_hash_and_bytes_are_stable(self): + with tempfile.TemporaryDirectory() as directory: + one = Path(directory) / "one.jsonl" + two = Path(directory) / "two.jsonl" + rows = [{"source_id": "x", "value": 2}, {"source_id": "x", "value": 1}] + _, first_hash, first_count = atomic_jsonl(one, rows) + _, second_hash, second_count = atomic_jsonl(two, rows) + self.assertEqual(first_hash, second_hash) + self.assertEqual(first_count, second_count) + self.assertEqual(one.read_bytes(), two.read_bytes()) + self.assertEqual([json.loads(line) for line in one.read_text().splitlines()], rows) + + def test_source_config_is_a_small_checked_in_interface(self): + config = SourceConfig("x", "https://example.test/x", "adapter-v1", "schema-v1") + self.assertEqual(config.as_mapping()["geocoding"], "disabled") + with self.assertRaisesRegex(ValueError, "source_id"): + SourceConfig("", "https://example.test/x", "adapter-v1", "schema-v1") + + def test_manifest_reconciles_counts_and_keeps_release_uncreated(self): + manifest = private_manifest( + source_id="x", adapter_version="a", schema_version="s", + artifact=self.artifact(), input_rows=3, normalized_rows=2, + quarantined_rows=1, normalized_sha256="b" * 64, + ) + self.assertEqual(manifest["contract_version"], "source-lifecycle-v1") + self.assertEqual(manifest["release_state"], "not-created") + with self.assertRaisesRegex(ValueError, "reconcile"): + validate_private_manifest({**manifest, "quarantined_rows": 2}) + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/run-denmark-pipeline.py b/pipeline/run-denmark-pipeline.py index 6355f5b..3daf3f3 100644 --- a/pipeline/run-denmark-pipeline.py +++ b/pipeline/run-denmark-pipeline.py @@ -1,154 +1,25 @@ #!/usr/bin/env python3 -"""Run the auditable Denmark staging pipeline from one command.""" - +"""Compatibility shim for Denmark's source-owned staging runner.""" from __future__ import annotations -import argparse -import hashlib -import json -import logging -import subprocess -import sys -import uuid -from datetime import datetime, timezone from pathlib import Path +import sys - -LOGGER = logging.getLogger("uec.denmark.pipeline") ROOT = Path(__file__).resolve().parent.parent -SHARED_STAGES = ROOT / "pipeline" / "scripts" / "stages" -DENMARK_STAGES = ROOT / "pipeline" / "sources" / "denmark" / "stages" - - -def utc_now() -> str: - return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) +from pipeline.sources.denmark import pipeline as _canonical -def sha256_file(path: Path) -> str: - digest = hashlib.sha256() - with path.open("rb") as source: - for chunk in iter(lambda: source.read(1024 * 1024), b""): - digest.update(chunk) - return digest.hexdigest() - - -def run_stage(name: str, script: Path, args: list[str]) -> None: - command = [sys.executable, str(script), *args] - LOGGER.info("stage=%s status=started command=%s", name, " ".join(command)) - subprocess.run(command, cwd=ROOT, check=True) - LOGGER.info("stage=%s status=completed", name) - - -def artifact_manifest(run_dir: Path, input_path: Path, started_at: str, completed_at: str, status: str = "success", error: str | None = None) -> Path: - artifacts = [] - for path in sorted(run_dir.rglob("*")): - if path.is_file() and path.name != "pipeline-manifest.json": - artifacts.append({ - "path": path.relative_to(ROOT).as_posix(), - "bytes": path.stat().st_size, - "sha256": sha256_file(path), - }) - manifest = { - "status": status, - "pipeline": "denmark-smiley-staging", - "started_at_utc": started_at, - "completed_at_utc": completed_at, - "input_path": input_path.relative_to(ROOT).as_posix() if input_path.is_relative_to(ROOT) else input_path.as_posix(), - "artifacts": artifacts, - "database_import": "not run; import is an explicit separate command", - } - if error: - manifest["error"] = error - path = run_dir / "pipeline-manifest.json" - path.write_text(json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") - return path +# Existing tests and operator scripts may patch these names. Real behavior +# lives in the source-owned module so Denmark has one canonical implementation. +run_stage = _canonical.run_stage +artifact_manifest = _canonical.artifact_manifest def main() -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("input", nargs="?", type=Path, help="Previously archived or local Smileydata.xml") - parser.add_argument("--fetch", action="store_true", help="Acquire dk.smiley first; requires --terms-review and does not import or promote.") - parser.add_argument("--terms-review", type=Path, help="Approved terms-review JSON required with --fetch.") - parser.add_argument("--source-url", help="Optional requested URL override for --fetch.") - parser.add_argument("--raw-output-root", type=Path, default=ROOT / "data/raw", help="Archive root for --fetch.") - parser.add_argument("--run-id", help="Acquisition run ID for --fetch.") - parser.add_argument("--rules", type=Path, default=ROOT / "pipeline/config/denmark-classification-v1.json") - parser.add_argument("--output-dir", type=Path, help="Run directory; defaults to data/staging/") - parser.add_argument("--expected-rows", type=int) - parser.add_argument("--geocode-limit", type=int, help="Optionally call DAWA for only this many queued records") - parser.add_argument("--geocode-delay", type=float, default=1.0) - parser.add_argument("--geocode-provider-config", type=Path, default=ROOT / "pipeline/config/geocoding-dev.json") - parser.add_argument("--geocode-terms-review", type=Path, help="Approved per-run terms review required to make bounded geocoding requests.") - parser.add_argument("--geocode-suppression-keys", type=Path, help="Payload-free source identity references excluded from geocoding.") - args = parser.parse_args() - - logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s", datefmt="%Y-%m-%dT%H:%M:%SZ") - if args.fetch and args.input is not None: - parser.error("input cannot be provided with --fetch") - if not args.fetch and args.input is None: - parser.error("input is required unless --fetch is used") - if args.fetch and args.terms_review is None: - parser.error("--terms-review is required with --fetch") - if args.geocode_limit is not None and args.geocode_terms_review is None: - parser.error("--geocode-terms-review is required with --geocode-limit") - if args.fetch: - terms_review_path = args.terms_review.resolve() - raw_output_root = args.raw_output_root.resolve() - acquisition_run_id = args.run_id or (datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + "-" + uuid.uuid4().hex[:8]) - acquisition_args = ["--fetch", "--output-root", str(raw_output_root), "--terms-review", str(terms_review_path), "--run-id", acquisition_run_id] - if args.source_url: - acquisition_args.extend(["--url", args.source_url]) - run_stage("acquire", DENMARK_STAGES / "acquire-denmark-smiley.py", acquisition_args) - acquisition_root = raw_output_root / "dk.smiley" - input_path = (acquisition_root / acquisition_run_id / "Smileydata.xml").resolve() - acquisition_metadata = json.loads((input_path.parent / "acquisition-metadata.json").read_text(encoding="utf-8")) - acquired_source_url = acquisition_metadata.get("final_url") or acquisition_metadata.get("requested_url") - else: - input_path = args.input.resolve() - acquired_source_url = None - if not input_path.is_file(): - LOGGER.error("pipeline status=failed reason=input_not_found input=%s", input_path) - return 2 - run_name = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") - run_dir = (args.output_dir or ROOT / "data/staging/denmark-smiley" / run_name).resolve() - run_dir.mkdir(parents=True, exist_ok=True) - started_at = utc_now() - LOGGER.info("pipeline=denmark-smiley status=started run_dir=%s", run_dir) - - try: - parse_dir = run_dir / "01-parse" - normalize_dir = run_dir / "02-normalize" - classify_dir = run_dir / "03-classify" - validate_dir = run_dir / "04-validate" - geocode_dir = run_dir / "05-geocode-queue" - parse_args = [str(input_path), "--output-dir", str(parse_dir)] - if acquired_source_url and acquired_source_url != "unknown": - parse_args.extend(["--source-url", acquired_source_url]) - run_stage("parse", DENMARK_STAGES / "parse-denmark-smiley.py", parse_args) - run_stage("normalize", DENMARK_STAGES / "normalize-denmark-smiley.py", [str(parse_dir / "parsed-rows.jsonl"), "--output-dir", str(normalize_dir)]) - run_stage("classify", DENMARK_STAGES / "classify-denmark.py", [str(normalize_dir / "normalized-records.jsonl"), "--rules", str(args.rules.resolve()), "--output-dir", str(classify_dir)]) - validation_args = [str(classify_dir / "classified-records.jsonl"), "--output-dir", str(validate_dir)] - if args.expected_rows is not None: - validation_args.extend(["--expected-rows", str(args.expected_rows)]) - run_stage("validate", DENMARK_STAGES / "validate-denmark.py", validation_args) - run_stage("geocode_queue", SHARED_STAGES / "create-geocode-queue.py", [str(classify_dir / "classified-records.jsonl"), "--output-dir", str(geocode_dir)]) - if args.geocode_limit is not None: - geocode_args = [str(geocode_dir / "geocode-queue.jsonl"), "--output", str(run_dir / "06-geocode-results.jsonl"), "--limit", str(args.geocode_limit), "--delay", str(args.geocode_delay), "--provider-config", str(args.geocode_provider_config.resolve()), "--terms-review", str(args.geocode_terms_review.resolve()), "--network"] - if args.geocode_suppression_keys: - geocode_args.extend(["--suppression-keys", str(args.geocode_suppression_keys.resolve())]) - run_stage("geocode_dawa", DENMARK_STAGES / "geocode-denmark-dawa.py", geocode_args) - manifest = artifact_manifest(run_dir, input_path, started_at, utc_now()) - LOGGER.info("pipeline=denmark-smiley status=success manifest=%s", manifest) - except subprocess.CalledProcessError as error: - manifest = artifact_manifest(run_dir, input_path, started_at, utc_now(), "failed", f"stage exited with code {error.returncode}") - LOGGER.error("pipeline=denmark-smiley status=failed stage_exit_code=%d manifest=%s", error.returncode, manifest) - return error.returncode or 1 - except Exception as error: - manifest = artifact_manifest(run_dir, input_path, started_at, utc_now(), "failed", str(error)) - LOGGER.exception("pipeline=denmark-smiley status=failed") - return 1 - return 0 + return _canonical.main(run_stage_fn=run_stage, artifact_manifest_fn=artifact_manifest) if __name__ == "__main__": - sys.exit(main()) + raise SystemExit(main()) diff --git a/pipeline/sources/denmark/README.md b/pipeline/sources/denmark/README.md index 4bbc7c7..06595c7 100644 --- a/pipeline/sources/denmark/README.md +++ b/pipeline/sources/denmark/README.md @@ -32,8 +32,11 @@ inspection records); the publisher's statistics page explicitly excludes wholesale businesses. This is source coverage, not a claim that the project dataset is complete or that records are current. A reviewed acquisition may be retained in ignored private storage for research and validation only. The -adapter emits a private candidate with `release_state: not-created`; no health, -approval, geocoding, or publication conclusion follows from a successful run. +adapter emits a private candidate with `release_state: not-created`. The +source-owned runner additionally emits row-free QA and private health evidence +when acquisition provenance is complete; `private-validated` is only an +evidence-contract result, not a currentness, approval, geocoding, or +publication conclusion. ## Full private refresh evidence diff --git a/pipeline/sources/denmark/adapter.py b/pipeline/sources/denmark/adapter.py index 05f6290..d37dfbb 100644 --- a/pipeline/sources/denmark/adapter.py +++ b/pipeline/sources/denmark/adapter.py @@ -4,12 +4,13 @@ publishes coordinates; records with no stable source key are quarantined. """ from __future__ import annotations -import hashlib, json, os +import hashlib, json import xml.etree.ElementTree as ET from pathlib import Path from typing import Any from pipeline.contracts.adapter_contract import SourceArtifact from pipeline.contracts.candidate_handoff import write_handoff +from pipeline.contracts.source_lifecycle import atomic_json, atomic_jsonl, private_manifest SOURCE_ID = "dk.smiley" ADAPTER_VERSION = "denmark-smiley-contract-v1" @@ -22,19 +23,6 @@ def check_refresh(previous: dict[str, Any], current: dict[str, Any], *, max_coun if old and abs(new - old) / old > max_count_delta: raise ValueError("Denmark refresh normalized row count drift exceeds threshold") -def _atomic(path: Path, payload: bytes) -> None: - """Publish one complete staging file, never a partially written artifact.""" - path.parent.mkdir(parents=True, exist_ok=True) - temporary = path.with_name(path.name + ".tmp") - temporary.write_bytes(payload) - os.replace(temporary, path) - -def _jsonl(path: Path, rows: list[dict[str, Any]]) -> str: - """Serialize with stable ordering so reruns can be compared byte-for-byte.""" - payload = b"".join((json.dumps(row, ensure_ascii=False, sort_keys=True, default=list) + "\n").encode() for row in rows) - _atomic(path, payload) - return hashlib.sha256(payload).hexdigest() - class DenmarkSmileyAdapter: source_id = SOURCE_ID adapter_version = ADAPTER_VERSION @@ -111,16 +99,22 @@ def run(self, raw_path: str | Path, run_dir: str | Path, artifact: SourceArtifac except ET.ParseError as exc: raise ValueError("invalid Denmark XML") from exc root = Path(run_dir) - parsed_hash = _jsonl(root / "parsed" / "records.jsonl", rows + [item["record"] for item in quarantined]) - normalized_hash = _jsonl(root / "normalized" / "records.jsonl", rows) - _jsonl(root / "quarantined" / "records.jsonl", quarantined) + parsed_rows = rows + [item["record"] for item in quarantined] + _, parsed_hash, _ = atomic_jsonl(root / "parsed" / "records.jsonl", parsed_rows) + _, normalized_hash, _ = atomic_jsonl(root / "normalized" / "records.jsonl", rows) + atomic_jsonl(root / "quarantined" / "records.jsonl", quarantined) # This state is deliberately private: validation cannot authorize release. - manifest = {"source_id": SOURCE_ID, "adapter_version": ADAPTER_VERSION, - "schema_version": ADAPTER_VERSION, "checksum_sha256": actual, - "byte_size": len(raw), "input_rows": len(rows) + len(quarantined), - "normalized_rows": len(rows), "quarantined_rows": len(quarantined), - "parsed_sha256": parsed_hash, "normalized_sha256": normalized_hash, - "release_state": "not-created", "publication_state": "private-candidate", - "acquisition": artifact.__dict__} - _atomic(root / "manifest.json", (json.dumps(manifest, ensure_ascii=False, sort_keys=True, indent=2) + "\n").encode()) + manifest = private_manifest( + source_id=SOURCE_ID, + adapter_version=ADAPTER_VERSION, + schema_version=ADAPTER_VERSION, + artifact=artifact, + input_rows=len(parsed_rows), + normalized_rows=len(rows), + quarantined_rows=len(quarantined), + normalized_sha256=normalized_hash, + parsed_sha256=parsed_hash, + anomaly_counts={"missing_source_key": len(quarantined)} if quarantined else {}, + ) + atomic_json(root / "manifest.json", manifest) return manifest diff --git a/pipeline/sources/denmark/cli.py b/pipeline/sources/denmark/cli.py index 4511d52..d3a4571 100644 --- a/pipeline/sources/denmark/cli.py +++ b/pipeline/sources/denmark/cli.py @@ -14,6 +14,12 @@ def run_stage(script_name: str) -> None: def main(script_name: str) -> None: + if script_name == "run-denmark-pipeline.py": + # Preserve the old helper API while routing the runner to the + # source-owned implementation. + from pipeline.sources.denmark.pipeline import main as canonical_main + canonical_main() + return base = LEGACY_PIPELINE if script_name == "run-denmark-pipeline.py" else LEGACY_STAGES script = (base / script_name).resolve() if not script.is_file(): diff --git a/pipeline/sources/denmark/pipeline.py b/pipeline/sources/denmark/pipeline.py new file mode 100644 index 0000000..4138044 --- /dev/null +++ b/pipeline/sources/denmark/pipeline.py @@ -0,0 +1,212 @@ +"""Canonical private staging runner for the Denmark Find Smiley source.""" +from __future__ import annotations + +import argparse +import hashlib +import json +import logging +import subprocess +import sys +import uuid +from datetime import datetime, timezone +from pathlib import Path +from typing import Callable + +from pipeline.contracts.private_run import write_private_run_report +from pipeline.contracts.source_health import build_health_snapshot, write_health_snapshot +from pipeline.contracts.source_lifecycle import atomic_json, validate_private_manifest + + +LOGGER = logging.getLogger("uec.denmark.pipeline") +PROJECT_ROOT = Path(__file__).resolve().parents[3] +SHARED_STAGES = PROJECT_ROOT / "pipeline" / "scripts" / "stages" +DENMARK_STAGES = PROJECT_ROOT / "pipeline" / "sources" / "denmark" / "stages" + + +def utc_now() -> str: + return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as source: + for chunk in iter(lambda: source.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def run_stage(name: str, script: Path, args: list[str]) -> None: + """Run one source stage; import and promotion are intentionally absent.""" + command = [sys.executable, str(script), *args] + LOGGER.info("stage=%s status=started command=%s", name, " ".join(command)) + subprocess.run(command, cwd=PROJECT_ROOT, check=True) + LOGGER.info("stage=%s status=completed", name) + + +def artifact_manifest(run_dir: Path, input_path: Path, started_at: str, + completed_at: str, status: str = "success", + error: str | None = None) -> Path: + """Retain the historical inventory manifest for existing operators.""" + artifacts = [] + for path in sorted(run_dir.rglob("*")): + if path.is_file() and path.name not in {"pipeline-manifest.json", "manifest.json"}: + artifacts.append({"path": path.relative_to(PROJECT_ROOT).as_posix() + if path.is_relative_to(PROJECT_ROOT) else path.as_posix(), + "bytes": path.stat().st_size, "sha256": sha256_file(path)}) + manifest = {"status": status, "pipeline": "denmark-smiley-staging", + "started_at_utc": started_at, "completed_at_utc": completed_at, + "input_path": input_path.relative_to(PROJECT_ROOT).as_posix() + if input_path.is_relative_to(PROJECT_ROOT) else input_path.as_posix(), + "artifacts": artifacts, + "database_import": "not run; import is an explicit separate command"} + if error: + manifest["error"] = error + return atomic_json(run_dir / "pipeline-manifest.json", manifest) + + +def _canonical_evidence(run_dir: Path, input_path: Path, metadata: dict, + started_at: str, completed_at: str) -> None: + """Bridge stage reports into the shared private-run contract. + + Validation findings are retained and counted as anomalies; they are not + silently removed from the normalized stage output. The shared manifest + therefore keeps ``normalized_rows`` equal to every parsed row and records + findings separately in ``anomaly_counts``. + """ + parse_meta = json.loads((run_dir / "01-parse" / "run-metadata.json").read_text(encoding="utf-8")) + validation = json.loads((run_dir / "04-validate" / "validation-report.json").read_text(encoding="utf-8")) + raw_hash = metadata.get("sha256") + raw_size = metadata.get("byte_size") + if not isinstance(raw_hash, str) or not isinstance(raw_size, int): + # A local file without acquisition metadata is still stageable, but + # cannot receive a credibility health claim. + raw_hash, raw_size = sha256_file(input_path), input_path.stat().st_size + retrieved = metadata.get("retrieved_at_utc") + source_url = metadata.get("final_url") or metadata.get("requested_url") + manifest = { + "contract_version": "source-lifecycle-v1", + "source_id": "dk.smiley", + "adapter_version": "denmark-smiley-contract-v1", + "schema_version": "denmark-smiley-contract-v1", + "source_url": source_url, + "retrieved_at_utc": retrieved, + "publication_date": (metadata.get("publication_metadata") or {}).get("Last-Modified"), + "effective_date": None, + "sha256": raw_hash, + "checksum_sha256": raw_hash, + "byte_size": raw_size, + "code_version": "denmark-smiley-contract-v1", + "config_version": "denmark-smiley-contract-v1", + "input_rows": int(parse_meta["rows_parsed"]), + "normalized_rows": int(parse_meta["rows_parsed"]), + "quarantined_rows": 0, + "release_state": "not-created", + "publication_state": "private-candidate", + "review_state": "review_required", + "privacy_gate": "pending", + "coordinate_gate": "review_required", + "parsed_sha256": sha256_file(run_dir / "01-parse" / "parsed-rows.jsonl"), + "normalized_sha256": sha256_file(run_dir / "03-classify" / "classified-records.jsonl"), + "anomaly_counts": {str(k): int(v) for k, v in validation.get("counts", {}).items()}, + "acquisition": metadata or {"source_url": source_url, "retrieved_at_utc": retrieved}, + "pipeline_started_at_utc": started_at, + "pipeline_completed_at_utc": completed_at, + } + validate_private_manifest(manifest) + atomic_json(run_dir / "manifest.json", manifest) + report = write_private_run_report(run_dir, manifest) + # Using the recorded observation time as the default makes local reruns + # byte-identical. Callers needing wall-clock freshness may override it. + if retrieved: + snapshot = build_health_snapshot(run_dir, as_of_utc=retrieved) + write_health_snapshot(run_dir / "source-health.json", snapshot) + LOGGER.info("private evidence source=dk.smiley rows=%d findings=%d qa=%s", + report["normalized_rows"], validation.get("finding_records", 0), run_dir / "qa.json") + + +def main(*, run_stage_fn: Callable[[str, Path, list[str]], None] | None = None, + artifact_manifest_fn: Callable[..., Path] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("input", nargs="?", type=Path, help="Previously archived or local Smileydata.xml") + parser.add_argument("--fetch", action="store_true", help="Acquire dk.smiley first; requires --terms-review and does not import or promote.") + parser.add_argument("--terms-review", type=Path, help="Approved terms-review JSON required with --fetch.") + parser.add_argument("--source-url", help="Optional requested URL override for --fetch.") + parser.add_argument("--raw-output-root", type=Path, default=PROJECT_ROOT / "data/raw") + parser.add_argument("--run-id", help="Acquisition run ID for --fetch.") + parser.add_argument("--rules", type=Path, default=PROJECT_ROOT / "pipeline/config/denmark-classification-v1.json") + parser.add_argument("--output-dir", type=Path, help="Run directory; defaults to data/staging/") + parser.add_argument("--expected-rows", type=int) + parser.add_argument("--geocode-limit", type=int) + parser.add_argument("--geocode-delay", type=float, default=1.0) + parser.add_argument("--geocode-provider-config", type=Path, default=PROJECT_ROOT / "pipeline/config/geocoding-dev.json") + parser.add_argument("--geocode-terms-review", type=Path) + parser.add_argument("--geocode-suppression-keys", type=Path) + args = parser.parse_args() + stage = run_stage_fn or run_stage + inventory = artifact_manifest_fn or artifact_manifest + logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s", datefmt="%Y-%m-%dT%H:%M:%SZ") + if args.fetch and args.input is not None: + parser.error("input cannot be provided with --fetch") + if not args.fetch and args.input is None: + parser.error("input is required unless --fetch is used") + if args.fetch and args.terms_review is None: + parser.error("--terms-review is required with --fetch") + if args.geocode_limit is not None and args.geocode_terms_review is None: + parser.error("--geocode-terms-review is required with --geocode-limit") + metadata: dict = {} + if args.fetch: + acquisition_run_id = args.run_id or (datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + "-" + uuid.uuid4().hex[:8]) + acquisition_args = ["--fetch", "--output-root", str(args.raw_output_root.resolve()), "--terms-review", str(args.terms_review.resolve()), "--run-id", acquisition_run_id] + if args.source_url: + acquisition_args.extend(["--url", args.source_url]) + stage("acquire", DENMARK_STAGES / "acquire-denmark-smiley.py", acquisition_args) + input_path = (args.raw_output_root.resolve() / "dk.smiley" / acquisition_run_id / "Smileydata.xml") + metadata = json.loads((input_path.parent / "acquisition-metadata.json").read_text(encoding="utf-8")) + else: + input_path = args.input.resolve() + sidecar = input_path.parent / "acquisition-metadata.json" + if sidecar.is_file(): + metadata = json.loads(sidecar.read_text(encoding="utf-8")) + if not input_path.is_file(): + LOGGER.error("pipeline status=failed reason=input_not_found input=%s", input_path) + return 2 + run_dir = (args.output_dir or PROJECT_ROOT / "data/staging/denmark-smiley" / datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")).resolve() + run_dir.mkdir(parents=True, exist_ok=True) + started_at = utc_now() + try: + parse_dir, normalize_dir = run_dir / "01-parse", run_dir / "02-normalize" + classify_dir, validate_dir = run_dir / "03-classify", run_dir / "04-validate" + geocode_dir = run_dir / "05-geocode-queue" + stage("parse", DENMARK_STAGES / "parse-denmark-smiley.py", [str(input_path), "--output-dir", str(parse_dir)]) + stage("normalize", DENMARK_STAGES / "normalize-denmark-smiley.py", [str(parse_dir / "parsed-rows.jsonl"), "--output-dir", str(normalize_dir)]) + stage("classify", DENMARK_STAGES / "classify-denmark.py", [str(normalize_dir / "normalized-records.jsonl"), "--rules", str(args.rules.resolve()), "--output-dir", str(classify_dir)]) + validation_args = [str(classify_dir / "classified-records.jsonl"), "--output-dir", str(validate_dir)] + if args.expected_rows is not None: + validation_args.extend(["--expected-rows", str(args.expected_rows)]) + stage("validate", DENMARK_STAGES / "validate-denmark.py", validation_args) + stage("geocode_queue", SHARED_STAGES / "create-geocode-queue.py", [str(classify_dir / "classified-records.jsonl"), "--output-dir", str(geocode_dir)]) + if args.geocode_limit is not None: + geo = [str(geocode_dir / "geocode-queue.jsonl"), "--output", str(run_dir / "06-geocode-results.jsonl"), "--limit", str(args.geocode_limit), "--delay", str(args.geocode_delay), "--provider-config", str(args.geocode_provider_config.resolve()), "--terms-review", str(args.geocode_terms_review.resolve()), "--network"] + if args.geocode_suppression_keys: + geo.extend(["--suppression-keys", str(args.geocode_suppression_keys.resolve())]) + stage("geocode_dawa", DENMARK_STAGES / "geocode-denmark-dawa.py", geo) + completed_at = utc_now() + inventory(run_dir, input_path, started_at, completed_at) + status = {"status": "private-candidate", "publication_state": "private-candidate", "candidate_created": False, "release_promoted": False, "public_surfaces": {"api": False, "map": False, "export": False, "cache": False, "history": False}, "run_dir": str(run_dir)} + atomic_json(run_dir / "run-status.json", status) + if (run_dir / "01-parse" / "run-metadata.json").is_file() and (run_dir / "04-validate" / "validation-report.json").is_file(): + _canonical_evidence(run_dir, input_path, metadata, started_at, completed_at) + # Refresh the inventory after shared evidence is written so operators + # can verify the complete private run from the historical manifest. + inventory(run_dir, input_path, started_at, completed_at) + LOGGER.info("pipeline=denmark-smiley status=success run_dir=%s", run_dir) + except subprocess.CalledProcessError as error: + inventory(run_dir, input_path, started_at, utc_now(), "failed", f"stage exited with code {error.returncode}") + atomic_json(run_dir / "run-status.json", {"status": "failed", "publication_state": "unchanged", "release_promoted": False, "error": str(error), "run_dir": str(run_dir)}) + return error.returncode or 1 + except Exception as error: + inventory(run_dir, input_path, started_at, utc_now(), "failed", str(error)) + atomic_json(run_dir / "run-status.json", {"status": "failed", "publication_state": "unchanged", "release_promoted": False, "error": str(error), "run_dir": str(run_dir)}) + LOGGER.exception("pipeline=denmark-smiley status=failed") + return 1 + return 0 diff --git a/pipeline/sources/denmark/run-denmark-pipeline.py b/pipeline/sources/denmark/run-denmark-pipeline.py index 3181404..2854560 100644 --- a/pipeline/sources/denmark/run-denmark-pipeline.py +++ b/pipeline/sources/denmark/run-denmark-pipeline.py @@ -5,8 +5,8 @@ PROJECT_ROOT = Path(__file__).resolve().parents[3] sys.path.insert(0, str(PROJECT_ROOT)) -from pipeline.sources.denmark.cli import main +from pipeline.sources.denmark.pipeline import main if __name__ == "__main__": - main("run-denmark-pipeline.py") + raise SystemExit(main()) diff --git a/pipeline/sources/denmark/stages/acquire-denmark-smiley.py b/pipeline/sources/denmark/stages/acquire-denmark-smiley.py index 8320509..b226bd2 100644 --- a/pipeline/sources/denmark/stages/acquire-denmark-smiley.py +++ b/pipeline/sources/denmark/stages/acquire-denmark-smiley.py @@ -14,6 +14,13 @@ import uuid from datetime import datetime, timezone from pathlib import Path +import sys + +for _candidate in Path(__file__).resolve().parents: + if (_candidate / "pipeline").is_dir(): + sys.path.insert(0, str(_candidate)) + break +from pipeline.contracts.source_lifecycle import atomic_json SOURCE_ID = "dk.smiley" @@ -93,7 +100,7 @@ def archive_stream(stream, artifact_path: Path, *, max_bytes: int) -> tuple[str, def write_metadata(path: Path, metadata: dict) -> None: - path.write_text(json.dumps(metadata, ensure_ascii=False, indent=2, sort_keys=True) + "\n", encoding="utf-8") + atomic_json(path, metadata) def archive_local_file(local_file: Path, output_root: Path, *, run_id: str, retrieved_at: str | None = None) -> dict: diff --git a/pipeline/sources/denmark/test_adapter.py b/pipeline/sources/denmark/test_adapter.py index 5c1c091..1e6a653 100644 --- a/pipeline/sources/denmark/test_adapter.py +++ b/pipeline/sources/denmark/test_adapter.py @@ -3,6 +3,7 @@ from pathlib import Path from .adapter import DenmarkSmileyAdapter, check_refresh from pipeline.contracts.adapter_contract import SourceArtifact +from pipeline.common.orchestrator import run_private_lifecycle XML = b'1TestUnkeyed' @@ -57,4 +58,31 @@ def test_refresh_guards_reject_schema_count_and_duplicate_drift(self): with self.assertRaisesRegex(ValueError, "duplicate"): DenmarkSmileyAdapter().write_candidate_handoff(Path(d), self.artifact(), [duplicate, duplicate]) + def test_canonical_lifecycle_emits_private_qa_status_and_deterministic_health(self): + with tempfile.TemporaryDirectory() as d: + root = Path(d); raw = root / "raw.xml"; raw.write_bytes(XML) + first = run_private_lifecycle(raw, root / "runs-one", self.artifact(), DenmarkSmileyAdapter()) + second = run_private_lifecycle(raw, root / "runs-two", self.artifact(), DenmarkSmileyAdapter()) + self.assertEqual(first["status"], "candidate-ready") + self.assertEqual(first["manifest"]["contract_version"], "source-lifecycle-v1") + self.assertEqual(first["manifest"], second["manifest"]) + first_dir, second_dir = Path(first["run_dir"]), Path(second["run_dir"]) + self.assertTrue((first_dir / "qa.json").is_file()) + self.assertTrue((first_dir / "run-status.json").is_file()) + self.assertTrue((first_dir / "source-health.json").is_file()) + self.assertEqual((first_dir / "source-health.json").read_bytes(), (second_dir / "source-health.json").read_bytes()) + health = json.loads((first_dir / "source-health.json").read_text()) + self.assertEqual(health["health_state"], "private-validated") + self.assertFalse(health["public_exposure"]) + self.assertNotIn("source_values", (first_dir / "source-health.json").read_text()) + + def test_canonical_lifecycle_rejects_malformed_xml_without_health(self): + with tempfile.TemporaryDirectory() as d: + root = Path(d); raw = root / "bad.xml"; raw.write_bytes(b"") + status = run_private_lifecycle(raw, root / "runs", self.artifact(raw.read_bytes()), DenmarkSmileyAdapter()) + self.assertEqual(status["status"], "failed") + run_dir = Path(status["run_dir"]) + self.assertFalse((run_dir / "source-health.json").exists()) + self.assertFalse((run_dir / "release-candidate" / "records.jsonl").exists()) + if __name__ == "__main__": unittest.main() From 73acbc0be0426b3f6024f8ec2a8c45bc9d06a03c Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Tue, 15 Sep 2026 10:28:05 -0700 Subject: [PATCH 111/311] Add end-to-end suppression re-exposure safeguards --- docs/governance/policy-implementation-todo.md | 8 +- docs/governance/suppression-runbook.md | 94 ++++++ .../024_suppression_case_events.sql | 91 ++++++ .../scripts/stages/enqueue-geocode-jobs.py | 15 +- pipeline/scripts/stages/geocode-worker.py | 7 +- pipeline/scripts/stages/lift-suppression.py | 51 ++++ pipeline/scripts/stages/restrict-record.py | 40 ++- .../sources/denmark/stages/import-denmark.py | 10 +- .../tests/e2e/test_suppression_lifecycle.py | 280 ++++++++++++++++++ pipeline/tests/test_lift_suppression.py | 30 ++ 10 files changed, 615 insertions(+), 11 deletions(-) create mode 100644 docs/governance/suppression-runbook.md create mode 100644 pipeline/migrations/024_suppression_case_events.sql create mode 100644 pipeline/scripts/stages/lift-suppression.py create mode 100644 pipeline/tests/e2e/test_suppression_lifecycle.py create mode 100644 pipeline/tests/test_lift_suppression.py diff --git a/docs/governance/policy-implementation-todo.md b/docs/governance/policy-implementation-todo.md index e4408e0..c363517 100644 --- a/docs/governance/policy-implementation-todo.md +++ b/docs/governance/policy-implementation-todo.md @@ -104,4 +104,10 @@ export, aggregate-count, historical/cache, or production operational controls. ## Current implementation boundary -The documentation is aligned with ETHICS.md. The current database migration rejects updates/deletes on evidence tables; a policy-compliant exceptional-removal path and end-to-end publication enforcement have not been established by this documentation task. Do not treat unchecked work as complete or launch affected public capabilities based on policy text alone. Independent acquisition work may continue within the governing retention/access rules. +The V2 checkpoint now includes a synthetic, tested urgent suppression path with +durable source-key references, append-only lift decisions, geocoding guards, +release gates, and backup replay verification. This is evidence for the +covered disposable V2 surfaces only. Production still needs an independently +operated restriction ledger and enforced service-start/deployment gate, plus +the remaining checklist items below. Do not treat unchecked work as complete +or launch affected public capabilities based on policy text alone. diff --git a/docs/governance/suppression-runbook.md b/docs/governance/suppression-runbook.md new file mode 100644 index 0000000..21295ff --- /dev/null +++ b/docs/governance/suppression-runbook.md @@ -0,0 +1,94 @@ +# Suppression and re-exposure runbook + +This is the operator procedure for a credible residential, private-location, +wrong-property, or harmful-identifying-information concern. Until Every Cage +currently has one authorized publication operator. Do not imply that a backup +reviewer, legal guarantee, or emergency-response service exists. + +## Urgent intake + +Use `untileverycageproject@protonmail.com` and ask the requester to provide the +public record ID/link, concern, requested action, and optional supporting +evidence. Ask for the least intrusive verification needed; do not request an +identity document or another private address. Keep the case ID and requester +details in restricted operator storage, not in a public issue. + +1. Assign a case ID and log only the reason category, affected opaque record + reference, decision, operator, timestamps, and affected systems. Never copy + an address, coordinates, private text, requester identity, or raw payload + into a suppression event or test output. +2. Urgently suppress the source record. The command below creates an active, + append-only case plus a source ID/record-key reference and a legacy access + revocation event. It does not mutate retained evidence: + + ```powershell + python pipeline/scripts/stages/restrict-record.py ` + --reason privacy --scope whole_record --policy-version ethics-v1 ` + --maintainer --database-url + ``` + + `whole_record` is the safe default for an unresolved residential or + wrong-location concern. Address/coordinate scopes are retained for the + case reference, but current public projections fail closed for the whole + source record because the current V2 API has no public field-level address + projection. +3. Verify list/detail API, map and facet queries, CSV exports, promoted and + historical release projections, candidate previews, and any known embeds or + caches. Record aggregate pass/fail results and opaque IDs only. + +## Review, retention, and lift + +Assess relevance, source evidence, residential/private overlap, accuracy, and +whether restricted evidence is still necessary to retain. A source being +government-published does not defeat the concern. Preserve non-sensitive +history; consider redaction or deletion of protected evidence separately and +assess preservation obligations before exceptional removal. + +If the concern remains unresolved, keep the case active or in review and set a +next review date. Closure or expiry does not restore access. Only an explicit, +documented review decision may lift a restriction: + +```powershell +python pipeline/scripts/stages/lift-suppression.py ` + --policy-version ethics-v1 --maintainer ` + --database-url +``` + +This appends a `lifted` case event and an explicit restoration event; it never +updates or deletes the original case, reference, source record, geocode result, +release member, or artifact. Reconsideration should use new evidence and a +second reviewer where one is actually available; do not invent one. + +## Propagation checklist + +The durable reference is keyed by `source_id` + `source_record_key`, so the +restriction follows same-source reimports and new snapshots. Before any +release is served, verify: + +- `/api/v2/locations`, detail, facets, CSV, and the loopback-only candidate + preview omit the restricted record; +- `map_facilities_public`, `map_facilities_display`, and + `map_facilities_display_history` return no restricted row; +- renewed geocoding is not queued or performed for a restricted source record, + and any already-retained geocode evidence remains private; +- validation and promotion reject a reconstructed release containing the + restricted record; +- the current restriction ledger digest is replayed into a restored database + before service start. An old backup must fail the pre-service gate; +- known distributed artifacts, previews, and caches are withdrawn or rebuilt. + +There is no V2 persistent application cache or raw-artifact preview endpoint in +this checkpoint. V2 responses are generated from the suppression-aware SQL +projections. The legacy embedded `/api/locations` route has no reviewed V1/V2 +identity crosswalk and is not a route for the maintained V2 release; no +crosswalk must be added without a suppression propagation design and test. +Independent third-party copies cannot be recalled by the project; disclose +that limitation and take feasible correction steps. + +## Evidence of completion + +Run the synthetic lifecycle test and the backup/restore drill with +`UEC_RUN_E2E=1`. Attach only aggregate results, commit ID, policy version, and +opaque case/record references to the operator log. If any controlled surface +fails, keep the affected publication capability stopped or restricted and +escalate to the responsible maintainer. diff --git a/pipeline/migrations/024_suppression_case_events.sql b/pipeline/migrations/024_suppression_case_events.sql new file mode 100644 index 0000000..42de5b7 --- /dev/null +++ b/pipeline/migrations/024_suppression_case_events.sql @@ -0,0 +1,91 @@ +-- Append-only suppression decisions. A case row is immutable, so an explicit +-- lift is represented as a new event rather than mutating the original case. +CREATE TABLE IF NOT EXISTS uec.suppression_case_events ( + suppression_case_event_id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + case_id UUID NOT NULL REFERENCES uec.suppression_cases(case_id), + event_type TEXT NOT NULL CHECK (event_type IN ('suppressed', 'lifted')), + reason_category TEXT NOT NULL CHECK (reason_category IN ('privacy', 'safety', 'legal', 'other')), + policy_version TEXT NOT NULL, + actor TEXT NOT NULL, + decision TEXT NOT NULL, + occurred_at TIMESTAMPTZ NOT NULL DEFAULT now(), + note TEXT +); + +CREATE INDEX IF NOT EXISTS suppression_case_events_current_idx + ON uec.suppression_case_events (case_id, occurred_at DESC, suppression_case_event_id DESC); + +CREATE TRIGGER suppression_case_events_append_only + BEFORE UPDATE OR DELETE ON uec.suppression_case_events + FOR EACH ROW EXECUTE FUNCTION uec.reject_evidence_mutation(); + +-- Access revocations/restorations are also retained as append-only decisions; +-- current access is derived from their latest event. +DROP TRIGGER IF EXISTS record_access_events_append_only ON uec.record_access_events; +CREATE TRIGGER record_access_events_append_only + BEFORE UPDATE OR DELETE ON uec.record_access_events + FOR EACH ROW EXECUTE FUNCTION uec.reject_evidence_mutation(); + +-- Existing cases receive a non-sensitive initial event. New cases are covered +-- by the trigger below, so direct SQL and the operator command share the same +-- lifecycle semantics. +INSERT INTO uec.suppression_case_events + (case_id, event_type, reason_category, policy_version, actor, decision, occurred_at) +SELECT case_id, 'suppressed', reason_category, policy_version, actor, decision, created_at +FROM uec.suppression_cases case_record +WHERE NOT EXISTS ( + SELECT 1 FROM uec.suppression_case_events event + WHERE event.case_id = case_record.case_id +); + +CREATE OR REPLACE FUNCTION uec.record_initial_suppression_case_event() +RETURNS trigger LANGUAGE plpgsql AS $$ +BEGIN + INSERT INTO uec.suppression_case_events + (case_id, event_type, reason_category, policy_version, actor, decision, occurred_at) + VALUES (NEW.case_id, 'suppressed', NEW.reason_category, NEW.policy_version, + NEW.actor, NEW.decision, NEW.created_at); + RETURN NEW; +END; +$$; + +DROP TRIGGER IF EXISTS suppression_cases_initial_event ON uec.suppression_cases; +CREATE TRIGGER suppression_cases_initial_event + AFTER INSERT ON uec.suppression_cases + FOR EACH ROW EXECUTE FUNCTION uec.record_initial_suppression_case_event(); + +CREATE OR REPLACE VIEW uec.suppression_case_current AS +SELECT DISTINCT ON (case_id) + case_id, event_type, reason_category, policy_version, actor, + decision, occurred_at +FROM uec.suppression_case_events +ORDER BY case_id, occurred_at DESC, suppression_case_event_id DESC; + +-- A lift is effective only through this explicit event. Closure, expiry, or +-- time passing remains restrictive until a documented lift is recorded. +CREATE OR REPLACE VIEW uec.public_access_restricted AS +SELECT source_record_id, reason_category, policy_version, occurred_at +FROM uec.record_access_current +WHERE action = 'public_access_revoked' +UNION +SELECT record.source_record_id, current_case.reason_category, + current_case.policy_version, current_case.occurred_at +FROM uec.suppression_case_current current_case +JOIN uec.suppression_cases case_record ON case_record.case_id = current_case.case_id +JOIN uec.suppression_references ref ON ref.case_id = case_record.case_id +JOIN uec.source_records record ON ( + ref.facility_id IS NOT NULL AND ( + EXISTS (SELECT 1 FROM uec.facility_source_links link + WHERE link.facility_id = ref.facility_id + AND link.source_record_id = record.source_record_id) + OR EXISTS (SELECT 1 FROM uec.observations observation + WHERE observation.facility_id = ref.facility_id + AND observation.source_record_id = record.source_record_id) + ) + OR (ref.source_id = record.source_id AND ref.source_record_key = record.source_record_key) +) +WHERE current_case.event_type = 'suppressed' + AND case_record.status IN ('active', 'review', 'closed', 'expired'); + +COMMENT ON VIEW uec.suppression_case_current IS + 'Current append-only suppression decision; only an explicit lifted event ends a case restriction.'; diff --git a/pipeline/scripts/stages/enqueue-geocode-jobs.py b/pipeline/scripts/stages/enqueue-geocode-jobs.py index 70a3e67..81d88c7 100644 --- a/pipeline/scripts/stages/enqueue-geocode-jobs.py +++ b/pipeline/scripts/stages/enqueue-geocode-jobs.py @@ -25,8 +25,19 @@ def enqueue(records_path: Path, database_url: str, provider_id: str, limit: int query = ", ".join(filter(None, [address["street"], address["postal_code"], address.get("city"), "Denmark"])) source_record_id = uuid.uuid5(uuid.NAMESPACE_URL, f"urn:uec:source-record:dk.smiley:{record['source_record_key']}:{record['source_artifact_sha256']}") job = connection.execute( - "INSERT INTO uec.geocode_jobs (source_record_id, provider_id, query) VALUES (%s, %s, %s) ON CONFLICT (source_record_id, provider_id, query) DO NOTHING RETURNING job_id", - (source_record_id, provider_id, query), + """ + INSERT INTO uec.geocode_jobs (source_record_id, provider_id, query) + SELECT record.source_record_id, %s, %s + FROM uec.source_records record + WHERE record.source_record_id=%s + AND NOT EXISTS ( + SELECT 1 FROM uec.public_access_restricted restricted + WHERE restricted.source_record_id=record.source_record_id + ) + ON CONFLICT (source_record_id, provider_id, query) DO NOTHING + RETURNING job_id + """, + (provider_id, query, source_record_id), ).fetchone() if job: connection.execute("INSERT INTO uec.geocode_job_events (job_id, event_type, attempt_number, occurred_at) VALUES (%s, 'queued', 1, %s)", (job[0], datetime.now(timezone.utc))) diff --git a/pipeline/scripts/stages/geocode-worker.py b/pipeline/scripts/stages/geocode-worker.py index 8c97388..ea3f7ff 100644 --- a/pipeline/scripts/stages/geocode-worker.py +++ b/pipeline/scripts/stages/geocode-worker.py @@ -26,7 +26,12 @@ def run(database_url: str, provider_id: str, limit: int | None, delay: float, re SELECT job.job_id, job.source_record_id, job.provider_id, job.query, COALESCE(current.attempt_number, 0) FROM uec.geocode_jobs AS job LEFT JOIN uec.geocode_job_current AS current ON current.job_id = job.job_id - WHERE job.provider_id = %s AND (current.event_type IS NULL OR current.event_type = 'queued' OR (current.event_type = 'failed' AND current.retryable)) + WHERE job.provider_id = %s + AND (current.event_type IS NULL OR current.event_type = 'queued' OR (current.event_type = 'failed' AND current.retryable)) + AND NOT EXISTS ( + SELECT 1 FROM uec.public_access_restricted restricted + WHERE restricted.source_record_id = job.source_record_id + ) ORDER BY job.created_at, job.job_id LIMIT 1 """, (provider_id,)).fetchone() if not job: diff --git a/pipeline/scripts/stages/lift-suppression.py b/pipeline/scripts/stages/lift-suppression.py new file mode 100644 index 0000000..77a693e --- /dev/null +++ b/pipeline/scripts/stages/lift-suppression.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 +"""Record an explicit review-authorized lift for an append-only suppression case.""" +import argparse +import os +import uuid + +import psycopg + +DEFAULT_DB = "postgresql://uec:uec-local-development-only@localhost:5433/uec" + + +def lift(database_url, case_id, policy_version, maintainer, note=None): + with psycopg.connect(database_url) as connection: + with connection.transaction(): + case = connection.execute( + "SELECT reason_category FROM uec.suppression_cases WHERE case_id=%s", + (case_id,), + ).fetchone() + if not case: + raise ValueError("suppression case not found") + # Do not persist operator notes: they can contain private evidence. + connection.execute(""" + INSERT INTO uec.suppression_case_events + (case_id, event_type, reason_category, policy_version, actor, decision) + VALUES (%s, 'lifted', %s, %s, %s, 'lift') + """, (case_id, case[0], policy_version, maintainer)) + connection.execute(""" + INSERT INTO uec.record_access_events + (access_event_id, source_record_id, action, reason_category, policy_version, maintainer) + SELECT gen_random_uuid(), source_record_id, 'public_access_restored', + %s, %s, %s + FROM uec.source_records record + JOIN uec.suppression_references ref + ON ref.case_id=%s + AND ((ref.source_id=record.source_id AND ref.source_record_key=record.source_record_key) + OR EXISTS (SELECT 1 FROM uec.observations observation + WHERE observation.facility_id=ref.facility_id + AND observation.source_record_id=record.source_record_id)) + """, (case[0], policy_version, maintainer, case_id)) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("case_id", type=uuid.UUID) + parser.add_argument("--policy-version", required=True) + parser.add_argument("--maintainer", required=True) + parser.add_argument("--note", help="accepted for CLI compatibility; sensitive intake notes are not stored") + parser.add_argument("--database-url", default=os.environ.get("UEC_DATABASE_URL", DEFAULT_DB)) + args = parser.parse_args() + lift(args.database_url, args.case_id, args.policy_version, args.maintainer, args.note) + print("Recorded explicit suppression lift; evidence was not modified.") diff --git a/pipeline/scripts/stages/restrict-record.py b/pipeline/scripts/stages/restrict-record.py index d48577c..403c80f 100644 --- a/pipeline/scripts/stages/restrict-record.py +++ b/pipeline/scripts/stages/restrict-record.py @@ -1,20 +1,47 @@ #!/usr/bin/env python3 -"""Record an append-only public-access safety restriction.""" +"""Urgently suppress a record and create a durable, payload-free case reference.""" import argparse, os, uuid import psycopg DEFAULT_DB = "postgresql://uec:uec-local-development-only@localhost:5433/uec" -def restrict(database_url, source_record_id, reason_category, policy_version, maintainer, note=None): +def restrict(database_url, source_record_id, reason_category, policy_version, maintainer, note=None, scope="whole_record"): if reason_category not in {"privacy", "safety", "legal", "other"}: raise ValueError("reason category must be privacy, safety, legal, or other") + if scope not in {"address", "coordinates", "whole_record"}: + raise ValueError("scope must be address, coordinates, or whole_record") with psycopg.connect(database_url) as connection: with connection.transaction(): + source = connection.execute(""" + SELECT source_id, source_record_key, + (SELECT facility_id FROM uec.observations + WHERE source_record_id = record.source_record_id + ORDER BY observed_at DESC, observation_id DESC LIMIT 1) + FROM uec.source_records record + WHERE source_record_id = %s + """, (source_record_id,)).fetchone() + if not source: + raise ValueError("source record not found") + source_id, source_record_key, facility_id = source + case_id = uuid.uuid4() + # The note is intentionally not retained: operator intake notes + # may contain private evidence. Keep audit metadata minimal. + connection.execute(""" + INSERT INTO uec.suppression_cases + (case_id, reason_category, status, policy_version, actor, decision) + VALUES (%s, %s, 'active', %s, %s, 'suppress') + """, (case_id, reason_category, policy_version, maintainer)) + connection.execute(""" + INSERT INTO uec.suppression_references + (case_id, facility_id, source_id, source_record_key, scope) + VALUES (%s, %s, %s, %s, %s) + """, (case_id, facility_id, source_id, source_record_key, scope)) connection.execute(""" INSERT INTO uec.record_access_events (access_event_id, source_record_id, action, reason_category, policy_version, maintainer, note) VALUES (%s, %s, 'public_access_revoked', %s, %s, %s, %s) - """, (uuid.uuid4(), source_record_id, reason_category, policy_version, maintainer, note)) + """, (uuid.uuid4(), source_record_id, reason_category, policy_version, maintainer, None)) + return case_id if __name__ == "__main__": p = argparse.ArgumentParser(description=__doc__) @@ -22,8 +49,9 @@ def restrict(database_url, source_record_id, reason_category, policy_version, ma p.add_argument("--reason", required=True, choices=["privacy", "safety", "legal", "other"]) p.add_argument("--policy-version", required=True) p.add_argument("--maintainer", required=True) - p.add_argument("--note") + p.add_argument("--note", help="accepted for CLI compatibility; sensitive intake notes are not stored") + p.add_argument("--scope", choices=["address", "coordinates", "whole_record"], default="whole_record") p.add_argument("--database-url", default=os.environ.get("UEC_DATABASE_URL", DEFAULT_DB)) a = p.parse_args() - restrict(a.database_url, a.source_record_id, a.reason, a.policy_version, a.maintainer, a.note) - print("Recorded public-access revocation; evidence was not modified.") + case_id = restrict(a.database_url, a.source_record_id, a.reason, a.policy_version, a.maintainer, a.note, a.scope) + print(f"Recorded public-access revocation case {case_id}; evidence was not modified.") diff --git a/pipeline/sources/denmark/stages/import-denmark.py b/pipeline/sources/denmark/stages/import-denmark.py index cff8185..203dae8 100644 --- a/pipeline/sources/denmark/stages/import-denmark.py +++ b/pipeline/sources/denmark/stages/import-denmark.py @@ -94,7 +94,15 @@ def run(classified_path: Path, artifact_metadata_path: Path, geocode_path: Path """, (source_record_id, record["source_record_key"], artifact_id, json.dumps(source_fields, ensure_ascii=False), checked_at)) existing = connection.execute("SELECT source_record_id FROM uec.source_records WHERE source_id='dk.smiley' AND source_record_key=%s AND artifact_id=%s", (record["source_record_key"], artifact_id)).fetchone() source_record_id = existing[0] - if geocode: + suppressed = connection.execute( + "SELECT EXISTS (SELECT 1 FROM uec.public_access_restricted WHERE source_record_id=%s)", + (source_record_id,), + ).fetchone()[0] + # A renewed geocoder request is not appropriate after an + # active residential/wrong-location restriction. Keep the + # source snapshot private, but do not create new coordinate + # evidence from its address. + if geocode and not suppressed: queried_at = geocode.get("queried_at_utc") or checked_at.isoformat() geocode_id = uuid.uuid5(uuid.NAMESPACE_URL, f"urn:uec:geocode:{geocode['queue_key']}:{geocode.get('provider', 'unknown')}:{queried_at}") connection.execute(""" diff --git a/pipeline/tests/e2e/test_suppression_lifecycle.py b/pipeline/tests/e2e/test_suppression_lifecycle.py new file mode 100644 index 0000000..bf7eff4 --- /dev/null +++ b/pipeline/tests/e2e/test_suppression_lifecycle.py @@ -0,0 +1,280 @@ +"""Synthetic suppression lifecycle coverage for every controlled V2 route.""" + +import csv +import hashlib +import importlib.util +import io +import json +import os +import unittest +import urllib.error +import urllib.request +import uuid +from datetime import datetime, timedelta, timezone +from pathlib import Path +from unittest.mock import patch + +import psycopg + +try: + from .fixture import E2EEnvironment +except ImportError: + from fixture import E2EEnvironment + + +ROOT = Path(__file__).parents[2] + + +def load_script(path): + spec = importlib.util.spec_from_file_location(path.stem.replace("-", "_"), path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +VALIDATE = load_script(ROOT / "scripts/stages/validate-release.py") +PROMOTE = load_script(ROOT / "scripts/stages/promote-release.py") +WORKER = load_script(ROOT / "scripts/stages/geocode-worker.py") + + +class SuppressionLifecycleE2ETests(unittest.TestCase): + @classmethod + def setUpClass(cls): + if os.environ.get("UEC_RUN_E2E") != "1": + raise unittest.SkipTest("set UEC_RUN_E2E=1 to run Docker-backed E2E tests") + cls.env = E2EEnvironment().start() + cls.seed_synthetic_fixture() + + @classmethod + def tearDownClass(cls): + cls.env.stop() + + @classmethod + def seed_synthetic_fixture(cls): + now = datetime.now(timezone.utc) + cls.private_marker = f"synthetic-private-{uuid.uuid4().hex}" + cls.source_record_id = uuid.uuid4() + cls.facility_id = uuid.uuid4() + cls.observation_id = uuid.uuid4() + artifact_id = uuid.uuid4() + cls.case_id = uuid.uuid4() + with psycopg.connect(cls.env.database_url) as db: + with db.transaction(): + db.execute( + "INSERT INTO uec.sources (source_id,country_code,name,official_url,access_method) " + "VALUES ('e2e.suppression','DK','Synthetic suppression source'," + "'https://example.invalid/suppression','fixture')" + ) + db.execute( + "INSERT INTO uec.releases (release_id,status,ruleset_version,profile,summary) " + "VALUES ('e2e-suppression-old','promoted','synthetic-v1','official','{}')" + ) + manifest = { + "eligible_record_count": 1, + "manifest_version": "v1", + "profile": "official", + "release_id": "e2e-suppression-old", + "ruleset_version": "synthetic-v1", + "source_ids": ["e2e.suppression"], + } + encoded = json.dumps(manifest, sort_keys=True, separators=(",", ":")) + db.execute( + "INSERT INTO uec.release_manifests (release_id,manifest,manifest_sha256) VALUES (%s,%s::jsonb,%s)", + ("e2e-suppression-old", encoded, hashlib.sha256(encoded.encode()).hexdigest()), + ) + db.execute( + "INSERT INTO uec.raw_artifacts (artifact_id,storage_key,sha256,byte_size,retrieved_at) " + "VALUES (%s,'e2e/suppression/old',%s,1,%s)", + (artifact_id, uuid.uuid4().hex * 2, now), + ) + db.execute( + "INSERT INTO uec.source_records " + "(source_record_id,source_id,source_record_key,artifact_id,raw_fields,parsed_at) " + "VALUES (%s,'e2e.suppression','same-source-key',%s,%s::jsonb,%s)", + (cls.source_record_id, artifact_id, json.dumps({"private": cls.private_marker}), now), + ) + db.execute( + "INSERT INTO uec.facilities " + "(facility_id,canonical_name,country_code,street_address,postal_code,city) " + "VALUES (%s,'Synthetic eligible target','DK',%s,'99999','Suppressionby')", + (cls.facility_id, cls.private_marker), + ) + db.execute( + "INSERT INTO uec.observations " + "(observation_id,facility_id,source_record_id,observed_at,observation,classification," + "ruleset_id,rule_id,classification_category,classification_review_status,default_visible," + "coordinate_review_status,first_observed_at) VALUES (%s,%s,%s,%s,'{}','{}','synthetic-v1'," + "'fixture','slaughter','approved',true,'approved',%s)", + (cls.observation_id, cls.facility_id, cls.source_record_id, now, now), + ) + db.execute( + "INSERT INTO uec.release_members (release_id,facility_id,observation_id,default_visible) " + "VALUES ('e2e-suppression-old',%s,%s,true)", + (cls.facility_id, cls.observation_id), + ) + db.execute( + "INSERT INTO uec.publication_review_events " + "(source_record_id,release_id,factual_review_status,privacy_screening_status," + "maintainer_approval,publication_eligible,reviewer_role) VALUES (%s,'e2e-suppression-old'," + "'reviewed','passed','approved',true,'maintainer')", + (cls.source_record_id,), + ) + db.execute( + "INSERT INTO uec.geocode_results " + "(source_record_id,provider_id,query,match_method,status,attempt_number,result,queried_at) " + "VALUES (%s,'synthetic',%s,'fixture','accepted',1,ST_SetSRID(ST_MakePoint(12,56),4326)::geography,%s)", + (cls.source_record_id, cls.private_marker, now), + ) + + def get_json(self, path): + with urllib.request.urlopen(f"http://localhost:{self.env.api_port}{path}", timeout=10) as response: + return response.status, json.loads(response.read()) + + def get_csv(self): + with urllib.request.urlopen( + f"http://localhost:{self.env.api_port}/api/v2/locations.csv?profile=official", timeout=10 + ) as response: + return response.status, response.read().decode("utf-8") + + def test_fixture_is_available_before_suppression(self): + status, body = self.get_json("/api/v2/locations?limit=100") + self.assertEqual(status, 200) + self.assertEqual([row["facility_id"] for row in body["data"]], [str(self.facility_id)]) + status, detail = self.get_json(f"/api/v2/locations/{self.facility_id}") + self.assertEqual(status, 200) + self.assertEqual(detail["data"]["facility_id"], str(self.facility_id)) + _, csv_body = self.get_csv() + self.assertIn("Synthetic eligible target", csv_body) + with psycopg.connect(self.env.database_url) as db: + self.assertEqual( + db.execute("SELECT count(*) FROM uec.map_facilities_public WHERE source_record_id=%s", (self.source_record_id,)).fetchone()[0], + 1, + ) + + def suppress(self): + now = datetime.now(timezone.utc) + with psycopg.connect(self.env.database_url) as db: + with db.transaction(): + if db.execute("SELECT 1 FROM uec.suppression_cases WHERE case_id=%s", (self.case_id,)).fetchone(): + return + db.execute( + "INSERT INTO uec.suppression_cases " + "(case_id,status,reason_category,policy_version,actor,decision,created_at) " + "VALUES (%s,'active','privacy','ethics-v1','synthetic-operator','suppress',%s)", + (self.case_id, now), + ) + db.execute( + "INSERT INTO uec.suppression_references " + "(case_id,source_id,source_record_key,scope) VALUES (%s,'e2e.suppression','same-source-key','whole_record')", + (self.case_id,), + ) + + def test_suppression_covers_api_map_facets_export_and_historical_view(self): + self.suppress() + status, body = self.get_json("/api/v2/locations?limit=100") + self.assertEqual(status, 200) + self.assertEqual(body["data"], []) + with self.assertRaises(urllib.error.HTTPError) as detail_error: + self.get_json(f"/api/v2/locations/{self.facility_id}") + self.assertEqual(detail_error.exception.code, 404) + status, facets = self.get_json("/api/v2/discovery/facets?category=slaughter") + self.assertEqual(status, 200) + self.assertEqual(facets["dimensions"]["category"], []) + _, csv_body = self.get_csv() + self.assertNotIn("Synthetic eligible target", csv_body) + self.assertNotIn(self.private_marker, csv_body) + with psycopg.connect(self.env.database_url) as db: + for view in ("map_facilities_public", "map_facilities_display", "map_facilities_display_history"): + self.assertEqual( + db.execute(f"SELECT count(*) FROM uec.{view} WHERE source_record_id=%s", (self.source_record_id,)).fetchone()[0], + 0, + ) + self.assertEqual( + db.execute("SELECT count(*) FROM uec.public_access_restricted WHERE source_record_id=%s", (self.source_record_id,)).fetchone()[0], + 1, + ) + + def test_reimport_new_geocode_and_release_reconstruction_stay_suppressed(self): + self.suppress() + artifact_id, record_id, observation_id = uuid.uuid4(), uuid.uuid4(), uuid.uuid4() + now = datetime.now(timezone.utc) + timedelta(seconds=1) + with psycopg.connect(self.env.database_url) as db: + with db.transaction(): + db.execute( + "INSERT INTO uec.raw_artifacts (artifact_id,storage_key,sha256,byte_size,retrieved_at) VALUES (%s,'e2e/suppression/new',%s,1,%s)", + (artifact_id, uuid.uuid4().hex * 2, now), + ) + db.execute( + "INSERT INTO uec.source_records (source_record_id,source_id,source_record_key,artifact_id,raw_fields,parsed_at) VALUES (%s,'e2e.suppression','same-source-key',%s,%s::jsonb,%s)", + (record_id, artifact_id, json.dumps({"private": self.private_marker}), now), + ) + db.execute( + "INSERT INTO uec.observations (observation_id,facility_id,source_record_id,observed_at,observation,classification,ruleset_id,rule_id,classification_category,classification_review_status,default_visible,coordinate_review_status,first_observed_at) VALUES (%s,%s,%s,%s,'{}','{}','synthetic-v2','fixture','slaughter','approved',true,'approved',%s)", + (observation_id, self.facility_id, record_id, now, now), + ) + db.execute( + "INSERT INTO uec.geocode_results (source_record_id,provider_id,query,match_method,status,attempt_number,result,queried_at) VALUES (%s,'synthetic',%s,'fixture','accepted',2,ST_SetSRID(ST_MakePoint(13,57),4326)::geography,%s)", + (record_id, self.private_marker, now), + ) + db.execute( + "INSERT INTO uec.releases (release_id,status,ruleset_version,profile,summary) VALUES ('e2e-suppression-rebuilt','candidate','synthetic-v2','official','{}')" + ) + db.execute( + "INSERT INTO uec.release_members (release_id,facility_id,observation_id,default_visible) VALUES ('e2e-suppression-rebuilt',%s,%s,true)", + (self.facility_id, observation_id), + ) + db.execute( + "INSERT INTO uec.publication_review_events (source_record_id,release_id,factual_review_status,privacy_screening_status,maintainer_approval,publication_eligible,reviewer_role) VALUES (%s,'e2e-suppression-rebuilt','reviewed','passed','approved',true,'maintainer')", + (record_id,), + ) + self.assertEqual( + db.execute("SELECT count(*) FROM uec.public_access_restricted WHERE source_record_id=%s", (record_id,)).fetchone()[0], + 1, + ) + self.assertEqual( + db.execute("SELECT count(*) FROM uec.map_facilities_display_history WHERE source_record_id=%s", (record_id,)).fetchone()[0], + 0, + ) + self.assertEqual( + db.execute("SELECT count(*) FROM uec.geocode_results WHERE source_record_id=%s", (record_id,)).fetchone()[0], + 1, + ) + report = VALIDATE.validate(self.env.database_url, "e2e-suppression-rebuilt", 1, False) + self.assertEqual(report["status"], "blocked") + self.assertEqual(report["metrics"]["active_suppression"], 1) + with psycopg.connect(self.env.database_url) as db: + db.execute("UPDATE uec.releases SET status='validated' WHERE release_id='e2e-suppression-rebuilt'") + with self.assertRaisesRegex(ValueError, "active_suppression=1"): + PROMOTE.promote(self.env.database_url, "e2e-suppression-rebuilt", []) + status, body = self.get_json("/api/v2/locations?limit=100") + self.assertEqual(status, 200) + self.assertEqual(body["data"], []) + + def test_queued_geocoding_is_not_renewed_after_suppression(self): + self.suppress() + job_id = uuid.uuid4() + with psycopg.connect(self.env.database_url) as db: + with db.transaction(): + db.execute( + "INSERT INTO uec.geocode_jobs (job_id,source_record_id,provider_id,query) VALUES (%s,%s,'synthetic',%s)", + (job_id, self.source_record_id, self.private_marker), + ) + db.execute( + "INSERT INTO uec.geocode_job_events (job_id,event_type,attempt_number) VALUES (%s,'queued',1)", + (job_id,), + ) + class Adapter: + calls = 0 + + def geocode(self, query): + self.calls += 1 + raise AssertionError("suppressed geocode must not be queried") + + adapter = Adapter() + with patch.object(WORKER, "get_adapter", return_value=adapter): + self.assertEqual(WORKER.run(self.env.database_url, "synthetic", 1, 0, 1), 0) + self.assertEqual(adapter.calls, 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/tests/test_lift_suppression.py b/pipeline/tests/test_lift_suppression.py new file mode 100644 index 0000000..e5ca70d --- /dev/null +++ b/pipeline/tests/test_lift_suppression.py @@ -0,0 +1,30 @@ +import importlib.util +import unittest +import uuid +from pathlib import Path +from unittest.mock import MagicMock, patch + + +SCRIPT = Path(__file__).parents[1] / "scripts" / "stages" / "lift-suppression.py" +SPEC = importlib.util.spec_from_file_location("lift_suppression", SCRIPT) +MODULE = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(MODULE) + + +class LiftSuppressionTests(unittest.TestCase): + def test_lift_is_an_append_only_case_event(self): + connection = MagicMock() + connection.__enter__.return_value = connection + connection.transaction.return_value.__enter__.return_value = connection + connection.execute.return_value.fetchone.side_effect = [("privacy",)] + case_id = uuid.uuid4() + with patch.object(MODULE.psycopg, "connect", return_value=connection): + MODULE.lift("postgresql://test", case_id, "ethics-v1", "maintainer") + queries = [call.args[0] for call in connection.execute.call_args_list] + self.assertTrue(any("event_type, reason_category" in query and "'lifted'" in query for query in queries)) + self.assertTrue(any("public_access_restored" in query for query in queries)) + self.assertFalse(any("UPDATE uec.suppression_cases" in query for query in queries)) + + +if __name__ == "__main__": + unittest.main() From 09e94bb9ecf541efa595a4b94855cf46451ec328 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Tue, 15 Sep 2026 10:56:30 -0700 Subject: [PATCH 112/311] Add private Italy 853/2004 lifecycle parity --- docs/architecture/source-registry.md | 4 +- docs/country-recon-it.md | 11 + docs/source-status.json | 3 +- docs/source-status.md | 5 +- pipeline/source-inventory.csv | 3 +- pipeline/source_registry.json | 30 +- pipeline/source_registry.py | 4 +- pipeline/sources/italy/README.md | 48 +++ pipeline/sources/italy/acquire.py | 315 ++++++++++++++++++ pipeline/sources/italy/it_853_adapter.py | 209 +++++++----- pipeline/sources/italy/refresh.py | 127 ++++++- pipeline/sources/italy/test_acquire.py | 107 ++++++ pipeline/sources/italy/test_it_853_adapter.py | 10 + .../tests/e2e/test_italy_candidate_import.py | 12 +- pipeline/tests/test_source_registry.py | 4 +- 15 files changed, 781 insertions(+), 111 deletions(-) create mode 100644 pipeline/sources/italy/README.md create mode 100644 pipeline/sources/italy/acquire.py create mode 100644 pipeline/sources/italy/test_acquire.py diff --git a/docs/architecture/source-registry.md b/docs/architecture/source-registry.md index 4b7759c..8a1a95c 100644 --- a/docs/architecture/source-registry.md +++ b/docs/architecture/source-registry.md @@ -2,7 +2,7 @@ [`pipeline/source_registry.json`](../../pipeline/source_registry.json) is the machine-readable inventory of source identities represented by the current legacy application data. It is an evidence register, not a claim that any upstream source is current, complete, licensed for redistribution, or approved for publication. -The registry contains one entry per source identity currently represented in `pipeline/source-inventory.csv`. A legacy path proves only that a repository artifact exists; it does not prove the artifact's upstream origin. URLs are populated only where repository scripts or documentation provide them. Every unresolved value is the literal `unknown`, and unresolved work is listed in `blockers`. +The registry contains one entry per source identity currently represented in `pipeline/source-inventory.csv`, plus separately evidenced current candidates that have no legacy row set yet. A legacy path proves only that a repository artifact exists; it does not prove the artifact's upstream origin. URLs are populated only where repository scripts or documentation provide them. Every unresolved value is the literal `unknown`, and unresolved work is listed in `blockers`. Validate it offline from the repository root: @@ -10,6 +10,6 @@ Validate it offline from the repository root: python -m unittest pipeline.tests.test_source_registry ``` -The loader checks the version, required fields, unique IDs, explicit unknowns, safe HTTP(S) URLs, recognized adapter statuses, and that each referenced legacy path exists inside the repository. It never downloads URLs or reads legacy facility contents. A future adapter must separately record retrieval time, byte size, checksum, publication/effective date, code/configuration version, and source terms for each acquired artifact. +The loader checks the version, required fields, unique IDs, explicit unknowns, safe HTTP(S) URLs, recognized adapter statuses, and that each referenced legacy path exists inside the repository. A source candidate may have no legacy paths yet. It never downloads URLs or reads legacy facility contents. A future adapter must separately record retrieval time, byte size, checksum, publication/effective date, code/configuration version, and source terms for each acquired artifact. Do not add raw or derived facility data to this registry. Split mixed or derived legacy material into separately evidenced source IDs before implementing acquisition. diff --git a/docs/country-recon-it.md b/docs/country-recon-it.md index 3ed5f72..0bc1045 100644 --- a/docs/country-recon-it.md +++ b/docs/country-recon-it.md @@ -57,6 +57,17 @@ Private aggregate QA of the 853 snapshot found 41,844 distinct recognition/activ Do not publish names, addresses, tax identifiers, or precise coordinates merely because the Ministry publishes them. Apply residential/private-location screening, source-origin labels, project approval, and publication profile independently. Government-sourced does not mean current, complete, project-approved, or safe to expose. +The private parity implementation is source-scoped as `it.853-2004`: the +catalog-linked acquisition records the catalog/download boundary, response +metadata, terms evidence, raw hash/bytes, and supplied file/catalog dates; +`pipeline.common.orchestrator.run_private_lifecycle` then emits parsed, +normalized, quarantined, QA, run-status, and private-health evidence. The +candidate importer and guarded test-only API are explicit development steps; +they do not promote a release or authorize publication. Regulation 1069/2009 +is registered as `it.1069-2009` but intentionally has no adapter, shared +counts, candidate release, or API integration. A future link between the two +must be a reviewed identity/link event, not a union by recognition number. + ## Limitations This reconnaissance and private candidate adapter do not certify completeness/current accuracy, rate limits/authentication, or publication eligibility. Both artifacts remain private; no public release or healthy-pipeline claim is made. Repeated-activity identity and safe publication treatment of addresses, identifiers, and OSM-derived coordinates remain to be reviewed. diff --git a/docs/source-status.json b/docs/source-status.json index bd92ec9..2a2f294 100644 --- a/docs/source-status.json +++ b/docs/source-status.json @@ -9,7 +9,8 @@ }, "sources": [ {"source_id":"fr.locations","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-fr.md","pipeline/source_registry.json"],"next_action":"Acquire and snapshot an official DGAL Section I/II artifact with terms, schema, and privacy review."}, - {"source_id":"it.locations","metadata":"verified","acquisition":"artifact_private_only","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-it.md","pipeline/source_registry.json","pipeline/sources/italy/it_853_adapter.py","pipeline/tests/e2e/test_italy_candidate_import.py"],"next_action":"Keep the implemented 853/2004 adapter and candidate-import path private and review-gated; validate the separate 1069/2009 variant, source terms, coverage, and privacy mappings before any release review."}, + {"source_id":"it.853-2004","metadata":"verified","acquisition":"artifact_private_only","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-it.md","pipeline/source_registry.json","pipeline/sources/italy/acquire.py","pipeline/sources/italy/it_853_adapter.py","pipeline/sources/italy/README.md","pipeline/tests/e2e/test_italy_candidate_import.py"],"next_action":"Keep catalog acquisition, lifecycle, candidate import, and guarded API evidence private; resolve repeated activity identity, coordinate/address privacy, coverage, and project approval before release review."}, + {"source_id":"it.1069-2009","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-it.md","pipeline/source_registry.json","pipeline/sources/italy/README.md"],"next_action":"Keep the by-products dataset separate; assess scope, schema, terms, identity links, privacy, and a dedicated adapter before any acquisition or integration."}, {"source_id":"mx.locations","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-mx.md","pipeline/source_registry.json"],"next_action":"Resolve DENUE token/terms and SENASICA current directory/schema before bounded acquisition."}, {"source_id":"nz.locations","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-nz.md","pipeline/source_registry.json"],"next_action":"Confirm MPI permitted acquisition route, list-specific terms, and schema before private staging."}, {"source_id":"uk.locations","metadata":"partial","acquisition":"artifact_private_only","runtime_health":"unknown","publication_eligibility":"blocked","evidence":["docs/country-recon-uk.md","docs/countries/uk/fss-approved-establishments-source-assessment.md","pipeline/sources/uk/fsa_approved/README.md","pipeline/sources/uk/fsa_approved/handoff.py","pipeline/sources/uk/fsa_approved/refresh.py","docs/architecture/disposable-candidate-import.md","pipeline/scripts/maintenance/import-candidate.py","pipeline/source_registry.json"],"next_action":"Repeatable source-local refresh dry-run passed on the retained snapshot (5,342 input, 4,300 normalized, 1,042 quarantined, expected schema fingerprint, no drift alarms); use refresh.py dry-run/handoff only in restricted staging. This does not establish production/runtime health or publication eligibility. Complete source-rights, privacy/coordinate, duplicate, coverage, and release review while keeping NI and Scotland separate."}, diff --git a/docs/source-status.md b/docs/source-status.md index c9e1c6b..342ba4f 100644 --- a/docs/source-status.md +++ b/docs/source-status.md @@ -16,7 +16,8 @@ No last-success timestamp is invented. Private artifacts are not proof of a publ | Source ID | Metadata | Acquisition | Runtime health | Publication | Evidence / next action | |---|---|---|---|---|---| | `fr.locations` | verified | blocked | not_run | blocked | DGAL/Alim’confiance/SIRENE/HVE/Agence Bio/Géorisques reconnaissance; acquire permitted official artifact and review terms/privacy | -| `it.locations` | verified | artifact_private_only | not_run | blocked | The 853/2004 adapter and private candidate-import path are covered by deterministic tests; keep them review-gated and validate the separate 1069/2009 variant, source terms, coverage, and privacy mappings before release review | +| `it.853-2004` | verified | artifact_private_only | not_run | blocked | Catalog acquisition, shared lifecycle, adapter, private candidate import, and guarded API checks remain review-gated; repeated activity identity, coordinate/address privacy, coverage, and project approval remain open | +| `it.1069-2009` | verified | not_run | not_run | blocked | Separate by-products catalog candidate; no adapter or integration decision; assess scope, schema, terms, identity links, and privacy | | `mx.locations` | verified | blocked | not_run | blocked | DENUE/SENASICA/DGSIAP reconnaissance; resolve token, directory, terms, and schema | | `nz.locations` | verified | blocked | not_run | blocked | MPI/Stats NZ reconnaissance; resolve 403/access and aggregate-vs-facility boundaries | | `uk.locations` | partial | artifact_private_only | not_run | blocked | Synthetic handoff passes importer pre-DB validation, but no real UK candidate has been imported or previewed; review-required/unapproved defaults, Docker E2E, privacy/coordinate, source-rights, duplicate, coverage, and release review remain open; NI/Scotland stay separate | @@ -28,4 +29,4 @@ No last-success timestamp is invented. Private artifacts are not proof of a publ | `us.aphis` | partial | not_run | not_run | blocked | APHIS export workflow and separate report/license provenance require review | | `us.inspections` | partial | not_run | not_run | blocked | Inspection observations require a current export and explicit identity matching | -The machine-readable file is the source of truth for these statuses. `.locations` IDs may represent composite legacy coverage rather than one upstream source. Candidate feeds mentioned in the France, Mexico, New Zealand, and Italy reconnaissance documents are not silently conflated into a single healthy source; source splitting remains a next action. Country reconnaissance documents provide evidence and next actions; they do not override this status vocabulary or authorize publication. +The machine-readable file is the source of truth for these statuses. Legacy `.locations` paths may represent composite coverage, but the Italy Ministry candidates are now split into explicit 853/2004 and 1069/2009 source IDs. Candidate feeds mentioned in the France, Mexico, and New Zealand reconnaissance documents are not silently conflated into a single healthy source. Country reconnaissance documents provide evidence and next actions; they do not override this status vocabulary or authorize publication. diff --git a/pipeline/source-inventory.csv b/pipeline/source-inventory.csv index b16e2fa..192b32e 100644 --- a/pipeline/source-inventory.csv +++ b/pipeline/source-inventory.csv @@ -4,7 +4,8 @@ de.locations,DE,static_data/de/locations.csv,BVL BLtU approved-establishments da dk.smiley,DK,static_data/dk/locations.csv;static_data/dk/Smiley_xml.xml,Fødevarestyrelsen Find Smiley bulk data,XML download,source_candidate,Prefer the official bulk XML/Excel endpoint over scraping detail pages. es.locations,ES,static_data/es/locations.csv;static_data/es/locations.csv.backup,Spanish competent-authority approved-establishment register,download or assisted export,source_candidate,Exact current authority and payload still need confirmation. fr.locations,FR,static_data/fr/locations.csv;france-data.kml,French DGAL approved-establishment lists,downloaded spreadsheet/text/KML,source_candidate,Prefer current DGAL list files and retain the raw file used for each run. -it.locations,IT,dirty-datasets/it/italy_locations.csv;Old CSVs/italian_facilities.csv,Italian Ministry of Health approved establishments,HTML table or official export,source_candidate,Existing scraper is a reference only; verify current URL and pagination. +it.853-2004,IT,dirty-datasets/it/italy_locations.csv;Old CSVs/italian_facilities.csv,Italian Ministry of Health 853/2004 establishments,catalog-discovered CSV,source_candidate,Private catalog acquisition and lifecycle implemented; repeated activity identity, privacy, and publication review remain open. +it.1069-2009,IT,,Italian Ministry of Health 1069/2009 animal by-products establishments,separate catalog candidate,source_candidate,Keep separate from 853/2004; schema, terms, identity links, and scope require assessment. mx.locations,MX,static_data/mx/locations.csv;Old CSVs/mexico, Mexican official registers and INEGI-derived work,download/API or assisted export,source_candidate,Separate official source records from derived aquaculture research files. nz.locations,NZ,static_data/nz/locations.csv,New Zealand MPI approved premises/registers,download/API,source_candidate,MPI country listings expose identifier, address, processes, species, and expiry fields. uk.locations,GB,static_data/uk/locations.csv;static_data/uk/locations.csv.backup,Food Standards Agency approved food establishments,monthly CSV download,source_candidate,Current FSA publication is split across England/Wales, Northern Ireland, and Scotland. diff --git a/pipeline/source_registry.json b/pipeline/source_registry.json index 05865bd..18be955 100644 --- a/pipeline/source_registry.json +++ b/pipeline/source_registry.json @@ -64,16 +64,28 @@ "blockers": ["Confirm current DGAL publication and whether the KML is an original source or a derived artifact."] }, { - "source_id": "it.locations", - "jurisdiction_scope": "Italy", + "source_id": "it.853-2004", + "jurisdiction_scope": "Italy; Ministry of Health establishments recognized under Regulation (EC) 853/2004", "legacy_paths": ["dirty-datasets/it/italy_locations.csv", "Old CSVs/italian_facilities.csv"], - "url": "https://www.salute.gov.it/consultazioneStabilimenti/ConsultazioneStabilimentiServlet?ACTION=gestioneSingolaCategoria&idNormativa=2&idCategoria=1", - "access_method": "HTML table or official export", - "cadence": "unknown", - "attribution_licensing_notes": "Legacy scraper identifies the Italian Ministry of Health; verify terms and attribution.", - "adapter_status": "reference_only", - "expected_artifact_schema": "HTML/table export -> preserved source rows; legacy CSV schema is not a canonical upstream schema", - "blockers": ["Confirm stable endpoint, pagination, retrieval policy, and source schema; geocoding scripts are separate derived processing."] + "url": "https://www.dati.salute.gov.it/it/dataset/stabilimenti-italiani-gli-alimenti-di-origine-animale/", + "access_method": "stable catalog page -> same-origin dated CSV discovery", + "cadence": "daily", + "attribution_licensing_notes": "Ministry of Health catalog identifies Italian Open Data Licence v2.0; terms evidence and privacy review remain required before publication.", + "adapter_status": "implemented_partial", + "expected_artifact_schema": "catalog-discovered UTF-8 semicolon CSV -> parsed/normalized/quarantined JSONL; source values preserved privately", + "blockers": ["Resolve repeated establishment/activity identity, coded values, coordinate provenance, address privacy, coverage, and project publication approval before release review."] + }, + { + "source_id": "it.1069-2009", + "jurisdiction_scope": "Italy; Ministry of Health establishments for animal by-products under Regulation (EC) 1069/2009", + "legacy_paths": [], + "url": "https://www.dati.salute.gov.it/it/dataset/stabilimenti-italiani-i-sottoprodotti-di-origine-animale/", + "access_method": "not acquired; separate catalog candidate", + "cadence": "daily", + "attribution_licensing_notes": "Separate Ministry catalog and Italian Open Data Licence v2.0 indication; no adapter or publication decision.", + "adapter_status": "not_started", + "expected_artifact_schema": "separate catalog CSV/XML/JSON schema; do not union with 853/2004", + "blockers": ["Assess scope, terms, schema, identity/link semantics, privacy, and whether this source belongs in the project before acquisition."] }, { "source_id": "mx.locations", diff --git a/pipeline/source_registry.py b/pipeline/source_registry.py index f462e4a..d2dc232 100644 --- a/pipeline/source_registry.py +++ b/pipeline/source_registry.py @@ -71,8 +71,8 @@ def validate_registry(payload: object, *, repository_root: Path | None = None) - raise SourceRegistryError(f"{prefix}.url must be an http(s) URL or 'unknown'") if source["adapter_status"] not in VALID_ADAPTER_STATUSES: raise SourceRegistryError(f"{prefix}.adapter_status is not recognized") - if not isinstance(source["legacy_paths"], list) or not source["legacy_paths"]: - raise SourceRegistryError(f"{prefix}.legacy_paths must be non-empty") + if not isinstance(source["legacy_paths"], list): + raise SourceRegistryError(f"{prefix}.legacy_paths must be a list") if not all(isinstance(item, str) and item for item in source["legacy_paths"]): raise SourceRegistryError(f"{prefix}.legacy_paths must contain non-empty strings") if not isinstance(source["blockers"], list) or not all(isinstance(item, str) and item for item in source["blockers"]): diff --git a/pipeline/sources/italy/README.md b/pipeline/sources/italy/README.md new file mode 100644 index 0000000..a7cceed --- /dev/null +++ b/pipeline/sources/italy/README.md @@ -0,0 +1,48 @@ +# Italy Ministry 853/2004 private source + +`it.853-2004` is the Ministry of Health open-data catalog for establishments +recognized under Regulation (EC) 853/2004. The catalog page is the stable +authority boundary; `acquire.py` discovers its current same-origin CSV link, +archives the raw bytes under ignored `data/raw/italy/`, and records the +requested catalog URL, final download URL, retrieval time, response metadata, +SHA-256, byte size, supplied catalog/file dates, source ID, code/configuration +versions, and terms-review evidence. Network acquisition is bounded and +requires an approved private terms-review JSON file. + +Run private acquisition and staging with: + +```text +python -m pipeline.sources.italy.acquire --fetch --terms-review --output-root data/raw --run-id +python -m pipeline.sources.italy.refresh --raw data/raw/it.853-2004//source.csv --run-dir data/staging/italy-853/ +``` + +The refresh uses `run_private_lifecycle`, producing parsed, normalized, +quarantined, row-free QA, restricted run status, and deterministic private +health artifacts. It ends at `candidate-ready`; candidate import and guarded +test-only API checks are separate explicit steps. No release is promoted and +default/public visibility remains zero. + +The adapter treats each establishment/activity row as an observation. It +preserves source values privately, retains source identifiers, represents +unknown dates/geography/coordinates explicitly, and quarantines malformed +rows, unknown statuses, missing identifiers/activity codes, invalid dates, +and repeated recognition/activity identities. Repeated rows are not merged. +Addresses, tax identifiers, and source coordinates are never copied into the +normalized public-shaped fields; any future coordinate or address use requires +separate privacy and project review. + +## 1069/2009 boundary + +The Ministry's animal-by-products catalog is a separate source with a separate +schema, recognition semantics, activity/product codes, and optional links to an +853 recognition number. It is not included in `it.853-2004`, its row counts, +identity rules, candidate release, health snapshot, or API filters. A future +`it.1069-2009` adapter must have its own source ID, acquisition evidence, +schema dictionary, normalization/quarantine rules, privacy review, and an +explicit reviewed identity/linking event before any cross-source relationship +is shown. It must never be silently unioned with 853/2004 facilities. + +This is private staging evidence, not a completeness, accuracy, project- +approval, or publication claim. The catalog notes that some coordinates came +from OpenStreetMap contributors; that provenance does not itself authorize +precise-coordinate publication. diff --git a/pipeline/sources/italy/acquire.py b/pipeline/sources/italy/acquire.py new file mode 100644 index 0000000..adc0317 --- /dev/null +++ b/pipeline/sources/italy/acquire.py @@ -0,0 +1,315 @@ +"""Controlled acquisition for the Italian Ministry 853/2004 catalog. + +The catalog page is the stable authority boundary. The linked CSV filename +contains a publisher date and changes over time, so the downloader discovers +the current same-origin CSV link instead of embedding a transient filename. +Acquisition only preserves evidence; it never parses, imports, or publishes +rows. +""" +from __future__ import annotations + +import argparse +import hashlib +import html.parser +import json +import os +import re +import tempfile +import urllib.error +import urllib.parse +import urllib.request +import uuid +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Callable + +from pipeline.contracts.source_lifecycle import atomic_json + + +SOURCE_ID = "it.853-2004" +ACQUISITION_VERSION = "it-853-acquisition-v1" +CATALOG_URL = "https://www.dati.salute.gov.it/it/dataset/stabilimenti-italiani-gli-alimenti-di-origine-animale/" +CATALOG_HOST = "www.dati.salute.gov.it" +DEFAULT_MAX_BYTES = 128 * 1024 * 1024 +SAFE_CONTENT_TYPES = {"text/csv", "application/csv", "application/octet-stream", "text/plain"} +CSV_LINK = re.compile(r"^/sites/default/files/opendata/STAB_POA_8_(?P\d{8})\.csv$", re.I) + + +class AcquisitionError(ValueError): + """The source could not be safely acquired into private storage.""" + + +def utc_now() -> str: + return datetime.now(timezone.utc).isoformat(timespec="seconds").replace("+00:00", "Z") + + +def default_run_id() -> str: + return datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + "-" + uuid.uuid4().hex[:8] + + +class _LinkParser(html.parser.HTMLParser): + def __init__(self) -> None: + super().__init__(convert_charrefs=True) + self.links: list[str] = [] + + def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: + if tag.lower() != "a": + return + href = dict(attrs).get("href") + if href: + self.links.append(href) + + +def require_terms_review(path: Path) -> dict[str, str]: + try: + review = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as error: + raise AcquisitionError(f"terms review cannot be read: {error}") from error + required = {"reviewer", "reference", "reviewed_at", "decision", "notes"} + if not isinstance(review, dict) or required - review.keys(): + raise AcquisitionError("terms review requires reviewer, reference, reviewed_at, decision, and notes") + for field in required: + if not isinstance(review[field], str) or not review[field].strip(): + raise AcquisitionError(f"terms review {field} must be a non-empty string") + try: + datetime.fromisoformat(review["reviewed_at"].replace("Z", "+00:00")) + except ValueError as error: + raise AcquisitionError("terms review reviewed_at must be ISO-8601") from error + if review["decision"] != "approved": + raise AcquisitionError("terms review decision must be 'approved'") + return {field: review[field] for field in sorted(required)} + + +def _headers(response: Any) -> dict[str, str]: + return { + name: response.headers[name] + for name in ("Content-Type", "Content-Length", "ETag", "Last-Modified") + if response.headers.get(name) is not None + } + + +def _safe_content_type(headers: Any, allowed: set[str]) -> bool: + content_type = headers.get("Content-Type") + return content_type is None or content_type.split(";", 1)[0].strip().lower() in allowed + + +def _archive_stream(stream: Any, destination: Path, *, max_bytes: int) -> tuple[str, int]: + if max_bytes <= 0: + raise AcquisitionError("max_bytes must be positive") + destination.parent.mkdir(parents=True, exist_ok=True) + digest = hashlib.sha256() + total = 0 + temporary: Path | None = None + try: + with tempfile.NamedTemporaryFile("wb", delete=False, dir=destination.parent, prefix=".download-", suffix=".part") as handle: + temporary = Path(handle.name) + while chunk := stream.read(1024 * 1024): + total += len(chunk) + if total > max_bytes: + raise AcquisitionError(f"download exceeds max_bytes={max_bytes}") + digest.update(chunk) + handle.write(chunk) + os.replace(temporary, destination) + return digest.hexdigest(), total + except Exception: + if temporary is not None: + temporary.unlink(missing_ok=True) + raise + + +def _request(url: str, *, timeout_seconds: float) -> Any: + if timeout_seconds <= 0: + raise AcquisitionError("timeout_seconds must be positive") + request = urllib.request.Request(url, headers={"User-Agent": "UntilEveryCage/controlled-acquisition"}) + try: + response = urllib.request.urlopen(request, timeout=timeout_seconds) + except urllib.error.HTTPError as error: + raise AcquisitionError(f"source returned HTTP {error.code}") from error + except urllib.error.URLError as error: + raise AcquisitionError(f"network error: {error.reason}") from error + if not 200 <= response.status < 300: + response.close() + raise AcquisitionError(f"source returned HTTP {response.status}") + return response + + +def discover_csv(catalog_bytes: bytes, catalog_url: str) -> tuple[str, str | None]: + """Return the latest same-origin 853 CSV link and supplied date, if any.""" + parser = _LinkParser() + try: + parser.feed(catalog_bytes.decode("utf-8", errors="strict")) + except UnicodeDecodeError as error: + raise AcquisitionError("catalog is not valid UTF-8 HTML") from error + catalog = urllib.parse.urlparse(catalog_url) + if catalog.hostname != CATALOG_HOST: + raise AcquisitionError("catalog host is not the authoritative Ministry host") + matches: list[tuple[str, str]] = [] + for link in parser.links: + absolute = urllib.parse.urljoin(catalog_url, link) + parsed = urllib.parse.urlparse(absolute) + match = CSV_LINK.fullmatch(parsed.path) + if parsed.scheme != "https" or parsed.hostname != CATALOG_HOST or not match: + continue + matches.append((match.group("date"), absolute)) + if not matches: + raise AcquisitionError("catalog has no supported 853/2004 CSV download link") + date, url = max(matches) + return url, f"{date[:4]}-{date[4:6]}-{date[6:]}" + + +def _catalog_metadata(catalog_bytes: bytes) -> dict[str, str]: + text = catalog_bytes.decode("utf-8", errors="replace") + # The catalog supplies this human-facing value; keep it as evidence and + # never treat it as an observation date for individual rows. + match = re.search(r"Data ultimo aggiornamento\s*]+>\s*[^<]*<[^>]+>\s*(\d{2}/\d{2}/\d{4})", text, re.I) + if not match: + match = re.search(r"Data ultimo aggiornamento.{0,200}?(\d{2}/\d{2}/\d{4})", text, re.I | re.S) + supplied = None + if match: + day, month, year = match.group(1).split("/") + supplied = f"{year}-{month}-{day}" + return {"catalog_last_updated": supplied or "unknown"} + + +def fetch( + *, + output_root: Path, + run_id: str, + terms_review_path: Path, + catalog_url: str = CATALOG_URL, + timeout_seconds: float = 60.0, + max_bytes: int = DEFAULT_MAX_BYTES, + opener: Callable[..., Any] | None = None, +) -> dict[str, Any]: + terms_review = require_terms_review(terms_review_path) + open_url = opener or urllib.request.urlopen + requested_at = utc_now() + catalog_request = urllib.request.Request(catalog_url, headers={"User-Agent": "UntilEveryCage/controlled-acquisition"}) + try: + with open_url(catalog_request, timeout=timeout_seconds) as catalog_response: + if not 200 <= catalog_response.status < 300: + raise AcquisitionError(f"catalog returned HTTP {catalog_response.status}") + if not _safe_content_type(catalog_response.headers, {"text/html", "application/xhtml+xml"}): + raise AcquisitionError(f"unexpected catalog content type: {catalog_response.headers.get('Content-Type')}") + catalog_bytes = catalog_response.read(4 * 1024 * 1024 + 1) + if len(catalog_bytes) > 4 * 1024 * 1024: + raise AcquisitionError("catalog exceeds bounded size") + catalog_final_url = catalog_response.geturl() + catalog_headers = _headers(catalog_response) + except urllib.error.HTTPError as error: + raise AcquisitionError(f"catalog returned HTTP {error.code}") from error + except urllib.error.URLError as error: + raise AcquisitionError(f"catalog network error: {error.reason}") from error + csv_url, filename_date = discover_csv(catalog_bytes, catalog_final_url) + try: + csv_request = urllib.request.Request(csv_url, headers={"User-Agent": "UntilEveryCage/controlled-acquisition"}) + with open_url(csv_request, timeout=timeout_seconds) as response: + if not 200 <= response.status < 300: + raise AcquisitionError(f"CSV returned HTTP {response.status}") + if not _safe_content_type(response.headers, SAFE_CONTENT_TYPES): + raise AcquisitionError(f"unexpected CSV content type: {response.headers.get('Content-Type')}") + run_dir = output_root / SOURCE_ID / run_id + artifact_path = run_dir / "source.csv" + digest, byte_size = _archive_stream(response, artifact_path, max_bytes=max_bytes) + response_headers = _headers(response) + final_url = response.geturl() + except urllib.error.HTTPError as error: + raise AcquisitionError(f"CSV returned HTTP {error.code}") from error + except urllib.error.URLError as error: + raise AcquisitionError(f"CSV network error: {error.reason}") from error + metadata = { + "acquisition_method": "catalog_discovered_network_fetch", + "adapter_version": ACQUISITION_VERSION, + "artifact": artifact_path.name, + "byte_size": byte_size, + "catalog_url": catalog_url, + "catalog_final_url": catalog_final_url, + "catalog_response_headers": catalog_headers, + "catalog_sha256": hashlib.sha256(catalog_bytes).hexdigest(), + "catalog_metadata": _catalog_metadata(catalog_bytes), + "config_version": ACQUISITION_VERSION, + "content_type": response_headers.get("Content-Type", "unknown").split(";", 1)[0].lower(), + "effective_date": "unknown", + "filename_publication_date": filename_date or "unknown", + "final_url": final_url, + "publication_metadata": { + "catalog_last_updated": _catalog_metadata(catalog_bytes).get("catalog_last_updated", "unknown"), + "filename_publication_date": filename_date or "unknown", + **{key: response_headers[key] for key in ("ETag", "Last-Modified") if key in response_headers}, + }, + "requested_at_utc": requested_at, + "requested_url": csv_url, + "response_headers": response_headers, + "retrieved_at_utc": utc_now(), + "run_id": run_id, + "sha256": digest, + "source_id": SOURCE_ID, + "terms_review": terms_review, + } + atomic_json(run_dir / "acquisition-metadata.json", metadata) + return metadata + + +def archive_local_file(local_file: Path, output_root: Path, *, run_id: str, retrieved_at: str | None = None) -> dict[str, Any]: + """Archive a local/synthetic CSV for tests without implying source access.""" + if not local_file.is_file(): + raise AcquisitionError(f"local file does not exist: {local_file}") + run_dir = output_root / SOURCE_ID / run_id + with local_file.open("rb") as stream: + digest, byte_size = _archive_stream(stream, run_dir / "source.csv", max_bytes=DEFAULT_MAX_BYTES) + metadata = { + "acquisition_method": "local_file", + "adapter_version": ACQUISITION_VERSION, + "artifact": "source.csv", + "byte_size": byte_size, + "catalog_url": "unknown", + "catalog_final_url": "unknown", + "catalog_metadata": {}, + "config_version": ACQUISITION_VERSION, + "content_type": "text/csv", + "effective_date": "unknown", + "filename_publication_date": "unknown", + "final_url": "unknown", + "publication_metadata": {}, + "requested_at_utc": retrieved_at or utc_now(), + "requested_url": "unknown", + "response_headers": {}, + "retrieved_at_utc": retrieved_at or utc_now(), + "run_id": run_id, + "sha256": digest, + "source_id": SOURCE_ID, + "terms_review": "not_required_for_local_file", + } + atomic_json(run_dir / "acquisition-metadata.json", metadata) + return metadata + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + mode = parser.add_mutually_exclusive_group(required=True) + mode.add_argument("--fetch", action="store_true") + mode.add_argument("--local-file", type=Path) + parser.add_argument("--terms-review", type=Path) + parser.add_argument("--catalog-url", default=CATALOG_URL) + parser.add_argument("--output-root", type=Path, default=Path("data/raw")) + parser.add_argument("--run-id", default=default_run_id()) + parser.add_argument("--timeout-seconds", type=float, default=60.0) + parser.add_argument("--max-bytes", type=int, default=DEFAULT_MAX_BYTES) + args = parser.parse_args() + try: + if args.fetch: + if args.terms_review is None: + raise AcquisitionError("--terms-review is required with --fetch") + metadata = fetch(output_root=args.output_root, run_id=args.run_id, terms_review_path=args.terms_review, + catalog_url=args.catalog_url, timeout_seconds=args.timeout_seconds, max_bytes=args.max_bytes) + else: + metadata = archive_local_file(args.local_file, args.output_root, run_id=args.run_id) + except AcquisitionError as error: + print(json.dumps({"status": "failed", "error": str(error)}, sort_keys=True)) + return 2 + print(json.dumps({"status": "archived", "metadata": metadata}, ensure_ascii=False, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/pipeline/sources/italy/it_853_adapter.py b/pipeline/sources/italy/it_853_adapter.py index 5c602c9..4d854c9 100644 --- a/pipeline/sources/italy/it_853_adapter.py +++ b/pipeline/sources/italy/it_853_adapter.py @@ -1,126 +1,181 @@ +"""Private adapter for the Ministry 853/2004 CSV contract. + +The source has one row per establishment/activity observation. This adapter +preserves the complete source row, keeps sensitive address/tax/coordinate +values private, and quarantines ambiguity or malformed input instead of +silently merging or repairing it. +""" from __future__ import annotations import csv import hashlib import json +from collections import Counter +from datetime import date from pathlib import Path +from typing import Any + from pipeline.contracts.adapter_contract import SourceArtifact from pipeline.contracts.candidate_handoff import write_handoff +from pipeline.contracts.source_lifecycle import atomic_json, atomic_jsonl, private_manifest + REQUIRED = tuple( "precedente_bollo_cee;num_identificativo_produzione_commercializzazione;ragione_sociale;indirizzo;comune;provincia;codice_regione;regione;classificazione_stabilimento;codice_impianto_attivita;descrizione_impianto_attivita;prodotti_abilitati;specifica_prodotti;paesi_export_autorizzato;longitudine;latitudine;stato_localizzazione;cod_fiscale;p_iva;codice_comune;data_inizio_attivita;data_fine_attivita;stato_attivita;data_ultimo_aggiornamento;num_identificativo_produzione_commercializzazione_2".split(";") ) STATUS = {"AUTORIZZATA": "Autorizzata", "REVOCATA": "Revocata", "SOSPESA": "Sospesa"} +DATE_FIELDS = ("data_inizio_attivita", "data_fine_attivita", "data_ultimo_aggiornamento") -def clean(v): - return v.strip() if v and v.strip() else None +def clean(value: Any) -> str | None: + return value.strip() if isinstance(value, str) and value.strip() else None -def row_id(row, occurrence): - payload = json.dumps(row, sort_keys=True, ensure_ascii=False, separators=(",", ":")) +def row_id(row: dict[str | None, Any], occurrence: int) -> str: + stable_row = {"__extra_columns" if key is None else str(key): value for key, value in row.items()} + payload = json.dumps(stable_row, sort_keys=True, ensure_ascii=False, default=list, separators=(",", ":")) return hashlib.sha256(f"{payload}|{occurrence}".encode()).hexdigest() -# Content identity plus occurrence preserves rerun stability without line-number identity. +def _date_state(value: str | None) -> str: + if not value: + return "unknown" + try: + date.fromisoformat(value) + except ValueError: + return "invalid" + return "known" + + class Italy853Adapter: source_id = "it.853-2004" - adapter_version = "it-853-candidate-v1" + adapter_version = "it-853-candidate-v2" schema_version = "it-853-csv-v2.0" - def write_candidate_handoff(self, run_dir, artifact: SourceArtifact, parsed): + def write_candidate_handoff(self, run_dir: str | Path, artifact: SourceArtifact, + parsed: dict[str, Any]) -> dict[str, Any]: + """Write the generic importer handoff for accepted private rows.""" rows = [item["record"] if "record" in item else item for item in parsed["accepted"]] - for row in rows: - row["normalized"]["establishment_id"] = row["normalized"]["recognition_number"] return write_handoff(run_dir, rows, artifact, source_id=self.source_id) - def parse_bytes(self, content): + def parse_bytes(self, content: bytes) -> dict[str, Any]: digest = hashlib.sha256(content).hexdigest() - rows = list(csv.DictReader(content.decode("utf-8-sig").splitlines(), delimiter=";")) - if not rows or tuple(rows[0]) != REQUIRED: + try: + text = content.decode("utf-8-sig") + reader = csv.DictReader(text.splitlines(), delimiter=";", strict=True) + headers = tuple(reader.fieldnames or ()) + rows = list(reader) + except (UnicodeDecodeError, csv.Error) as error: + raise ValueError("malformed or unsupported UTF-8 CSV") from error + if headers != REQUIRED: raise ValueError("schema drift") - seen = set() - occurrences = {} - accepted = [] - quarantined = [] + + occurrences: Counter[tuple[str | None, str | None]] = Counter() + accepted: list[dict[str, Any]] = [] + quarantined: list[dict[str, Any]] = [] for line, row in enumerate(rows, 2): + if None in row: + # An extra delimited field changes the source shape rather + # than merely making one observation incomplete; fail closed + # so schema drift cannot enter a candidate run. + raise ValueError("schema drift: row has extra columns") rec = clean(row.get(REQUIRED[1])) - act = clean(row.get("codice_impianto_attivita")) - key = (rec, act) - reasons = [] - occurrences[key] = occurrences.get(key, 0) + 1 - if None in row or None in row.values() or any(isinstance(v, list) for v in row.values()): + activity = clean(row.get("codice_impianto_attivita")) + key = (rec, activity) + occurrences[key] += 1 + reasons: list[str] = [] + if any(value is None or isinstance(value, list) for value in row.values()): reasons.append("malformed_row_shape") if not rec: reasons.append("missing_recognition_number") - if not act: + if not activity: reasons.append("missing_activity_code") - # Repeated recognition/activity rows are quarantined instead of silently collapsed. - if key in seen: + if occurrences[key] > 1: reasons.append("ambiguous_repeated_recognition_activity") - seen.add(key) - status = clean(row.get("stato_attivita")) - if status and status.upper() not in STATUS: + raw_status = clean(row.get("stato_attivita")) + if raw_status and raw_status.upper() not in STATUS: reasons.append("unknown_status") - status = STATUS.get(status.upper()) if status else None - # Sensitive source_values remain private; normalized exposure is screened. - out = { + status = STATUS.get(raw_status.upper()) if raw_status else None + for field in DATE_FIELDS: + value = clean(row.get(field)) + if _date_state(value) == "invalid": + reasons.append(f"invalid_{field}") + municipality_code = clean(row.get("codice_comune")) + geography_precision = "municipality-code" if municipality_code and len(municipality_code) == 6 else "unknown" + normalized = { + "establishment_id": rec, + "recognition_number": rec, + "source_activity_code": activity, + "facility_grouping": "provisional-recognition-number", + "name": clean(row.get("ragione_sociale")), + "trading_name": clean(row.get("ragione_sociale")), + "address": None, + "municipality": clean(row.get("comune")), + "city": clean(row.get("comune")), + "province": clean(row.get("provincia")), + "region": clean(row.get("regione")), + "country_code": "IT", + "nation": "Italy", + "geography_precision": geography_precision, + "geography_state": "source-municipality-code" if geography_precision != "unknown" else "unknown", + "classification": clean(row.get("classificazione_stabilimento")), + "activity_code": activity, + "activity_description": clean(row.get("descrizione_impianto_attivita")), + "products": clean(row.get("prodotti_abilitati")), + "status": status, + "status_state": "known" if status else "unknown", + "dates": {field: clean(row.get(field)) for field in DATE_FIELDS}, + "date_state": {field: _date_state(clean(row.get(field))) for field in DATE_FIELDS}, + "coordinates": None, + "coordinate_state": "source-value-present-pending-review" if clean(row.get("longitudine")) or clean(row.get("latitudine")) else "unknown", + "privacy_gate": "pending-review", + "coordinate_gate": "review_required", + "publication_gate": "blocked", + } + record = { "source_id": self.source_id, "source_row": line, "source_row_id": row_id(row, occurrences[key]), - "source_values": dict(row), - "normalized": { - "establishment_id": rec, - "recognition_number": rec, - "facility_grouping": "provisional-recognition-number", - "name": clean(row.get("ragione_sociale")), - "trading_name": clean(row.get("ragione_sociale")), - "address": None, - "municipality": clean(row.get("comune")), - "city": clean(row.get("comune")), - "country_code": "IT", - "nation": "Italy", - "region": clean(row.get("regione")), - "activity_code": act, - "activity_description": clean(row.get("descrizione_impianto_attivita")), - "products": clean(row.get("prodotti_abilitati")), - "status": status, - "coordinates": None, - "privacy_gate": "pending-review", - "publication_gate": "blocked", - }, + "source_record_key": f"{rec or 'unknown'}|{activity or 'unknown'}|{occurrences[key]}", + "source_values": {"__extra_columns" if key is None else str(key): value for key, value in row.items()}, + "normalized": normalized, } - (quarantined if reasons else accepted).append( - {"reasons": tuple(reasons), "record": out} if reasons else out - ) + if reasons: + quarantined.append({"reasons": tuple(dict.fromkeys(reasons)), "record": record}) + else: + accepted.append(record) return {"accepted": accepted, "quarantined": quarantined, "source_sha256": digest, "input_rows": len(rows)} - def parse_file(self, path): + def parse_file(self, path: str | Path) -> dict[str, Any]: return self.parse_bytes(Path(path).read_bytes()) - def run(self, raw_path, run_dir, artifact): + def run(self, raw_path: str | Path, run_dir: str | Path, artifact: SourceArtifact) -> dict[str, Any]: raw = Path(raw_path).read_bytes() - result = self.parse_bytes(raw) - if artifact.sha256 != result["source_sha256"] or artifact.byte_size != len(raw): + digest = hashlib.sha256(raw).hexdigest() + if artifact.sha256 != digest or artifact.byte_size != len(raw): raise ValueError("artifact provenance mismatch") + result = self.parse_bytes(raw) root = Path(run_dir) - (root / "normalized").mkdir(parents=True, exist_ok=True) - (root / "quarantined").mkdir(parents=True, exist_ok=True) - normalized = "".join(json.dumps(x, sort_keys=True, default=list) + "\n" for x in result["accepted"]) - quarantined = "".join(json.dumps(x, sort_keys=True, default=list) + "\n" for x in result["quarantined"]) - (root / "normalized" / "records.jsonl").write_text(normalized, encoding="utf-8") - (root / "quarantined" / "records.jsonl").write_text(quarantined, encoding="utf-8") - manifest = { - "source_id": self.source_id, "schema_version": self.schema_version, - "source_url": artifact.source_url, "retrieved_at_utc": artifact.retrieved_at_utc, - "sha256": artifact.sha256, "checksum_sha256": artifact.sha256, - "byte_size": artifact.byte_size, "code_version": artifact.code_version, - "config_version": artifact.config_version, "publication_state": "private-candidate", - "release_state": "not-created", "input_rows": result["input_rows"], - "normalized_rows": len(result["accepted"]), "quarantined_rows": len(result["quarantined"]), - "normalized_sha256": hashlib.sha256(normalized.encode()).hexdigest(), - "acquisition": {"source_url": artifact.source_url, "retrieved_at_utc": artifact.retrieved_at_utc, "sha256": artifact.sha256, "byte_size": artifact.byte_size}, - "handoff_contract": "candidate_handoff-v1", - } - (root / "manifest.json").write_text(json.dumps(manifest, sort_keys=True, indent=2) + "\n", encoding="utf-8") + accepted = result["accepted"] + quarantined = result["quarantined"] + parsed = accepted + [item["record"] for item in quarantined] + _, parsed_sha256, _ = atomic_jsonl(root / "parsed" / "records.jsonl", parsed) + _, normalized_sha256, _ = atomic_jsonl(root / "normalized" / "records.jsonl", accepted) + atomic_jsonl(root / "quarantined" / "records.jsonl", quarantined) + anomaly_counts = Counter(reason for item in quarantined for reason in item["reasons"]) + manifest = private_manifest( + source_id=self.source_id, + adapter_version=self.adapter_version, + schema_version=self.schema_version, + artifact=artifact, + input_rows=result["input_rows"], + normalized_rows=len(accepted), + quarantined_rows=len(quarantined), + normalized_sha256=normalized_sha256, + parsed_sha256=parsed_sha256, + anomaly_counts=dict(sorted(anomaly_counts.items())), + ) + manifest["coverage"] = "Italian Ministry 853/2004 CSV; one source row per establishment/activity; 1069/2009 excluded" + manifest["geocoding"] = "disabled" + atomic_json(root / "manifest.json", manifest) return manifest diff --git a/pipeline/sources/italy/refresh.py b/pipeline/sources/italy/refresh.py index 3a56d26..0804ad2 100644 --- a/pipeline/sources/italy/refresh.py +++ b/pipeline/sources/italy/refresh.py @@ -1,18 +1,119 @@ -"""Run a registered Italian 853 snapshot into private candidate staging.""" +"""Acquire and stage one Italian 853/2004 snapshot privately. + +The command ends at the shared lifecycle's candidate-ready boundary. Database +candidate import and guarded API checks remain explicit, separate commands. +""" from __future__ import annotations -import argparse, hashlib, json + +import argparse +import json from pathlib import Path -from pipeline.common.orchestrator import run_registered_typed_input + +from pipeline.common.orchestrator import run_private_lifecycle +from pipeline.contracts.adapter_contract import SourceArtifact +from pipeline.contracts.source_lifecycle import atomic_json + +from .acquire import CATALOG_URL, DEFAULT_MAX_BYTES, fetch from .it_853_adapter import Italy853Adapter + +def _metadata_for_local(raw_path: Path, metadata: dict, *, url: str, retrieved_at: str | None, adapter: Italy853Adapter) -> dict: + import hashlib + + raw = raw_path.read_bytes() + return { + "source_id": adapter.source_id, + "source_url": next((value for value in (metadata.get("final_url"), metadata.get("requested_url"), url) if value and value != "unknown"), url), + "retrieved_at_utc": metadata.get("retrieved_at_utc") or retrieved_at, + "checksum_sha256": hashlib.sha256(raw).hexdigest(), + "byte_size": len(raw), + "publication_date": (metadata.get("publication_metadata") or {}).get("catalog_last_updated") or metadata.get("filename_publication_date"), + "effective_date": metadata.get("effective_date"), + "code_version": adapter.adapter_version, + "config_version": adapter.schema_version, + "rights_caveat": "Italian Open Data Licence v2.0; terms evidence retained in acquisition metadata; publication review remains separate", + "privacy_caveat": "restricted private staging; address, tax, and coordinate exposure pending privacy review", + "coverage": "Italian Ministry 853/2004 CSV only; separate 1069/2009 by-products dataset excluded", + } + + +def refresh( + *, + run_dir: str | Path, + raw_path: str | Path | None = None, + fetch_source: bool = False, + catalog_url: str = CATALOG_URL, + output_root: str | Path = "data/raw", + run_id: str | None = None, + terms_review: str | Path | None = None, + retrieved_at_utc: str | None = None, + timeout_seconds: float = 60.0, + max_bytes: int = DEFAULT_MAX_BYTES, +) -> dict: + """Run the shared private lifecycle from a preserved or acquired artifact.""" + if fetch_source == (raw_path is not None): + raise ValueError("specify exactly one of raw_path or fetch_source") + adapter = Italy853Adapter() + raw_root = Path(output_root) + if fetch_source: + if terms_review is None: + raise ValueError("terms_review is required for network acquisition") + metadata = fetch(output_root=raw_root, run_id=run_id or "manual", terms_review_path=Path(terms_review), + catalog_url=catalog_url, timeout_seconds=timeout_seconds, max_bytes=max_bytes) + input_path = raw_root / adapter.source_id / (run_id or "manual") / metadata["artifact"] + else: + input_path = Path(raw_path).resolve() # type: ignore[arg-type] + sidecar = input_path.parent / "acquisition-metadata.json" + metadata = json.loads(sidecar.read_text(encoding="utf-8")) if sidecar.is_file() else {} + if not input_path.is_file(): + raise ValueError(f"raw artifact does not exist: {input_path}") + facts = _metadata_for_local(input_path, metadata, url=catalog_url, retrieved_at=retrieved_at_utc, adapter=adapter) + if not facts["retrieved_at_utc"]: + raise ValueError("retrieved_at_utc is required for private health evidence") + artifact = SourceArtifact( + source_url=str(facts["source_url"]), retrieved_at_utc=str(facts["retrieved_at_utc"]), + sha256=str(facts["checksum_sha256"]), byte_size=int(facts["byte_size"]), + publication_date=facts.get("publication_date"), effective_date=facts.get("effective_date"), + code_version=str(facts["code_version"]), config_version=str(facts["config_version"]), + rights_caveat=facts["rights_caveat"], privacy_caveat=facts["privacy_caveat"], coverage=facts["coverage"], + ) + status = run_private_lifecycle(input_path, run_dir, artifact, adapter) + # Keep catalog/response/terms evidence beside the lifecycle run without + # copying row payloads into QA, health, or API-shaped artifacts. + if metadata: + atomic_json(Path(run_dir) / "acquisition-metadata.json", metadata) + status["acquisition"] = metadata + return status + + def main() -> int: - parser=argparse.ArgumentParser() - parser.add_argument("raw", type=Path); parser.add_argument("--runs", type=Path, required=True) - parser.add_argument("--url", required=True); parser.add_argument("--retrieved-at-utc", required=True) - parser.add_argument("--publication-date", required=True) - args=parser.parse_args(); raw=args.raw.read_bytes(); adapter=Italy853Adapter() - config={"source_url":args.url,"retrieved_at_utc":args.retrieved_at_utc,"publication_date":args.publication_date,"checksum_sha256":hashlib.sha256(raw).hexdigest(),"byte_size":len(raw),"code_version":adapter.adapter_version,"config_version":adapter.schema_version,"source_id":adapter.source_id,"terms_status":"pending_confirmation"} - status=run_registered_typed_input(args.raw,args.runs,config,adapter) - print(json.dumps({"status":status["status"],"run_dir":status["run_dir"],"input_rows":status["manifest"]["input_rows"],"normalized_rows":status["manifest"]["normalized_rows"],"quarantined_rows":status["manifest"]["quarantined_rows"]},sort_keys=True)) - return 0 if status["status"] in {"candidate-ready","staged-restricted"} else 1 -if __name__=="__main__": raise SystemExit(main()) + parser = argparse.ArgumentParser(description=__doc__) + source = parser.add_mutually_exclusive_group(required=True) + source.add_argument("--raw", type=Path) + source.add_argument("--fetch", action="store_true") + parser.add_argument("--run-dir", type=Path, required=True) + parser.add_argument("--catalog-url", default=CATALOG_URL) + parser.add_argument("--output-root", type=Path, default=Path("data/raw")) + parser.add_argument("--run-id") + parser.add_argument("--terms-review", type=Path) + parser.add_argument("--retrieved-at-utc") + parser.add_argument("--timeout-seconds", type=float, default=60.0) + parser.add_argument("--max-bytes", type=int, default=DEFAULT_MAX_BYTES) + args = parser.parse_args() + try: + status = refresh(run_dir=args.run_dir, raw_path=args.raw, fetch_source=args.fetch, + catalog_url=args.catalog_url, output_root=args.output_root, run_id=args.run_id, + terms_review=args.terms_review, retrieved_at_utc=args.retrieved_at_utc, + timeout_seconds=args.timeout_seconds, max_bytes=args.max_bytes) + except (OSError, ValueError) as error: + print(json.dumps({"status": "failed", "error": str(error)}, sort_keys=True)) + return 2 + print(json.dumps({"status": status["status"], "run_dir": status["run_dir"], + "input_rows": status.get("manifest", {}).get("input_rows"), + "normalized_rows": status.get("manifest", {}).get("normalized_rows"), + "quarantined_rows": status.get("manifest", {}).get("quarantined_rows")}, sort_keys=True)) + return 0 if status["status"] in {"candidate-ready", "staged-restricted"} else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/pipeline/sources/italy/test_acquire.py b/pipeline/sources/italy/test_acquire.py new file mode 100644 index 0000000..4769872 --- /dev/null +++ b/pipeline/sources/italy/test_acquire.py @@ -0,0 +1,107 @@ +import hashlib +import json +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +from .acquire import AcquisitionError, discover_csv, fetch +from .it_853_adapter import REQUIRED +from .refresh import refresh + + +class FakeResponse: + def __init__(self, body: bytes, url: str, content_type: str): + self.body = body + self.status = 200 + self.headers = {"Content-Type": content_type, "Content-Length": str(len(body))} + self._url = url + self._offset = 0 + + def __enter__(self): + return self + + def __exit__(self, *_): + return False + + def read(self, limit=-1): + if limit < 0: + chunk = self.body[self._offset:] + else: + chunk = self.body[self._offset:self._offset + limit] + self._offset += len(chunk) + return chunk + + def geturl(self): + return self._url + + +HEADER = ";".join(REQUIRED) +CSV = (HEADER + "\n;IT-1;Test;Address;Town;XX;010;Piemonte;Class;A1;Activity;P;S;IT;12;45;1;tax;vat;001001;;;AUTORIZZATA;2026-09-15;\n").encode() +CATALOG = b'Scarica

Data ultimo aggiornamento

15/09/2026' + + +class ItalyAcquisitionTests(unittest.TestCase): + def review(self, root: Path) -> Path: + path = root / "terms.json" + path.write_text(json.dumps({"reviewer": "test", "reference": "iodl", "reviewed_at": "2026-09-15T00:00:00Z", "decision": "approved", "notes": "synthetic fixture"}), encoding="utf-8") + return path + + def test_discovery_uses_catalog_link_and_rejects_other_hosts(self): + url, date = discover_csv(CATALOG, "https://www.dati.salute.gov.it/catalog") + self.assertEqual(url, "https://www.dati.salute.gov.it/sites/default/files/opendata/STAB_POA_8_20260915.csv") + self.assertEqual(date, "2026-09-15") + with self.assertRaisesRegex(AcquisitionError, "no supported"): + discover_csv(b'x', "https://www.dati.salute.gov.it/catalog") + + def test_fetch_archives_catalog_discovered_csv_and_provenance(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + calls = [] + + def opener(request, timeout): + calls.append((request.full_url, timeout)) + if len(calls) == 1: + return FakeResponse(CATALOG, "https://www.dati.salute.gov.it/catalog", "text/html; charset=utf-8") + return FakeResponse(CSV, request.full_url, "text/csv; charset=utf-8") + + metadata = fetch(output_root=root / "raw", run_id="run-1", terms_review_path=self.review(root), opener=opener) + artifact = root / "raw" / "it.853-2004" / "run-1" / "source.csv" + self.assertEqual(calls[1][0], "https://www.dati.salute.gov.it/sites/default/files/opendata/STAB_POA_8_20260915.csv") + self.assertEqual(metadata["source_id"], "it.853-2004") + self.assertEqual(metadata["filename_publication_date"], "2026-09-15") + self.assertEqual(metadata["sha256"], hashlib.sha256(CSV).hexdigest()) + self.assertEqual(metadata["byte_size"], len(CSV)) + self.assertEqual(json.loads((artifact.parent / "acquisition-metadata.json").read_text())["terms_review"]["decision"], "approved") + self.assertEqual(artifact.read_bytes(), CSV) + + def test_fetch_requires_approved_terms_and_bounds_artifact(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + review = root / "terms.json" + review.write_text(json.dumps({"reviewer": "test", "reference": "x", "reviewed_at": "2026-09-15T00:00:00Z", "decision": "pending", "notes": "no"}), encoding="utf-8") + with self.assertRaisesRegex(AcquisitionError, "approved"): + fetch(output_root=root / "raw", run_id="run", terms_review_path=review, opener=lambda *_: None) + with patch("pipeline.sources.italy.acquire._archive_stream", side_effect=AcquisitionError("download exceeds")): + calls = iter([FakeResponse(CATALOG, "https://www.dati.salute.gov.it/catalog", "text/html"), FakeResponse(CSV, "https://www.dati.salute.gov.it/file.csv", "text/csv")]) + with self.assertRaisesRegex(AcquisitionError, "exceeds"): + fetch(output_root=root / "raw", run_id="run", terms_review_path=self.review(root), opener=lambda *args, **kwargs: next(calls), max_bytes=1) + + def test_local_archived_artifact_runs_shared_lifecycle(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + raw = root / "fixture.csv" + raw.write_bytes(CSV) + # The local archive path is test-only; it records unknown source + # URLs rather than implying that the fixture was downloaded. + from .acquire import archive_local_file + metadata = archive_local_file(raw, root / "raw", run_id="local", retrieved_at="2026-09-15T00:00:00Z") + status = refresh(raw_path=root / "raw" / "it.853-2004" / "local" / "source.csv", run_dir=root / "staging") + self.assertEqual(status["status"], "candidate-ready") + self.assertEqual(status["manifest"]["input_rows"], 1) + self.assertEqual(status["manifest"]["normalized_rows"], 1) + self.assertEqual(status["acquisition"]["sha256"], metadata["sha256"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/sources/italy/test_it_853_adapter.py b/pipeline/sources/italy/test_it_853_adapter.py index b6d58b8..150de8e 100644 --- a/pipeline/sources/italy/test_it_853_adapter.py +++ b/pipeline/sources/italy/test_it_853_adapter.py @@ -2,6 +2,7 @@ import tempfile, json, hashlib from .it_853_adapter import Italy853Adapter from pipeline.contracts.adapter_contract import SourceArtifact +from pipeline.common.orchestrator import run_private_lifecycle H="precedente_bollo_cee;num_identificativo_produzione_commercializzazione;ragione_sociale;indirizzo;comune;provincia;codice_regione;regione;classificazione_stabilimento;codice_impianto_attivita;descrizione_impianto_attivita;prodotti_abilitati;specifica_prodotti;paesi_export_autorizzato;longitudine;latitudine;stato_localizzazione;cod_fiscale;p_iva;codice_comune;data_inizio_attivita;data_fine_attivita;stato_attivita;data_ultimo_aggiornamento;num_identificativo_produzione_commercializzazione_2" def row(n="A",a="10",s="Autorizzata"): return f";{n};Name;;Town;;010;Piemonte;X;{a};Activity;P;S;IT;12;45;1;tax;vat;001001;;;{s};2026-09-13;\n" class Test(unittest.TestCase): @@ -13,6 +14,11 @@ def test_sensitive_and_deterministic_identity(self): content=(H+"\n"+row()).encode(); a=Italy853Adapter(); x=a.parse_bytes(content)["accepted"][0]; y=a.parse_bytes(content)["accepted"][0]; self.assertEqual(x["source_row_id"],y["source_row_id"]); self.assertNotIn("p_iva",x["normalized"]); self.assertIsNone(x["normalized"]["coordinates"]) def test_shape_drift_quarantine(self): content=(H+"\n"+row().replace("Name","Name;extra")).encode(); self.assertRaises(ValueError,Italy853Adapter().parse_bytes,content) + def test_missing_date_and_geography_are_explicit(self): + r=Italy853Adapter().parse_bytes((H+"\n"+row().replace("001001","001").replace("2026-09-13","")).encode())["accepted"][0] + self.assertEqual(r["normalized"]["date_state"]["data_inizio_attivita"],"unknown"); self.assertEqual(r["normalized"]["geography_precision"],"unknown"); self.assertEqual(r["normalized"]["coordinate_state"],"source-value-present-pending-review") + def test_repeated_activity_quarantines_collision_without_merge(self): + result=Italy853Adapter().parse_bytes((H+"\n"+row()+row()).encode()); self.assertEqual(len(result["accepted"]),1); self.assertEqual(len(result["quarantined"]),1); self.assertIn("ambiguous_repeated_recognition_activity",result["quarantined"][0]["reasons"]) def test_run_writes_contract_manifest_and_row_quarantine(self): from pipeline.contracts.adapter_contract import assert_manifest content=(H+"\n"+row()+row("","10")).encode() @@ -22,3 +28,7 @@ def test_candidate_handoff_uses_shared_writer(self): content=(H+"\n"+row()).encode(); sha=hashlib.sha256(content).hexdigest(); a=Italy853Adapter() with tempfile.TemporaryDirectory() as d, tempfile.NamedTemporaryFile(suffix=".csv", delete=False) as f: f.write(content); f.flush(); m=a.write_candidate_handoff(d,SourceArtifact("u","2026-09-14T00:00:00Z",sha,len(content)),a.parse_bytes(content)); self.assertEqual(m["contract_version"],"candidate-handoff-v1"); self.assertEqual(m["normalized_rows"],1) + def test_shared_lifecycle_emits_private_health_and_candidate(self): + content=(H+"\n"+row()+row("B","11")).encode(); sha=hashlib.sha256(content).hexdigest(); a=Italy853Adapter() + with tempfile.TemporaryDirectory() as d, tempfile.NamedTemporaryFile(suffix=".csv", delete=False) as f: + f.write(content); f.flush(); artifact=SourceArtifact("https://example.invalid/it.csv","2026-09-14T00:00:00Z",sha,len(content),publication_date="2026-09-14",code_version=a.adapter_version,config_version=a.schema_version); status=run_private_lifecycle(f.name,__import__('pathlib').Path(d)/"runs",artifact,a); root=__import__('pathlib').Path(status["run_dir"]); self.assertEqual(status["status"],"candidate-ready"); self.assertEqual(status["manifest"]["contract_version"],"source-lifecycle-v1"); self.assertTrue((root/"qa.json").exists()); self.assertTrue((root/"source-health.json").exists()); self.assertTrue((root/"release-candidate/records.jsonl").exists()); self.assertNotIn("source_values",(root/"qa.json").read_text()) diff --git a/pipeline/tests/e2e/test_italy_candidate_import.py b/pipeline/tests/e2e/test_italy_candidate_import.py index aca966b..d4ec625 100644 --- a/pipeline/tests/e2e/test_italy_candidate_import.py +++ b/pipeline/tests/e2e/test_italy_candidate_import.py @@ -3,6 +3,7 @@ import psycopg from .fixture import E2EEnvironment from pipeline.contracts.adapter_contract import SourceArtifact +from pipeline.common.orchestrator import run_private_lifecycle from pipeline.sources.italy.it_853_adapter import Italy853Adapter ROOT=Path(__file__).resolve().parents[3] @@ -15,7 +16,7 @@ def setUpClass(cls): if os.environ.get("UEC_RUN_E2E")!="1": raise unittest.SkipTest("set UEC_RUN_E2E=1") cls.env=E2EEnvironment(); cls.env.test_release_id="e2e-private-candidate"; cls.env=cls.env.start(); cls.temp=tempfile.TemporaryDirectory(); root=Path(cls.temp.name); cls.raw=root/"it.csv"; cls.raw.write_text(HEADER+"\n"+ROW+"\n",encoding="utf-8") a=Italy853Adapter(); raw=cls.raw.read_bytes(); artifact=SourceArtifact("https://example.invalid/it-853.csv","2026-09-14T00:00:00Z",hashlib.sha256(raw).hexdigest(),len(raw),code_version=a.adapter_version,config_version=a.schema_version) - parsed=a.parse_bytes(raw); cls.run_dir=root/"handoff"; a.write_candidate_handoff(cls.run_dir,artifact,parsed) + cls.run_dir=root/"lifecycle"; lifecycle=run_private_lifecycle(cls.raw,root/"runs",artifact,a); assert lifecycle["status"]=="candidate-ready", lifecycle; cls.run_dir=Path(lifecycle["run_dir"]) cls.release_id="candidate-italy-e2e"; cmd=[sys.executable,str(ROOT/"pipeline/scripts/maintenance/import-candidate.py"),"--manifest",str(cls.run_dir/"manifest.json"),"--normalized",str(cls.run_dir/"normalized/records.jsonl"),"--raw",str(cls.raw),"--release-id",cls.release_id,"--database-url",cls.env.database_url,"--disposable-db"] for _ in range(2): result=subprocess.run(cmd,cwd=ROOT,capture_output=True,text=True); assert result.returncode==0,result.stderr @@ -32,11 +33,18 @@ def tearDownClass(cls): if hasattr(cls,"temp"): cls.temp.cleanup() if hasattr(cls,"env"): cls.env.stop() def test_import_is_single_private_candidate(self): self.assertEqual(self.counts,(1,1,0)); self.assertEqual(self.linked,1) + def test_failed_batch_rolls_back_then_valid_import_is_idempotent(self): + bad_manifest=json.loads((self.run_dir/"manifest.json").read_text()); normalized=(self.run_dir/"normalized/records.jsonl").read_bytes(); bad=self.run_dir/"normalized/bad-records.jsonl"; bad.write_bytes(normalized+b'{"source_id":"it.853-2004","source_row":99}\n'); bad_manifest["normalized_rows"]=2; bad_manifest["normalized_sha256"]=hashlib.sha256(bad.read_bytes()).hexdigest(); bad_manifest_path=self.run_dir/"bad-manifest.json"; bad_manifest_path.write_text(json.dumps(bad_manifest),encoding="utf-8") + bad_cmd=[sys.executable,str(ROOT/"pipeline/scripts/maintenance/import-candidate.py"),"--manifest",str(bad_manifest_path),"--normalized",str(bad),"--raw",str(self.raw),"--release-id","candidate-italy-recovery","--database-url",self.env.database_url,"--disposable-db","--batch-size","2"]; failed=subprocess.run(bad_cmd,cwd=ROOT,capture_output=True,text=True); self.assertNotEqual(failed.returncode,0) + with psycopg.connect(self.env.database_url) as db: self.assertEqual(db.execute("SELECT count(*) FROM uec.releases WHERE release_id='candidate-italy-recovery'").fetchone()[0],0) + good_cmd=[sys.executable,str(ROOT/"pipeline/scripts/maintenance/import-candidate.py"),"--manifest",str(self.run_dir/"manifest.json"),"--normalized",str(self.run_dir/"normalized/records.jsonl"),"--raw",str(self.raw),"--release-id","candidate-italy-recovery","--database-url",self.env.database_url,"--disposable-db"] + first=subprocess.run(good_cmd,cwd=ROOT,capture_output=True,text=True); second=subprocess.run(good_cmd,cwd=ROOT,capture_output=True,text=True); self.assertEqual(first.returncode,0,first.stderr); self.assertEqual(second.returncode,0,second.stderr); self.assertIn("imported 0 candidate rows",first.stdout); self.assertIn("imported 0 candidate rows",second.stdout) + with psycopg.connect(self.env.database_url) as db: self.assertEqual(db.execute("SELECT count(*) FROM uec.release_members WHERE release_id='candidate-italy-recovery'").fetchone()[0],1) def test_guarded_preview_and_public_exclusion(self): base=f"http://127.0.0.1:{self.env.api_port}" with urllib.request.urlopen(base+"/api/v2/locations?profile=official") as r: self.assertEqual(json.loads(r.read())["data"],[]) h={"X-UEC-Dev-Preview-Token":self.env.dev_preview_token} - req=urllib.request.Request(base+"/api/dev/preview/test-release/locations?profile=official",headers=h) + req=urllib.request.Request(base+"/api/dev/preview/test-release/locations?profile=official&country_code=IT&limit=1",headers=h) with urllib.request.urlopen(req) as r: body=json.loads(r.read()) self.assertTrue(body["meta"]["test_only"]); self.assertTrue(body["meta"]["private_preview"]); self.assertEqual(body["meta"]["release_status"],"candidate"); self.assertIn("preview_label",body["meta"]); self.assertEqual(len(body["data"]),1); self.assertIn("Italy",json.dumps(body)); self.assertNotIn("source_values",json.dumps(body)); self.assertIsNone(body["data"][0]["latitude"]) fid=body["data"][0]["facility_id"] diff --git a/pipeline/tests/test_source_registry.py b/pipeline/tests/test_source_registry.py index 995e8c5..c5a02f9 100644 --- a/pipeline/tests/test_source_registry.py +++ b/pipeline/tests/test_source_registry.py @@ -9,8 +9,8 @@ class SourceRegistryTests(unittest.TestCase): def test_repository_registry_loads_and_references_existing_legacy_paths(self): registry = load_registry() - self.assertEqual(len(registry["sources"]), 12) - self.assertEqual(len({source["source_id"] for source in registry["sources"]}), 12) + self.assertEqual(len(registry["sources"]), 13) + self.assertEqual(len({source["source_id"] for source in registry["sources"]}), 13) def test_unknowns_are_explicit(self): registry = json.loads(REGISTRY_PATH.read_text(encoding="utf-8")) From f3c35e433bd6c35077e1c085925bb9e026de0d29 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Tue, 15 Sep 2026 11:15:30 -0700 Subject: [PATCH 113/311] Complete UK private V2 lifecycle --- docs/country-recon-uk.md | 52 ++++- pipeline/adapter-capabilities.json | 4 +- pipeline/common/acquisition.py | 181 +++++++++++++++++ pipeline/common/activity.py | 41 ++++ pipeline/common/test_acquisition.py | 31 +++ pipeline/common/test_activity.py | 19 ++ pipeline/contracts/adapter_contract.py | 4 +- pipeline/contracts/candidate_handoff.py | 1 + pipeline/contracts/source_health.py | 2 +- .../scripts/maintenance/import-candidate.py | 19 +- pipeline/sources/uk/fsa_approved/README.md | 17 +- pipeline/sources/uk/fsa_approved/adapter.py | 10 +- pipeline/sources/uk/fsa_approved/refresh.py | 81 +++++--- pipeline/sources/uk/fss_approved/README.md | 26 ++- pipeline/sources/uk/fss_approved/adapter.py | 131 ++++++++++--- pipeline/sources/uk/fss_approved/config.json | 10 +- pipeline/sources/uk/fss_approved/refresh.py | 185 ++++++++++++++++++ .../sources/uk/fss_approved/test_adapter.py | 12 +- .../sources/uk/fss_approved/test_refresh.py | 42 ++++ pipeline/sources/uk/registry.json | 2 +- src/lib.rs | 35 +++- 21 files changed, 831 insertions(+), 74 deletions(-) create mode 100644 pipeline/common/acquisition.py create mode 100644 pipeline/common/activity.py create mode 100644 pipeline/common/test_acquisition.py create mode 100644 pipeline/common/test_activity.py create mode 100644 pipeline/sources/uk/fss_approved/refresh.py create mode 100644 pipeline/sources/uk/fss_approved/test_refresh.py diff --git a/docs/country-recon-uk.md b/docs/country-recon-uk.md index 44b76c9..aa4c977 100644 --- a/docs/country-recon-uk.md +++ b/docs/country-recon-uk.md @@ -69,7 +69,7 @@ drop-in approved-establishment feed. private restricted staging, using `AppNo` only as a candidate identifier after the duplicate review; preserve all source values, activity strings, withheld-address flags, source coordinates, and coverage values. -2. Add Scotland as the next candidate after inspecting its CSV schema and exact terms. +2. Add Scotland as a restricted candidate after inspecting its CSV schema and exact terms. 3. Keep Northern Ireland metadata-gated and do not represent the four UK feeds as one source until identity, coverage, update, terms, and suppression behavior are explicitly reconciled. @@ -168,6 +168,56 @@ This reconciliation is a design and test record, not source approval or legal clearance. No live row data is included, and the private artifact remains outside Git and public outputs. +### Private lifecycle implementation status (2026-09-15) + +The FSA England/Wales profile and the FSS Scotland profile now use the shared +bounded acquisition and private lifecycle seams. A network fetch requires an +operator terms-review record, is byte-bounded, and preserves requested/final URLs, +redirects, response headers, retrieval/effective metadata, hashes, byte size, and +code/configuration versions under ignored private storage. Both adapters emit +deterministic parsed/normalized/quarantined outputs, row-free QA, and private +source-health evidence. Activity categories are limited to slaughter, cutting, +processing, and logistics/storage; unknown activity text remains unresolved or +quarantined rather than guessed. + +The FSA feed remains explicitly England/Wales in the monthly source profile; +Northern Ireland is retained as a separate source scope and is accepted only by +the synthetic contract until its distinct catalogue resource is independently +inspected. FSS is Scotland-only. Composition keeps source IDs, nation keys, and +identities separate and emits possible-match review signals without merging. +No release, promotion, public API/export exposure, or production acquisition is +authorized by this implementation. + +### Bounded live-source validation (2026-09-15) + +The explicitly authorized private fetches completed under ignored storage. The +following are aggregate validation results; no raw rows, addresses, coordinates, +or derived records are committed: + +| Source/profile | Effective date | Bytes / SHA-256 | Input | Normalized | Quarantined | +| --- | --- | ---: | ---: | ---: | ---: | +| FSA monthly England/Wales profile | 2026-09-01 | 1,774,417 / `d5cfec048b0f4dc4a8594b0597982f3788f10eb1b4270f9593ead8abce33b61f` | 5,342 | 4,300 | 1,042 | +| FSS live Scotland export | 2026-08-11 | 245,871 / `b95b66afb112636c09f6de401054c7ea3d11e5058f34522d900c60435a125246` | 725 | 586 | 139 | + +The FSA anomaly counts were 999 `remarks_present`, 31 `unknown_nation`, 11 +`address_privacy_risk`, and 4 `duplicate_id_within_nation`. The FSS anomaly +counts were 125 `no_relevant_activity`, 18 `address_privacy_risk`, 2 +`malformed_row`, 2 `missing_activity`, and 2 `missing_approval_number`. +Both runs emitted `health_state: private-validated`, `public_exposure: false`, +and import evidence with zero publication-eligible and zero default-visible +rows. Composition contained 4,886 source-preserving reviewable rows and created +no candidate release. + +On a disposable PostGIS stack, both source manifests imported idempotently: +4,300 FSA rows and 586 FSS rows on first pass, zero rows on each rerun, 4,886 +source records/observations/release members/review events, and zero +default-visible rows. The guarded test-only API returned list, detail, category +filter, cursor pagination, and facets successfully; public list returned zero +rows. The full 4,886-row test export was rejected by the bounded +`export_too_large` guard, while the existing small-candidate E2E covers a +successful private CSV response. The disposable database was destroyed after +validation. + ### Repeatable refresh QA (2026-09-14) The source-local refresh command was validated in aggregate-only dry-run mode against diff --git a/pipeline/adapter-capabilities.json b/pipeline/adapter-capabilities.json index 35f5631..f413f79 100644 --- a/pipeline/adapter-capabilities.json +++ b/pipeline/adapter-capabilities.json @@ -2,7 +2,7 @@ "schema_version": "adapter-capabilities-v1", "adapters": [ {"country_code": "de", "source_id": "de-bvl-bltu", "adapter_version": "de-v2-foundation-1", "schema_version": "location-v2-foundation-1", "adapter_path": "pipeline/germany/adapter.py", "acquisition": "restricted_pending_terms", "geocoding": "disabled", "publication": "human_gate_required"}, - {"country_code": "gb", "source_id": "fss_approved_establishments", "adapter_version": "fss-scotland-v2-1", "schema_version": "fss-scotland-approved-v1", "adapter_path": "pipeline/sources/uk/fss_approved/adapter.py", "acquisition": "synthetic_only", "geocoding": "disabled", "publication": "human_gate_required"}, - {"country_code": "gb", "source_id": "fsa_approved_establishments", "adapter_version": "fsa-uk-v2-1", "schema_version": "fsa-uk-approved-v1", "adapter_path": "pipeline/sources/uk/fsa_approved/adapter.py", "acquisition": "synthetic_only", "geocoding": "disabled", "publication": "human_gate_required"} + {"country_code": "gb", "source_id": "fss_approved_establishments", "adapter_version": "fss-scotland-v2-1", "schema_version": "fss-scotland-approved-v1", "adapter_path": "pipeline/sources/uk/fss_approved/adapter.py", "acquisition": "bounded_private_fetch", "geocoding": "disabled", "publication": "human_gate_required"}, + {"country_code": "gb", "source_id": "fsa_approved_establishments", "adapter_version": "fsa-uk-v2-1", "schema_version": "fsa-uk-approved-v1", "adapter_path": "pipeline/sources/uk/fsa_approved/adapter.py", "acquisition": "bounded_private_fetch", "geocoding": "disabled", "publication": "human_gate_required"} ] } diff --git a/pipeline/common/acquisition.py b/pipeline/common/acquisition.py new file mode 100644 index 0000000..6005bcf --- /dev/null +++ b/pipeline/common/acquisition.py @@ -0,0 +1,181 @@ +"""Bounded, provenance-preserving acquisition primitives shared by sources. + +The acquisition layer only preserves bytes and response facts. It does not +parse, classify, import, or publish a source. Network fetches require an +operator-authored terms record and always write to a caller-selected private +root; source packages provide only source identity and content-type policy. +""" +from __future__ import annotations + +import hashlib +import json +import os +import tempfile +import urllib.error +import urllib.request +import uuid +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Iterable + + +class AcquisitionError(ValueError): + """An acquisition was not bounded, authorized, valid, or complete.""" + + +def utc_now() -> str: + return datetime.now(timezone.utc).isoformat(timespec="seconds").replace("+00:00", "Z") + + +def default_run_id() -> str: + return datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + "-" + uuid.uuid4().hex[:8] + + +def require_terms_review(path: Path) -> dict[str, str]: + try: + review = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as error: + raise AcquisitionError(f"terms review cannot be read: {error}") from error + required = {"reviewer", "reference", "reviewed_at", "decision", "notes"} + if not isinstance(review, dict) or required - review.keys(): + raise AcquisitionError("terms review requires reviewer, reference, reviewed_at, decision, and notes") + for field in required: + if not isinstance(review[field], str) or not review[field].strip(): + raise AcquisitionError(f"terms review {field} must be a non-empty string") + try: + datetime.fromisoformat(review["reviewed_at"].replace("Z", "+00:00")) + except ValueError as error: + raise AcquisitionError("terms review reviewed_at must be ISO-8601") from error + if review["decision"] != "approved": + raise AcquisitionError("terms review decision must be 'approved'") + return {field: review[field] for field in sorted(required)} + + +def selected_headers(headers: Any) -> dict[str, str]: + """Keep response metadata useful for reproducibility, never credentials.""" + names = ("Content-Type", "Content-Length", "Content-Disposition", "ETag", "Last-Modified", "Date") + return {name: headers[name] for name in names if headers.get(name) is not None} + + +def _atomic_bytes(path: Path, payload: bytes) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + fd, temporary = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent) + try: + with os.fdopen(fd, "wb") as handle: + handle.write(payload) + os.replace(temporary, path) + except Exception: + try: + os.unlink(temporary) + except FileNotFoundError: + pass + raise + + +def archive_stream(stream: Any, artifact_path: Path, *, max_bytes: int) -> tuple[str, int]: + if max_bytes <= 0: + raise AcquisitionError("max_bytes must be positive") + artifact_path.parent.mkdir(parents=True, exist_ok=True) + temporary: Path | None = None + digest = hashlib.sha256() + total = 0 + try: + with tempfile.NamedTemporaryFile("wb", delete=False, dir=artifact_path.parent, prefix=".download-", suffix=".part") as handle: + temporary = Path(handle.name) + while chunk := stream.read(1024 * 1024): + total += len(chunk) + if total > max_bytes: + raise AcquisitionError(f"download exceeds max_bytes={max_bytes}") + digest.update(chunk) + handle.write(chunk) + os.replace(temporary, artifact_path) + return digest.hexdigest(), total + except Exception: + if temporary is not None: + temporary.unlink(missing_ok=True) + raise + + +class _RedirectRecorder(urllib.request.HTTPRedirectHandler): + def __init__(self) -> None: + super().__init__() + self.redirects: list[dict[str, Any]] = [] + + def redirect_request(self, req, fp, code, msg, headers, newurl): # type: ignore[no-untyped-def] + self.redirects.append({"status": code, "from_url": fp.geturl(), "to_url": newurl}) + return super().redirect_request(req, fp, code, msg, headers, newurl) + + +def fetch_source( + *, + source_id: str, + url: str, + output_root: str | Path, + artifact_name: str, + terms_review_path: str | Path, + run_id: str | None = None, + timeout_seconds: float = 60.0, + max_bytes: int = 64 * 1024 * 1024, + user_agent: str = "UntilEveryCage/controlled-acquisition", + allowed_content_types: Iterable[str] = ("text/csv", "application/csv", "application/octet-stream"), + effective_date: str | None = None, + publication_date: str | None = None, + code_version: str = "unknown", + config_version: str = "unknown", + coverage: str | None = None, + rights_caveat: str | None = None, + privacy_caveat: str | None = None, +) -> dict[str, Any]: + if not source_id or not url: + raise AcquisitionError("source_id and url are required") + if timeout_seconds <= 0: + raise AcquisitionError("timeout_seconds must be positive") + terms_review = require_terms_review(Path(terms_review_path)) + run_id = run_id or default_run_id() + run_dir = Path(output_root) / source_id / run_id + artifact_path = run_dir / artifact_name + requested_at = utc_now() + recorder = _RedirectRecorder() + try: + opener = urllib.request.build_opener(recorder) + request = urllib.request.Request(url, headers={"User-Agent": user_agent}) + with opener.open(request, timeout=timeout_seconds) as response: + if not 200 <= response.status < 300: + raise AcquisitionError(f"source returned HTTP {response.status}") + content_type = response.headers.get("Content-Type") + allowed = {item.lower() for item in allowed_content_types} + if content_type and content_type.split(";", 1)[0].strip().lower() not in allowed: + raise AcquisitionError(f"unexpected content type: {content_type}") + sha256, byte_size = archive_stream(response, artifact_path, max_bytes=max_bytes) + headers = selected_headers(response.headers) + final_url = response.geturl() + except urllib.error.HTTPError as error: + raise AcquisitionError(f"source returned HTTP {error.code}") from error + except urllib.error.URLError as error: + raise AcquisitionError(f"network error: {error.reason}") from error + metadata = { + "acquisition_method": "network_fetch", + "source_id": source_id, + "artifact": artifact_name, + "artifact_path": str(artifact_path), + "run_id": run_id, + "requested_url": url, + "final_url": final_url, + "redirects": recorder.redirects, + "response_headers": headers, + "requested_at_utc": requested_at, + "retrieved_at_utc": utc_now(), + "effective_date": effective_date or headers.get("Last-Modified") or "unknown", + "publication_date": publication_date, + "sha256": sha256, + "byte_size": byte_size, + "adapter_version": code_version, + "code_version": code_version, + "config_version": config_version, + "coverage": coverage, + "rights_caveat": rights_caveat, + "privacy_caveat": privacy_caveat, + "terms_review": terms_review, + } + _atomic_bytes(run_dir / "acquisition-metadata.json", (json.dumps(metadata, ensure_ascii=False, sort_keys=True, indent=2) + "\n").encode()) + return metadata diff --git a/pipeline/common/activity.py b/pipeline/common/activity.py new file mode 100644 index 0000000..545fa3d --- /dev/null +++ b/pipeline/common/activity.py @@ -0,0 +1,41 @@ +"""Deterministic, non-geocoding activity classification shared by UK adapters.""" +from __future__ import annotations + +import re +from collections.abc import Iterable + + +_RULES: tuple[tuple[str, tuple[str, ...]], ...] = ( + ("slaughter", ("slaughter", "abattoir", "killing")), + ("cutting", ("cutting", "butchery", "deboning")), + ("processing", ("processing", "meat product", "minced", "preparation", "manufactur")), + ("logistics_and_storage", ("cold store", "cold-store", "coldstorage", "storage", "warehouse", "freezer", "refrigerat")), +) + + +def classify_activities(values: Iterable[str | None]) -> tuple[str, ...]: + """Return stable API categories while leaving original activity text intact.""" + categories: list[str] = [] + for value in values: + if not value: + continue + text = re.sub(r"\s+", " ", value).strip().casefold() + # The FSA/FSS exports use these short activity codes in some monthly + # snapshots. Keep the raw code in source_values; this is only the + # stable project interpretation used by candidate/API plumbing. + aliases = {"sh": "slaughter", "cp": "cutting", "mp": "processing", "cs": "logistics_and_storage"} + if text in aliases and aliases[text] not in categories: + categories.append(aliases[text]) + continue + if re.search(r"\bSH\s*\(", value, re.I) and "slaughter" not in categories: + categories.append("slaughter") + if re.search(r"\bCP\s*\(", value, re.I) and "cutting" not in categories: + categories.append("cutting") + if re.search(r"\b(?:PP|MP|MM|RPM|MMP)\s*\(", value, re.I) and "processing" not in categories: + categories.append("processing") + if re.search(r"\bCS\s*\(", value, re.I) and "logistics_and_storage" not in categories: + categories.append("logistics_and_storage") + for category, needles in _RULES: + if any(needle in text for needle in needles) and category not in categories: + categories.append(category) + return tuple(categories) diff --git a/pipeline/common/test_acquisition.py b/pipeline/common/test_acquisition.py new file mode 100644 index 0000000..f9c2ff0 --- /dev/null +++ b/pipeline/common/test_acquisition.py @@ -0,0 +1,31 @@ +import json +import tempfile +import unittest +from pathlib import Path + +from .acquisition import AcquisitionError, archive_stream, require_terms_review + + +class AcquisitionContractTests(unittest.TestCase): + def test_terms_review_requires_explicit_approved_record(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "terms.json" + path.write_text(json.dumps({"reviewer": "operator", "reference": "OGL", "reviewed_at": "2026-09-15T00:00:00Z", "decision": "pending", "notes": "restricted"}), encoding="utf-8") + with self.assertRaises(AcquisitionError): + require_terms_review(path) + + def test_archive_stream_enforces_bound_and_removes_partial(self): + class Stream: + def read(self, _size): + return b"123456" + + with tempfile.TemporaryDirectory() as directory: + target = Path(directory) / "artifact.csv" + with self.assertRaises(AcquisitionError): + archive_stream(Stream(), target, max_bytes=5) + self.assertFalse(target.exists()) + self.assertFalse(list(Path(directory).glob("*.part"))) + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/common/test_activity.py b/pipeline/common/test_activity.py new file mode 100644 index 0000000..9c7b055 --- /dev/null +++ b/pipeline/common/test_activity.py @@ -0,0 +1,19 @@ +import unittest + +from .activity import classify_activities + + +class ActivityClassificationTests(unittest.TestCase): + def test_known_words_and_monthly_codes_are_stable(self): + self.assertEqual( + classify_activities(("Slaughterhouse", "Cutting plant", "Meat products", "Cold Store")), + ("slaughter", "cutting", "processing", "logistics_and_storage"), + ) + self.assertEqual(classify_activities(("CP", "SH", "CS")), ("cutting", "slaughter", "logistics_and_storage")) + + def test_unknown_activity_is_not_guessed(self): + self.assertEqual(classify_activities(("rendering", "other")), ()) + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/contracts/adapter_contract.py b/pipeline/contracts/adapter_contract.py index 5ad1ccf..5bbf880 100644 --- a/pipeline/contracts/adapter_contract.py +++ b/pipeline/contracts/adapter_contract.py @@ -20,6 +20,7 @@ class SourceArtifact: rights_caveat: str | None = None privacy_caveat: str | None = None coverage: str | None = None + redirects: tuple[dict[str, Any], ...] = () def source_artifact_from_mapping(values: dict[str, Any]) -> SourceArtifact: @@ -33,7 +34,8 @@ def source_artifact_from_mapping(values: dict[str, Any]) -> SourceArtifact: sha256=str(values["checksum_sha256"]), byte_size=int(values["byte_size"]), publication_date=values.get("publication_date"), effective_date=values.get("effective_date"), code_version=str(values.get("code_version", "unknown")), config_version=str(values.get("config_version", "unknown")), - rights_caveat=values.get("rights_caveat"), privacy_caveat=values.get("privacy_caveat"), coverage=values.get("coverage")) + rights_caveat=values.get("rights_caveat"), privacy_caveat=values.get("privacy_caveat"), coverage=values.get("coverage"), + redirects=tuple(values.get("redirects") or ())) class SourceAdapter(Protocol): diff --git a/pipeline/contracts/candidate_handoff.py b/pipeline/contracts/candidate_handoff.py index c15181d..1e45f39 100644 --- a/pipeline/contracts/candidate_handoff.py +++ b/pipeline/contracts/candidate_handoff.py @@ -26,6 +26,7 @@ def write_handoff(run_dir: str | Path, rows: list[dict[str, Any]], artifact: Sou manifest = {"contract_version": CONTRACT_VERSION, "profile": profile, "source_id": source_id, "source_url": artifact.source_url, "retrieved_at_utc": artifact.retrieved_at_utc, "checksum_sha256": artifact.sha256, "byte_size": artifact.byte_size, + "redirects": list(artifact.redirects), "code_version": artifact.code_version, "config_version": artifact.config_version, "coverage": artifact.coverage, "normalized_rows": len(rows), "normalized_sha256": hashlib.sha256(payload).hexdigest(), diff --git a/pipeline/contracts/source_health.py b/pipeline/contracts/source_health.py index 9985f47..34ef339 100644 --- a/pipeline/contracts/source_health.py +++ b/pipeline/contracts/source_health.py @@ -149,7 +149,7 @@ def build_health_snapshot( provenance = { key: manifest.get(key) - for key in ("source_url", "retrieved_at_utc", "publication_date", "effective_date", "sha256", "checksum_sha256", "byte_size", "code_version", "config_version") + for key in ("source_url", "retrieved_at_utc", "publication_date", "effective_date", "sha256", "checksum_sha256", "byte_size", "code_version", "config_version", "redirects") if manifest.get(key) is not None } snapshot = { diff --git a/pipeline/scripts/maintenance/import-candidate.py b/pipeline/scripts/maintenance/import-candidate.py index 2c642f6..e3d2e7f 100644 --- a/pipeline/scripts/maintenance/import-candidate.py +++ b/pipeline/scripts/maintenance/import-candidate.py @@ -94,7 +94,19 @@ def _country_code(manifest: dict, normalized: dict) -> str: return str(explicit)[:2].upper() # Do not turn a country name into an invented code. These are the only # source-country names currently present in the validated adapters. - return {"Denmark": "DK", "England": "GB", "Wales": "GB"}.get(normalized.get("nation"), "ZZ") + return {"Denmark": "DK", "England": "GB", "Wales": "GB", "Scotland": "GB", "Northern Ireland": "GB"}.get(normalized.get("nation"), "ZZ") + + +def _classification_category(normalized: dict) -> str: + """Use an adapter's explicit classification without inventing one.""" + categories = normalized.get("activity_categories") or normalized.get("classification_categories") or () + if isinstance(categories, str): + categories = (categories,) + allowed = {"slaughter", "cutting", "processing", "logistics_and_storage"} + for category in categories: + if category in allowed: + return category + return "unclassified" def _stable_uuid(*parts: object) -> uuid.UUID: @@ -178,10 +190,11 @@ def import_candidate(database_url: str, manifest: dict, rows: list[dict], releas db.execute("""INSERT INTO uec.facilities(facility_id,canonical_name,country_code,city) VALUES (%s,%s,%s,%s) ON CONFLICT (facility_id) DO NOTHING""", (facility_id, name, country, city)) + category = _classification_category(normalized) created = db.execute("""INSERT INTO uec.observations(observation_id,facility_id,source_record_id,observed_at,observation,classification,ruleset_id,rule_id,classification_category,classification_review_status,default_visible,coordinate_review_status,first_observed_at) - VALUES (%s,%s,%s,%s,%s,'{}',%s,'candidate','unclassified','review_required',false,'review_required',%s) + VALUES (%s,%s,%s,%s,%s,%s,%s,'candidate',%s,'review_required',false,'review_required',%s) ON CONFLICT (facility_id,source_record_id,observed_at) DO NOTHING RETURNING observation_id""", - (observation_id, facility_id, record_id, now, json.dumps(normalized), ruleset, now)).fetchone() + (observation_id, facility_id, record_id, now, json.dumps(normalized), json.dumps({"activity_categories": list(normalized.get("activity_categories") or normalized.get("classification_categories") or [])}), ruleset, category, now)).fetchone() observation_ref = created[0] if created else db.execute("""SELECT observation_id FROM uec.observations WHERE facility_id=%s AND source_record_id=%s AND observed_at=%s""", (facility_id, record_id, now)).fetchone()[0] diff --git a/pipeline/sources/uk/fsa_approved/README.md b/pipeline/sources/uk/fsa_approved/README.md index b8b82b1..338af66 100644 --- a/pipeline/sources/uk/fsa_approved/README.md +++ b/pipeline/sources/uk/fsa_approved/README.md @@ -23,14 +23,17 @@ whose `release_state` is always `not-created` and whose publication state is private-candidate. The source-local refresh command provides the repeatable acquisition boundary. It -can fetch the configured official URL or accept a preserved raw artifact, records +can fetch the configured official URL or accept a preserved raw artifact. Network +fetches require a terms-review JSON and use the shared bounded acquisition +primitive, recording requested/final URLs, redirects, response headers, URL/retrieval/effective dates, hash, byte size, code/config versions, schema -fingerprint, coverage, counts, and quarantine reasons, and writes an aggregate -`refresh.json`. Dry-run is the default; `--mode handoff` is required to emit the -private candidate-handoff contract. A changed header fingerprint or substantial -unbounded count change raises a drift alarm and blocks handoff. A comparison with a -prior normalized run reports disappeared identifiers as `not-observed`, never as -closure. `--bounded-sample` is only for explicitly labeled private test samples. +fingerprint, coverage, counts, and quarantine reasons. It also runs the shared +lifecycle and emits row-free QA/source-health evidence. Dry-run is the default; +`--mode handoff` additionally emits the private candidate-handoff contract. A +changed header fingerprint or substantial unbounded count change raises a drift +alarm and blocks handoff. A comparison with a prior normalized run reports +disappeared identifiers as `not-observed`, never as closure. `--bounded-sample` +is only for explicitly labeled private test samples. Before a registered or fetched run, maintainers must verify the current official URL, effective/publication date, ownership, terms/licence, attribution, rate limits, diff --git a/pipeline/sources/uk/fsa_approved/adapter.py b/pipeline/sources/uk/fsa_approved/adapter.py index 3c642d1..82c436f 100644 --- a/pipeline/sources/uk/fsa_approved/adapter.py +++ b/pipeline/sources/uk/fsa_approved/adapter.py @@ -4,6 +4,8 @@ from dataclasses import asdict, dataclass from pathlib import Path from typing import Any +from pipeline.contracts.adapter_contract import SourceArtifact +from pipeline.common.activity import classify_activities ROOT=Path(__file__).parent CONFIG=json.loads((ROOT/"config.json").read_text(encoding="utf-8")) @@ -59,7 +61,7 @@ def _csv(content): except csv.Error as exc:raise FsaContractError("malformed CSV") from exc raise FsaContractError("unsupported CSV encoding") def _synthetic_record(row,line): - nation=_clean(row.get("nation"));return {"source_id":CONFIG["source_id"],"source_row":line,"source_values":dict(row),"normalized":{"establishment_id":_clean(row.get("establishment_id")),"trading_name":_clean(row.get("trading_name")),"address_lines":tuple(_clean(row.get(f"address_line_{n}")) for n in range(1,4)),"postcode":_clean(row.get("postcode")),"activities":_split(row.get("activities")),"species":_clean(row.get("species")),"competent_authority":_clean(row.get("competent_authority")),"nation":nation,"authority_nation_key":nation,"status":_clean(row.get("status")),"remarks":_clean(row.get("remarks")),"published_date":_clean(row.get("published_date")),"coordinates":None}} + nation=_clean(row.get("nation"));acts=_split(row.get("activities"));return {"source_id":CONFIG["source_id"],"source_row":line,"source_values":dict(row),"normalized":{"establishment_id":_clean(row.get("establishment_id")),"trading_name":_clean(row.get("trading_name")),"address_lines":tuple(_clean(row.get(f"address_line_{n}")) for n in range(1,4)),"postcode":_clean(row.get("postcode")),"activities":acts,"activity_categories":classify_activities(acts),"species":_clean(row.get("species")),"competent_authority":_clean(row.get("competent_authority")),"nation":nation,"authority_nation_key":nation,"status":_clean(row.get("status")),"remarks":_clean(row.get("remarks")),"published_date":_clean(row.get("published_date")),"coordinates":None}} def _coords(row): try:x,y=float(row.get("X","").strip()),float(row.get("Y","").strip()) except ValueError:return None,None,"unresolved-nonnumeric" @@ -71,7 +73,7 @@ def _monthly_record(row,line): acts=tuple(x for x in (_clean(row.get("All_Activities")),_clean(row.get("Part_A__All_sections_")),_clean(row.get("Part B All sections "))) if x) privacy_gate="restricted-withheld-address" if withheld else "privacy-review-required" coordinate_gate="restricted-withheld-address" if withheld else "privacy-review-required" - return {"source_id":CONFIG["source_id"],"source_row":line,"source_values":dict(row),"normalized":{"establishment_id":_clean(row.get("AppNo")),"trading_name":_clean(row.get("TradingName")),"address_lines":None if withheld else tuple(_clean(row.get(k)) for k in ("Address1","Address2","Address3")),"postcode":_clean(row.get("Postcode")),"activities":acts,"species":_clean(row.get("Species")),"competent_authority":_clean(row.get("CompetentAuthority")),"nation":_clean(row.get("Country")),"authority_nation_key":_clean(row.get("Country")),"status":None,"remarks":_clean(row.get("Remarks")),"published_date":None,"coordinates":None,"coordinate_gate":coordinate_gate,"privacy_gate":privacy_gate,"publication_gate":"blocked"}} + return {"source_id":CONFIG["source_id"],"source_row":line,"source_values":dict(row),"normalized":{"establishment_id":_clean(row.get("AppNo")),"trading_name":_clean(row.get("TradingName")),"address_lines":None if withheld else tuple(_clean(row.get(k)) for k in ("Address1","Address2","Address3")),"postcode":_clean(row.get("Postcode")),"activities":acts,"activity_categories":classify_activities(acts),"species":_clean(row.get("Species")),"competent_authority":_clean(row.get("CompetentAuthority")),"nation":_clean(row.get("Country")),"authority_nation_key":_clean(row.get("Country")),"status":None,"remarks":_clean(row.get("Remarks")),"published_date":None,"coordinates":None,"coordinate_gate":coordinate_gate,"privacy_gate":privacy_gate,"publication_gate":"blocked"}} class FsaApprovedEstablishmentsAdapter: source_id=CONFIG["source_id"];schema_version=CONFIG["contract_version"];adapter_version=CONFIG["adapter_version"] @@ -125,6 +127,8 @@ def _monthly(self,headers,rows,digest,fp): return ValidationResult(tuple(accepted),tuple(quarantined),digest,profile="monthly",schema_fingerprint=fp,coverage_counts=coverage,anomaly_counts=anomalies) def parse_file(self,path):return self.parse_bytes(Path(path).read_bytes()) def run(self,raw_path,run_dir,artifact=None): + if isinstance(artifact, SourceArtifact): + artifact = {**asdict(artifact), "checksum_sha256": artifact.sha256} if artifact is None:raise FsaContractError("SourceArtifact metadata is required") required={"source_url","retrieved_at_utc","checksum_sha256","byte_size"};missing=sorted(required-set(artifact)) if missing:raise FsaContractError(f"missing SourceArtifact fields: {', '.join(missing)}") @@ -132,7 +136,7 @@ def run(self,raw_path,run_dir,artifact=None): if digest!=artifact["checksum_sha256"] or len(raw)!=int(artifact["byte_size"]):raise FsaContractError("source checksum or byte size mismatch") result=self.parse_bytes(raw);accepted=list(result.accepted);quarantined=list(result.quarantined);root=Path(run_dir);parsed=accepted+[x["record"] for x in quarantined] normalized_sha=_jsonl(root/"normalized"/"records.jsonl",accepted);_jsonl(root/"parsed"/"records.jsonl",parsed);_jsonl(root/"quarantined"/"records.jsonl",quarantined);(root/"released").mkdir(parents=True,exist_ok=True) - manifest={**artifact,"source_id":self.source_id,"adapter_version":self.adapter_version,"schema_version":self.schema_version,"checksum_sha256":digest,"byte_size":len(raw),"input_rows":len(parsed),"normalized_rows":len(accepted),"normalized_sha256":normalized_sha,"quarantined_rows":len(quarantined),"profile":result.profile,"schema_fingerprint":result.schema_fingerprint,"coverage_counts":result.coverage_counts or {},"anomaly_counts":result.anomaly_counts or {},"geocoding":"disabled","release_state":"not-created","publication_state":"private-candidate"} + manifest={**artifact,"source_id":self.source_id,"country_code":"GB","adapter_version":self.adapter_version,"schema_version":self.schema_version,"checksum_sha256":digest,"byte_size":len(raw),"input_rows":len(parsed),"normalized_rows":len(accepted),"normalized_sha256":normalized_sha,"quarantined_rows":len(quarantined),"profile":result.profile,"schema_fingerprint":result.schema_fingerprint,"coverage_counts":result.coverage_counts or {},"anomaly_counts":result.anomaly_counts or {},"geocoding":"disabled","release_state":"not-created","publication_state":"private-candidate"} _atomic(root/"manifest.json",(json.dumps(manifest,ensure_ascii=False,sort_keys=True,indent=2,default=list)+"\n").encode());return manifest def run_registered(raw_path,run_dir,config):return FsaApprovedEstablishmentsAdapter().run(raw_path,run_dir,config) diff --git a/pipeline/sources/uk/fsa_approved/refresh.py b/pipeline/sources/uk/fsa_approved/refresh.py index 086c353..7dd07bc 100644 --- a/pipeline/sources/uk/fsa_approved/refresh.py +++ b/pipeline/sources/uk/fsa_approved/refresh.py @@ -2,15 +2,14 @@ from __future__ import annotations import argparse -import csv import hashlib import json -import urllib.request -from datetime import datetime, timezone from pathlib import Path from typing import Any from pipeline.contracts.adapter_contract import SourceArtifact +from pipeline.common.acquisition import AcquisitionError, fetch_source, utc_now +from pipeline.common.orchestrator import run_private_lifecycle from .adapter import CONFIG, FsaApprovedEstablishmentsAdapter, _csv from .handoff import write_private_monthly_handoff @@ -20,10 +19,6 @@ class RefreshError(ValueError): """The source refresh cannot safely continue.""" -def _utc_now() -> str: - return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z") - - def _header_fingerprint(raw: bytes) -> tuple[str, int]: headers, _, _ = _csv(raw) return hashlib.sha256(json.dumps(headers, ensure_ascii=False, separators=(",", ":")).encode()).hexdigest(), len(headers) @@ -48,14 +43,26 @@ def _write_json(path: Path, value: dict[str, Any]) -> None: path.write_text(json.dumps(value, ensure_ascii=False, sort_keys=True, indent=2, default=list) + "\n", encoding="utf-8") -def _fetch(url: str, destination: Path) -> tuple[bytes, str]: - request = urllib.request.Request(url, headers={"User-Agent": "UntilEveryCage/uk-fsa-refresh"}) - with urllib.request.urlopen(request, timeout=60) as response: - raw = response.read() - effective = response.headers.get("Last-Modified") or "unknown" - destination.parent.mkdir(parents=True, exist_ok=True) - destination.write_bytes(raw) - return raw, effective +def _local_acquisition_metadata(path: Path, *, source_url: str, retrieved_at_utc: str, effective_date: str | None) -> dict[str, Any]: + raw = path.read_bytes() + existing = path.parent / "acquisition-metadata.json" + if existing.exists(): + retained = json.loads(existing.read_text(encoding="utf-8")) + if retained.get("sha256") == hashlib.sha256(raw).hexdigest() and int(retained.get("byte_size", -1)) == len(raw): + return retained + return { + "acquisition_method": "preserved_local_artifact", "source_id": CONFIG["source_id"], + "artifact": path.name, "artifact_path": str(path), "requested_url": source_url, + "final_url": source_url, "redirects": [], "response_headers": {}, + "requested_at_utc": retrieved_at_utc, "retrieved_at_utc": retrieved_at_utc, + "effective_date": effective_date or "unknown", "publication_date": None, + "sha256": hashlib.sha256(raw).hexdigest(), "byte_size": len(raw), + "code_version": CONFIG["adapter_version"], "config_version": CONFIG["contract_version"], + "coverage": "England and Wales profile; Northern Ireland remains a separate source scope", + "rights_caveat": "OGL v3 indicated by catalogue; project terms/attribution review remains recorded separately", + "privacy_caveat": "restricted private staging; privacy and coordinate review pending", + "terms_review": "not_required_for_already-preserved-local-artifact", + } def refresh_monthly( @@ -71,6 +78,8 @@ def refresh_monthly( mode: str = "dry-run", previous_normalized: str | Path | None = None, bounded_sample: bool = False, + terms_review_path: str | Path | None = None, + max_bytes: int = 64 * 1024 * 1024, ) -> dict[str, Any]: """Run a private refresh; ``mode=handoff`` also emits candidate-handoff-v1.""" if mode not in {"dry-run", "handoff"}: @@ -79,14 +88,31 @@ def refresh_monthly( raise RefreshError("specify exactly one of raw_path or fetch") root = Path(run_dir) if fetch: - raw, observed_effective = _fetch(source_url, root / "raw" / "source.csv") - effective_date = effective_date or observed_effective - input_path = root / "raw" / "source.csv" + if terms_review_path is None: + raise RefreshError("terms_review_path is required for network acquisition") + try: + acquisition = fetch_source( + source_id=CONFIG["source_id"], url=source_url, output_root=root / "acquisition", + artifact_name="source.csv", terms_review_path=terms_review_path, max_bytes=max_bytes, + code_version=CONFIG["adapter_version"], config_version=CONFIG["contract_version"], + coverage="England and Wales profile; Northern Ireland remains a separate source scope", + rights_caveat="OGL v3 indicated by catalogue; project terms/attribution review is retained with this run", + privacy_caveat="restricted private staging; privacy and coordinate review pending", + effective_date=effective_date, + ) + except AcquisitionError as exc: + raise RefreshError(str(exc)) from exc + effective_date = effective_date or acquisition.get("effective_date") + input_path = Path(acquisition["artifact_path"]) + raw = input_path.read_bytes() else: input_path = Path(raw_path) # type: ignore[arg-type] raw = input_path.read_bytes() - retrieved_at_utc = retrieved_at_utc or _utc_now() + retrieved_at_utc = retrieved_at_utc or utc_now() effective_date = effective_date or "unknown" + if not fetch: + acquisition = _local_acquisition_metadata(input_path, source_url=source_url, retrieved_at_utc=retrieved_at_utc, effective_date=effective_date) + _write_json(root / "acquisition-metadata.json", acquisition) adapter = FsaApprovedEstablishmentsAdapter() result = adapter.parse_bytes(raw) if result.profile != "monthly": @@ -117,15 +143,17 @@ def refresh_monthly( effective_date=effective_date, code_version=code_version, config_version=config_version, - rights_caveat="metadata-indicated-open-government-licence-v3-pending-project-review", - privacy_caveat="restricted-private-staging; privacy and coordinate review pending", - coverage="England and Wales profile; other nations remain quarantined", + rights_caveat=acquisition.get("rights_caveat") or "metadata-indicated-open-government-licence-v3-pending-project-review", + privacy_caveat=acquisition.get("privacy_caveat") or "restricted-private-staging; privacy and coordinate review pending", + coverage=acquisition.get("coverage") or "England and Wales profile; Northern Ireland remains a separate source scope", + redirects=tuple(acquisition.get("redirects") or ()), ) if alarms and mode == "handoff": raise RefreshError("refresh drift alarm blocks handoff: " + ", ".join(alarms)) handoff = None if mode == "handoff": handoff = write_private_monthly_handoff(input_path, root / "handoff", artifact) + lifecycle = run_private_lifecycle(input_path, root / "lifecycle", artifact, adapter, health_as_of_utc=retrieved_at_utc) report = { "source_url": source_url, "retrieved_at_utc": retrieved_at_utc, @@ -149,6 +177,12 @@ def refresh_monthly( "mode": mode, "release_state": "not-created", "publication_state": "private-candidate" if handoff else "not-staged", + "requested_url": acquisition.get("requested_url"), + "final_url": acquisition.get("final_url"), + "redirects": acquisition.get("redirects", []), + "acquisition_metadata": str(root / "acquisition-metadata.json"), + "lifecycle_run_dir": lifecycle.get("run_dir"), + "health_path": str(Path(lifecycle["run_dir"]) / "source-health.json"), } _write_json(root / "refresh.json", report) return {"report": report, "handoff": handoff} @@ -168,12 +202,15 @@ def main() -> int: parser.add_argument("--mode", choices=("dry-run", "handoff"), default="dry-run") parser.add_argument("--previous-normalized", type=Path) parser.add_argument("--bounded-sample", action="store_true") + parser.add_argument("--terms-review", type=Path) + parser.add_argument("--max-bytes", type=int, default=64 * 1024 * 1024) args = parser.parse_args() result = refresh_monthly( run_dir=args.run_dir, raw_path=args.raw, fetch=args.fetch, source_url=args.source_url, retrieved_at_utc=args.retrieved_at_utc, effective_date=args.effective_date, code_version=args.code_version, config_version=args.config_version, mode=args.mode, previous_normalized=args.previous_normalized, bounded_sample=args.bounded_sample, + terms_review_path=args.terms_review, max_bytes=args.max_bytes, ) print(json.dumps(result["report"], sort_keys=True)) return 0 diff --git a/pipeline/sources/uk/fss_approved/README.md b/pipeline/sources/uk/fss_approved/README.md index 5cb201b..d211ed1 100644 --- a/pipeline/sources/uk/fss_approved/README.md +++ b/pipeline/sources/uk/fss_approved/README.md @@ -1,15 +1,23 @@ # FSS Scotland approved establishments -This source-first adapter is synthetic-fixture-only. It accepts caller-supplied -CSV bytes and never downloads, geocodes, promotes, publishes, exports, or -contacts FSS/FSA. It preserves source cells and approval IDs as strings, +This source-first adapter accepts a preserved CSV artifact or a bounded, +explicitly authorized fetch from the FSS open-data URL. Acquisition uses the +shared private acquisition primitive: raw bytes and acquisition metadata remain +under ignored private storage, with requested/final URLs, redirect chain, +response headers, retrieval/effective dates, hashes, byte size, code/config +versions, coverage, privacy/rights caveats, and the operator terms record. +The adapter never promotes, publishes, exports, or geocodes. It preserves source cells and approval IDs as strings, including leading zeroes, while representing absent coordinates as `null`. Rows with duplicate or missing approval IDs, missing/unknown activities, unknown statuses, malformed cells, remarks, or privacy-risk address tokens are quarantined. Header and row-shape drift fails closed. Every run records a -checksum, byte size, source metadata, adapter/schema versions, counts, and -the explicit human-gated/non-release state. +checksum, byte size, source metadata, adapter/schema versions, activity +categories, counts, and the explicit private-candidate/non-release state. + +The source-owned refresh command runs the shared lifecycle and emits row-free +QA and `source-health.json`. `--mode handoff` additionally emits the shared +candidate-handoff contract; it does not approve or publish a release. Before acquisition, a maintainer must verify the current FSS artifact URL, schema, publication/effective date, licence and attribution terms in an @@ -17,3 +25,11 @@ approved environment. England/Wales FSA and Northern Ireland sources require separate evidence and adapters. Privacy/suppression review, human factual review, project approval, release authorization, and any legal/terms decision remain gates; no real artifact or facility record belongs in this repository. + +Private refresh example: + +```text +python -m pipeline.sources.uk.fss_approved.refresh \ + --fetch --terms-review \ + --run-dir --mode dry-run +``` diff --git a/pipeline/sources/uk/fss_approved/adapter.py b/pipeline/sources/uk/fss_approved/adapter.py index 50e1bfd..b843b02 100644 --- a/pipeline/sources/uk/fss_approved/adapter.py +++ b/pipeline/sources/uk/fss_approved/adapter.py @@ -12,6 +12,8 @@ from dataclasses import asdict, dataclass from pathlib import Path from typing import Any +from pipeline.contracts.adapter_contract import SourceArtifact +from pipeline.common.activity import classify_activities ROOT = Path(__file__).parent CONFIG = json.loads((ROOT / "config.json").read_text(encoding="utf-8")) @@ -32,6 +34,8 @@ class ValidationResult: source_sha256: str contract_version: str = CONFIG["contract_version"] release_allowed: bool = False + coverage_counts: dict[str, int] | None = None + anomaly_counts: dict[str, int] | None = None def as_dict(self) -> dict[str, Any]: return asdict(self) @@ -52,9 +56,14 @@ def _record(row: dict[str, str], line: int) -> dict[str, Any]: return {"source_id": CONFIG["source_id"], "source_row": line, "source_values": dict(row), "normalized": { "approval_number": _clean(row.get("approval_number")), + # Keep the source-native identifier and expose the shared + # importer identity explicitly; neither value is inferred or + # used to merge records across authorities. + "establishment_id": _clean(row.get("approval_number")), "trading_name": _clean(row.get("trading_name")), "address_lines": tuple(_clean(row.get(f"address_line_{n}")) for n in range(1, 4)), "postcode": _clean(row.get("postcode")), "activities": _split(row.get("activities")), + "activity_categories": classify_activities(_split(row.get("activities"))), "species": _clean(row.get("species")), "competent_authority": _clean(row.get("competent_authority")), "nation": _clean(row.get("nation")), "status": _clean(row.get("status")), @@ -71,65 +80,141 @@ def parse_bytes(self, content: bytes) -> ValidationResult: digest = hashlib.sha256(content).hexdigest() try: text = content.decode("utf-8-sig") - reader = csv.DictReader(text.splitlines(), strict=True) - if tuple(reader.fieldnames or ()) != REQUIRED_COLUMNS: - raise FssContractError("schema drift: expected pinned FSS columns in exact order") - rows = list(reader) - except UnicodeDecodeError as exc: - raise FssContractError("source is not UTF-8 CSV") from exc + except UnicodeDecodeError: + try: + text = content.decode("cp1252") + except UnicodeDecodeError as exc: + raise FssContractError("source is neither UTF-8 nor Windows-1252 CSV") from exc + try: + rows = list(csv.reader(text.splitlines(), strict=True)) except csv.Error as exc: raise FssContractError("malformed CSV") from exc - if any(None in row for row in rows): - raise FssContractError("schema drift: a row has extra columns") - values = [_clean(row.get("approval_number")) for row in rows] + + # The live FSS export has a four-row title/published-date preamble and + # a blank first column. The synthetic contract remains supported for + # deterministic tests, but live parsing is pinned to the inspected + # header rather than guessing from column positions. + live_header = tuple(CONFIG.get("live_columns", ())) + live_header_index = next((index for index, row in enumerate(rows) if tuple(row) == live_header), None) + live = live_header_index is not None + if not live and rows and tuple(rows[0]) == REQUIRED_COLUMNS: + headers = rows[0] + data_rows = rows[1:] + mapped_rows = [{header: row[index] if index < len(row) else None for index, header in enumerate(headers)} for row in data_rows] + line_numbers = range(2, len(rows) + 1) + elif live: + assert live_header_index is not None + headers = rows[live_header_index] + data_rows = rows[live_header_index + 1:] + mapped_rows = [] + for row in data_rows: + if len(row) != len(headers): + raise FssContractError("schema drift: a live row has the wrong column count") + mapped_rows.append({(header if header else "source_column_0"): row[index] for index, header in enumerate(headers)}) + line_numbers = range(live_header_index + 2, len(rows) + 1) + else: + raise FssContractError("schema drift: expected pinned FSS or inspected live header") + if any(len(row) != len(headers) for row in data_rows): + raise FssContractError("schema drift: a row has the wrong column count") + values = [_clean(row.get("approval_number") or row.get("Approval Number")) for row in mapped_rows] duplicates = {value for value in values if value and values.count(value) > 1} accepted: list[dict[str, Any]] = [] quarantined: list[dict[str, Any]] = [] - for line, row in enumerate(rows, 2): + anomalies: dict[str, int] = {} + coverage: dict[str, int] = {} + for line, row in zip(line_numbers, mapped_rows): reasons: list[str] = [] - if any(value is None for value in row.values()): + if live: + approval = _clean(row.get("Approval Number")) + activities = tuple(value for key, value in row.items() if key in {"All Activities Approved", "Associated Activities"} or key.startswith("Part A - ") or key.startswith("Part B - ") if _clean(value)) + record_row = { + "approval_number": approval, "trading_name": row.get("Trading Name"), + "address_line_1": row.get("Address 1"), "address_line_2": row.get("Address 2"), + "address_line_3": row.get("Address 3"), "address_line_4": row.get("Address 4"), + "postcode": row.get("Post Code"), "activities": ";".join(activities), + "species": row.get("Species"), "competent_authority": row.get("Competent Authority"), + "nation": "Scotland", "status": None, "remarks": row.get("Remarks"), + "published_date": None, "source_url": None, "source_licence": None, + **{f"live_{key}": value for key, value in row.items()}, + } + activities = tuple(_clean(value) for value in activities if _clean(value)) + missing_values = not approval or not _clean(row.get("Trading Name")) + else: + approval = _clean(row.get("approval_number")) + activities = _split(row.get("activities")) + record_row = row + missing_values = any(value is None for value in row.values()) + if missing_values: reasons.append("malformed_row") - approval = _clean(row.get("approval_number")) if not approval: reasons.append("missing_approval_number") if approval in duplicates: reasons.append("duplicate_id") - activities = _split(row.get("activities")) if not activities: reasons.append("missing_activity") - elif any(activity not in ALLOWED_ACTIVITIES for activity in activities): + elif not live and any(activity not in ALLOWED_ACTIVITIES for activity in activities): reasons.append("unknown_activity") - status = _clean(row.get("status")) + elif live and not classify_activities(activities): + reasons.append("no_relevant_activity") + status = _clean(row.get("status")) if not live else None if status and status.lower() not in ALLOWED_STATUSES: reasons.append("unknown_status") if _clean(row.get("remarks")): reasons.append("remarks_present") - address = " ".join(_clean(row.get(f"address_line_{n}")) or "" for n in range(1, 4)) + address = " ".join(_clean(record_row.get(f"address_line_{n}")) or "" for n in range(1, 5 if live else 4)) if ADDRESS_RISK.search(address): reasons.append("address_privacy_risk") - record = _record(row, line) + nation = "Scotland" if live else (_clean(row.get("nation")) or "") + coverage[nation] = coverage.get(nation, 0) + 1 + for reason in reasons: + anomalies[reason] = anomalies.get(reason, 0) + 1 + record = _record(record_row, line) (quarantined if reasons else accepted).append({"reasons": tuple(reasons), "record": record} if reasons else record) - return ValidationResult(tuple(accepted), tuple(quarantined), digest) + return ValidationResult(tuple(accepted), tuple(quarantined), digest, coverage_counts=coverage, anomaly_counts=anomalies) def parse_file(self, path: str | Path) -> ValidationResult: return self.parse_bytes(Path(path).read_bytes()) - def run(self, raw_path: str | Path, run_dir: str | Path, config: dict[str, Any] | None = None) -> dict[str, Any]: + def run(self, raw_path: str | Path, run_dir: str | Path, config: SourceArtifact | dict[str, Any] | None = None) -> dict[str, Any]: raw = Path(raw_path).read_bytes() + if isinstance(config, SourceArtifact): + config = asdict(config) + config["checksum_sha256"] = config["sha256"] + elif config is None: + # Retain the fixture-only convenience used by the original unit + # tests; real lifecycle runs must supply a typed artifact. + config = {"source_url": "synthetic-fixture://fss", "retrieved_at_utc": "2026-01-01T00:00:00Z", + "checksum_sha256": hashlib.sha256(raw).hexdigest(), "byte_size": len(raw), + "code_version": self.adapter_version, "config_version": self.schema_version, + "coverage": "Scotland synthetic fixture"} + if config.get("checksum_sha256") or "byte_size" in config: + if config.get("checksum_sha256") != hashlib.sha256(raw).hexdigest() or int(config.get("byte_size", -1)) != len(raw): + raise FssContractError("source checksum or byte size mismatch") + else: + # Legacy fixture callers supplied only a URL/timestamp. Fill the + # facts locally for compatibility; typed lifecycle callers always + # arrive with the shared SourceArtifact integrity fields above. + config["checksum_sha256"] = hashlib.sha256(raw).hexdigest() + config["byte_size"] = len(raw) result = self.parse_bytes(raw) root = Path(run_dir) _write_jsonl(root / "parsed" / "records.jsonl", list(result.accepted) + [item["record"] for item in result.quarantined]) normalized_sha256 = _write_jsonl(root / "normalized" / "records.jsonl", list(result.accepted)) _write_jsonl(root / "quarantined" / "records.jsonl", list(result.quarantined)) (root / "released").mkdir(parents=True, exist_ok=True) - manifest = {"source_id": self.source_id, "adapter_version": self.adapter_version, + manifest = {"source_id": self.source_id, "country_code": "GB", "adapter_version": self.adapter_version, "schema_version": self.schema_version, "checksum_sha256": result.source_sha256, + "sha256": result.source_sha256, "byte_size": len(raw), "input_rows": len(result.accepted) + len(result.quarantined), "normalized_rows": len(result.accepted), "normalized_sha256": normalized_sha256, "quarantined_rows": len(result.quarantined), - "release_state": "not-created", "publication_state": "human-gate-required", - "acquisition": "synthetic-fixture-only", "source_url": (config or {}).get("source_url"), - "retrieved_at": (config or {}).get("retrieved_at")} + "coverage_counts": result.coverage_counts or {}, "anomaly_counts": result.anomaly_counts or {}, + "release_state": "not-created", "publication_state": "private-candidate", + "acquisition": dict(config), "source_url": config.get("source_url"), + "retrieved_at_utc": config.get("retrieved_at_utc") or config.get("retrieved_at"), "effective_date": config.get("effective_date"), + "publication_date": config.get("publication_date"), "code_version": config.get("code_version", self.adapter_version), + "config_version": config.get("config_version", self.schema_version), "coverage": config.get("coverage"), + "rights_caveat": config.get("rights_caveat"), "privacy_caveat": config.get("privacy_caveat")} _atomic(root / "manifest.json", (json.dumps(manifest, ensure_ascii=False, sort_keys=True, indent=2) + "\n").encode()) return manifest diff --git a/pipeline/sources/uk/fss_approved/config.json b/pipeline/sources/uk/fss_approved/config.json index c6124a0..75978f3 100644 --- a/pipeline/sources/uk/fss_approved/config.json +++ b/pipeline/sources/uk/fss_approved/config.json @@ -3,14 +3,20 @@ "adapter_version": "fss-scotland-v2-1", "source_id": "fss_approved_establishments", "authority": "Food Standards Scotland", + "source_url": "https://www.foodstandards.gov.scot/sites/default/files/2026-08/Approved%20Establishments%20in%20Scotland.csv", + "catalog_url": "https://www.foodstandards.gov.scot/open-data-portal/approved-establishments-in-scotland", + "terms_url": "https://www.nationalarchives.gov.uk/doc/open-government-licence/version/3/", "country": "GB", "nation": "Scotland", "format": "csv", "update_frequency": "monthly", "licence": "Open Government Licence v3.0", - "acquisition": "synthetic-fixture-only", + "acquisition": "bounded-private-fetch", + "schema_status": "pinned-fixture-contract-pending-live-verification", "release_allowed_by_default": false, "allowed_activities": ["slaughter", "cutting", "processing", "storage"], "allowed_statuses": ["active", "inactive", "suspended", "closed"], - "required_columns": ["approval_number", "trading_name", "address_line_1", "address_line_2", "address_line_3", "postcode", "activities", "species", "competent_authority", "nation", "status", "remarks", "published_date", "source_url", "source_licence"] + "required_columns": ["approval_number", "trading_name", "address_line_1", "address_line_2", "address_line_3", "postcode", "activities", "species", "competent_authority", "nation", "status", "remarks", "published_date", "source_url", "source_licence"], + "live_header_row": 5, + "live_columns": ["", "Approval Number", "Trading Name", "Address 1", "Address 2", "Address 3", "Address 4", "Post Code", "All Activities Approved", "Part A - Section 0 - General Activity Establishment", "Part A - Section I - Meat of Domestic Ungulates", "Part A - Section II - Meat from poultry and lagomorphs", "Part A - Section III - Meat of farmed game", "Part A - Section IV - Wild game meat", "Part A - Section V - Minced meat; Meat preparations and Mechanically separated meat", "Part A - Section VI - Meat Products", "Part A - Section VII - Live Bivalve Molluscs", "Part A - Section VIII - Fishery Products", "Part A - Section IX - Raw milk and Dairy products", "Part A - Section X - Eggs and egg products", "Part A - Section XI - Frog's Legs and Snails", "Part A - Section XII - Rendered animal fats and greaves", "Part A - Section XIII - Treated stomachs; bladders and intestines", "Part A - Section XIV - Gelatine", "Part A - Section XV - Collagen", "Part A - Section XVI - Highly refined products", "Part B - Section I - Sprouts", "Associated Activities", "Species", "Remarks", "Competent Authority", "Geographic Local Authority", "EU Export List"] } diff --git a/pipeline/sources/uk/fss_approved/refresh.py b/pipeline/sources/uk/fss_approved/refresh.py new file mode 100644 index 0000000..819eb02 --- /dev/null +++ b/pipeline/sources/uk/fss_approved/refresh.py @@ -0,0 +1,185 @@ +"""Bounded private acquisition and shared-lifecycle refresh for FSS Scotland.""" +from __future__ import annotations + +import argparse +import hashlib +import json +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from pipeline.common.acquisition import AcquisitionError, fetch_source, utc_now +from pipeline.common.orchestrator import run_private_lifecycle +from pipeline.contracts.adapter_contract import SourceArtifact + +from .adapter import CONFIG, FssApprovedEstablishmentsAdapter +from .handoff import write_private_handoff + + +def _write_json(path: Path, value: dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(value, ensure_ascii=False, sort_keys=True, indent=2, default=list) + "\n", encoding="utf-8") + + +def _local_metadata(raw_path: Path, *, source_url: str, retrieved_at_utc: str, effective_date: str | None) -> dict[str, Any]: + raw = raw_path.read_bytes() + existing = raw_path.parent / "acquisition-metadata.json" + if existing.exists(): + retained = json.loads(existing.read_text(encoding="utf-8")) + if retained.get("sha256") == hashlib.sha256(raw).hexdigest() and int(retained.get("byte_size", -1)) == len(raw): + return retained + return { + "acquisition_method": "preserved_local_artifact", + "source_id": CONFIG["source_id"], + "artifact": raw_path.name, + "artifact_path": str(raw_path), + "requested_url": source_url, + "final_url": source_url, + "redirects": [], + "response_headers": {}, + "requested_at_utc": retrieved_at_utc, + "retrieved_at_utc": retrieved_at_utc, + "effective_date": effective_date or "unknown", + "publication_date": None, + "sha256": hashlib.sha256(raw).hexdigest(), + "byte_size": len(raw), + "adapter_version": CONFIG["adapter_version"], + "code_version": CONFIG["adapter_version"], + "config_version": CONFIG["contract_version"], + "coverage": "Scotland only; FSA England/Wales and Northern Ireland remain separate source scopes", + "rights_caveat": "OGL v3 indicated by source metadata; project terms/attribution review remains recorded separately", + "privacy_caveat": "restricted private staging; personal-data and precise-location screening pending", + "terms_review": "not_required_for_already-preserved-local-artifact", + } + + +def _read_previous_ids(path: Path | None) -> set[str]: + if path is None or not path.exists(): + return set() + values: set[str] = set() + for line in path.read_text(encoding="utf-8").splitlines(): + if line: + normalized = json.loads(line).get("normalized", {}) + value = normalized.get("establishment_id") or normalized.get("approval_number") + if isinstance(value, str) and value: + values.add(value) + return values + + +def refresh_scotland( + *, + run_dir: str | Path, + raw_path: str | Path | None = None, + fetch: bool = False, + source_url: str = CONFIG["source_url"], + terms_review_path: str | Path | None = None, + retrieved_at_utc: str | None = None, + effective_date: str | None = None, + publication_date: str | None = None, + mode: str = "dry-run", + previous_normalized: str | Path | None = None, + max_bytes: int = 64 * 1024 * 1024, +) -> dict[str, Any]: + if mode not in {"dry-run", "handoff"}: + raise AcquisitionError("mode must be dry-run or handoff") + if fetch == (raw_path is not None): + raise AcquisitionError("specify exactly one of raw_path or fetch") + root = Path(run_dir) + if fetch: + if terms_review_path is None: + raise AcquisitionError("terms_review_path is required for network acquisition") + acquisition = fetch_source( + source_id=CONFIG["source_id"], url=source_url, output_root=root / "acquisition", + artifact_name="source.csv", terms_review_path=terms_review_path, max_bytes=max_bytes, + code_version=CONFIG["adapter_version"], config_version=CONFIG["contract_version"], + coverage="Scotland only; England/Wales/Northern Ireland remain separate source scopes", + rights_caveat="OGL v3 indicated; project attribution/terms review is retained with this run", + privacy_caveat="restricted private staging; personal-data and precise-location screening pending", + effective_date=effective_date, publication_date=publication_date, + ) + input_path = Path(acquisition["artifact_path"]) + else: + input_path = Path(raw_path) # type: ignore[arg-type] + if not input_path.is_file(): + raise AcquisitionError(f"raw artifact does not exist: {input_path}") + retrieved_at_utc = retrieved_at_utc or utc_now() + acquisition = _local_metadata(input_path, source_url=source_url, retrieved_at_utc=retrieved_at_utc, effective_date=effective_date) + raw = input_path.read_bytes() + _write_json(root / "acquisition-metadata.json", acquisition) + retrieved_at_utc = acquisition["retrieved_at_utc"] + artifact = SourceArtifact( + source_url=source_url, retrieved_at_utc=retrieved_at_utc, + sha256=acquisition["sha256"], byte_size=acquisition["byte_size"], + publication_date=publication_date or acquisition.get("publication_date"), + effective_date=effective_date or acquisition.get("effective_date"), + code_version=CONFIG["adapter_version"], config_version=CONFIG["contract_version"], + rights_caveat=acquisition.get("rights_caveat"), privacy_caveat=acquisition.get("privacy_caveat"), + coverage=acquisition.get("coverage"), redirects=tuple(acquisition.get("redirects") or ()), + ) + adapter = FssApprovedEstablishmentsAdapter() + result = adapter.parse_bytes(raw) + current_ids = { + record["normalized"].get("approval_number") + for record in result.accepted + } | { + item["record"]["normalized"].get("approval_number") + for item in result.quarantined + } + current_ids.discard(None) + previous_ids = _read_previous_ids(Path(previous_normalized) if previous_normalized else None) + disappeared = len(previous_ids - current_ids) if previous_ids else 0 + lifecycle = run_private_lifecycle(input_path, root / "lifecycle", artifact, adapter, health_as_of_utc=retrieved_at_utc) + handoff = None + if mode == "handoff": + handoff = write_private_handoff(input_path, root / "handoff", artifact) + report = { + "source_id": CONFIG["source_id"], "source_url": source_url, + "requested_url": acquisition.get("requested_url"), "final_url": acquisition.get("final_url"), + "redirects": acquisition.get("redirects", []), "retrieved_at_utc": retrieved_at_utc, + "effective_date": effective_date or acquisition.get("effective_date"), + "publication_date": publication_date or acquisition.get("publication_date"), + "sha256": artifact.sha256, "byte_size": artifact.byte_size, + "code_version": artifact.code_version, "config_version": artifact.config_version, + "input_rows": len(result.accepted) + len(result.quarantined), + "normalized_rows": len(result.accepted), "quarantined_rows": len(result.quarantined), + "coverage_counts": result.coverage_counts or {}, "anomaly_counts": result.anomaly_counts or {}, + "disappeared_not_observed_count": disappeared, + "disappearance_semantics": "not-observed; never inferred as closure", + "geocoding": "disabled", "release_state": "not-created", + "publication_state": "private-candidate", "mode": mode, + "acquisition_metadata": str(root / "acquisition-metadata.json"), + "lifecycle_run_dir": str(lifecycle["run_dir"]), + "health_path": str(Path(lifecycle["run_dir"]) / "source-health.json"), + "handoff": handoff, + } + _write_json(root / "refresh.json", report) + return {"report": report, "lifecycle": lifecycle, "handoff": handoff} + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + source = parser.add_mutually_exclusive_group(required=True) + source.add_argument("--raw", type=Path) + source.add_argument("--fetch", action="store_true") + parser.add_argument("--run-dir", type=Path, required=True) + parser.add_argument("--source-url", default=CONFIG["source_url"]) + parser.add_argument("--terms-review", type=Path) + parser.add_argument("--retrieved-at-utc") + parser.add_argument("--effective-date") + parser.add_argument("--publication-date") + parser.add_argument("--mode", choices=("dry-run", "handoff"), default="dry-run") + parser.add_argument("--previous-normalized", type=Path) + parser.add_argument("--max-bytes", type=int, default=64 * 1024 * 1024) + args = parser.parse_args() + result = refresh_scotland( + run_dir=args.run_dir, raw_path=args.raw, fetch=args.fetch, source_url=args.source_url, + terms_review_path=args.terms_review, retrieved_at_utc=args.retrieved_at_utc, + effective_date=args.effective_date, publication_date=args.publication_date, mode=args.mode, + previous_normalized=args.previous_normalized, max_bytes=args.max_bytes, + ) + print(json.dumps(result["report"], ensure_ascii=False, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/pipeline/sources/uk/fss_approved/test_adapter.py b/pipeline/sources/uk/fss_approved/test_adapter.py index b7eba3d..c318c09 100644 --- a/pipeline/sources/uk/fss_approved/test_adapter.py +++ b/pipeline/sources/uk/fss_approved/test_adapter.py @@ -3,7 +3,7 @@ import unittest from pathlib import Path -from .adapter import FssApprovedEstablishmentsAdapter, FssContractError +from .adapter import CONFIG, FssApprovedEstablishmentsAdapter, FssContractError FIXTURES = Path(__file__).parent / "fixtures" @@ -32,6 +32,16 @@ def test_schema_drift_fails_closed(self): with self.assertRaises(FssContractError): self.adapter.parse_file(FIXTURES / "schema_drift.csv") + def test_inspected_live_export_profile_skips_preamble_and_classifies(self): + row = ["", "FSS-TEST-1", "Synthetic Scotland Foods", "Industrial Estate", "", "", "", "AB1 2CD", "CP (Cutting Plant); CS (Cold Store)"] + [""] * 19 + ["pig", "", "Food Standards Scotland", "Aberdeen City", "No"] + content = ("\n\nApproved Establishments in Scotland\n11 August 2026\n\n" + ",".join(CONFIG["live_columns"]) + "\n" + ",".join(row) + "\n").encode("cp1252") + result = self.adapter.parse_bytes(content) + self.assertEqual(len(result.accepted), 1) + self.assertEqual(result.accepted[0]["source_row"], 7) + self.assertEqual(result.accepted[0]["normalized"]["nation"], "Scotland") + self.assertEqual(result.accepted[0]["normalized"]["activity_categories"], ("cutting", "logistics_and_storage")) + self.assertEqual(result.coverage_counts, {"Scotland": 1}) + def test_run_is_deterministic_and_has_no_release(self): with tempfile.TemporaryDirectory() as directory: first = Path(directory) / "one" diff --git a/pipeline/sources/uk/fss_approved/test_refresh.py b/pipeline/sources/uk/fss_approved/test_refresh.py new file mode 100644 index 0000000..b4dd2d2 --- /dev/null +++ b/pipeline/sources/uk/fss_approved/test_refresh.py @@ -0,0 +1,42 @@ +import json +import tempfile +import unittest +from pathlib import Path + +from .refresh import refresh_scotland + + +FIXTURE = Path(__file__).parent / "fixtures" / "valid.csv" + + +class FssRefreshTests(unittest.TestCase): + def test_shared_lifecycle_emits_health_and_not_observed_semantics(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + previous = root / "previous.jsonl" + previous.write_text(json.dumps({"normalized": {"approval_number": "001234"}}) + "\n" + json.dumps({"normalized": {"approval_number": "missing"}}) + "\n", encoding="utf-8") + result = refresh_scotland( + raw_path=FIXTURE, run_dir=root / "run", retrieved_at_utc="2026-09-14T00:00:00Z", + effective_date="2026-09-01", mode="handoff", previous_normalized=previous, + ) + report = result["report"] + self.assertEqual(report["coverage_counts"], {"Scotland": 2}) + self.assertEqual(report["disappeared_not_observed_count"], 1) + self.assertIn("not-observed", report["disappearance_semantics"]) + health_path = Path(report["health_path"]) + health = json.loads(health_path.read_text(encoding="utf-8")) + self.assertEqual(health["health_state"], "private-validated") + self.assertFalse(health["public_exposure"]) + self.assertEqual(result["handoff"]["release_state"], "not-created") + + def test_handoff_keeps_quarantine_and_source_scope(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + result = refresh_scotland(raw_path=Path(__file__).parent / "fixtures" / "quarantine.csv", run_dir=root / "run", retrieved_at_utc="2026-09-14T00:00:00Z", mode="handoff") + self.assertEqual(result["report"]["normalized_rows"], 0) + self.assertEqual(result["report"]["quarantined_rows"], 4) + self.assertEqual(result["handoff"]["normalized_rows"], 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/sources/uk/registry.json b/pipeline/sources/uk/registry.json index 3f4e44f..c7343a3 100644 --- a/pipeline/sources/uk/registry.json +++ b/pipeline/sources/uk/registry.json @@ -1 +1 @@ -{"version": "1", "sources": {"fss_approved_establishments": {"adapter": "pipeline.sources.uk.fss_approved.adapter:FssApprovedEstablishmentsAdapter", "authority": "Food Standards Scotland", "nation": "Scotland", "format": "csv", "acquisition": "synthetic-fixture-only", "publication": "human-gated"}}} +{"version": "1", "sources": {"fsa_approved_establishments": {"adapter": "pipeline.sources.uk.fsa_approved.adapter:FsaApprovedEstablishmentsAdapter", "authority": "Food Standards Agency", "scope": ["England", "Wales", "Northern Ireland (source-specific scope only)"], "format": "csv", "acquisition": "bounded-private-fetch", "publication": "human-gated"}, "fss_approved_establishments": {"adapter": "pipeline.sources.uk.fss_approved.adapter:FssApprovedEstablishmentsAdapter", "authority": "Food Standards Scotland", "scope": ["Scotland"], "format": "csv", "acquisition": "bounded-private-fetch", "publication": "human-gated"}}} diff --git a/src/lib.rs b/src/lib.rs index f8f1568..cc3368d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -375,7 +375,28 @@ pub async fn get_dev_test_release_locations_handler( "limit must be between 1 and 1000", ); } - let rows = match client.query(r#"SELECT f.facility_id,f.canonical_name,f.country_code,f.city,o.classification_category, + if params.cursor.is_some() && params.offset.is_some() { + return v2_error( + StatusCode::BAD_REQUEST, + "cursor_offset_conflict", + "cursor and offset cannot be combined", + ); + } + let cursor = match params.cursor.as_deref() { + Some(value) => match value.parse::() { + Ok(value) => Some(value), + Err(_) => { + return v2_error( + StatusCode::BAD_REQUEST, + "invalid_cursor", + "cursor must be a facility UUID", + ); + } + }, + None => None, + }; + let query_limit = limit + 1; + let mut rows = match client.query(r#"SELECT f.facility_id,f.canonical_name,f.country_code,f.city,o.classification_category, CASE WHEN o.coordinate_review_status='approved' AND g.result IS NOT NULL THEN 'exact' ELSE 'unmapped' END, CASE WHEN o.coordinate_review_status='approved' AND g.result IS NOT NULL THEN ST_Y(g.result::geometry) ELSE NULL END, CASE WHEN o.coordinate_review_status='approved' AND g.result IS NOT NULL THEN ST_X(g.result::geometry) ELSE NULL END, @@ -391,12 +412,22 @@ pub async fn get_dev_test_release_locations_handler( AND COALESCE(review.privacy_screening_status,'pending') <> 'failed' AND COALESCE(review.factual_review_status,'unreviewed') <> 'rejected' AND NOT EXISTS (SELECT 1 FROM uec.public_access_restricted restricted WHERE restricted.source_record_id=o.source_record_id) AND ($2::text IS NULL OR f.country_code=$2) AND ($3::text IS NULL OR o.classification_category=$3) - ORDER BY f.facility_id LIMIT $4"#, &[&release_id,¶ms.country_code,¶ms.category,&limit]).await { + AND ($4::uuid IS NULL OR f.facility_id > $4) + ORDER BY f.facility_id LIMIT $5"#, &[&release_id,¶ms.country_code,¶ms.category,&cursor,&query_limit]).await { Ok(rows)=>rows, Err(_)=>return v2_error(StatusCode::SERVICE_UNAVAILABLE,"test_release_query_failed","test release unavailable") }; + let has_next = rows.len() > limit as usize; + if has_next { + rows.truncate(limit as usize); + } let data = rows.into_iter().map(|row| json!({"facility_id":row.get::<_,uuid::Uuid>(0),"canonical_name":row.get::<_,Option>(1),"country_code":row.get::<_,String>(2),"city":row.get::<_,Option>(3),"category":row.get::<_,String>(4),"publication_profile":profile,"factual_review_status":row.get::<_,Option>(8).unwrap_or("unreviewed".into()),"privacy_screening_status":row.get::<_,Option>(9).unwrap_or("pending".into()),"project_approval":"not-approved","reviewer_role":row.get::<_,Option>(11),"publication_warning":"Disposable test release — not project-approved or published","display_precision":row.get::<_,String>(5),"latitude":row.get::<_,Option>(6),"longitude":row.get::<_,Option>(7),"first_observed_at":null,"last_observed_at":null,"observation_count":null,"lifecycle_status":"status_unknown","source_type":row.get::<_,String>(12),"release_id":release_id,"release_ruleset_version":row.get::<_,String>(16),"provenance_source_id":row.get::<_,String>(13),"provenance_source_name":row.get::<_,String>(14),"provenance_source_url":row.get::<_,String>(15),"provenance_retrieved_at":row.get::<_,chrono::DateTime>(17)})).collect::>(); let mut meta = test_release_meta(release_id, profile); meta["result_count"] = json!(data.len()); + meta["next_cursor"] = json!(if has_next { + data.last().and_then(|row| row.get("facility_id")).cloned() + } else { + None:: + }); Json(json!({"data":data,"meta":meta})).into_response() } From b98bbf9cf0fb17b53f45fc93161238f43c7a497d Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Tue, 15 Sep 2026 10:08:34 -0700 Subject: [PATCH 114/311] Document Belgium FASFC source readiness --- docs/country-recon-be.md | 45 ++++++++++++++++++++++++++ docs/source-status.json | 1 + pipeline/source_registry.json | 14 +++++++- pipeline/tests/test_source_registry.py | 5 +++ 4 files changed, 64 insertions(+), 1 deletion(-) create mode 100644 docs/country-recon-be.md diff --git a/docs/country-recon-be.md b/docs/country-recon-be.md new file mode 100644 index 0000000..7c44bb6 --- /dev/null +++ b/docs/country-recon-be.md @@ -0,0 +1,45 @@ +# Belgium source reconnaissance + +Status: reconnaissance only. No adapter, release, publication, or row-level fixture was created. No facility rows, names, addresses, contacts, coordinates, or private artifacts are retained here. + +Last checked: 2026-09-15 UTC under `docs/ETHICS.md`, policy version 1.0, last reviewed 2026-09-12. This is source-status evidence, not publication approval or a runtime-health claim. + +## Readiness + +| Candidate | Evidence | Acquisition | Terms / privacy | Readiness / next action | +|---|---|---|---|---| +| FASFC operator list | Official [data.gov.be dataset](https://data.gov.be/en/datasets/favv-afsca-operators) and published [English CSV](https://www.static.favv.be/bo-documents/inter_actieve_actoren_EN.csv) verified | Published CSV route; bounded header retrieval timed out from the static host in this environment | CC Attribution 4.0; FASFC says attribute source and last-update date, do not imply FASFC affiliation/approval, and do not mislead | Candidate for a private adapter after header/schema capture, delimiter/encoding test, and category mapping | +| FASFC activity-code list | Official [data.gov.be dataset](https://data.gov.be/en/datasets/fasfc-activity-codes) and [English CSV](https://www.static.favv.be/bo-documents/inter_PAP_omschrijving_EN.csv) verified | In-memory bounded retrieval succeeded; bytes discarded | CC Attribution 4.0; weekly metadata on data.gov.be | Companion codebook; not a facility list and not sufficient by itself | +| EU-approved food-establishment PDF lists | Official [FASFC approved-establishments page](https://www.foodweb.favv-afsca.be/professionals/foodstuffs/establishments/default.asp) verified | PDF links are a separate publication surface | Scope and currentness differ from open-data CSV; PDF schema/version handling required | Useful cross-check only; do not merge with operator rows without explicit identity/version evidence | +| Animal by-products lists | Official [FASFC animal-by-products page](https://www.foodweb.favv-afsca.be/professionals/animalbyproducts/approvedoperators/) verified | Sectioned PDF lists | Separate legal scope under Regulation (EC) 1069/2009 | Keep separate from food slaughter/processing coverage | + +## What the authoritative open-data files mean + +The operator dataset describes enterprises and establishments that currently have an FASFC registration, approval, or authorization, with LAP/PAP activity codes, activity descriptions, and approval/authorization numbers. Data.gov.be reports identifier `favv-afsca-operators`, Belgium coverage, CSV format, weekly frequency, CC BY 4.0, and an update date of 2026-08-05 at the checked page revision. + +The activity-code dataset is a codebook, not a location dataset. Its published description says codes are grouped by Place/Activity/Product and include linked approval codes. The fetched English CSV used 13 delimited header fields (accented labels redacted here), 395 non-empty data rows, and includes fields corresponding to PAP ID, place code/description, activity code/description, product code/description, approval form/code/description, language, and a currentness/date field. Preserve source labels and codes; do not infer slaughtering from broad food-sector categories. + +The operator page does not establish that every row is a slaughterhouse. It covers all FASFC-regulated operators with a current registration, approval, or authorization, potentially including retail, restaurants, transport, storage, processing, feed, primary production, and other activities. Filtering must use the codebook and retain every original activity/category value. Slaughterhouses, cutting plants, processing plants, cold stores, animal-by-product facilities, exports, inspection outcomes, and aggregate statistics are separate concepts and must not be silently combined. + +## Identity, geography, and privacy risks + +The dataset description confirms enterprise/establishment scope and approval identifiers, but the operator CSV header was not captured in this reconnaissance. Address, postal-code, municipality, establishment-versus-enterprise identifiers, coordinates, effective dates, and status fields therefore remain unverified. Treat any address as a facility claim requiring source-field preservation and privacy screening; a registered office or mixed residential/business address is not automatically an operating site. Do not geocode until provider/query/time/precision/review metadata and the ETHICS.md residential/private-location rules are implemented. + +Foodweb is an interactive lookup and inspection-results surface, not a bulk inspection dataset. Its published FAQ says inspection-result publication is limited to B2C operators and cannot produce a complete list by municipality or activity. It must not be used as a substitute for the operator master or treated as slaughterhouse evidence. + +Coverage is Belgium-wide according to data.gov.be, but completeness is bounded by FASFC registrations/approvals/authorizations currently represented in that feed. It is not evidence of operating status beyond the publisher's stated current eligibility, nor a census of all animal-agriculture facilities. + +## Acquisition provenance (private, no raw artifact retained) + +The activity-code request was performed read-only in memory and response bytes were discarded. The operator request was attempted but did not yield bytes; no operator rows were retained. + +| Artifact | Retrieval UTC | HTTP | Content type | Bytes | SHA-256 | Supplied update/effective date | +|---|---|---:|---|---:|---|---| +| `inter_PAP_omschrijving_EN.csv` | 2026-09-15T17:05:26.4902745Z | 200 | `text/csv` | 101,987 | `c50d5ff8db705664a56bef73aec88c94159ff7f810cf6125e978e0e16c5a0812` | data.gov.be page: 2026-08-05; no artifact-level effective date observed | +| `inter_actieve_actoren_EN.csv` | 2026-09-15; no response body | unavailable | unavailable | unavailable | unavailable | data.gov.be page: 2026-08-05 | + +## Adapter readiness and recommended next step + +Readiness: medium difficulty, not ready for implementation. The authoritative source pair and reuse terms are clear, and the activity codebook is machine-readable. The main work is schema capture for the operator CSV, deterministic delimiter/encoding handling, codebook versioning, establishment/enterprise identity semantics, multilingual labels, status/effective-date interpretation, and explicit mappings for slaughterhouse versus cutting/processing/storage and other PAP categories. Expect one operator row per activity or repeated establishment identifiers; this must be verified rather than assumed. + +Next step: obtain an authorized, bounded operator-CSV retrieval; record its redirect chain, headers, byte size, SHA-256, supplied date, and sanitized schema; then build synthetic fixtures for repeated activities, missing identifiers, multilingual text, category ambiguity, and mixed residential/business addresses. Keep acquisition private and publication blocked pending human privacy and release review. diff --git a/docs/source-status.json b/docs/source-status.json index 2a2f294..a0f8bb6 100644 --- a/docs/source-status.json +++ b/docs/source-status.json @@ -8,6 +8,7 @@ "publication_eligibility": ["not_assessed", "blocked", "eligible_pending_release_approval"] }, "sources": [ + {"source_id":"be.locations","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-be.md","pipeline/source_registry.json"],"next_action":"Obtain an authorized bounded operator CSV response, record provenance and sanitized schema, then implement codebook/category/privacy validation before any release review."}, {"source_id":"fr.locations","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-fr.md","pipeline/source_registry.json"],"next_action":"Acquire and snapshot an official DGAL Section I/II artifact with terms, schema, and privacy review."}, {"source_id":"it.853-2004","metadata":"verified","acquisition":"artifact_private_only","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-it.md","pipeline/source_registry.json","pipeline/sources/italy/acquire.py","pipeline/sources/italy/it_853_adapter.py","pipeline/sources/italy/README.md","pipeline/tests/e2e/test_italy_candidate_import.py"],"next_action":"Keep catalog acquisition, lifecycle, candidate import, and guarded API evidence private; resolve repeated activity identity, coordinate/address privacy, coverage, and project approval before release review."}, {"source_id":"it.1069-2009","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-it.md","pipeline/source_registry.json","pipeline/sources/italy/README.md"],"next_action":"Keep the by-products dataset separate; assess scope, schema, terms, identity links, privacy, and a dedicated adapter before any acquisition or integration."}, diff --git a/pipeline/source_registry.json b/pipeline/source_registry.json index 18be955..4ed47c8 100644 --- a/pipeline/source_registry.json +++ b/pipeline/source_registry.json @@ -1,8 +1,20 @@ { "registry_version": "1.0", - "evidence_basis": "Repository evidence only; no live source verification was performed.", + "evidence_basis": "Repository evidence; source-specific live verification is documented in the linked evidence files.", "unknown_value": "unknown", "sources": [ + { + "source_id": "be.locations", + "jurisdiction_scope": "Belgium; FASFC-registered, approved, or authorized operators, including animal-origin food and other food-chain activities", + "legacy_paths": [], + "url": "https://data.gov.be/en/datasets/favv-afsca-operators", + "access_method": "published weekly CSV download; companion PAP/LAP activity-code CSV", + "cadence": "weekly", + "attribution_licensing_notes": "CC Attribution 4.0; attribute FASFC and the last update date, do not imply FASFC affiliation/approval, and do not mislead", + "adapter_status": "not_started", + "expected_artifact_schema": "CSV operator rows with establishment/operator identity, address/geography, PAP/LAP activity codes, descriptions, and approval/authorization identifiers; exact header and field semantics require capture", + "blockers": ["Capture and verify the current operator CSV header, delimiter/encoding, identifiers, address/geography fields, status/date semantics, and slaughterhouse/cutting/processing/storage category mappings before implementation or acquisition staging."] + }, { "source_id": "ca.locations", "jurisdiction_scope": "Canada; federal and Ontario legacy coverage", diff --git a/pipeline/tests/test_source_registry.py b/pipeline/tests/test_source_registry.py index c5a02f9..4585b38 100644 --- a/pipeline/tests/test_source_registry.py +++ b/pipeline/tests/test_source_registry.py @@ -17,6 +17,11 @@ def test_unknowns_are_explicit(self): self.assertEqual(registry["unknown_value"], "unknown") self.assertIn("unknown", {source["url"] for source in registry["sources"]}) + def test_allows_source_without_legacy_paths(self): + registry = load_registry() + belgium = next(source for source in registry["sources"] if source["source_id"] == "be.locations") + self.assertEqual(belgium["legacy_paths"], []) + def test_rejects_duplicate_ids(self): registry = load_registry() registry["sources"].append(dict(registry["sources"][0])) From bccf610b46c72bceac665245fabc260d7aaaedad Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Tue, 15 Sep 2026 11:11:08 -0700 Subject: [PATCH 115/311] Integrate frontend with V2 profiles and states --- frontend/README.md | 8 +- frontend/src/api/LocalCsvExportRepository.ts | 17 +- frontend/src/api/LocalLocationRepository.ts | 8 +- frontend/src/api/errors.ts | 2 +- frontend/src/api/wireSchema.ts | 2 +- frontend/src/app/App.svelte | 192 ++++++++++++------ frontend/src/app/routeState.ts | 2 +- frontend/src/domain/publication.ts | 13 +- frontend/src/map/LeafletMapAdapter.ts | 2 +- frontend/src/map/MapAdapter.ts | 2 +- frontend/src/map/MapView.svelte | 6 +- frontend/src/ui/ExportControl.svelte | 5 +- frontend/tests/e2e/local-safety.spec.ts | 35 +++- .../unit/localCsvExportRepository.test.ts | 6 + .../unit/localLocationRepository.test.ts | 15 +- frontend/tests/unit/routeState.test.ts | 1 + 16 files changed, 227 insertions(+), 89 deletions(-) diff --git a/frontend/README.md b/frontend/README.md index f6cf82d..d4765a6 100644 --- a/frontend/README.md +++ b/frontend/README.md @@ -1,6 +1,8 @@ -# V2 frontend preview +# V2 frontend preview and local integration -Fixture-only Svelte 5 demonstrator. Run `npm install`, then `npm run dev`. It uses no live API, external assets, analytics, or map tiles. The ethics link intentionally targets the existing `/ethics.html` page. +Svelte 5/TypeScript frontend preview. Run `npm install`, then `npm run dev`. Fixture mode remains the safe default and uses no live API, external assets, analytics, or map tiles. The ethics link intentionally targets the existing `/ethics.html` page. + +The explicit `?mode=local-v2` path uses the V2 API client for the official, secondary, and community profiles, controlled filters, cursor pagination, map/detail navigation, release/provenance context, and the profile-scoped public CSV route. A community profile keeps its persistent screened-but-unreviewed warning; it is not merged into official or secondary counts. Requests are cancellable and generation-checked so stale list/detail responses cannot replace newer state. No V1 fallback is used. Phase 2 gate notes: staging is explicit (`npm run stage`) and copies only `frontend/dist` to the resolved ignored `static/v2-preview` destination. The Leaflet adapter is isolated and uses a blank local background; no tile provider is configured. Export previews retain profile, release, limitations, source, and observation context. @@ -21,7 +23,7 @@ These are documented gaps, not frontend claims or invented DTO fields. Phase 3 c Local V2 integration (opt-in only): run the backend with `cargo run` (port 8000), then run the frontend with `npm run dev` (port 5173). In development/preview, Vite proxies `/api` to `http://127.0.0.1:8000`; use `http://127.0.0.1:5173/v2-preview/?mode=local-v2#/` to opt in. The `LocalLocationRepository` still targets `/api/v2/locations?profile=...` only when explicitly invoked; fixture mode remains the default and there is no V1 fallback. The proxy is development-only configuration and production builds do not enable a backend connection. -The opt-in local view loads one API page at a time. When the response includes `next_cursor`, the UI labels its counts, search, and map as partial; search runs only against loaded records. Full pagination and backend search remain future integration work. Record context is limited to fields in the current V2 wire response and does not imply that review events, evidence hashes, or scoped approvals are available. +The opt-in local view loads one API page at a time. When the response includes `next_cursor`, the UI labels its counts, search, and map as partial; text search runs only against loaded records because the current public contract intentionally has no server-side free-text query. Record context is limited to fields in the current V2 wire response and does not imply that review events, evidence hashes, or scoped approvals are available. The private scale/story prototype begins with a neutral individual-animal representation and uses only bounded synthetic values. It labels model arithmetic separately from measured facility evidence; no biography, live counter, global animal total, or sourced aggregate is embedded in the production build. Candidate sourced scale figures remain outside this UI until maintainer publication approval. diff --git a/frontend/src/api/LocalCsvExportRepository.ts b/frontend/src/api/LocalCsvExportRepository.ts index 798e9a9..3d26be1 100644 --- a/frontend/src/api/LocalCsvExportRepository.ts +++ b/frontend/src/api/LocalCsvExportRepository.ts @@ -1,4 +1,5 @@ -import type { FetchLike } from './LocalLocationRepository'; +import type { FetchLike, LocalProfile } from './LocalLocationRepository'; +import type { ApiError } from './errors'; export type CsvExport = Readonly<{ body: string; @@ -10,16 +11,22 @@ export type CsvExport = Readonly<{ export class LocalCsvExportRepository { constructor(private readonly fetcher: FetchLike = globalThis.fetch, private readonly baseUrl = '') {} - async download(profile: 'official' | 'secondary' | 'community' = 'official'): Promise { - const response = await this.fetcher.call(globalThis, `${this.baseUrl}/api/v2/locations.csv?profile=${profile}`, { cache: 'no-store' }); + async download(profile: LocalProfile = 'official', signal?: AbortSignal): Promise { + const init: RequestInit = { cache: 'no-store' }; if (signal) init.signal = signal; + const response = await this.fetcher.call(globalThis, `${this.baseUrl}/api/v2/locations.csv?profile=${profile}`, init); if (!response.ok) { let message = `Local V2 export request failed with status ${response.status}.`; + let code: string | undefined; try { - const payload = await response.json() as { error?: { message?: string } }; + const payload = await response.json() as { error?: { code?: string; message?: string } }; + code = payload.error?.code; if (payload.error?.message) message = payload.error.message; } catch { /* Preserve the status-based safe message. */ } - throw Object.assign(new Error(message), { status: response.status }); + const kind: ApiError['kind'] = response.status === 404 ? 'no-release' : response.status === 429 ? 'rate-limited' : response.status >= 500 ? 'unavailable' : 'http'; + throw Object.assign(new Error(message), { kind, status: response.status, code }); } + const contentType = response.headers.get('content-type'); + if (contentType?.toLowerCase().includes('application/json')) throw Object.assign(new Error('The local V2 export response was not CSV.'), { kind: 'invalid-contract' as const }); const body = await response.text(); const releaseId = response.headers.get('x-uec-release-id'); const responseProfile = profile; diff --git a/frontend/src/api/LocalLocationRepository.ts b/frontend/src/api/LocalLocationRepository.ts index ada77bf..b5203bd 100644 --- a/frontend/src/api/LocalLocationRepository.ts +++ b/frontend/src/api/LocalLocationRepository.ts @@ -4,10 +4,10 @@ import type { Location } from '../domain/location'; export type FetchLike = (input: RequestInfo | URL, init?: RequestInit) => Promise; export type LocalProfile = 'official' | 'secondary' | 'community'; -export type LocationFilters = Readonly<{ country_code?: string | undefined; category?: string | undefined; source_type?: string | undefined; display_precision?: string | undefined; lifecycle_status?: string | undefined; cursor?: string | undefined }>; +export type LocationFilters = Readonly<{ country_code?: string | undefined; category?: string | undefined; source_type?: string | undefined; display_precision?: string | undefined; lifecycle_status?: string | undefined; cursor?: string | undefined; limit?: number | undefined }>; export type LocalListResult = Readonly<{ locations: readonly Location[]; releaseId: string; profile: LocalProfile; coverageNote: string; coverageScope?: string; countSemantics?: string; nextCursor: string | null; ruleset?: string }>; export const localOrigin = (value: string | undefined): string | undefined => { if (!value) return undefined; const url = new URL(value); if (url.protocol !== 'http:' || !['127.0.0.1', 'localhost', '::1'].includes(url.hostname)) throw new Error('Local API origin must be loopback HTTP.'); return url.origin; }; -const fail = (kind: ApiError['kind'], message: string, status?: number): ApiError => Object.assign(new Error(message), status === undefined ? { kind } : { kind, status }); +const fail = (kind: ApiError['kind'], message: string, status?: number, code?: string): ApiError => Object.assign(new Error(message), { kind, ...(status === undefined ? {} : { status }), ...(code ? { code } : {}) }); export const mapWireLocation = (r: WireLocation): Location => ({ id: r.facility_id, name: r.canonical_name ?? 'Unnamed candidate record', region: r.city ?? r.country_code, category: r.category, lat: r.latitude, lon: r.longitude, observed: r.last_observed_at ?? r.first_observed_at ?? 'unknown', source: r.provenance_source_name, @@ -19,7 +19,7 @@ export const mapWireLocation = (r: WireLocation): Location => ({ retrievedAt: r.provenance_retrieved_at, displayPrecision: r.display_precision, lifecycleStatus: r.lifecycle_status, observationCount: r.observation_count, }, }); -const query = (profile: LocalProfile, filters: LocationFilters) => { const params = new URLSearchParams({ profile }); for (const [key, value] of Object.entries(filters)) if (value) params.set(key, value); return `/api/v2/locations?${params}`; }; +const query = (profile: LocalProfile, filters: LocationFilters) => { const params = new URLSearchParams({ profile }); for (const [key, value] of Object.entries(filters)) if (value !== undefined && value !== '') params.set(key, String(value)); return `/api/v2/locations?${params}`; }; // Eligibility is a conservative client-side check, not a publication decision; // the server's current public projection and suppression rules remain authoritative. const eligible = (row: WireLocation, profile: LocalProfile, releaseId: string, ruleset: string): boolean => @@ -30,7 +30,7 @@ const eligible = (row: WireLocation, profile: LocalProfile, releaseId: string, r export class LocalLocationRepository { readonly #base: string | undefined; constructor(private readonly fetcher: FetchLike = globalThis.fetch, baseUrl?: string) { this.#base = localOrigin(baseUrl); } - private async json(path: string, signal?: AbortSignal) { const init: RequestInit = { cache: 'no-store' }; if (signal) init.signal = signal; const response = await this.fetcher.call(globalThis, `${this.#base ?? ''}${path}`, init); if (!response.ok) throw fail('http', `Local V2 request failed with status ${response.status}`, response.status); try { return await response.json(); } catch { throw fail('invalid-contract', 'Local V2 response was not valid JSON.'); } } + private async json(path: string, signal?: AbortSignal) { const init: RequestInit = { cache: 'no-store' }; if (signal) init.signal = signal; const response = await this.fetcher.call(globalThis, `${this.#base ?? ''}${path}`, init); if (!response.ok) { let code: string | undefined; let message = `Local V2 request failed with status ${response.status}.`; try { const payload = await response.json() as { error?: { code?: string; message?: string } }; code = payload.error?.code; message = payload.error?.message ?? message; } catch { /* Keep the status-safe message. */ } const kind: ApiError['kind'] = response.status === 404 ? 'restricted' : response.status === 429 ? 'rate-limited' : response.status >= 500 ? 'unavailable' : 'http'; throw fail(kind, message, response.status, code); } try { return await response.json(); } catch { throw fail('invalid-contract', 'Local V2 response was not valid JSON.'); } } async list(profile: LocalProfile = 'official', filters: LocationFilters = {}, signal?: AbortSignal): Promise { try { const b = envelopeSchema.safeParse(await this.json(query(profile, filters), signal)); diff --git a/frontend/src/api/errors.ts b/frontend/src/api/errors.ts index 2732f65..110898c 100644 --- a/frontend/src/api/errors.ts +++ b/frontend/src/api/errors.ts @@ -1 +1 @@ -export type ApiError=Readonly<{kind:'aborted'|'network'|'http'|'invalid-contract'|'no-release';message:string;status?:number}>; +export type ApiError=Readonly<{kind:'aborted'|'network'|'http'|'invalid-contract'|'no-release'|'unavailable'|'restricted'|'rate-limited';message:string;status?:number;code?:string}>; diff --git a/frontend/src/api/wireSchema.ts b/frontend/src/api/wireSchema.ts index 76fa01c..88f405b 100644 --- a/frontend/src/api/wireSchema.ts +++ b/frontend/src/api/wireSchema.ts @@ -2,7 +2,7 @@ import { z } from 'zod'; const textOrNull=z.string().nullable(); // This is the Rust-shaped boundary. Optional coverage/count metadata is additive; // older valid list envelopes remain readable with safe UI fallbacks. -const locationShape={facility_id:z.string().uuid(),canonical_name:z.string().min(1),city:textOrNull,country_code:z.string().regex(/^[A-Z]{2}$/),category:z.string(),source_type:z.enum(['official','secondary','user_submitted']),publication_profile:z.enum(['official','secondary','community']),factual_review_status:z.enum(['unreviewed','reviewed','rejected']),privacy_screening_status:z.literal('passed'),project_approval:z.enum(['pending','approved']),reviewer_role:textOrNull,publication_warning:textOrNull,display_precision:z.enum(['exact','city','unmapped']),latitude:z.number().finite().nullable(),longitude:z.number().finite().nullable(),first_observed_at:textOrNull,last_observed_at:textOrNull,observation_count:z.number().int().nonnegative().nullable(),lifecycle_status:z.enum(['active_observed','explicitly_closed','not_seen_recently','status_unknown']),provenance_source_id:z.string(),provenance_source_name:z.string(),provenance_source_url:z.string().url().refine(value=>{try{return ['http:','https:'].includes(new URL(value).protocol);}catch{return false;}},'source URL must use HTTP or HTTPS'),provenance_retrieved_at:z.string(),release_id:z.string(),release_ruleset_version:z.string()}; +const locationShape={facility_id:z.string().uuid(),canonical_name:z.string().nullable(),city:textOrNull,country_code:z.string().regex(/^[A-Z]{2}$/),category:z.string(),source_type:z.enum(['official','secondary','user_submitted']),publication_profile:z.enum(['official','secondary','community']),factual_review_status:z.enum(['unreviewed','reviewed','rejected']),privacy_screening_status:z.literal('passed'),project_approval:z.enum(['pending','approved']),reviewer_role:textOrNull,publication_warning:textOrNull,display_precision:z.enum(['exact','city','unmapped']),latitude:z.number().finite().nullable(),longitude:z.number().finite().nullable(),first_observed_at:textOrNull,last_observed_at:textOrNull,observation_count:z.number().int().nonnegative().nullable(),lifecycle_status:z.enum(['active_observed','explicitly_closed','not_seen_recently','status_unknown']),provenance_source_id:z.string(),provenance_source_name:z.string(),provenance_source_url:z.string().url().refine(value=>{try{return ['http:','https:'].includes(new URL(value).protocol);}catch{return false;}},'source URL must use HTTP or HTTPS'),provenance_retrieved_at:z.string(),release_id:z.string(),release_ruleset_version:z.string()}; const coordinateRules=(row:{latitude:number|null;longitude:number|null;display_precision:string},ctx:z.RefinementCtx)=>{if((row.latitude===null)!==(row.longitude===null))ctx.addIssue({code:z.ZodIssueCode.custom,message:'coordinate pair must be complete'});if(row.display_precision==='unmapped'&&(row.latitude!==null||row.longitude!==null))ctx.addIssue({code:z.ZodIssueCode.custom,message:'unmapped record cannot have coordinates'});if(row.display_precision!=='unmapped'&&(row.latitude===null||row.longitude===null))ctx.addIssue({code:z.ZodIssueCode.custom,message:'mapped record requires coordinates'});}; export const locationSchema=z.object(locationShape).superRefine(coordinateRules); export const testReleaseLocationSchema=z.object({...locationShape,canonical_name:z.string().nullable(),publication_profile:z.enum(['official','secondary','community']).nullable(),privacy_screening_status:z.enum(['pending','passed','failed']),project_approval:z.union([z.enum(['pending','approved']),z.literal(false),z.literal('not-approved')]),release_ruleset_version:z.string().nullable()}).superRefine(coordinateRules); diff --git a/frontend/src/app/App.svelte b/frontend/src/app/App.svelte index eec1275..e4df6e5 100644 --- a/frontend/src/app/App.svelte +++ b/frontend/src/app/App.svelte @@ -3,6 +3,7 @@ import { locations } from '../fixtures/locations'; import type { Location } from '../domain/location'; import type { Profile } from '../domain/publication'; + import { API_PROFILES, profileLabel, toApiProfile } from '../domain/publication'; import { parseRoute } from './routeState'; import { filterLocations, initialFilters, type FilterState } from '../features/locations/filterState'; import MapView from '../map/MapView.svelte'; @@ -14,18 +15,25 @@ import ReleaseContext from '../ui/ReleaseContext.svelte'; import ExportControl from '../ui/ExportControl.svelte'; import ScaleNarrative from '../features/scale/ScaleNarrative.svelte'; - import { canOpenDevPreview, DEV_PREVIEW_LABEL, TEST_RELEASE_LABEL, devPreviewExportLabel, canMountPublicExport } from '../features/devPreview/devPreviewContract'; + import { canOpenDevPreview, DEV_PREVIEW_LABEL, TEST_RELEASE_LABEL, devPreviewExportLabel } from '../features/devPreview/devPreviewContract'; import type { DevCandidate } from '../api/DevCandidatePreviewRepository'; import { TestReleaseRepository } from '../api/TestReleaseRepository'; import { TestReleaseCsvExportRepository } from '../api/TestReleaseCsvExportRepository'; import DevReviewPanel from '../features/devPreview/DevReviewPanel.svelte'; + import type { ApiError } from '../api/errors'; let profile: Profile = 'curated'; let selected: Location | undefined = locations[0]; - let search = ''; let region = 'all'; let category = 'all'; - let sourceType = 'all'; let displayPrecision = 'all'; let lifecycleStatus = 'all'; + let search = ''; + let region = 'all'; + let category = 'all'; + let sourceType = 'all'; + let displayPrecision = 'all'; + let lifecycleStatus = 'all'; let filters: FilterState = initialFilters; - let showMap = false; let showExport = false; let showGuidance = false; + let showMap = false; + let showExport = false; + let showGuidance = false; let localMode = false; let devPreviewMode = false; let testReleaseMode = false; @@ -36,31 +44,55 @@ let testCsvBusy = false; let testCsvError = ''; let localStatus: 'idle' | 'loading' | 'ready' | 'error' | 'no-release' = 'idle'; + let localFailure: ApiError['kind'] | 'unknown' = 'unknown'; let detailStatus: 'idle' | 'loading' | 'error' = 'idle'; - let localError = ''; let exportError = ''; let exportBusy = false; - let loaded: readonly Location[] = []; let release = 'synthetic-2026.09'; - let ruleset: string | undefined; let manifestSha256: string | undefined; - let repo = new LocalLocationRepository(); let csvRepo = new LocalCsvExportRepository(); - let metadata: FilterMetadata | undefined; let metadataStatus: 'idle' | 'loading' | 'ready' | 'error' = 'idle'; - let nextCursor: string | null = null; let coverageNote = ''; let coverageScope = ''; let countSemantics = ''; + let detailFailure: ApiError['kind'] | 'unknown' = 'unknown'; + let localError = ''; + let exportError = ''; + let exportBusy = false; + let loaded: readonly Location[] = []; + let release = 'synthetic-2026.09'; + let ruleset: string | undefined; + let manifestSha256: string | undefined; + let repo = new LocalLocationRepository(); + let csvRepo = new LocalCsvExportRepository(); + let metadata: FilterMetadata | undefined; + let metadataStatus: 'idle' | 'loading' | 'ready' | 'error' = 'idle'; + let nextCursor: string | null = null; + let coverageNote = ''; + let coverageScope = ''; + let countSemantics = ''; let paging = false; - let listGeneration = 0; let detailGeneration = 0; - let listAbort: AbortController | undefined; let detailAbort: AbortController | undefined; + let listGeneration = 0; + let detailGeneration = 0; + let listAbort: AbortController | undefined; + let detailAbort: AbortController | undefined; let detailRequestKey = ''; let activeListKey = ''; + let lastRemoteQuery = ''; let detailErrorHeading: HTMLHeadingElement; + $: filters = { search, region, category }; + $: apiProfile = toApiProfile(profile); $: source = devPreviewMode ? previewRows : localMode ? loaded : (profile === 'community' ? locations.slice(0, 1) : locations); $: visibleLocations = filterLocations(source, filters); $: exportPreview = devPreviewExportLabel(devPreviewMode) ?? previewExport(makeExportModel(visibleLocations, profile, release)); - $: eligibleExport = !devPreviewMode && localMode && profile === 'curated' && localStatus === 'ready' && Boolean(release); - $: profileLabel = profile === 'curated' ? 'Curated release' : 'Community claims'; - let lastRemoteQuery = ''; - // Search is intentionally excluded: the API has no q contract, so it filters - // only the loaded page while controlled dimensions refetch a new snapshot. - $: remoteQuery = `${profile}|${region}|${category}|${sourceType}|${displayPrecision}|${lifecycleStatus}`; - const currentRemoteQuery = () => `${profile}|${region}|${category}|${sourceType}|${displayPrecision}|${lifecycleStatus}|${search}`; - $: if (localMode && (localStatus === 'ready' || localStatus === 'loading') && remoteQuery !== lastRemoteQuery) { pushFilterUrl(); void loadLocal(); } + $: eligibleExport = !devPreviewMode && localMode && localStatus === 'ready' && Boolean(release); + $: profileLabelText = profileLabel(profile); + // V2 deliberately has no q parameter. Search never refetches a page. + $: remoteQuery = `${apiProfile}|${region}|${category}|${sourceType}|${displayPrecision}|${lifecycleStatus}`; + $: if (localMode && (localStatus === 'ready' || localStatus === 'loading') && remoteQuery !== lastRemoteQuery) { + pushFilterUrl(); + void loadLocal(); + } + + const failureTitle = (failure: ApiError['kind'] | 'unknown', fallback: string): string => ({ + network: 'Network error', unavailable: 'V2 service unavailable', restricted: 'Access restricted or record unavailable', 'invalid-contract': 'V2 data contract unavailable', 'rate-limited': 'Temporarily rate-limited', + } as Record)[failure] ?? fallback; + const sourceLabel = (value: string): string => value === 'user_submitted' ? 'Community-submitted' : value === 'official' ? 'Government-sourced' : 'Secondary-sourced'; + const reviewLabel = (location: Location): string => location.evidence?.factualReviewStatus === 'unreviewed' ? 'Factually unreviewed' : 'Factually reviewed'; + const isUnreviewedCommunity = (location: Location): boolean => location.evidence?.publicationProfile === 'community' && location.evidence.factualReviewStatus === 'unreviewed'; + const profileForUrl = (): string => localMode ? toApiProfile(profile) : profile; const pushFilterUrl = () => { const url = new URL(window.location.href); @@ -79,65 +111,102 @@ const syncRoute = async () => { const route = parseRoute(window.location.hash); if (route.kind === 'location' && localMode) { - const requestKey = `${route.facilityId}|${route.profile}|${profile}`; + const requestedProfile = toApiProfile(route.profile); + const requestKey = `${route.facilityId}|${requestedProfile}`; if (detailRequestKey === requestKey && detailStatus === 'loading') return; - detailRequestKey = requestKey; - invalidateDetail(); + detailRequestKey = requestKey; invalidateDetail(); const generation = detailGeneration; - const requestedProfile = profile === 'community' ? 'community' : 'official'; - const controller = new AbortController(); detailAbort = controller; - detailStatus = 'loading'; - try { const result = await repo.detail(route.facilityId, requestedProfile, controller.signal); const currentRoute = parseRoute(window.location.hash); if (generation !== detailGeneration || currentRoute.kind !== 'location' || currentRoute.facilityId !== route.facilityId || currentRoute.profile !== route.profile || profile !== route.profile) return; if (result.releaseId !== release) throw new Error('Local V2 detail belongs to a different release.'); selected = result.location; detailStatus = 'idle'; } - catch (error) { if (generation !== detailGeneration) return; selected = undefined; detailStatus = 'error'; localError = error instanceof Error ? error.message : 'The local detail response was rejected safely.'; await tick(); detailErrorHeading?.focus(); } + const controller = new AbortController(); detailAbort = controller; detailStatus = 'loading'; detailFailure = 'unknown'; + try { + const result = await repo.detail(route.facilityId, requestedProfile, controller.signal); + const currentRoute = parseRoute(window.location.hash); + if (generation !== detailGeneration || currentRoute.kind !== 'location' || currentRoute.facilityId !== route.facilityId || toApiProfile(currentRoute.profile) !== requestedProfile || toApiProfile(profile) !== requestedProfile) return; + if (result.releaseId !== release) throw Object.assign(new Error('The detail response belongs to a different promoted release.'), { kind: 'invalid-contract' as const }); + selected = result.location; detailStatus = 'idle'; + } catch (error) { + if (generation !== detailGeneration) return; + const kind = error && typeof error === 'object' && 'kind' in error ? (error as { kind: ApiError['kind'] }).kind : 'unknown'; + if (kind === 'aborted') return; + selected = undefined; detailFailure = kind; detailStatus = 'error'; localError = error instanceof Error ? error.message : 'The V2 detail response was rejected safely.'; + await tick(); detailErrorHeading?.focus(); + } } else if (route.kind === 'location') selected = source.find((item) => item.id === route.facilityId) ?? selected; else if (route.kind === 'home') { detailRequestKey = ''; invalidateDetail(); selected = localMode ? loaded[0] : source[0]; } }; + const loadLocal = async (cursor?: string, append = false) => { - const queryKey = currentRemoteQuery(); + const requestProfile = toApiProfile(profile); + const queryKey = `${requestProfile}|${region}|${category}|${sourceType}|${displayPrecision}|${lifecycleStatus}`; if (!append && activeListKey === queryKey) return; if (!append) activeListKey = queryKey; listGeneration += 1; const generation = listGeneration; - listAbort?.abort(); const controller = new AbortController(); listAbort = controller; - lastRemoteQuery = queryKey; - if (!append) { invalidateDetail(); localStatus = 'loading'; localError = ''; manifestSha256 = undefined; loaded = []; selected = undefined; nextCursor = null; coverageNote = ''; coverageScope = ''; countSemantics = ''; } else paging = true; - try { const result = await repo.list(profile === 'community' ? 'community' : 'official', { country_code: region === 'all' ? undefined : region, category: category === 'all' ? undefined : category, source_type: sourceType === 'all' ? undefined : sourceType, display_precision: displayPrecision === 'all' ? undefined : displayPrecision, lifecycle_status: lifecycleStatus === 'all' ? undefined : lifecycleStatus, cursor }, controller.signal); if (generation !== listGeneration) return; loaded = append ? [...loaded, ...result.locations] : result.locations; if (!selected) selected = result.locations[0]; release = result.releaseId; ruleset = result.ruleset; nextCursor = result.nextCursor; coverageNote = result.coverageNote; coverageScope = result.coverageScope ?? ''; countSemantics = result.countSemantics ?? ''; localStatus = 'ready'; paging = false; if (!append) await syncRoute(); } - catch (error) { paging = false; if (!append) activeListKey = ''; if (generation !== listGeneration) return; const kind = error && typeof error === 'object' && 'kind' in error ? (error as { kind: string }).kind : 'error'; localStatus = kind === 'no-release' ? 'no-release' : 'error'; localError = error instanceof Error ? error.message : 'Local V2 response was rejected safely.'; loaded = []; selected = undefined; } + listAbort?.abort(); const controller = new AbortController(); listAbort = controller; lastRemoteQuery = queryKey; + if (!append) { invalidateDetail(); localStatus = 'loading'; localFailure = 'unknown'; localError = ''; manifestSha256 = undefined; loaded = []; selected = undefined; nextCursor = null; coverageNote = ''; coverageScope = ''; countSemantics = ''; } else paging = true; + try { + const result = await repo.list(requestProfile, { country_code: region === 'all' ? undefined : region, category: category === 'all' ? undefined : category, source_type: sourceType === 'all' ? undefined : sourceType, display_precision: displayPrecision === 'all' ? undefined : displayPrecision, lifecycle_status: lifecycleStatus === 'all' ? undefined : lifecycleStatus, cursor }, controller.signal); + if (generation !== listGeneration) return; + loaded = append ? [...loaded, ...result.locations] : result.locations; + if (!selected) selected = result.locations[0]; + release = result.releaseId; ruleset = result.ruleset; nextCursor = result.nextCursor; coverageNote = result.coverageNote; coverageScope = result.coverageScope ?? ''; countSemantics = result.countSemantics ?? ''; localStatus = 'ready'; localFailure = 'unknown'; paging = false; + if (!append) await syncRoute(); + } catch (error) { + paging = false; if (!append) activeListKey = ''; if (generation !== listGeneration) return; + const kind = error && typeof error === 'object' && 'kind' in error ? (error as { kind: ApiError['kind'] }).kind : 'unknown'; + if (kind === 'aborted') return; + localStatus = kind === 'no-release' ? 'no-release' : 'error'; localFailure = kind; localError = error instanceof Error ? error.message : 'The V2 list response was rejected safely.'; loaded = []; selected = undefined; + } }; + const downloadCsv = async () => { - if (devPreviewMode || !eligibleExport || exportBusy) return; exportBusy = true; exportError = ''; - try { const result = await csvRepo.download('official'); release = result.releaseId; manifestSha256 = result.manifestSha256; const url = URL.createObjectURL(new Blob([result.body], { type: 'text/csv;charset=utf-8' })); const anchor = document.createElement('a'); anchor.href = url; anchor.download = `uec-v2-${result.releaseId}.csv`; anchor.click(); URL.revokeObjectURL(url); } - catch (error) { const status = error && typeof error === 'object' && 'status' in error ? (error as { status: number }).status : undefined; exportError = status === 400 ? 'Choose an explicit supported profile before exporting.' : status === 404 ? 'No eligible promoted release with a manifest is available.' : status === 429 ? 'Export is temporarily rate-limited; try again later.' : error instanceof Error ? error.message : 'The CSV export could not be prepared safely.'; } + if (devPreviewMode || !eligibleExport || exportBusy) return; + exportBusy = true; exportError = ''; + try { + const result = await csvRepo.download(apiProfile); + release = result.releaseId; manifestSha256 = result.manifestSha256; + const url = URL.createObjectURL(new Blob([result.body], { type: 'text/csv;charset=utf-8' })); const anchor = document.createElement('a'); anchor.href = url; anchor.download = `uec-v2-${result.profile}-${result.releaseId}.csv`; anchor.click(); URL.revokeObjectURL(url); + } catch (error) { const kind = error && typeof error === 'object' && 'kind' in error ? (error as { kind: ApiError['kind'] }).kind : 'unknown'; exportError = kind === 'no-release' ? 'No eligible promoted release with a manifest is available.' : kind === 'rate-limited' ? 'Export is temporarily rate-limited; try again later.' : error instanceof Error ? error.message : 'The CSV export could not be prepared safely.'; } finally { exportBusy = false; } }; - const select = (id: string) => { selected = source.find((item) => item.id === id) ?? selected; window.location.hash = `/locations/${id}?profile=${profile}`; }; - const profileChanged = () => { history.pushState(null, '', `#/?profile=${profile}`); if (localMode) void loadLocal(); else void syncRoute(); }; + + const select = (id: string) => { selected = source.find((item) => item.id === id) ?? selected; window.location.hash = `/locations/${encodeURIComponent(id)}?profile=${encodeURIComponent(profileForUrl())}`; }; + const profileChanged = () => { const url = new URL(window.location.href); url.hash = `/?profile=${encodeURIComponent(profileForUrl())}`; history.pushState(null, '', url); if (localMode) void tick().then(() => loadLocal()); else void syncRoute(); }; + const filterChanged = () => { if (localMode) void tick().then(() => loadLocal()); }; + const loadDevPreview = async () => { if (!canOpenDevPreview(import.meta.env.DEV, devPreviewMode ? 'dev-candidates' : null)) { previewStatus = 'blocked'; previewError = 'Private candidate preview is unavailable in production builds.'; return; } if (!previewToken.trim()) { previewStatus = 'error'; previewError = 'Enter the operator token for this development session.'; return; } previewStatus = 'loading'; previewError = ''; - try { if (testReleaseMode) previewRows = (await new TestReleaseRepository().list(profile === 'community' ? 'community' : 'official', previewToken)).locations; else { const { DevCandidatePreviewRepository } = await import('../api/DevCandidatePreviewRepository'); previewRows = await new DevCandidatePreviewRepository().list(previewToken); } previewStatus = 'ready'; selected = previewRows[0]; } - catch (error) { previewStatus = 'error'; previewError = error instanceof Error ? error.message : 'Private candidate preview was rejected safely.'; previewRows = []; selected = undefined; } + try { + if (testReleaseMode) previewRows = (await new TestReleaseRepository().list(apiProfile, previewToken)).locations; + else { const { DevCandidatePreviewRepository } = await import('../api/DevCandidatePreviewRepository'); previewRows = await new DevCandidatePreviewRepository().list(previewToken); } + previewStatus = 'ready'; selected = previewRows[0]; + } catch (error) { previewStatus = 'error'; previewError = error instanceof Error ? error.message : 'Private candidate preview was rejected safely.'; previewRows = []; selected = undefined; } + }; + + const downloadTestCsv = async () => { + if (!testReleaseMode || previewStatus !== 'ready' || testCsvBusy) return; + testCsvBusy = true; testCsvError = ''; + try { const result = await new TestReleaseCsvExportRepository().download(apiProfile, previewToken); const url = URL.createObjectURL(new Blob([result.body], { type: 'text/csv;charset=utf-8' })); const anchor = document.createElement('a'); anchor.href = url; anchor.download = `uec-test-release-${result.profile}-${result.releaseId}.csv`; anchor.click(); URL.revokeObjectURL(url); } + catch (error) { testCsvError = error instanceof Error ? error.message : 'Private test-release CSV was rejected safely.'; } + finally { testCsvBusy = false; } }; - const downloadTestCsv = async () => { if (!testReleaseMode || previewStatus !== 'ready' || testCsvBusy) return; testCsvBusy = true; testCsvError = ''; try { const result = await new TestReleaseCsvExportRepository().download(profile === 'community' ? 'community' : 'official', previewToken); const url = URL.createObjectURL(new Blob([result.body], { type: 'text/csv;charset=utf-8' })); const anchor = document.createElement('a'); anchor.href = url; anchor.download = `uec-test-release-${result.releaseId}.csv`; anchor.click(); URL.revokeObjectURL(url); } catch (error) { testCsvError = error instanceof Error ? error.message : 'Private test-release CSV was rejected safely.'; } finally { testCsvBusy = false; } }; + const clearFilters = () => { search = ''; region = 'all'; category = 'all'; sourceType = 'all'; displayPrecision = 'all'; lifecycleStatus = 'all'; }; - const searchChanged = () => { if (localMode) { const url = new URL(window.location.href); if (search.trim()) url.searchParams.set('q', search.trim()); else url.searchParams.delete('q'); history.replaceState(null, '', url); } }; + const searchChanged = () => { const url = new URL(window.location.href); if (search.trim()) url.searchParams.set('q', search.trim()); else url.searchParams.delete('q'); history.replaceState(null, '', url); }; + onMount(() => { - const params = new URLSearchParams(window.location.search); - localMode = params.get('mode') === 'local-v2'; - testReleaseMode = params.get('preview') === 'test-release'; - devPreviewMode = testReleaseMode || params.get('preview') === 'dev-candidates'; - const route = parseRoute(window.location.hash); - if (route.kind !== 'not-found') profile = route.profile; + const params = new URLSearchParams(window.location.search); localMode = params.get('mode') === 'local-v2'; testReleaseMode = params.get('preview') === 'test-release'; devPreviewMode = testReleaseMode || params.get('preview') === 'dev-candidates'; + const route = parseRoute(window.location.hash); if (route.kind !== 'not-found') profile = localMode ? toApiProfile(route.profile) : route.profile; if (localMode) { search = params.get('q') ?? ''; region = params.get('country_code') ?? 'all'; category = params.get('category') ?? 'all'; sourceType = params.get('source_type') ?? 'all'; displayPrecision = params.get('display_precision') ?? 'all'; lifecycleStatus = params.get('lifecycle_status') ?? 'all'; - try { const api = params.get('api') ?? undefined; repo = new LocalLocationRepository(globalThis.fetch, api); csvRepo = new LocalCsvExportRepository(globalThis.fetch, api ?? ''); metadataStatus = 'loading'; void new FilterMetadataRepository(globalThis.fetch, api ?? '').get().then((value) => { metadata = value; metadataStatus = 'ready'; }).catch(() => { metadataStatus = 'error'; }); } catch (error) { localStatus = 'error'; localError = error instanceof Error ? error.message : 'Local API origin was rejected safely.'; } + try { const api = params.get('api') ?? undefined; repo = new LocalLocationRepository(globalThis.fetch, api); csvRepo = new LocalCsvExportRepository(globalThis.fetch, api ?? ''); metadataStatus = 'loading'; void new FilterMetadataRepository(globalThis.fetch, api ?? '').get().then((value) => { metadata = value; metadataStatus = 'ready'; }).catch(() => { metadataStatus = 'error'; }); } + catch (error) { localStatus = 'error'; localFailure = 'network'; localError = error instanceof Error ? error.message : 'Local API origin was rejected safely.'; } } if (devPreviewMode) { previewStatus = import.meta.env.DEV ? 'idle' : 'blocked'; if (!import.meta.env.DEV) previewError = 'Private candidate preview is unavailable in production builds.'; } else if (localMode && localStatus !== 'error') void loadLocal(); else if (!localMode) void syncRoute(); - const onHashChange = () => { const next = parseRoute(window.location.hash); if (next.kind !== 'not-found' && next.profile !== profile) { profile = next.profile; if (localMode) void loadLocal(); else void syncRoute(); } else void syncRoute(); }; + const onHashChange = () => { const next = parseRoute(window.location.hash); if (next.kind !== 'not-found' && toApiProfile(next.profile) !== toApiProfile(profile)) { profile = localMode ? toApiProfile(next.profile) : next.profile; if (!localMode) void syncRoute(); } else void syncRoute(); }; const onPopState = () => { const current = new URLSearchParams(window.location.search); if (localMode) { search = current.get('q') ?? ''; region = current.get('country_code') ?? 'all'; category = current.get('category') ?? 'all'; sourceType = current.get('source_type') ?? 'all'; displayPrecision = current.get('display_precision') ?? 'all'; lifecycleStatus = current.get('lifecycle_status') ?? 'all'; } onHashChange(); }; - window.addEventListener('hashchange', onHashChange); window.addEventListener('popstate', onPopState); - return () => { listAbort?.abort(); detailAbort?.abort(); window.removeEventListener('hashchange', onHashChange); window.removeEventListener('popstate', onPopState); }; + window.addEventListener('hashchange', onHashChange); window.addEventListener('popstate', onPopState); return () => { listAbort?.abort(); detailAbort?.abort(); window.removeEventListener('hashchange', onHashChange); window.removeEventListener('popstate', onPopState); }; }); @@ -150,19 +219,18 @@ {#if testReleaseMode}
TEST-ONLY CSV — NOT PROJECT-APPROVED OR PUBLISHEDComplete bounded test-release rows only; this action never uses the public export route.{#if testCsvError}

{testCsvError}

{/if}
{/if} {#if !devPreviewMode || previewStatus === 'ready'} -

01 / DISCOVER

Choose the evidence lane

Profiles stay separate. A community claim is never silently promoted into a curated result.

{#if localMode && metadataStatus === 'loading'}

Loading available filter values…

{:else if localMode && metadataStatus === 'error'}

Filter metadata is unavailable; only unscoped discovery is shown.

{/if}
+

01 / DISCOVER

Choose the evidence lane

Profiles stay separate. A community claim is never silently promoted into a curated result.

{#if localMode && metadataStatus === 'loading'}

Loading available filter values…

{:else if localMode && metadataStatus === 'error'}

Filter metadata is unavailable; controlled filter values could not be loaded.

{/if}
{search || region !== 'all' || category !== 'all' || sourceType !== 'all' || displayPrecision !== 'all' || lifecycleStatus !== 'all' ? `Filters applied: ${[search && `name “${search}”`, region !== 'all' && region, category !== 'all' && category, sourceType !== 'all' && sourceType, displayPrecision !== 'all' && displayPrecision, lifecycleStatus !== 'all' && lifecycleStatus].filter(Boolean).join(' · ')}` : 'No additional filters applied'}
+ {#if localMode && search.trim()}

Text search is limited to the currently loaded V2 page; the public contract does not provide a server-side free-text query. {nextCursor ? 'Later pages may contain additional matches.' : 'All rows in this response are loaded.'}

{/if} {#if profile === 'community'}
Community claimsUnreviewed community claims: Not verified by Until Every Cage. Check each record’s factual review status before relying on it.
{/if} - {#if localMode && localStatus === 'loading'}
Loading the {profileLabel.toLowerCase()}…
{:else if localMode && (localStatus === 'error' || localStatus === 'no-release')}{:else if localMode && detailStatus === 'loading'}
Loading the selected local record…
{:else if localMode && detailStatus === 'error'}{:else} -

02 / FILTER & COMPARE

Results {visibleLocations.length}{localMode && nextCursor ? ' on first page' : ''}

{localMode ? 'Current eligible response' : 'Fictional demonstration data'} · {profileLabel}

- - {#if localMode}
VISIBLE FACILITY RECORDS{visibleLocations.length}{nextCursor ? '+' : ''}
DENOMINATORNot available

{countSemantics || 'Counts refer to eligible public facility projection rows, not animals or a story-wide total.'} Scope: {coverageScope || 'selected promoted release public facilities'}. {nextCursor ? 'This is a partial page.' : 'This response has no further page.'} Legacy status is not inferred: the current V2 record contract has no legacy field.

{/if} - {#if localMode}

{coverageNote} {nextCursor ? 'Only the first page is loaded. Search and filters below may miss later records; counts and map points are partial.' : 'All records in this response are loaded; search applies to those records.'}

{/if} -
{#if selected}

03 / RECORD DETAIL

{selected.name}

{selected.region} · {selected.category}

{#if selected.evidence?.publicationProfile === 'community' && selected.evidence.factualReviewStatus === 'unreviewed'}

{selected.evidence.publicationWarning ?? 'Unreviewed community claim — not verified by Until Every Cage'}

{/if}
OBSERVED{selected.observed}
MAP STATUS{selected.lat === null ? 'No publishable map location' : selected.evidence?.displayPrecision === 'exact' ? 'Exact display point' : 'Approximate display point'}
{#if selected.evidence}
Source origin
{selected.evidence.sourceType === 'user_submitted' ? 'Community-submitted' : selected.evidence.sourceType === 'official' ? 'Government-sourced' : 'Secondary-sourced'}
Factual review
{selected.evidence.factualReviewStatus}{selected.evidence.reviewerRole ? ` · ${selected.evidence.reviewerRole}` : ' · reviewer role unavailable'}
Privacy screening
{selected.evidence.privacyScreeningStatus}
Project approval
{selected.evidence.projectApproval}
Published profile
{selected.evidence.publicationProfile} · release {release}
Source
{selected.source} · {selected.evidence.sourceId}
Retrieved
{selected.evidence.retrievedAt}
{/if}

READ WITH CARE

This is an observation in a particular release, not a guarantee of current operation. Source origin, factual review, privacy screening, and project approval are separate signals.

{/if}
+ {#if localMode && localStatus === 'loading'}
Loading the {profileLabelText.toLowerCase()}…
{:else if localMode && (localStatus === 'error' || localStatus === 'no-release')}{:else if localMode && detailStatus === 'loading'}
Loading the selected local record…
{:else if localMode && detailStatus === 'error'}{:else} +

02 / FILTER & COMPARE

Results {visibleLocations.length}{localMode && nextCursor ? ' on first page' : ''}

{localMode ? 'Current eligible response' : 'Fictional demonstration data'} · {profileLabelText}

+ {#if localMode}
VISIBLE FACILITY RECORDS{visibleLocations.length}{nextCursor ? '+' : ''}
DENOMINATORNot available

{countSemantics || 'Counts refer to eligible public facility projection rows, not animals or a story-wide total.'} Scope: {coverageScope || 'selected promoted release public facilities'}. {nextCursor ? 'This is a partial page.' : 'This response has no further page.'} Legacy status is not inferred: the current V2 record contract has no legacy field.

{coverageNote} {nextCursor ? 'Only the first page is loaded. Search and filters below may miss later records; counts and map points are partial.' : 'All records in this response are loaded; search applies to those records.'}

{/if} +
{#if selected}

03 / RECORD DETAIL

{selected.name}

{selected.region} · {selected.category}

{#if isUnreviewedCommunity(selected)}

{selected.evidence?.publicationWarning ?? 'Unreviewed community claim — not verified by Until Every Cage'}

{/if}
OBSERVED{selected.observed}
MAP STATUS{selected.lat === null ? 'No publishable map location' : selected.evidence?.displayPrecision === 'exact' ? 'Exact display point' : 'Approximate display point'}
{#if selected.evidence}
Source origin
{sourceLabel(selected.evidence.sourceType)}
Factual review
{selected.evidence.factualReviewStatus}{selected.evidence.reviewerRole ? ` · ${selected.evidence.reviewerRole}` : ' · reviewer role unavailable'}
Privacy screening
{selected.evidence.privacyScreeningStatus}
Project approval
{selected.evidence.projectApproval}
Published profile
{selected.evidence.publicationProfile ?? 'unavailable'} · release {release}
Source
{selected.source} · {selected.evidence.sourceId}
Retrieved
{selected.evidence.retrievedAt}
Lifecycle
{selected.evidence.lifecycleStatus}
Observation count
{selected.evidence.observationCount ?? 'unavailable'}
{/if}

READ WITH CARE

This is an observation in a particular release, not a guarantee of current operation. Source origin, factual review, privacy screening, and project approval are separate signals.

{:else}

Select a record to inspect its evidence context.

{/if}
{#if selected}

RECORD / {selected.id}

{/if} {/if} - {#if localMode && localStatus === 'ready'}{#if canMountPublicExport(devPreviewMode)}{/if}{/if} + {#if localMode && localStatus === 'ready'}{/if}
{#if showGuidance}

Do not infer closure, identity, or permission from a map point. For a correction, privacy concern, or suppression request, preserve the record ID and contact the project maintainer through the reporting channel on the ethics page. Do not include sensitive personal details in a public issue.

Read reporting guidance ↗
{/if}
- {#if !devPreviewMode}
{#if showMap}{/if}{#if showExport}

IN-MEMORY EXPORT PREVIEW

{profile} · {release}

Loaded results only. This preview is not a live or complete export.

{exportPreview}
{/if}{/if}
{localMode ? 'Local V2 mode · no fixture fallback' : 'Method preview · no live requests'}
+ {#if !devPreviewMode}
{#if showMap}{/if}{#if showExport}

IN-MEMORY EXPORT PREVIEW

{apiProfile} · {release}

Loaded results only. This preview is not a live or complete export.

{exportPreview}
{/if}{/if}
{localMode ? 'Local V2 mode · no fixture fallback' : 'Method preview · no live requests'}
{/if}
diff --git a/frontend/src/app/routeState.ts b/frontend/src/app/routeState.ts index e5c6123..5bb1c2e 100644 --- a/frontend/src/app/routeState.ts +++ b/frontend/src/app/routeState.ts @@ -1,6 +1,6 @@ import type { Profile } from '../domain/publication'; export type RouteState = Readonly<{kind:'home';profile:Profile}> | Readonly<{kind:'location';facilityId:string;profile:Profile}> | Readonly<{kind:'not-found';fragment:string}>; -const profiles: readonly Profile[]=['curated','community']; +const profiles: readonly Profile[]=['curated','official','secondary','community']; const profileOf=(value:string|null):Profile=>profiles.includes(value as Profile)?value as Profile:'curated'; export function parseRoute(hash:string):RouteState { const raw=hash.startsWith('#')?hash.slice(1):hash; const [pathPart,query='']=raw.split('?'); const path=pathPart ?? ''; const profile=profileOf(new URLSearchParams(query).get('profile')); if(path===''||path==='/') return {kind:'home',profile}; const match=/^\/locations\/([a-z0-9-]+)$/.exec(path); if(match?.[1]) return {kind:'location',facilityId:match[1],profile}; return {kind:'not-found',fragment:raw}; } export function serializeRoute(route:Exclude):string { const path=route.kind==='home'?'/':`/locations/${encodeURIComponent(route.facilityId)}`; return `#${path}?profile=${route.profile}`; } diff --git a/frontend/src/domain/publication.ts b/frontend/src/domain/publication.ts index 8bc2b57..b5b9e43 100644 --- a/frontend/src/domain/publication.ts +++ b/frontend/src/domain/publication.ts @@ -1,2 +1,13 @@ -export type Profile='curated'|'community'; +/** `curated` is retained only as the fixture-mode label. It maps to the V2 + * `official` profile before a request is made. */ +export type Profile = 'curated' | 'official' | 'secondary' | 'community'; +export type ApiProfile = Exclude; +export const API_PROFILES: readonly ApiProfile[] = ['official', 'secondary', 'community']; +export const toApiProfile = (profile: Profile): ApiProfile => profile === 'curated' ? 'official' : profile; +export const profileLabel = (profile: Profile): string => ({ + curated: 'Official / curated release', + official: 'Official profile', + secondary: 'Secondary sources', + community: 'Community claims', +}[profile]); export type Publication = Readonly<{origin:'government-sourced'|'community-submitted';review:'project-approved'|'community-unreviewed';profile:Profile;published:boolean}>; diff --git a/frontend/src/map/LeafletMapAdapter.ts b/frontend/src/map/LeafletMapAdapter.ts index 9886ea6..b353822 100644 --- a/frontend/src/map/LeafletMapAdapter.ts +++ b/frontend/src/map/LeafletMapAdapter.ts @@ -4,7 +4,7 @@ import type { DisplayFeature } from './mapProjection'; type MarkerFactory = (feature: DisplayFeature, map: LeafletMap) => Marker; export class LeafletMapAdapter implements MapAdapter { #map: LeafletMap | null = null; #markers = new Map(); #disposed = false; #makeMarker: MarkerFactory | null = null; - async mount(container: HTMLElement): Promise { const leaflet = await import('leaflet'); if (this.#disposed) return; this.#makeMarker = (feature, map) => leaflet.marker([feature.lat, feature.lon], { title: feature.label }).addTo(map); this.#map = leaflet.map(container, { attributionControl: false, zoomControl: true }).setView([55, 10], 6); this.#map.getContainer().style.background = '#ded8cc'; } + async mount(container: HTMLElement, onSelect?: (id:string) => void): Promise { const leaflet = await import('leaflet'); if (this.#disposed) return; this.#makeMarker = (feature, map) => { const marker = leaflet.marker([feature.lat, feature.lon], { title: feature.label, keyboard: true }).addTo(map); if (onSelect) marker.on('click', () => onSelect(feature.id)); return marker; }; this.#map = leaflet.map(container, { attributionControl: false, zoomControl: true }).setView([55, 10], 6); this.#map.getContainer().style.background = '#ded8cc'; } update(features: readonly DisplayFeature[], selectedId: string | null): void { if (!this.#map || !this.#makeMarker) return; const active = new Set(features.map((feature) => feature.id)); for (const [id, marker] of this.#markers) { if (!active.has(id)) { marker.remove(); this.#markers.delete(id); } } for (const feature of features) { const marker = this.#markers.get(feature.id) ?? this.#makeMarker(feature, this.#map); this.#markers.set(feature.id, marker); marker.setOpacity(selectedId === null || selectedId === feature.id ? 1 : 0.55); } } destroy(): void { this.#disposed = true; for (const marker of this.#markers.values()) marker.remove(); this.#markers.clear(); this.#map?.remove(); this.#map = null; this.#makeMarker = null; } } diff --git a/frontend/src/map/MapAdapter.ts b/frontend/src/map/MapAdapter.ts index 2fbf6ff..0ec4295 100644 --- a/frontend/src/map/MapAdapter.ts +++ b/frontend/src/map/MapAdapter.ts @@ -1,2 +1,2 @@ import type { DisplayFeature } from './mapProjection'; -export interface MapAdapter { mount(container:HTMLElement):Promise; update(features:readonly DisplayFeature[],selectedId:string|null):void; destroy():void; } +export interface MapAdapter { mount(container:HTMLElement, onSelect?: (id:string) => void):Promise; update(features:readonly DisplayFeature[],selectedId:string|null):void; destroy():void; } diff --git a/frontend/src/map/MapView.svelte b/frontend/src/map/MapView.svelte index 4b225e9..8409479 100644 --- a/frontend/src/map/MapView.svelte +++ b/frontend/src/map/MapView.svelte @@ -5,6 +5,8 @@ import { LeafletMapAdapter } from './LeafletMapAdapter'; export let items: readonly Location[] = []; export let selectedId: string | null = null; + export let onSelect: ((id: string) => void) | undefined = undefined; + export let synthetic = true; let container: HTMLDivElement; let adapter: LeafletMapAdapter | null = null; $: features = projectLocations(items); @@ -13,11 +15,11 @@ let disposed = false; const map = new LeafletMapAdapter(); adapter = map; - map.mount(container).then(() => { if (!disposed) map.update(features, selectedId); }); + map.mount(container, onSelect).then(() => { if (!disposed) map.update(features, selectedId); }); return () => { disposed = true; map.destroy(); adapter = null; }; }); $: adapter?.update(features, selectedId); -
{#if hasUnreviewedClaims}

Unreviewed community claims — not verified by Until Every Cage

{/if}

Facility pins only, not animal counts. The results list is the accessible equivalent. Blank local background · {features.length} display points · no external tiles

+
{#if hasUnreviewedClaims}

Unreviewed community claims — not verified by Until Every Cage

{/if}

Facility pins only, not animal counts. The results list is the accessible equivalent. Blank local background · {features.length} display points · no external tiles

diff --git a/frontend/src/ui/ExportControl.svelte b/frontend/src/ui/ExportControl.svelte index 581fea4..1dba4c4 100644 --- a/frontend/src/ui/ExportControl.svelte +++ b/frontend/src/ui/ExportControl.svelte @@ -2,11 +2,12 @@ export let enabled = false; export let busy = false; export let error = ''; + export let profile = 'official'; export let onExport: () => void;
- - {#if !enabled}

CSV export is available only for an explicit official profile with a selected eligible release.

{/if} + + {#if !enabled}

CSV export is available only when the selected V2 profile has an eligible promoted release.

{/if} {#if error}{/if}
diff --git a/frontend/tests/e2e/local-safety.spec.ts b/frontend/tests/e2e/local-safety.spec.ts index 14dc57c..e545cfa 100644 --- a/frontend/tests/e2e/local-safety.spec.ts +++ b/frontend/tests/e2e/local-safety.spec.ts @@ -2,9 +2,9 @@ import { test, expect, type Page } from '@playwright/test'; const firstId = '550e8400-e29b-41d4-a716-446655440000'; const secondId = '550e8400-e29b-41d4-a716-446655440001'; -const row = (id = firstId, name = 'First local record', profile: 'official' | 'community' = 'official') => ({ +const row = (id = firstId, name = 'First local record', profile: 'official' | 'secondary' | 'community' = 'official') => ({ facility_id: id, canonical_name: name, city: 'North Coast', country_code: 'DK', category: 'dairy', - source_type: profile === 'community' ? 'user_submitted' : 'official', publication_profile: profile, + source_type: profile === 'community' ? 'user_submitted' : profile, publication_profile: profile, factual_review_status: profile === 'community' ? 'unreviewed' : 'reviewed', privacy_screening_status: 'passed', project_approval: profile === 'community' ? 'pending' : 'approved', reviewer_role: null, publication_warning: profile === 'community' ? 'Unreviewed community claim — not verified by Until Every Cage' : null, @@ -14,10 +14,10 @@ const row = (id = firstId, name = 'First local record', profile: 'official' | 'c provenance_source_url: 'https://example.test/source', provenance_retrieved_at: '2026-01-01T00:00:00Z', release_id: 'rel-1', release_ruleset_version: 'rules-1', }); -const list = (profile: 'official' | 'community', data = [row(firstId, 'First local record', profile)], nextCursor: string | null = null) => ({ +const list = (profile: 'official' | 'secondary' | 'community', data = [row(firstId, 'First local record', profile)], nextCursor: string | null = null) => ({ data, api_version: 'v2', meta: { release_id: 'rel-1', ruleset_version: 'rules-1', profile, next_cursor: nextCursor, coverage_note: 'Selected promoted release only.' }, }); -const detail = (data: ReturnType, profile: 'official' | 'community') => ({ +const detail = (data: ReturnType, profile: 'official' | 'secondary' | 'community') => ({ data, api_version: 'v2', meta: { release_id: 'rel-1', ruleset_version: 'rules-1', release_created_at: '2026-01-01T00:00:00Z', profile }, }); const mockMetadata = async (page: Page) => page.route('**/api/v2/discovery/filters', route => route.fulfill({ status: 503, body: 'unavailable' })); @@ -110,3 +110,30 @@ test('detail failure focuses the error heading and offers recovery', async ({ pa await page.getByRole('button', { name: 'Back to results' }).click(); await expect(page.getByRole('heading', { name: 'First local record' })).toBeVisible(); }); + +test('secondary profile remains distinct through list, detail, and export requests', async ({ page }) => { + await mockMetadata(page); + const secondaryPayload = row(firstId, 'Secondary source record', 'secondary'); + const seen: string[] = []; + await page.route('**/api/v2/locations*', async route => { + const url = new URL(route.request().url()); + seen.push(`${url.pathname}|${url.searchParams.get('profile') ?? ''}`); + await route.fulfill({ json: url.pathname.endsWith(firstId) ? detail(secondaryPayload, 'secondary') : list('secondary', [secondaryPayload]) }); + }); + await page.route('**/api/v2/locations.csv*', route => route.fulfill({ headers: { 'content-type': 'text/csv', 'x-uec-release-id': 'rel-1' }, body: 'facility_id\nsecondary\n' })); + await page.goto('./?mode=local-v2#/'); + await page.getByLabel('Profile').selectOption('secondary'); + await expect(page.getByRole('heading', { name: 'Secondary source record' })).toBeVisible(); + expect(seen.some(value => value.endsWith('|secondary'))).toBeTruthy(); + await page.getByRole('button', { name: /Download secondary CSV/ }).click(); + await expect(page.locator('.release-panel')).toContainText('secondary'); +}); + +test('malformed V2 envelopes fail closed with a contract-specific state', async ({ page }) => { + await mockMetadata(page); + await page.route('**/api/v2/locations**', route => route.fulfill({ json: { api_version: 'v1', data: [] } })); + await page.goto('./?mode=local-v2#/'); + await expect(page.getByRole('heading', { name: 'Could not load local V2 data' })).toBeVisible(); + await expect(page.getByRole('alert')).toContainText('V2 data contract unavailable'); + await expect(page.getByRole('button', { name: /North Star Cooperative/ })).toHaveCount(0); +}); diff --git a/frontend/tests/unit/localCsvExportRepository.test.ts b/frontend/tests/unit/localCsvExportRepository.test.ts index e949988..5287b69 100644 --- a/frontend/tests/unit/localCsvExportRepository.test.ts +++ b/frontend/tests/unit/localCsvExportRepository.test.ts @@ -14,4 +14,10 @@ describe('LocalCsvExportRepository', () => { it('rejects an empty response or missing release context', async () => { await expect(new LocalCsvExportRepository(vi.fn().mockResolvedValue(response(200, ''))).download('official')).rejects.toThrow(/eligible release context/); }); + it('requests the selected non-official profile and requires CSV content', async () => { + const fetcher = vi.fn().mockResolvedValue(response(200, 'facility_id\nsecondary-1\n')); + await expect(new LocalCsvExportRepository(fetcher).download('secondary')).resolves.toMatchObject({ profile: 'secondary' }); + expect(String(fetcher.mock.calls[0]?.[0])).toContain('profile=secondary'); + await expect(new LocalCsvExportRepository(vi.fn().mockResolvedValue(new Response('{}', { status: 200, headers: { 'x-uec-release-id': 'release-1', 'content-type': 'application/json' } }))).download('official')).rejects.toMatchObject({ kind: 'invalid-contract' }); + }); }); diff --git a/frontend/tests/unit/localLocationRepository.test.ts b/frontend/tests/unit/localLocationRepository.test.ts index fa89f00..213cea0 100644 --- a/frontend/tests/unit/localLocationRepository.test.ts +++ b/frontend/tests/unit/localLocationRepository.test.ts @@ -1,7 +1,7 @@ import{describe,expect,it,vi}from'vitest';import{LocalLocationRepository}from'../../src/api/LocalLocationRepository'; const row={facility_id:'550e8400-e29b-41d4-a716-446655440000',canonical_name:'Local V2 Fixture',city:'North Coast',country_code:'DK',category:'dairy',source_type:'official',publication_profile:'official',factual_review_status:'reviewed',privacy_screening_status:'passed',project_approval:'approved',reviewer_role:null,publication_warning:null,display_precision:'city',latitude:55,longitude:10,first_observed_at:null,last_observed_at:'2026-01-01T00:00:00Z',observation_count:1,lifecycle_status:'active_observed',provenance_source_id:'source-1',provenance_source_name:'Synthetic local source',provenance_source_url:'https://example.test/source',provenance_retrieved_at:'2026-01-01T00:00:00Z',release_id:'rel-1',release_ruleset_version:'rules-1'}; const response=(body:unknown,status=200)=>new Response(JSON.stringify(body),{status,headers:{'content-type':'application/json'}});const envelope=(data=[row],meta={release_id:'rel-1',ruleset_version:'rules-1',profile:'official',next_cursor:null,coverage_note:'Local promoted release.'})=>({data,api_version:'v2',meta}); -describe('LocalLocationRepository',()=>{it('maps a valid Rust-shaped envelope',async()=>{const result=await new LocalLocationRepository(vi.fn().mockResolvedValue(response(envelope()))).list();expect(result.locations[0]).toMatchObject({id:row.facility_id,name:'Local V2 Fixture',lat:55});});it('fails closed when no release is promoted',async()=>{await expect(new LocalLocationRepository(vi.fn().mockResolvedValue(response(envelope([],{release_id:null,profile:'official',coverage_note:'No promoted release.'})))).list()).rejects.toMatchObject({kind:'no-release'});});it('classifies HTTP failures',async()=>{await expect(new LocalLocationRepository(vi.fn().mockResolvedValue(response({},503))).list()).rejects.toMatchObject({kind:'http',status:503});});it('rejects malformed or restricted payloads',async()=>{await expect(new LocalLocationRepository(vi.fn().mockResolvedValue(response({...envelope(),api_version:'v1'}))).list()).rejects.toMatchObject({kind:'invalid-contract'});await expect(new LocalLocationRepository(vi.fn().mockResolvedValue(response(envelope([{...row,privacy_screening_status:'failed'}])))).list()).rejects.toMatchObject({kind:'invalid-contract'});});}); +describe('LocalLocationRepository',()=>{it('maps a valid Rust-shaped envelope',async()=>{const result=await new LocalLocationRepository(vi.fn().mockResolvedValue(response(envelope()))).list();expect(result.locations[0]).toMatchObject({id:row.facility_id,name:'Local V2 Fixture',lat:55});});it('fails closed when no release is promoted',async()=>{await expect(new LocalLocationRepository(vi.fn().mockResolvedValue(response(envelope([],{release_id:null,profile:'official',coverage_note:'No promoted release.'})))).list()).rejects.toMatchObject({kind:'no-release'});});it('classifies server failures as unavailable',async()=>{await expect(new LocalLocationRepository(vi.fn().mockResolvedValue(response({},503))).list()).rejects.toMatchObject({kind:'unavailable',status:503});});it('rejects malformed or restricted payloads',async()=>{await expect(new LocalLocationRepository(vi.fn().mockResolvedValue(response({...envelope(),api_version:'v1'}))).list()).rejects.toMatchObject({kind:'invalid-contract'});await expect(new LocalLocationRepository(vi.fn().mockResolvedValue(response(envelope([{...row,privacy_screening_status:'failed'}])))).list()).rejects.toMatchObject({kind:'invalid-contract'});});}); describe('LocalLocationRepository query contract', () => { it('passes supported filters and the opaque cursor without inventing search semantics', async () => { const fetcher = vi.fn().mockResolvedValue(response(envelope([], { ...envelope().meta, next_cursor: 'cursor-2' }))); @@ -18,6 +18,19 @@ describe('LocalLocationRepository query contract', () => { }); }); +describe('current V2 wire edge cases', () => { + it('accepts a null canonical name and exposes an explicit safe display label', async () => { + const fetcher = vi.fn().mockResolvedValue(response(envelope([{ ...row, canonical_name: null }]))); + await expect(new LocalLocationRepository(fetcher).list()).resolves.toMatchObject({ locations: [{ name: 'Unnamed candidate record' }] }); + }); + it('classifies structured server failures for first-class UI states', async () => { + const unavailable = vi.fn().mockResolvedValue(new Response(JSON.stringify({ api_version: 'v2', error: { code: 'database_pool_unavailable', message: 'database unavailable' } }), { status: 503 })); + await expect(new LocalLocationRepository(unavailable).list()).rejects.toMatchObject({ kind: 'unavailable', code: 'database_pool_unavailable', status: 503 }); + const restricted = vi.fn().mockResolvedValue(new Response(JSON.stringify({ api_version: 'v2', error: { code: 'location_not_found', message: 'location not found' } }), { status: 404 })); + await expect(new LocalLocationRepository(restricted).detail(row.facility_id)).rejects.toMatchObject({ kind: 'restricted', code: 'location_not_found', status: 404 }); + }); +}); + describe('community list safety', () => { const community = { ...row, source_type: 'user_submitted', publication_profile: 'community', factual_review_status: 'unreviewed', project_approval: 'pending', publication_warning: 'Unreviewed community claim — not verified by Until Every Cage' }; const communityMeta = { ...envelope().meta, profile: 'community' }; diff --git a/frontend/tests/unit/routeState.test.ts b/frontend/tests/unit/routeState.test.ts index 5db5d5e..ae6c9dc 100644 --- a/frontend/tests/unit/routeState.test.ts +++ b/frontend/tests/unit/routeState.test.ts @@ -1,2 +1,3 @@ import {describe,expect,it} from 'vitest'; import {parseRoute,serializeRoute} from '../../src/app/routeState'; describe('routeState',()=>{it('parses home and defaults safely',()=>expect(parseRoute('#/')).toEqual({kind:'home',profile:'curated'}));it('parses location profile',()=>expect(parseRoute('#/locations/syn-north-star?profile=community')).toEqual({kind:'location',facilityId:'syn-north-star',profile:'community'}));it('round-trips routes',()=>{expect(serializeRoute({kind:'home',profile:'curated'})).toBe('#/?profile=curated');expect(parseRoute(serializeRoute({kind:'location',facilityId:'syn-river-meadow',profile:'community'}))).toEqual({kind:'location',facilityId:'syn-river-meadow',profile:'community'});});it('does not guess unsupported paths',()=>expect(parseRoute('#/search?q=eggs')).toEqual({kind:'not-found',fragment:'/search?q=eggs'}));}); +describe('V2 profile routes',()=>it('keeps official, secondary, and community profile names explicit',()=>{for(const profile of ['official','secondary','community'] as const) expect(parseRoute(serializeRoute({kind:'home',profile}))).toEqual({kind:'home',profile});})); From a4f9930b0bc2ff1177b7a75e21d617e395489939 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Tue, 15 Sep 2026 12:02:57 -0700 Subject: [PATCH 116/311] Finalize v2 integration validation record --- docs/V2-IMPLEMENTATION-TODO.md | 23 +++++++++++++++++++++++ docs/V2-SPRINT-2026-09-13.md | 22 ++++++++++++++++++++++ docs/source-status.md | 5 +++-- pipeline/tests/e2e/backup-restore.ps1 | 9 ++++++++- pipeline/tests/test_source_registry.py | 4 ++-- 5 files changed, 58 insertions(+), 5 deletions(-) diff --git a/docs/V2-IMPLEMENTATION-TODO.md b/docs/V2-IMPLEMENTATION-TODO.md index 2b4c870..2aafc2d 100644 --- a/docs/V2-IMPLEMENTATION-TODO.md +++ b/docs/V2-IMPLEMENTATION-TODO.md @@ -29,6 +29,29 @@ The [review cleanup record](V2-REVIEW-CLEANUP-2026-09-13.md) distinguishes fixes Post-fix A/B regression checks now cover release-scoped review in migration 022 and validation/promotion, plus Rust list/detail/CSV behavior through a two-profile synthetic HTTP E2E: B's later denial cannot revoke or relabel promoted A, and B cannot inherit A's approval. The local synthetic launcher explicitly declares `--no-distributed-artifacts`, with a narrow contract test; the portless two-stage backup drill passed twice in scoped local verification and in the final gate. The independent verifier reports a no-retry final local pass: standard 72 Rust and 102 Python (5 skipped), four sequential API E2E modules 36/36, root Jest 19/19, frontend unit 41/41, Playwright 42/42 across three browsers, plus frontend check/lint/boundary/build, `cargo fmt`, `git diff --check`, and PowerShell gate self-test 3/3. No disposable Docker project remained; persistent databases were untouched. **Remote CI remains pending.** Deploy migration 022 and the updated V2 API together with V2 public access paused during the transition, then verify compatibility and current restrictions before any eligible access resumes. See the [review cleanup record](V2-REVIEW-CLEANUP-2026-09-13.md); production crosswalk, independent suppression replay, and the other release blockers remain open. +### 2026-09-15 final integration evidence + +The dependency-ordered integration of shared private lifecycle handling, +suppression/re-exposure migration 024, Italy 853/2004, UK FSA/FSS, Belgium +reconnaissance metadata, and the frontend V2 profile/state wiring is locally +validated. The standard gate passed 65 Rust library tests, 10 Rust binary tests, +85 Python tests with 10 expected skips, and 60 country/contract tests. The +disposable Docker API modules passed public 10/10, community 6/6, seeded 19/19, +public-surface 3/3, candidate-import 4/4, Italy 4/4, and suppression lifecycle +4/4. Backup/restore passed stale-ledger rejection before replay and current +restriction verification after replay. Root Jest passed 19/19, frontend unit +tests 50/50, Playwright passed 54/54 across Chromium, Firefox, and WebKit, and +frontend check/lint/build, `cargo fmt -- --check`, and `git diff --check` passed. + +These are local/disposable validation results, not production authorization. +Belgium is deliberately non-runtime and reconnaissance-only; Italy +853/2004/1069/2009 and UK FSA/FSS remain separate source scopes. The remaining +ethics and implementation gates include an independently operated durable +restriction ledger with enforced service-start/deployment integration, a +reviewed V1↔V2 suppression crosswalk, source terms/privacy and visitor/provider +audits, authorized human release authority, and remote CI verification. Keep +V2 publication paused until those decisions have evidence. + ## Phase 0 — Close and certify the backend foundation Goal: make the current backend/data-platform branch independently runnable, reviewable, and safe for frontend integration. diff --git a/docs/V2-SPRINT-2026-09-13.md b/docs/V2-SPRINT-2026-09-13.md index 0fa6c41..2d5e89d 100644 --- a/docs/V2-SPRINT-2026-09-13.md +++ b/docs/V2-SPRINT-2026-09-13.md @@ -4,6 +4,28 @@ This is a development-branch integration record, not publication approval or a claim that V2 is deployed. Real raw and derived records remain private and are not included here. +## 2026-09-15 final integration validation + +The dependency-ordered country/frontend integration is complete on the local +development branch. The standard gate passed 65 Rust library tests, 10 Rust +binary tests, 85 Python tests with 10 expected policy skips, and the additional +60 country/contract tests. Sequential disposable Docker API E2E passed public +10/10, community 6/6, seeded 19/19, public-surface safety 3/3, +candidate-import 4/4, Italy 4/4, and suppression lifecycle 4/4. The +backup/restore drill passed stale-ledger rejection before replay and current +restriction acceptance after replay. Root Jest passed 19/19; frontend unit, +check, lint, build, and the three-browser Playwright matrix passed 50/50, +clean, clean, clean, and 54/54 respectively. `cargo fmt -- --check` and +`git diff --check` also passed. + +This evidence covers disposable/local behavior only. Belgium remains +reconnaissance-only and non-runtime; Italy 853/2004 remains distinct from +1069/2009; and UK FSA remains distinct from FSS. No source is thereby +publication-approved. Production still requires an independently operated +restriction ledger and enforced startup/deployment gate, source terms/privacy +decisions, V1↔V2 suppression crosswalk, visitor/provider audit, and authorized +human release review. Remote CI remains pending. + ## Verified local evidence - The canonical disposable standard runner passed 63 Rust library tests, 9 diff --git a/docs/source-status.md b/docs/source-status.md index 342ba4f..9777660 100644 --- a/docs/source-status.md +++ b/docs/source-status.md @@ -15,13 +15,14 @@ No last-success timestamp is invented. Private artifacts are not proof of a publ | Source ID | Metadata | Acquisition | Runtime health | Publication | Evidence / next action | |---|---|---|---|---|---| +| `be.locations` | verified | blocked | not_run | blocked | FASFC operator/codebook reconnaissance; obtain an authorized bounded operator CSV, capture its schema and provenance, then implement category/privacy validation | | `fr.locations` | verified | blocked | not_run | blocked | DGAL/Alim’confiance/SIRENE/HVE/Agence Bio/Géorisques reconnaissance; acquire permitted official artifact and review terms/privacy | | `it.853-2004` | verified | artifact_private_only | not_run | blocked | Catalog acquisition, shared lifecycle, adapter, private candidate import, and guarded API checks remain review-gated; repeated activity identity, coordinate/address privacy, coverage, and project approval remain open | | `it.1069-2009` | verified | not_run | not_run | blocked | Separate by-products catalog candidate; no adapter or integration decision; assess scope, schema, terms, identity links, and privacy | | `mx.locations` | verified | blocked | not_run | blocked | DENUE/SENASICA/DGSIAP reconnaissance; resolve token, directory, terms, and schema | | `nz.locations` | verified | blocked | not_run | blocked | MPI/Stats NZ reconnaissance; resolve 403/access and aggregate-vs-facility boundaries | -| `uk.locations` | partial | artifact_private_only | not_run | blocked | Synthetic handoff passes importer pre-DB validation, but no real UK candidate has been imported or previewed; review-required/unapproved defaults, Docker E2E, privacy/coordinate, source-rights, duplicate, coverage, and release review remain open; NI/Scotland stay separate | -| `dk.smiley` | partial | artifact_private_only | not_run | blocked | Current official artifact and registered adapter are privately staged for validation; coverage/effective-date uncertainty and terms/privacy/release review remain open | +| `uk.locations` | partial | artifact_private_only | unknown | blocked | FSA and FSS private V2 lifecycle paths and synthetic handoff tests pass; no real UK candidate has been imported or previewed; privacy/coordinate, source-rights, duplicate, coverage, and release review remain open; NI/Scotland stay separate | +| `dk.smiley` | verified | verified | unknown | blocked | Shared private lifecycle and registered adapter are validated on synthetic/retained evidence; coverage/effective-date uncertainty and terms/privacy/release review remain open | | `de.locations` | partial | not_run | not_run | blocked | Legacy session URL and current BVL endpoint/schema/terms remain unresolved | | `ca.locations` | verified | not_run | not_run | blocked | Federal/export and Ontario candidates are documented separately; verify permitted artifact, terms, schema, coverage, and privacy before acquisition | | `es.locations` | partial | blocked | not_run | blocked | AESAN RGSEAA and MAPA sector routes are documented, but direct acquisition was refused; rights, export/schema, effective dates, sector coverage, privacy, and legacy/source boundaries remain unresolved | diff --git a/pipeline/tests/e2e/backup-restore.ps1 b/pipeline/tests/e2e/backup-restore.ps1 index a81b0f4..4fc26b8 100644 --- a/pipeline/tests/e2e/backup-restore.ps1 +++ b/pipeline/tests/e2e/backup-restore.ps1 @@ -95,8 +95,15 @@ try { if ($LASTEXITCODE -ne 0) { throw "Restore failed (exit $LASTEXITCODE)." } Assert-Snapshot 'old backup restored, before replay' '1,1,0,0,0,1,1' if (Test-SyntheticServiceGate) { throw 'Unsafe drill gate accepted an old backup before current restriction replay.' } + # The verifier's nonzero exit is the expected result for the stale snapshot. + # Capture it while temporarily allowing native stderr so PowerShell's Stop + # policy does not turn the expected rejection into a harness failure. + $gateErrorAction = $ErrorActionPreference + $ErrorActionPreference = 'Continue' & python $ledgerGate --ledger $ledgerFile --snapshot $oldSnapshot *> $null - if ($LASTEXITCODE -eq 0) { throw 'Pre-service ledger gate accepted an old restriction snapshot.' } + $oldGateExitCode = $LASTEXITCODE + $ErrorActionPreference = $gateErrorAction + if ($oldGateExitCode -eq 0) { throw 'Pre-service ledger gate accepted an old restriction snapshot.' } Write-Host '[backup-restore] PASS: synthetic pre-service gate rejects the old backup before replay.' Invoke-FixtureSql $suppressionFile diff --git a/pipeline/tests/test_source_registry.py b/pipeline/tests/test_source_registry.py index 4585b38..6856582 100644 --- a/pipeline/tests/test_source_registry.py +++ b/pipeline/tests/test_source_registry.py @@ -9,8 +9,8 @@ class SourceRegistryTests(unittest.TestCase): def test_repository_registry_loads_and_references_existing_legacy_paths(self): registry = load_registry() - self.assertEqual(len(registry["sources"]), 13) - self.assertEqual(len({source["source_id"] for source in registry["sources"]}), 13) + self.assertEqual(len(registry["sources"]), 14) + self.assertEqual(len({source["source_id"] for source in registry["sources"]}), 14) def test_unknowns_are_explicit(self): registry = json.loads(REGISTRY_PATH.read_text(encoding="utf-8")) From ba255e447669ff123f3a2c3d5dddb13d25ea28d8 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Tue, 15 Sep 2026 13:46:57 -0700 Subject: [PATCH 117/311] Add deterministic V1 V2 reconciliation reports --- docs/architecture/v1-v2-reconciliation.md | 50 +++++++ pipeline/reconciliation/__init__.py | 5 + pipeline/reconciliation/crosswalk.py | 125 ++++++++++++++++++ pipeline/reconciliation/test_crosswalk.py | 40 ++++++ .../scripts/diagnostics/crosswalk-report.py | 30 +++++ 5 files changed, 250 insertions(+) create mode 100644 docs/architecture/v1-v2-reconciliation.md create mode 100644 pipeline/reconciliation/__init__.py create mode 100644 pipeline/reconciliation/crosswalk.py create mode 100644 pipeline/reconciliation/test_crosswalk.py create mode 100644 pipeline/scripts/diagnostics/crosswalk-report.py diff --git a/docs/architecture/v1-v2-reconciliation.md b/docs/architecture/v1-v2-reconciliation.md new file mode 100644 index 0000000..b72d56e --- /dev/null +++ b/docs/architecture/v1-v2-reconciliation.md @@ -0,0 +1,50 @@ +# V1-to-V2 reconciliation + +`pipeline/reconciliation/crosswalk.py` produces a row-free comparison of a +legacy V1 snapshot and one private V2 adapter run. It is a review instrument, +not an identity resolver and not a release gate by itself. + +## Matching rules + +- A source-specific stable key must be supplied for both inputs. +- Matching is exact after trimming whitespace. Names, addresses, coordinates, + geocoder output, and fuzzy similarity are never used as identity evidence. +- A repeated key on either side is `ambiguous`; it is excluded from matches and + requires a separately recorded identity decision. +- A V1 key absent from V2 is `not_observed_in_v2`, never “closed”, deleted, or + suppressed. Source disappearance is only an observation boundary. +- Quarantined rows remain counted in the report but are not treated as accepted + V2 observations. + +The report contains aggregate counts and field-presence/category summaries only. +It must not be written to Git when generated from real local data. Raw rows, +addresses, coordinates, names, and geocoder responses stay in ignored private +staging. + +## Example + +```powershell +python pipeline/scripts/diagnostics/crosswalk-report.py ` + static_data/dk/locations.csv ` + data/staging/dk/run/normalized/records.jsonl ` + --v1-key establishment_id ` + --v2-key source_record_key ` + --country DK ` + --source-id dk.smiley ` + --output data/reports/dk-v1-v2-crosswalk.json +``` + +This command is intentionally explicit about the two source keys. It should be +run only after the V2 run has produced a private normalized artifact and its +manifest has been reviewed. It does not create facility links, releases, API +records, or publication approval. + +## Remaining country work + +Denmark, the UK, and Italy still need source-specific key mappings and private +current snapshots before a substantive comparison can be claimed. For Italy, +the 853/2004 and 1069/2009 populations remain separate. For the UK, England / +Wales, Scotland, and Northern Ireland remain separate until coverage and +identity semantics are evidenced. Legacy rows with unknown provenance or +unresolved keys remain explicitly unmatched rather than being repaired by +heuristic matching. diff --git a/pipeline/reconciliation/__init__.py b/pipeline/reconciliation/__init__.py new file mode 100644 index 0000000..dc177e6 --- /dev/null +++ b/pipeline/reconciliation/__init__.py @@ -0,0 +1,5 @@ +"""Deterministic, privacy-safe reconciliation helpers.""" + +from .crosswalk import CrosswalkError, compare_v1_v2 + +__all__ = ["CrosswalkError", "compare_v1_v2"] diff --git a/pipeline/reconciliation/crosswalk.py b/pipeline/reconciliation/crosswalk.py new file mode 100644 index 0000000..a3cf53e --- /dev/null +++ b/pipeline/reconciliation/crosswalk.py @@ -0,0 +1,125 @@ +"""Aggregate V1-to-V2 comparison without identity guesses. + +This module deliberately accepts only an explicit, source-qualified key. It +does not fuzzy-match names, addresses, geocoder results, or coordinates, and +it does not interpret a missing V2 observation as closure. +""" + +from __future__ import annotations + +import csv +import json +from collections import Counter +from pathlib import Path +from typing import Any, Iterable + + +class CrosswalkError(ValueError): + """Raised when a comparison input violates the crosswalk contract.""" + + +def _rows(path: Path, kind: str) -> list[dict[str, Any]]: + if not path.exists(): + raise CrosswalkError(f"missing {kind} input: {path}") + try: + if path.suffix.lower() == ".csv": + with path.open(newline="", encoding="utf-8-sig") as handle: + return list(csv.DictReader(handle)) + return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line.strip()] + except (OSError, UnicodeError, json.JSONDecodeError, csv.Error) as exc: + raise CrosswalkError(f"could not read {kind} input: {path}") from exc + + +def _value(row: dict[str, Any], path: str) -> Any: + value: Any = row + for part in path.split("."): + if not isinstance(value, dict): + return None + value = value.get(part) + return value + + +def _keys(rows: Iterable[dict[str, Any]], field: str, label: str) -> tuple[set[str], set[str], int]: + keys: list[str] = [] + missing = 0 + for row in rows: + value = _value(row, field) + if value is None or not str(value).strip(): + missing += 1 + continue + keys.append(str(value).strip()) + counts = Counter(keys) + ambiguous = {key for key, count in counts.items() if count > 1} + return set(keys), ambiguous, missing + + +def _coordinate_stats(rows: Iterable[dict[str, Any]], path: str) -> dict[str, int]: + states = Counter() + for row in rows: + value = _value(row, path) + if value is None or value == "": + states["missing"] += 1 + else: + states["present"] += 1 + return dict(sorted(states.items())) + + +def _field_counts(rows: Iterable[dict[str, Any]], path: str) -> dict[str, int]: + values = Counter() + for row in rows: + value = _value(row, path) + if value is not None and str(value).strip(): + values[str(value).strip()] += 1 + return dict(sorted(values.items())) + + +def compare_v1_v2( + v1_path: str | Path, + v2_path: str | Path, + *, + v1_key: str, + v2_key: str, + v1_country: str, + v2_source_id: str, + v1_coordinate_field: str | None = None, + v2_coordinate_field: str = "normalized.coordinates", + v2_classification_field: str = "normalized.classification", + v2_effective_date_field: str = "normalized.effective_date", +) -> dict[str, Any]: + """Return a row-free, deterministic crosswalk report. + + Matching is exact and key-based only. Duplicate keys are excluded from + matches and reported as ``ambiguous``. ``not_observed_in_v2`` is an + observation difference, never a closure or deletion decision. + """ + v1 = _rows(Path(v1_path), "V1") + v2 = _rows(Path(v2_path), "V2") + if not v1_country or not v2_source_id: + raise CrosswalkError("country and source_id are required") + v1_keys, v1_ambiguous, v1_missing = _keys(v1, v1_key, "V1") + v2_keys, v2_ambiguous, v2_missing = _keys(v2, v2_key, "V2") + ambiguous = v1_ambiguous | v2_ambiguous + matched = (v1_keys & v2_keys) - ambiguous + return { + "report_version": "v1-v2-crosswalk-1", + "country_code": v1_country, + "v2_source_id": v2_source_id, + "matching": {"strategy": "exact_key_only", "fuzzy_matching": False, "coordinate_matching": False}, + "counts": { + "v1_rows": len(v1), "v2_rows": len(v2), "matched": len(matched), + "ambiguous": len(ambiguous), "v1_missing_key": v1_missing, + "v2_missing_key": v2_missing, "v2_only": len(v2_keys - v1_keys - ambiguous), + "not_observed_in_v2": len(v1_keys - v2_keys - ambiguous), + }, + "interpretation": { + "not_observed_in_v2_is_closure": False, + "ambiguous_keys_are_matched": False, + "identity_decisions_created": False, + "raw_rows_in_report": False, + }, + "v1_coordinates": _coordinate_stats(v1, v1_coordinate_field) if v1_coordinate_field else {"not_compared": len(v1)}, + "v2_coordinates": _coordinate_stats(v2, v2_coordinate_field), + "v2_classifications": _field_counts(v2, v2_classification_field), + "v2_effective_dates": _field_counts(v2, v2_effective_date_field), + "quarantine": {"v2_rows": sum(1 for row in v2 if row.get("quarantine_reason") or row.get("reasons"))}, + } diff --git a/pipeline/reconciliation/test_crosswalk.py b/pipeline/reconciliation/test_crosswalk.py new file mode 100644 index 0000000..9a5bcc3 --- /dev/null +++ b/pipeline/reconciliation/test_crosswalk.py @@ -0,0 +1,40 @@ +import json +import tempfile +import unittest +from pathlib import Path + +from .crosswalk import compare_v1_v2 + + +class CrosswalkTests(unittest.TestCase): + def test_exact_matches_and_ambiguous_keys_are_row_free(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / "v1.csv").write_text("id,lat\nA,1\nB,\nB,2\nC,3\n", encoding="utf-8") + v2 = [ + {"source_id": "dk.smiley", "source_record_key": "A", "normalized": {"coordinates": [1, 2], "classification": "facility", "effective_date": "2026-01-01"}}, + {"source_id": "dk.smiley", "source_record_key": "B", "normalized": {"coordinates": None}}, + {"source_id": "dk.smiley", "source_record_key": "B", "normalized": {"coordinates": None}}, + {"source_id": "dk.smiley", "source_record_key": "D", "normalized": {"coordinates": None}, "quarantine_reason": "review"}, + ] + (root / "v2.jsonl").write_text("\n".join(json.dumps(row) for row in v2), encoding="utf-8") + report = compare_v1_v2(root / "v1.csv", root / "v2.jsonl", v1_key="id", v2_key="source_record_key", v1_country="DK", v2_source_id="dk.smiley", v1_coordinate_field="lat") + self.assertEqual(report["counts"], {"v1_rows": 4, "v2_rows": 4, "matched": 1, "ambiguous": 1, "v1_missing_key": 0, "v2_missing_key": 0, "v2_only": 1, "not_observed_in_v2": 1}) + self.assertFalse(report["interpretation"]["not_observed_in_v2_is_closure"]) + self.assertFalse(report["interpretation"]["raw_rows_in_report"]) + self.assertEqual(report["quarantine"]["v2_rows"], 1) + + def test_missing_keys_and_unknown_semantics_are_explicit(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / "v1.csv").write_text("id\n\n", encoding="utf-8") + (root / "v2.jsonl").write_text(json.dumps({"normalized": {}}) + "\n", encoding="utf-8") + report = compare_v1_v2(root / "v1.csv", root / "v2.jsonl", v1_key="id", v2_key="source_record_key", v1_country="IT", v2_source_id="it.853-2004") + self.assertEqual(report["counts"]["v1_missing_key"], 0) # csv blank lines are not data rows + self.assertEqual(report["counts"]["v2_missing_key"], 1) + self.assertEqual(report["v1_coordinates"], {"not_compared": 0}) + self.assertEqual(report["interpretation"]["identity_decisions_created"], False) + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/scripts/diagnostics/crosswalk-report.py b/pipeline/scripts/diagnostics/crosswalk-report.py new file mode 100644 index 0000000..f611630 --- /dev/null +++ b/pipeline/scripts/diagnostics/crosswalk-report.py @@ -0,0 +1,30 @@ +"""Write a row-free V1/V2 crosswalk report for a private review run.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +from pipeline.reconciliation.crosswalk import compare_v1_v2 + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("v1_path", type=Path) + parser.add_argument("v2_path", type=Path) + parser.add_argument("--v1-key", required=True) + parser.add_argument("--v2-key", required=True) + parser.add_argument("--country", required=True) + parser.add_argument("--source-id", required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + report = compare_v1_v2(args.v1_path, args.v2_path, v1_key=args.v1_key, v2_key=args.v2_key, + v1_country=args.country, v2_source_id=args.source_id) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(json.dumps(report, indent=2, sort_keys=True)) + + +if __name__ == "__main__": + main() From 3970f71c8012487373164f2014de217a6504c9c0 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Tue, 15 Sep 2026 13:50:26 -0700 Subject: [PATCH 118/311] Inventory legacy country reconciliation coverage --- docs/architecture/v1-v2-reconciliation.md | 16 +++++++++ pipeline/reconciliation/test_v1_inventory.py | 21 +++++++++++ pipeline/reconciliation/v1_inventory.py | 35 ++++++++++++++++++ .../scripts/diagnostics/crosswalk-report.py | 3 ++ .../diagnostics/inventory-v1-countries.py | 36 +++++++++++++++++++ 5 files changed, 111 insertions(+) create mode 100644 pipeline/reconciliation/test_v1_inventory.py create mode 100644 pipeline/reconciliation/v1_inventory.py create mode 100644 pipeline/scripts/diagnostics/inventory-v1-countries.py diff --git a/docs/architecture/v1-v2-reconciliation.md b/docs/architecture/v1-v2-reconciliation.md index b72d56e..0ba993c 100644 --- a/docs/architecture/v1-v2-reconciliation.md +++ b/docs/architecture/v1-v2-reconciliation.md @@ -48,3 +48,19 @@ Wales, Scotland, and Northern Ireland remain separate until coverage and identity semantics are evidenced. Legacy rows with unknown provenance or unresolved keys remain explicitly unmatched rather than being repaired by heuristic matching. + +## Legacy-country inventory + +Before a private V2 artifact exists, use the row-free inventory command: + +```powershell +python pipeline/scripts/diagnostics/inventory-v1-countries.py ` + --output data/reports/v1-country-inventory.json +``` + +It counts rows, missing and duplicate legacy keys, and coordinate-pair +presence for the V1 country directories represented by the checkout. It +explicitly reports `blocked_no_private_v2_artifact`; it does not pretend that +every legacy row is unmatched against a current source, and it does not infer +currentness, closure, or identity. Generated reports belong in ignored local +data when run against real snapshots. diff --git a/pipeline/reconciliation/test_v1_inventory.py b/pipeline/reconciliation/test_v1_inventory.py new file mode 100644 index 0000000..549327d --- /dev/null +++ b/pipeline/reconciliation/test_v1_inventory.py @@ -0,0 +1,21 @@ +import tempfile +import unittest +from pathlib import Path + +from .v1_inventory import inventory_v1 + + +class V1InventoryTests(unittest.TestCase): + def test_reports_aggregate_legacy_quality_without_claiming_currentness(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "locations.csv" + path.write_text("establishment_id,latitude,longitude\nA,1,2\nA,,\n,3,4\n", encoding="utf-8") + report = inventory_v1(path) + self.assertEqual(report["counts"], {"rows": 3, "keys_present": 2, "keys_missing": 1, "duplicate_keys": 1, "coordinate_pairs_present": 2}) + self.assertEqual(report["comparison_status"], "blocked_no_private_v2_artifact") + self.assertFalse(report["interpretation"]["v1_rows_are_current"]) + self.assertFalse(report["interpretation"]["missing_v2_means_closed"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/reconciliation/v1_inventory.py b/pipeline/reconciliation/v1_inventory.py new file mode 100644 index 0000000..47aa22f --- /dev/null +++ b/pipeline/reconciliation/v1_inventory.py @@ -0,0 +1,35 @@ +"""Row-free inventory of checked-in V1 country snapshots.""" + +from __future__ import annotations + +import csv +from collections import Counter +from pathlib import Path +from typing import Any + +from .crosswalk import CrosswalkError + +DEFAULT_COUNTRIES = ("ca", "de", "dk", "es", "fr", "it", "mx", "nz", "uk", "us") + + +def inventory_v1(path: str | Path, *, key_field: str = "establishment_id", coordinate_fields: tuple[str, str] = ("latitude", "longitude")) -> dict[str, Any]: + source = Path(path) + if not source.exists(): + raise CrosswalkError(f"missing V1 inventory input: {source}") + try: + with source.open(newline="", encoding="utf-8-sig") as handle: + rows = list(csv.DictReader(handle)) + except (OSError, UnicodeError, csv.Error) as exc: + raise CrosswalkError(f"could not read V1 inventory input: {source}") from exc + values = [str(row.get(key_field) or "").strip() for row in rows] + present = [value for value in values if value] + duplicate_keys = {key for key, count in Counter(present).items() if count > 1} + coords_present = sum(1 for row in rows if all(str(row.get(field) or "").strip() for field in coordinate_fields)) + return { + "report_version": "v1-inventory-1", + "source_path": source.as_posix(), + "comparison_status": "blocked_no_private_v2_artifact", + "counts": {"rows": len(rows), "keys_present": len(present), "keys_missing": len(rows) - len(present), "duplicate_keys": len(duplicate_keys), "coordinate_pairs_present": coords_present}, + "key": {"field": key_field, "strategy": "exact_key_only", "ambiguous_keys_are_not_resolved": True}, + "interpretation": {"v1_rows_are_current": False, "missing_v2_means_closed": False, "identity_decisions_created": False, "raw_rows_in_report": False}, + } diff --git a/pipeline/scripts/diagnostics/crosswalk-report.py b/pipeline/scripts/diagnostics/crosswalk-report.py index f611630..f873630 100644 --- a/pipeline/scripts/diagnostics/crosswalk-report.py +++ b/pipeline/scripts/diagnostics/crosswalk-report.py @@ -5,6 +5,9 @@ import argparse import json from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[3])) from pipeline.reconciliation.crosswalk import compare_v1_v2 diff --git a/pipeline/scripts/diagnostics/inventory-v1-countries.py b/pipeline/scripts/diagnostics/inventory-v1-countries.py new file mode 100644 index 0000000..f8bdc44 --- /dev/null +++ b/pipeline/scripts/diagnostics/inventory-v1-countries.py @@ -0,0 +1,36 @@ +"""Emit a row-free aggregate inventory for every V1 locations.csv snapshot.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[3])) + +from pipeline.reconciliation.v1_inventory import DEFAULT_COUNTRIES, inventory_v1 + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--static-data", type=Path, default=Path("static_data")) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + countries = [] + for code in DEFAULT_COUNTRIES: + path = args.static_data / code / "locations.csv" + item = {"country_code": code} + if path.exists(): + item.update(inventory_v1(path)) + else: + item.update({"comparison_status": "blocked_missing_v1_snapshot", "counts": None}) + countries.append(item) + report = {"report_version": "v1-country-inventory-1", "countries": countries, "raw_rows_in_report": False} + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(json.dumps(report, indent=2, sort_keys=True)) + + +if __name__ == "__main__": + main() From 8daf8aade870534ce66669c6ff627a96a67eae56 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Tue, 15 Sep 2026 14:05:21 -0700 Subject: [PATCH 119/311] Add private environment safety gates --- Cargo.lock | 1 + Cargo.toml | 1 + docs/api/v2-contract.json | 1 + docs/architecture/api-location-contract.md | 10 + docs/deployment/private-environment.md | 122 ++++++++ docs/governance/policy-implementation-todo.md | 15 +- docs/governance/visitor-privacy-inventory.md | 36 ++- pipeline/scripts/README.md | 14 + .../maintenance/private-environment-gate.py | 195 ++++++++++++ .../maintenance/replay-restriction-ledger.py | 172 +++++++++++ pipeline/tests/e2e/BACKUP-RESTORE.md | 4 +- pipeline/tests/e2e/backup-restore.ps1 | 28 +- pipeline/tests/e2e/test_readiness.py | 6 + .../tests/test_private_environment_gate.py | 114 +++++++ src/main.rs | 153 ++++++++-- src/private_environment.rs | 289 ++++++++++++++++++ 16 files changed, 1119 insertions(+), 42 deletions(-) create mode 100644 docs/deployment/private-environment.md create mode 100644 pipeline/scripts/maintenance/private-environment-gate.py create mode 100644 pipeline/scripts/maintenance/replay-restriction-ledger.py create mode 100644 pipeline/tests/test_private_environment_gate.py create mode 100644 src/private_environment.rs diff --git a/Cargo.lock b/Cargo.lock index 193a163..0576923 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1992,6 +1992,7 @@ dependencies = [ "deadpool-postgres", "futures-util", "include_dir", + "ipnet", "once_cell", "reqwest", "rustls", diff --git a/Cargo.toml b/Cargo.toml index 3142b94..774f9ab 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,6 +23,7 @@ tower = { version = "=0.5.2", features = ["util"] } reqwest = { version = "=0.12.24", features = ["json", "rustls-tls"], default-features = false } serde_json = "=1.0.140" sha2 = "=0.10.9" +ipnet = "=2.12.0" # Shuttle dependencies serde-xml-rs = "0.8.1" diff --git a/docs/api/v2-contract.json b/docs/api/v2-contract.json index c7212fe..215d4a3 100644 --- a/docs/api/v2-contract.json +++ b/docs/api/v2-contract.json @@ -4,6 +4,7 @@ "endpoints": { "GET /health/live": {"success": {"status": "ok", "service": "uec-api"}}, "GET /health/ready": {"success": {"status": "ready", "database": "ok"}, "unavailable_status": 503}, + "GET /health/diagnostics": {"success": {"status": "ok", "privacy": {"diagnostic_identifiers": "excluded"}}, "payloads": "coarse status only"}, "GET /api/v2/locations": {"success": {"api_version": "v2", "data": [], "meta": {"profile": "official", "next_cursor": null, "coverage_scope": "selected_promoted_release_public_facilities", "count_semantics": "public facility projection rows, not animal counts"}}, "no_release": 200}, "GET /api/v2/locations/{facility_id}": {"success": {"api_version": "v2", "data": {}, "meta": {"coverage_scope": "selected_promoted_release_public_facilities", "count_semantics": "public facility projection, not an animal count"}}, "not_found": 404}, "GET /api/v2/locations.csv": {"success_content_type": "text/csv", "bounded_rows": 1000, "profile_required_for_nonofficial": true, "not_found_without_promoted_manifest": 404}, diff --git a/docs/architecture/api-location-contract.md b/docs/architecture/api-location-contract.md index d3396f9..735028b 100644 --- a/docs/architecture/api-location-contract.md +++ b/docs/architecture/api-location-contract.md @@ -9,6 +9,16 @@ with `Retry-After: 60`. The V2 public API must read from curated database projections, never raw evidence tables. The default query returns only records in a promoted release that are eligible for public access. It is exposed under `/api/v2/locations`; the legacy `/api/locations` endpoint remains separate during migration. +Production startup is fail-closed until the separately stored restriction +ledger has been replayed into the restored database and matches its +payload-free snapshot. It also verifies the trusted release manifest digest. +See [private environment and recovery](../deployment/private-environment.md). + +`GET /health/diagnostics` is intentionally coarse: it reports mode, boolean +configuration state, control names, and privacy-safe status labels. It does not +return request URLs, query values, paths, forwarded addresses, source rows, or +restriction references. + Each location response includes the stable location ID, name, `category`, source origin, `publication_profile`, independent `factual_review_status`, `privacy_screening_status`, `project_approval`, optional `reviewer_role`, optional `publication_warning`, display precision, and provenance. These fields are not inferred from source origin. List responses use `{data: [...], api_version: "v2", meta: {...}}`; detail responses use `{data: {...}, api_version: "v2", meta: {...}}`. Provenance includes `first_observed_at`, `last_observed_at`, and `observation_count`; these describe the project's retained observations, not guaranteed opening or operating dates. Lifecycle is independent from observation history. Valid states are `active_observed`, `explicitly_closed`, `not_seen_recently`, and `status_unknown`. A record disappearing from a later source snapshot must not be labeled closed. `explicitly_closed` requires traceable closure evidence and a recorded lifecycle event. diff --git a/docs/deployment/private-environment.md b/docs/deployment/private-environment.md new file mode 100644 index 0000000..6c85151 --- /dev/null +++ b/docs/deployment/private-environment.md @@ -0,0 +1,122 @@ +# Private environment and recovery gate + +Status: implemented as a production-shaped, private-only control. It has not +been deployed, and it is not publication approval. The only publication +operator remains the authorized project maintainer; this repository does not +appoint a backup reviewer, assert legal coverage, or claim staffing capacity. + +## Control-plane boundary + +The current restriction ledger is an independently stored, durable JSON +control-plane artifact. It is not inside a PostgreSQL backup and must be kept +on a separately access-controlled/retained volume or object-store key. The +ledger contains only opaque `source_id`, `source_record_key`, `scope`, and +`action: suppress` references. It must not contain names, addresses, +coordinates, requester details, source text, or a copy of restricted evidence. + +A restore snapshot is produced from the restored database after replay. The +service-start gate requires the ledger and snapshot to be different files and +requires matching revision, digest, and reference sets. An old backup therefore +cannot serve until the current ledger has been replayed successfully. Missing, +stale, malformed, duplicated, or ambiguous references fail closed. + +Production-shaped startup also requires a trusted release manifest and its +trusted lowercase SHA-256 reference. The manifest is versioned and includes a +declared distributed-artifact inventory. Checksums detect alteration relative +to a trusted reference; they do not prove factual accuracy, privacy +eligibility, or source correctness. Release signing is not claimed. + +## Required production configuration + +The Rust service validates these before binding its listener: + +```text +UEC_RUNTIME_MODE=production +UEC_DATABASE_URL= +UEC_CORS_ORIGINS=https:// +UEC_TRUST_PROXY=true|false +UEC_TRUSTED_PROXY_CIDRS= +UEC_RESTRICTION_LEDGER_PATH=/current-ledger.json +UEC_RESTORED_RESTRICTION_SNAPSHOT_PATH=/restriction-snapshot.json +UEC_RELEASE_MANIFEST_PATH=/manifest.json +UEC_RELEASE_MANIFEST_SHA256=<64 lowercase hex characters> +``` + +`UEC_TRUST_PROXY` is explicit in production. If enabled, forwarded addresses +are accepted only from peers inside `UEC_TRUSTED_PROXY_CIDRS`; a forwarded +header from any other peer is ignored and the socket peer remains the rate-limit +key. Wildcard CORS origins, malformed origins, missing proxy boundaries, and +unexpected boolean values are rejected. + +The diagnostics endpoint (`/health/diagnostics`) reports only status, counts, +mode, and control names. It does not report URLs, paths, request data, +forwarded addresses, source rows, or restriction references. Readiness remains +separate from liveness and continues to fail when the required schema is not +migrated. + +## Recovery rehearsal + +Use synthetic fixtures only. The portless drill in +[`pipeline/tests/e2e/backup-restore.ps1`](../../pipeline/tests/e2e/backup-restore.ps1) +performs this order: + +1. Apply every migration to an isolated PostGIS container and seed a promoted + synthetic release. +2. Take a custom-format backup while the record is eligible. +3. Apply a later synthetic suppression and verify public projections exclude + the record. +4. Roll back to the older backup. The stale snapshot is rejected before a + service can start. +5. Validate the independent current ledger, replay it with + [`replay-restriction-ledger.py`](../../pipeline/scripts/maintenance/replay-restriction-ledger.py), + write a row-free post-replay snapshot, and verify both map projections + remain suppressed. The private recovery command is: + + ```powershell + python pipeline/scripts/maintenance/replay-restriction-ledger.py ` + --database-url ` + --ledger /current-ledger.json ` + --snapshot-output /restriction-snapshot.json + ``` +6. Run the startup/deployment gate only after replay. Any failed migration, + incomplete replay, manifest mismatch, or unsafe proxy configuration leaves + the service stopped. + +The replay is append-only and idempotent. It is not a substitute for the +ordinary suppression case workflow or for a maintainer's decision to lift a +restriction. A lift must remain an explicit, separately recorded decision. + +## Clean-checkout/deployment check + +From a clean checkout, the operator/deployment job should run the gate with +the independently mounted ledger, post-restore snapshot, and trusted manifest: + +```powershell +$env:UEC_RUNTIME_MODE = "production" +$env:UEC_DATABASE_URL = "postgresql://redacted.invalid/uec" +$env:UEC_CORS_ORIGINS = "https://verified.example" +$env:UEC_TRUST_PROXY = "false" +python pipeline/scripts/maintenance/private-environment-gate.py ` + --repository . ` + --ledger /current-ledger.json ` + --snapshot /restriction-snapshot.json ` + --manifest /manifest.json ` + --manifest-sha256 ` + --clean-checkout +``` + +The example URL and origin are placeholders, not deployment values. The gate +does not fetch data, promote a release, or publish anything. It validates the +migration inventory, runtime configuration, replay state, and manifest, and +prints only safe metadata. + +## Verification record + +- Hosted CI status at checkpoint `a4f9930`: **user-reported green**. This is a + record of the supplied report, not an independently inspected job result or + a claim about this follow-up commit. +- This lane's focused evidence: Rust binary tests, Python private-gate tests, + and the synthetic portless backup/restore/replay drill when Docker is + available. +- No deployment, publication, real-data acquisition, or real requester data + handling is part of this change. diff --git a/docs/governance/policy-implementation-todo.md b/docs/governance/policy-implementation-todo.md index c363517..f130823 100644 --- a/docs/governance/policy-implementation-todo.md +++ b/docs/governance/policy-implementation-todo.md @@ -104,10 +104,13 @@ export, aggregate-count, historical/cache, or production operational controls. ## Current implementation boundary -The V2 checkpoint now includes a synthetic, tested urgent suppression path with +The V2 checkpoint includes a synthetic, tested urgent suppression path with durable source-key references, append-only lift decisions, geocoding guards, -release gates, and backup replay verification. This is evidence for the -covered disposable V2 surfaces only. Production still needs an independently -operated restriction ledger and enforced service-start/deployment gate, plus -the remaining checklist items below. Do not treat unchecked work as complete -or launch affected public capabilities based on policy text alone. +release gates, and backup replay verification. This lane adds a +production-shaped independently mounted ledger contract, fail-closed Rust +startup validation, a replay command, migration/config checks, and a recovery +runbook. That is implementation evidence for private staging only: no +production instance has been configured or deployed, and the remaining +deployment audit, authorized review, provider/log verification, and policy +checklist items below remain open. Do not launch affected public capabilities +based on this code or policy text alone. diff --git a/docs/governance/visitor-privacy-inventory.md b/docs/governance/visitor-privacy-inventory.md index ff4f160..998eb31 100644 --- a/docs/governance/visitor-privacy-inventory.md +++ b/docs/governance/visitor-privacy-inventory.md @@ -1,12 +1,34 @@ # Visitor privacy and retention inventory -This inventory is a deployment gate. It must be updated from browser/network and infrastructure observations before publishing a privacy statement. +This inventory is a deployment gate. The repository observations below were +made from the checked-in application on 2026-09-15; they do not establish what +the deployed host, CDN, reverse proxy, error reporter, or providers retain. +Those settings require a deployment-specific browser/network and +infrastructure audit before a privacy statement or affected v2 capability is +published. -| Surface | Data sent | Identifier/log fields | Retention/access owner | Status | +| Surface/provider evidence | Data sent | Identifier/log fields | Retention/access owner | Status | |---|---|---|---|---| -| V2 API | Query filters and pagination only | Web-server and provider logs must be confirmed | Maintainer to assign | Verify in deployment | -| Map tiles/assets | Browser requests and approximate map viewport | Provider policy/configuration must be confirmed | Maintainer to assign | Verify in deployment | -| Geocoding pipeline | Restricted operational queries, never visitor location | Restricted pipeline logs | Pipeline maintainer | Development-only provider gate | -| Error/reporting services | Must be confirmed from deployment configuration | Must exclude precise visitor location and source payloads | Maintainer to assign | Not claimed until audited | +| V2 API (`src/main.rs`, frontend V2 clients) | Profile, filters, pagination, and cursor in the request URL | Application/proxy/provider request logs are not configured in this repository | Maintainer to assign | Code shape observed; deployment logging unknown | +| Legacy API origin (`static/modules/constants.js`) | Browser requests to the configured Railway API, including legacy endpoints | Railway/web-server logs and retention are unknown | Hosting operator/provider to confirm | Legacy surface remains an audit item | +| Legacy Leaflet CDN (`static/index.html`) | Browser IP/referrer/user-agent and asset request metadata; Leaflet JS/CSS | unpkg retention/control is unknown | Hosting operator/provider to confirm | External dependency present in legacy page | +| Legacy marker assets (`static/modules/constants.js`) | Browser asset requests to raw GitHub and cdnjs | Provider request metadata/retention unknown | Hosting operator/provider to confirm | External dependency present in legacy page | +| Legacy map tiles (`static/modules/MapManager.js`) | Tile requests containing the map tile coordinates/viewport; no device geolocation code observed | OpenStreetMap and ArcGIS provider logging/retention unknown | Hosting operator/provider to confirm | Must be audited or removed before a visitor-data claim | +| External directions (`static/modules/popupBuilder.js`) | A user click can send a displayed latitude/longitude to Google Maps | Google retention/control outside project control | Visitor chooses provider; project must disclose | Legacy click-through disclosure required | +| Geocoding pipeline (`pipeline/geocoding`, `pipeline/config`) | Restricted operational address queries only; not visitor location | Restricted pipeline logs; provider retention and deletion limits unresolved | Pipeline maintainer | Development-only provider gate | +| Analytics/error reporting | No analytics or error-reporting integration found by repository search | Deployment/CDN/provider behavior still unknown | Maintainer to audit | No “no tracking” or “no logging” claim follows | +| Application diagnostics (`/health/diagnostics`) | No request payload, address, path, or forwarded address in response | Returns coarse control status only | Service operator | Implemented and covered by Rust contract; host logs still unknown | -Visitor location entry is optional. Device geolocation must not be required. Precise visitor locations and home addresses must not enter shared links, analytics, or public diagnostics. Unknown provider retention or logging behavior is an unresolved deployment limitation, not evidence of no logging. +Repository search found no `navigator.geolocation` use. Visitor location entry is +optional and device geolocation must not be required. Precise visitor +locations and home addresses must not enter shared links, analytics, or public +diagnostics. Unknown provider retention, CDN behavior, proxy configuration, +and log deletion are unresolved deployment limitations, not evidence of no +logging. The legacy page's external map/directions requests are separate from +the Svelte V2 preview's blank local map background and must not be conflated. + +Before launch, capture a browser network trace for each supported route and +the reverse-proxy/hosting/error-reporting configuration. Record actual request +fields, access owner, retention/deletion behavior, and any provider control +limits here. A missing analytics script is not proof that infrastructure logs +are absent. diff --git a/pipeline/scripts/README.md b/pipeline/scripts/README.md index 1a948e8..b7ea017 100644 --- a/pipeline/scripts/README.md +++ b/pipeline/scripts/README.md @@ -31,3 +31,17 @@ python pipeline/scripts/stages/geocode-worker.py --provider dawa --limit 5 --del ``` The worker writes append-only job events and geocode attempts. It does not modify source records or observations. New providers should implement the adapter contract in `pipeline/geocoding/` and reuse the worker’s lifecycle, retry, logging, and persistence behavior. +# Private environment controls + +`maintenance/private-environment-gate.py` is the clean-checkout and +deployment-shaped gate. In production it requires explicit runtime/CORS/proxy +configuration, a separate durable restriction ledger and post-restore +snapshot, a complete migration inventory, and a trusted release-manifest +digest. It is validation only: it does not acquire, promote, or publish data. + +`maintenance/replay-restriction-ledger.py` applies current payload-free +suppression references to a restored database in one transaction. It rejects +missing or ambiguous source keys, supports only whole-record suppression, +writes a row-free post-replay snapshot, and can emit idempotent SQL for a +portless recovery container. Run the service startup gate after replay; never +start public service on a stale restore. diff --git a/pipeline/scripts/maintenance/private-environment-gate.py b/pipeline/scripts/maintenance/private-environment-gate.py new file mode 100644 index 0000000..834a70e --- /dev/null +++ b/pipeline/scripts/maintenance/private-environment-gate.py @@ -0,0 +1,195 @@ +"""Fail-closed validation for the private production-shaped environment. + +This is an operator/deployment check, not publication approval. It validates +configuration, migration inventory, an independently stored restriction ledger, +the post-restore replay snapshot, and a trusted release manifest. Diagnostics +contain only counts, versions, and status; never print database URLs or rows. +""" +from __future__ import annotations + +import argparse +import hashlib +import ipaddress +import json +import os +import re +import subprocess +import sys +from pathlib import Path +from typing import Any + + +class PrivateEnvironmentError(ValueError): + """The private environment is unavailable or unsafe to start.""" + + +def canonical_json(value: Any) -> str: + return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + + +def ledger_digest(ledger: dict[str, Any]) -> str: + return hashlib.sha256(canonical_json({ + "schema_version": ledger.get("schema_version"), + "revision": ledger.get("revision"), + "active_restrictions": ledger.get("active_restrictions"), + }).encode("utf-8")).hexdigest() + + +def read_json(path: Path, label: str) -> dict[str, Any]: + if not path.is_file(): + raise PrivateEnvironmentError(f"{label} is unavailable") + try: + value = json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeError, json.JSONDecodeError) as exc: + raise PrivateEnvironmentError(f"{label} is invalid") from exc + if not isinstance(value, dict): + raise PrivateEnvironmentError(f"{label} must be an object") + return value + + +def verify_ledger_replay(ledger_path: Path, snapshot_path: Path) -> dict[str, Any]: + if ledger_path.resolve() == snapshot_path.resolve(): + raise PrivateEnvironmentError("restriction ledger and restored snapshot must be separate files") + ledger = read_json(ledger_path, "restriction ledger") + if ledger.get("schema_version") != 1 or not isinstance(ledger.get("revision"), str) or not ledger["revision"]: + raise PrivateEnvironmentError("restriction ledger schema or revision is unsupported") + if ledger.get("ledger_sha256") != ledger_digest(ledger): + raise PrivateEnvironmentError("restriction ledger digest is invalid") + restrictions = ledger.get("active_restrictions") + if not isinstance(restrictions, list) or any( + not isinstance(row, dict) + or any(not isinstance(row.get(key), str) or not row[key] for key in ("source_id", "source_record_key", "scope")) + or row.get("action") != "suppress" + for row in restrictions + ): + raise PrivateEnvironmentError("restriction ledger references are invalid") + snapshot = read_json(snapshot_path, "restored restriction snapshot") + if snapshot.get("ledger_revision") != ledger["revision"] or snapshot.get("ledger_sha256") != ledger["ledger_sha256"]: + raise PrivateEnvironmentError("restored restriction state is stale") + expected = {canonical_json(row) for row in restrictions} + applied = snapshot.get("active_restrictions") + if not isinstance(applied, list) or {canonical_json(row) for row in applied} != expected or len(applied) != len(expected): + raise PrivateEnvironmentError("restored restrictions do not match current ledger") + if any(not isinstance(row, dict) or row.get("action") != "suppress" for row in applied): + raise PrivateEnvironmentError("restored restriction action is unsupported") + return {"revision": ledger["revision"], "active_restriction_count": len(restrictions)} + + +def validate_proxy_config(mode: str, trust_proxy: str | None, trusted_cidrs: str | None) -> dict[str, Any]: + if trust_proxy is None and mode == "development": + trust = False + elif trust_proxy in {"true", "false"}: + trust = trust_proxy == "true" + elif trust_proxy is None: + raise PrivateEnvironmentError("UEC_TRUST_PROXY must be explicitly set in production") + else: + raise PrivateEnvironmentError("UEC_TRUST_PROXY must be true or false") + networks = [] + for raw in (trusted_cidrs or "").split(","): + raw = raw.strip() + if raw: + try: + networks.append(ipaddress.ip_network(raw, strict=False)) + except ValueError as exc: + raise PrivateEnvironmentError("UEC_TRUSTED_PROXY_CIDRS contains an invalid network") from exc + if trust and not networks: + raise PrivateEnvironmentError("UEC_TRUSTED_PROXY_CIDRS is required when proxy trust is enabled") + if not trust and networks: + raise PrivateEnvironmentError("UEC_TRUSTED_PROXY_CIDRS requires UEC_TRUST_PROXY=true") + return {"trust_forwarded_for": trust, "trusted_proxy_network_count": len(networks)} + + +def validate_runtime_config(values: dict[str, str | None], production: bool = True) -> dict[str, Any]: + mode = values.get("UEC_RUNTIME_MODE") + if production and mode != "production": + raise PrivateEnvironmentError("UEC_RUNTIME_MODE must be production for this gate") + if mode not in {"development", "production"}: + raise PrivateEnvironmentError("UEC_RUNTIME_MODE must be development or production") + if mode == "production" and not (values.get("UEC_DATABASE_URL") or "").strip(): + raise PrivateEnvironmentError("UEC_DATABASE_URL is required in production") + origins = [origin.strip() for origin in (values.get("UEC_CORS_ORIGINS") or "").split(",") if origin.strip()] + if mode == "production" and not origins: + raise PrivateEnvironmentError("UEC_CORS_ORIGINS is required in production") + for origin in origins: + if "*" in origin or not re.match(r"^https?://[^/?#]+$", origin): + raise PrivateEnvironmentError("UEC_CORS_ORIGINS must contain bare http(s) origins") + proxy = validate_proxy_config(mode, values.get("UEC_TRUST_PROXY"), values.get("UEC_TRUSTED_PROXY_CIDRS")) + return {"runtime_mode": mode, "cors_origin_count": len(origins), **proxy} + + +def validate_migrations(directory: Path) -> dict[str, Any]: + paths = sorted(directory.glob("*.sql")) + if not paths: + raise PrivateEnvironmentError("no SQL migrations found") + stems = [path.stem for path in paths] + if len(stems) != len(set(stems)): + raise PrivateEnvironmentError("migration identifiers are duplicated") + if any(not path.read_text(encoding="utf-8").strip() for path in paths): + raise PrivateEnvironmentError("migration is empty") + inventory = [{"name": path.name, "sha256": hashlib.sha256(path.read_bytes()).hexdigest()} for path in paths] + return {"migration_count": len(paths), "migration_inventory_sha256": hashlib.sha256(canonical_json(inventory).encode()).hexdigest()} + + +def validate_release_manifest(path: Path, expected_digest: str) -> dict[str, Any]: + manifest = read_json(path, "release manifest") + actual = hashlib.sha256(canonical_json(manifest).encode("utf-8")).hexdigest() + if expected_digest != actual or len(expected_digest) != 64 or expected_digest.lower() != expected_digest: + raise PrivateEnvironmentError("release manifest digest does not match trusted reference") + for field in ("manifest_version", "release_id", "profile", "ruleset_version"): + if not isinstance(manifest.get(field), str) or not manifest[field]: + raise PrivateEnvironmentError(f"release manifest field is missing: {field}") + artifacts = manifest.get("distributed_artifacts", []) + if not isinstance(artifacts, list): + raise PrivateEnvironmentError("release manifest artifact inventory is invalid") + names: list[str] = [] + for item in artifacts: + if not isinstance(item, dict) or not isinstance(item.get("name"), str): + raise PrivateEnvironmentError("release manifest artifact inventory is invalid") + if not re.match(r"^[^/\\]+$", item["name"]): + raise PrivateEnvironmentError("release manifest artifact name is unsafe") + if not re.match(r"^[0-9a-f]{64}$", str(item.get("sha256", ""))) or not isinstance(item.get("byte_size"), int) or item["byte_size"] < 0: + raise PrivateEnvironmentError("release manifest artifact checksum is invalid") + names.append(item["name"]) + if len(names) != len(set(names)): + raise PrivateEnvironmentError("release manifest artifact inventory is invalid") + return {"manifest_version": manifest["manifest_version"], "release_id": manifest["release_id"], "profile": manifest["profile"], "artifact_count": len(artifacts), "manifest_sha256": actual} + + +def assert_clean_checkout(repository: Path) -> None: + result = subprocess.run(["git", "status", "--porcelain", "--untracked-files=all"], cwd=repository, capture_output=True, text=True, check=False) + if result.returncode != 0: + raise PrivateEnvironmentError("clean-checkout status could not be read") + if result.stdout.strip(): + raise PrivateEnvironmentError("checkout contains uncommitted or untracked files") + + +def run_gate(repository: Path, values: dict[str, str | None], ledger: Path, snapshot: Path, manifest: Path, manifest_digest: str, clean_checkout: bool = False) -> dict[str, Any]: + if clean_checkout: + assert_clean_checkout(repository) + config = validate_runtime_config(values) + migrations = validate_migrations(repository / "pipeline" / "migrations") + replay = verify_ledger_replay(ledger, snapshot) + release = validate_release_manifest(manifest, manifest_digest) + return {"status": "pass", "checks": {**config, **migrations, "restriction_ledger": replay, "release_manifest": release}} + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--repository", type=Path, default=Path.cwd()) + parser.add_argument("--ledger", type=Path, required=True) + parser.add_argument("--snapshot", type=Path, required=True) + parser.add_argument("--manifest", type=Path, required=True) + parser.add_argument("--manifest-sha256", required=True) + parser.add_argument("--clean-checkout", action="store_true") + args = parser.parse_args() + values = {key: os.environ.get(key) for key in ("UEC_RUNTIME_MODE", "UEC_DATABASE_URL", "UEC_CORS_ORIGINS", "UEC_TRUST_PROXY", "UEC_TRUSTED_PROXY_CIDRS")} + try: + print(json.dumps(run_gate(args.repository, values, args.ledger, args.snapshot, args.manifest, args.manifest_sha256, args.clean_checkout), sort_keys=True)) + return 0 + except PrivateEnvironmentError as exc: + print(json.dumps({"status": "blocked", "reason": str(exc)}), file=sys.stderr) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/pipeline/scripts/maintenance/replay-restriction-ledger.py b/pipeline/scripts/maintenance/replay-restriction-ledger.py new file mode 100644 index 0000000..1de38ed --- /dev/null +++ b/pipeline/scripts/maintenance/replay-restriction-ledger.py @@ -0,0 +1,172 @@ +"""Replay an external, payload-free restriction ledger into a restored DB. + +Only opaque source_id/source_record_key references are read from the ledger. +The transaction fails if a reference is absent or ambiguous, so a partial +replay can never be mistaken for a safe restore. This command is for private +staging/recovery; the service-start gate must run after it. +""" +from __future__ import annotations + +import argparse +import importlib.util +import json +import os +import sys +import tempfile +from pathlib import Path + +import psycopg + + +def _ledger_module(): + path = Path(__file__).with_name("restriction-ledger-gate.py") + spec = importlib.util.spec_from_file_location("restriction_ledger_gate", path) + if spec is None or spec.loader is None: + raise RuntimeError("restriction ledger verifier is unavailable") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def replay(database_url: str, ledger_path: Path, actor: str = "external-ledger-replay") -> dict[str, int | str]: + verifier = _ledger_module() + ledger = verifier.load_ledger(ledger_path) + restrictions = ledger["active_restrictions"] + if any(item.get("scope") != "whole_record" for item in restrictions): + raise ValueError("ledger replay supports only whole_record suppression references") + applied = 0 + with psycopg.connect(database_url) as connection: + with connection.transaction(): + for item in restrictions: + rows = connection.execute( + "SELECT source_record_id FROM uec.source_records WHERE source_id = %s AND source_record_key = %s", + (item["source_id"], item["source_record_key"]), + ).fetchall() + if len(rows) != 1: + raise ValueError("restriction reference is absent or ambiguous in restored database") + record_id = rows[0][0] + inserted = connection.execute( + """ + INSERT INTO uec.record_access_events + (source_record_id, action, reason_category, policy_version, maintainer, note) + SELECT %s, 'public_access_revoked', 'privacy', 'ethics-v1', %s, 'External restriction ledger replay' + WHERE NOT EXISTS ( + SELECT 1 FROM uec.record_access_current + WHERE source_record_id = %s AND action = 'public_access_revoked' + ) + RETURNING access_event_id + """, + (record_id, actor, record_id), + ).fetchone() + applied += int(inserted is not None) + return {"status": "pass", "ledger_revision": ledger["revision"], "reference_count": len(restrictions), "new_events": applied} + + +def _sql_literal(value: str) -> str: + return "'" + value.replace("'", "''") + "'" + + +def replay_sql(ledger_path: Path, actor: str = "external-ledger-replay") -> str: + """Emit safe, idempotent SQL for a portless recovery container.""" + verifier = _ledger_module() + ledger = verifier.load_ledger(ledger_path) + restrictions = ledger["active_restrictions"] + if any(item.get("scope") != "whole_record" for item in restrictions): + raise ValueError("ledger replay supports only whole_record suppression references") + statements = ["BEGIN;"] + for item in restrictions: + source_id = _sql_literal(item["source_id"]) + source_key = _sql_literal(item["source_record_key"]) + actor_sql = _sql_literal(actor) + statements.append(f""" +DO $$ +DECLARE matched_count integer; +BEGIN + SELECT count(*) INTO matched_count + FROM uec.source_records + WHERE source_id = {source_id} AND source_record_key = {source_key}; + IF matched_count <> 1 THEN + RAISE EXCEPTION 'restriction reference is absent or ambiguous in restored database'; + END IF; + INSERT INTO uec.record_access_events + (source_record_id, action, reason_category, policy_version, maintainer, note) + SELECT source_record_id, 'public_access_revoked', 'privacy', 'ethics-v1', {actor_sql}, 'External restriction ledger replay' + FROM uec.source_records + WHERE source_id = {source_id} AND source_record_key = {source_key} + AND NOT EXISTS ( + SELECT 1 FROM uec.record_access_current current_access + WHERE current_access.source_record_id = uec.source_records.source_record_id + AND current_access.action = 'public_access_revoked' + ); +END $$;""".strip()) + statements.append("COMMIT;") + return "\n".join(statements) + "\n" + + +def write_replayed_snapshot(database_url: str, ledger_path: Path, output: Path) -> dict[str, int | str]: + """Write a row-free snapshot after confirming each reference is suppressed.""" + verifier = _ledger_module() + ledger = verifier.load_ledger(ledger_path) + restrictions = ledger["active_restrictions"] + if any(item.get("scope") != "whole_record" for item in restrictions): + raise ValueError("snapshot export supports only whole_record suppression references") + with psycopg.connect(database_url) as connection: + with connection.transaction(): + for item in restrictions: + count = connection.execute( + """ + SELECT count(*) + FROM uec.source_records source_record + WHERE source_record.source_id = %s + AND source_record.source_record_key = %s + AND EXISTS ( + SELECT 1 FROM uec.public_access_restricted restricted + WHERE restricted.source_record_id = source_record.source_record_id + ) + """, + (item["source_id"], item["source_record_key"]), + ).fetchone()[0] + if count != 1: + raise ValueError("restriction reference is not currently suppressed in restored database") + snapshot = { + "ledger_revision": ledger["revision"], + "ledger_sha256": ledger["ledger_sha256"], + "active_restrictions": restrictions, + } + if not output.parent.is_dir(): + raise ValueError("snapshot output directory is unavailable") + with tempfile.NamedTemporaryFile("w", encoding="utf-8", dir=output.parent, prefix=f".{output.name}.", delete=False) as handle: + handle.write(json.dumps(snapshot, sort_keys=True, separators=(",", ":")) + "\n") + temporary = Path(handle.name) + os.replace(temporary, output) + return {"status": "pass", "ledger_revision": ledger["revision"], "reference_count": len(restrictions)} + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--database-url", default=os.environ.get("UEC_DATABASE_URL")) + parser.add_argument("--ledger", type=Path, required=True) + parser.add_argument("--actor", default="external-ledger-replay") + parser.add_argument("--emit-sql", action="store_true", help="emit idempotent SQL for a portless recovery container") + parser.add_argument("--snapshot-output", type=Path, help="write a row-free post-replay snapshot after database verification") + args = parser.parse_args() + if not args.database_url and not args.emit_sql: + parser.error("--database-url or UEC_DATABASE_URL is required") + try: + if args.emit_sql and args.snapshot_output: + parser.error("--emit-sql and --snapshot-output cannot be combined") + if args.emit_sql: + print(replay_sql(args.ledger, args.actor), end="") + else: + result = replay(args.database_url, args.ledger, args.actor) + if args.snapshot_output: + result = write_replayed_snapshot(args.database_url, args.ledger, args.snapshot_output) + print(json.dumps(result, sort_keys=True)) + return 0 + except Exception as exc: + print(json.dumps({"status": "blocked", "reason": str(exc)}), file=sys.stderr) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/pipeline/tests/e2e/BACKUP-RESTORE.md b/pipeline/tests/e2e/BACKUP-RESTORE.md index 9333d13..3e26c3b 100644 --- a/pipeline/tests/e2e/BACKUP-RESTORE.md +++ b/pipeline/tests/e2e/BACKUP-RESTORE.md @@ -1,6 +1,6 @@ # Synthetic backup/restore verification -This check uses only disposable PostGIS containers and synthetic records. It applies every migration, seeds a promoted release plus an active suppression case, creates a custom-format `pg_dump`, restores it, and verifies the promoted release survives while restricted records remain excluded by the current public restriction view. +This check uses only disposable PostGIS containers and synthetic records. It applies every migration, seeds a promoted release plus an active suppression case, creates a custom-format `pg_dump`, rolls back to that older backup, rejects the stale restriction snapshot, replays the independent payload-free ledger, and verifies the promoted release survives while restricted records remain excluded by the current public restriction view. From PowerShell: @@ -9,4 +9,4 @@ $env:UEC_RUN_E2E = "1" pwsh -NoProfile -ExecutionPolicy Bypass -File pipeline/tests/e2e/backup-restore.ps1 ``` -The script uses a unique Compose project and removes its volume in `finally`. `-KeepArtifacts` retains the temporary dump for local inspection only. It never reads project data or credentials. +The script uses a unique Compose project and removes its volume in `finally`. `-KeepArtifacts` retains the temporary dump for local inspection only. It never reads project data or credentials. No application service is started during the drill; the point of the gate is to prove that an old restore stays stopped until current restrictions are replayed. diff --git a/pipeline/tests/e2e/backup-restore.ps1 b/pipeline/tests/e2e/backup-restore.ps1 index 4fc26b8..91bf7e0 100644 --- a/pipeline/tests/e2e/backup-restore.ps1 +++ b/pipeline/tests/e2e/backup-restore.ps1 @@ -14,12 +14,20 @@ $ledgerGate = Join-Path $root 'pipeline\scripts\maintenance\restriction-ledger-g $ledgerFile = Join-Path $PSScriptRoot 'restriction-ledger.json' $oldSnapshot = Join-Path $PSScriptRoot 'restriction-ledger-old-snapshot.json' $currentSnapshot = Join-Path $PSScriptRoot 'restriction-ledger-current-snapshot.json' +$ledgerReplayFile = Join-Path ([IO.Path]::GetTempPath()) "$project-ledger-replay.sql" function Invoke-FixtureSql([string]$path) { Get-Content -LiteralPath $path -Raw | & docker compose @composeArgs exec -T postgres psql -1 -v ON_ERROR_STOP=1 -U uec -d uec if ($LASTEXITCODE -ne 0) { throw "Fixture SQL failed: $path (exit $LASTEXITCODE)." } } +function Invoke-LedgerReplaySql([string]$path) { + # The emitted SQL owns its BEGIN/COMMIT so the zero/multiple-match exception + # rolls back the complete replay. Do not add psql's outer -1 transaction. + Get-Content -LiteralPath $path -Raw | & docker compose @composeArgs exec -T postgres psql -v ON_ERROR_STOP=1 -U uec -d uec + if ($LASTEXITCODE -ne 0) { throw "Restriction ledger replay failed: $path (exit $LASTEXITCODE)." } +} + function Get-Snapshot { $sql = @" SELECT count(*) FROM uec.releases WHERE release_id='e2e-promoted' AND status='promoted'; @@ -41,10 +49,10 @@ function Assert-Snapshot([string]$phase, [string]$expected) { Write-Host "[backup-restore] ${phase}: $actual" } -function Test-SyntheticServiceGate { +function Test-SyntheticServiceGate([string]$expected = '1,1,1,1,1,0,0') { # The external synthetic restriction is required in the restored DB and both # public projections must exclude it. A query error stops the drill. - return ((Get-Snapshot) -join ',') -eq '1,1,1,1,1,0,0' + return ((Get-Snapshot) -join ',') -eq $expected } try { @@ -106,13 +114,16 @@ try { if ($oldGateExitCode -eq 0) { throw 'Pre-service ledger gate accepted an old restriction snapshot.' } Write-Host '[backup-restore] PASS: synthetic pre-service gate rejects the old backup before replay.' - Invoke-FixtureSql $suppressionFile - Assert-Snapshot 'current restriction replayed' '1,1,1,1,1,0,0' - if (-not (Test-SyntheticServiceGate)) { throw 'Synthetic pre-service gate rejected the replayed current restriction.' } - & python $ledgerGate --ledger $ledgerFile --snapshot $currentSnapshot + & python $ledgerGate --ledger $ledgerFile --snapshot $currentSnapshot *> $null if ($LASTEXITCODE -ne 0) { throw 'Pre-service ledger gate rejected the current restriction replay.' } - Write-Host 'PASS: synthetic old-backup restore remains gated until current restriction is replayed and both public projections exclude it.' - Write-Host 'TEST ONLY: production still needs an independent durable restriction ledger and an enforced service-start gate.' + $replaySql = & python (Join-Path $root 'pipeline\scripts\maintenance\replay-restriction-ledger.py') --ledger $ledgerFile --emit-sql + if ($LASTEXITCODE -ne 0) { throw 'Restriction ledger replay SQL generation failed.' } + Set-Content -LiteralPath $ledgerReplayFile -Value ($replaySql -join "`n") -Encoding UTF8 + Invoke-LedgerReplaySql $ledgerReplayFile + Assert-Snapshot 'current restriction replayed' '1,1,0,0,1,0,0' + if (-not (Test-SyntheticServiceGate '1,1,0,0,1,0,0')) { throw 'Synthetic pre-service gate rejected the replayed current restriction.' } + Write-Host 'PASS: synthetic old-backup rollback remains gated until the independent current ledger is replayed and both public projections exclude it.' + Write-Host 'TEST ONLY: production still requires separately operated ledger storage, trusted references, and deployment-specific review.' } finally { $savedPreference = $ErrorActionPreference $ErrorActionPreference = 'Continue' @@ -120,4 +131,5 @@ try { $ErrorActionPreference = $savedPreference if (-not $KeepArtifacts -and (Test-Path -LiteralPath $dump)) { Remove-Item -LiteralPath $dump -Force } if (Test-Path -LiteralPath $migrationFile) { Remove-Item -LiteralPath $migrationFile -Force } + if (Test-Path -LiteralPath $ledgerReplayFile) { Remove-Item -LiteralPath $ledgerReplayFile -Force } } diff --git a/pipeline/tests/e2e/test_readiness.py b/pipeline/tests/e2e/test_readiness.py index eaee7e3..8ec5e6b 100644 --- a/pipeline/tests/e2e/test_readiness.py +++ b/pipeline/tests/e2e/test_readiness.py @@ -32,6 +32,12 @@ def test_full_schema_is_ready(self): with urllib.request.urlopen(f"http://127.0.0.1:{env.api_port}/health/ready", timeout=2) as response: self.assertEqual(response.status, 200) self.assertEqual(json.load(response)["schema"], "migrated") + with urllib.request.urlopen(f"http://127.0.0.1:{env.api_port}/health/diagnostics", timeout=2) as response: + diagnostics = json.load(response) + self.assertEqual(response.status, 200) + self.assertEqual(diagnostics["privacy"]["diagnostic_identifiers"], "excluded") + self.assertNotIn("database_url", json.dumps(diagnostics)) + self.assertNotIn("forwarded", json.dumps(diagnostics).lower()) finally: env.stop() diff --git a/pipeline/tests/test_private_environment_gate.py b/pipeline/tests/test_private_environment_gate.py new file mode 100644 index 0000000..0f2ae5b --- /dev/null +++ b/pipeline/tests/test_private_environment_gate.py @@ -0,0 +1,114 @@ +"""Synthetic contract tests for the private startup/deployment gate.""" +import importlib.util +import hashlib +import json +import tempfile +import unittest +from pathlib import Path + +ROOT = Path(__file__).parents[1] +REPOSITORY = Path(__file__).parents[2] +SPEC = importlib.util.spec_from_file_location( + "private_environment_gate", + ROOT / "scripts" / "maintenance" / "private-environment-gate.py", +) +MODULE = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(MODULE) + + +class PrivateEnvironmentGateTests(unittest.TestCase): + def setUp(self): + self.values = { + "UEC_RUNTIME_MODE": "production", + "UEC_DATABASE_URL": "postgresql://redacted.invalid/uec", + "UEC_CORS_ORIGINS": "https://example.invalid", + "UEC_TRUST_PROXY": "true", + "UEC_TRUSTED_PROXY_CIDRS": "192.0.2.0/24", + } + + def fixtures(self, directory): + ledger = { + "schema_version": 1, + "revision": "synthetic-r1", + "active_restrictions": [{ + "source_id": "synthetic", + "source_record_key": "opaque-1", + "scope": "whole_record", + "action": "suppress", + }], + } + ledger["ledger_sha256"] = MODULE.ledger_digest(ledger) + snapshot = { + "ledger_revision": ledger["revision"], + "ledger_sha256": ledger["ledger_sha256"], + "active_restrictions": ledger["active_restrictions"], + } + manifest = { + "manifest_version": "v1", + "release_id": "synthetic-release", + "profile": "official", + "ruleset_version": "synthetic-v1", + "distributed_artifacts": [], + } + paths = {name: Path(directory) / name for name in ("ledger.json", "snapshot.json", "manifest.json")} + paths["ledger.json"].write_text(json.dumps(ledger), encoding="utf-8") + paths["snapshot.json"].write_text(json.dumps(snapshot), encoding="utf-8") + paths["manifest.json"].write_text(MODULE.canonical_json(manifest), encoding="utf-8") + return paths, MODULE.canonical_json(manifest) + + def test_full_gate_reports_only_safe_metadata(self): + with tempfile.TemporaryDirectory() as directory: + paths, manifest_json = self.fixtures(directory) + report = MODULE.run_gate( + REPOSITORY, self.values, paths["ledger.json"], paths["snapshot.json"], + paths["manifest.json"], hashlib.sha256(manifest_json.encode()).hexdigest(), + ) + self.assertEqual(report["status"], "pass") + self.assertGreater(report["checks"]["migration_count"], 0) + self.assertEqual(report["checks"]["restriction_ledger"]["active_restriction_count"], 1) + self.assertNotIn("opaque-1", json.dumps(report["checks"]["restriction_ledger"])) + + def test_stale_replay_and_missing_proxy_boundary_fail_closed(self): + with tempfile.TemporaryDirectory() as directory: + paths, _ = self.fixtures(directory) + stale = json.loads(paths["snapshot.json"].read_text(encoding="utf-8")) + stale["ledger_revision"] = "old" + paths["snapshot.json"].write_text(json.dumps(stale), encoding="utf-8") + with self.assertRaises(MODULE.PrivateEnvironmentError): + MODULE.verify_ledger_replay(paths["ledger.json"], paths["snapshot.json"]) + invalid = dict(self.values, UEC_TRUSTED_PROXY_CIDRS=None) + with self.assertRaises(MODULE.PrivateEnvironmentError): + MODULE.validate_runtime_config(invalid) + + def test_manifest_rejects_path_traversal_and_migration_inventory_is_versioned(self): + with tempfile.TemporaryDirectory() as directory: + paths, _ = self.fixtures(directory) + manifest = json.loads(paths["manifest.json"].read_text(encoding="utf-8")) + manifest["distributed_artifacts"] = [{"name": "../private.txt", "sha256": "0" * 64, "byte_size": 1}] + serialized = MODULE.canonical_json(manifest) + paths["manifest.json"].write_text(serialized, encoding="utf-8") + with self.assertRaises(MODULE.PrivateEnvironmentError): + MODULE.validate_release_manifest(paths["manifest.json"], hashlib.sha256(serialized.encode()).hexdigest()) + inventory = MODULE.validate_migrations(ROOT / "migrations") + self.assertGreater(inventory["migration_count"], 0) + self.assertEqual(len(inventory["migration_inventory_sha256"]), 64) + + def test_replay_sql_is_idempotent_and_does_not_embed_sensitive_columns(self): + replay_spec = importlib.util.spec_from_file_location( + "replay_restriction_ledger", + ROOT / "scripts" / "maintenance" / "replay-restriction-ledger.py", + ) + replay = importlib.util.module_from_spec(replay_spec) + replay_spec.loader.exec_module(replay) + with tempfile.TemporaryDirectory() as directory: + paths, _ = self.fixtures(directory) + sql = replay.replay_sql(paths["ledger.json"]) + self.assertIn("BEGIN;", sql) + self.assertIn("RAISE EXCEPTION", sql) + self.assertIn("record_access_events", sql) + self.assertNotIn("address", sql.lower()) + self.assertNotIn("coordinate", sql.lower()) + + +if __name__ == "__main__": + unittest.main() diff --git a/src/main.rs b/src/main.rs index c142abd..224657a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -32,11 +32,14 @@ use tokio_postgres::NoTls; use tokio_postgres_rustls::MakeRustlsConnect; use tower_http::services::ServeDir; -pub fn app(state: uec_api::ApiState) -> Router { +mod private_environment; + +pub fn app(state: uec_api::ApiState, proxy: private_environment::ProxyConfig) -> Router { let cors = cors_layer().expect("CORS configuration must be validated before app startup"); Router::new() .route("/health/live", get(liveness)) .route("/health/ready", get(readiness)) + .route("/health/diagnostics", get(diagnostics)) .route("/api/locations", get(uec_api::get_locations_handler)) .route("/api/v2/locations", get(uec_api::get_v2_locations_handler)) .route( @@ -93,7 +96,7 @@ pub fn app(state: uec_api::ApiState) -> Router { .layer(axum::middleware::from_fn_with_state( RateLimitState { limiter: Limiter::default(), - trust_proxy: std::env::var("UEC_TRUST_PROXY").as_deref() == Ok("true"), + proxy, }, rate_limit, )) @@ -221,7 +224,7 @@ struct Limiter(Arc>>); #[derive(Clone)] struct RateLimitState { limiter: Limiter, - trust_proxy: bool, + proxy: private_environment::ProxyConfig, } impl Limiter { @@ -237,8 +240,17 @@ impl Limiter { } } -fn client_key(request: &Request, trust_proxy: bool) -> Option { - if trust_proxy { +fn client_key( + request: &Request, + proxy: &private_environment::ProxyConfig, +) -> Option { + let trusted_peer = request + .extensions() + .get::>() + .is_some_and(|ConnectInfo(peer)| { + private_environment::peer_is_trusted_proxy(peer.ip(), proxy) + }); + if proxy.trust_forwarded_for && trusted_peer { if let Some(ip) = request .headers() .get("x-forwarded-for") @@ -263,7 +275,7 @@ async fn rate_limit( if request.uri().path().starts_with("/health/") { return next.run(request).await; } - let Some(key) = client_key(&request, config.trust_proxy) else { + let Some(key) = client_key(&request, &config.proxy) else { return StatusCode::SERVICE_UNAVAILABLE.into_response(); }; if !config.limiter.allow(key, Instant::now()) { @@ -283,6 +295,34 @@ async fn liveness() -> impl IntoResponse { Json(serde_json::json!({"status": "ok", "service": "uec-api"})) } +async fn diagnostics() -> impl IntoResponse { + let mode = std::env::var("UEC_RUNTIME_MODE").unwrap_or_else(|_| "development".into()); + let database_configured = std::env::var("UEC_DATABASE_URL") + .ok() + .is_some_and(|url| !url.trim().is_empty()); + let proxy_trust = match std::env::var("UEC_TRUST_PROXY").as_deref() { + Ok("true") => "enabled_with_configured_boundary", + Ok("false") | Err(_) => "disabled", + Ok(_) => "invalid_configuration", + }; + Json(serde_json::json!({ + "status": "ok", + "service": "uec-api", + "runtime_mode": mode, + "database_configured": database_configured, + "proxy_trust": proxy_trust, + "startup_gates": { + "restriction_ledger": if mode == "production" { "verified" } else { "not_required_development" }, + "release_manifest": if mode == "production" { "verified" } else { "not_required_development" } + }, + "privacy": { + "request_payloads": "not_reported", + "visitor_location": "not_reported", + "diagnostic_identifiers": "excluded" + } + })) +} + async fn readiness( axum::extract::State(state): axum::extract::State, ) -> impl IntoResponse { @@ -404,6 +444,34 @@ async fn main() { ); std::process::exit(2); } + let proxy = private_environment::parse_proxy_config( + mode.as_str(), + std::env::var("UEC_TRUST_PROXY").ok().as_deref(), + std::env::var("UEC_TRUSTED_PROXY_CIDRS").ok().as_deref(), + ) + .unwrap_or_else(|error| { + eprintln!( + "{{\"event\":\"configuration_error\",\"reason\":\"{}\"}}", + error + ); + std::process::exit(2) + }); + let startup_gate = private_environment::validate_startup( + mode.as_str(), + std::env::var("UEC_RESTRICTION_LEDGER_PATH").ok().as_deref(), + std::env::var("UEC_RESTORED_RESTRICTION_SNAPSHOT_PATH") + .ok() + .as_deref(), + std::env::var("UEC_RELEASE_MANIFEST_PATH").ok().as_deref(), + std::env::var("UEC_RELEASE_MANIFEST_SHA256").ok().as_deref(), + ) + .unwrap_or_else(|error| { + eprintln!( + "{{\"event\":\"configuration_error\",\"reason\":\"{}\"}}", + error + ); + std::process::exit(2) + }); let dev_preview_token = preview_config( mode.as_str(), std::env::var("UEC_DEV_PREVIEW").ok().as_deref(), @@ -453,8 +521,8 @@ async fn main() { }); let addr = format!("{}:{}", bind_host, port); println!( - "{{\"event\":\"server_starting\",\"service\":\"uec-api\",\"mode\":\"{}\",\"port\":{}}}", - mode, port + "{{\"event\":\"server_starting\",\"service\":\"uec-api\",\"mode\":\"{}\",\"port\":{},\"restriction_ledger\":\"{}\",\"release_manifest\":\"{}\"}}", + mode, port, startup_gate.restriction_ledger, startup_gate.release_manifest ); let (dev_test_release_id, dev_test_release_token) = match ( std::env::var("UEC_TEST_RELEASE_ID").ok(), @@ -483,12 +551,15 @@ async fn main() { let listener = tokio::net::TcpListener::bind(addr).await.unwrap(); axum::serve( listener, - app(uec_api::ApiState { - database, - dev_preview_token, - dev_test_release_id, - dev_test_release_token, - }) + app( + uec_api::ApiState { + database, + dev_preview_token, + dev_test_release_id, + dev_test_release_token, + }, + proxy, + ) .into_make_service_with_connect_info::(), ) .await @@ -633,7 +704,8 @@ mod rate_limit_tests { .layer(axum::middleware::from_fn_with_state( RateLimitState { limiter: Limiter::default(), - trust_proxy: false, + proxy: private_environment::parse_proxy_config("development", None, None) + .unwrap(), }, rate_limit, )); @@ -687,11 +759,24 @@ mod rate_limit_tests { .unwrap(); request.extensions_mut().insert(ConnectInfo(peer)); assert_eq!( - client_key(&request, false).as_deref(), + client_key( + &request, + &private_environment::parse_proxy_config("development", None, None).unwrap(), + ) + .as_deref(), Some("peer:192.0.2.10") ); assert_eq!( - client_key(&request, true).as_deref(), + client_key( + &request, + &private_environment::parse_proxy_config( + "production", + Some("true"), + Some("192.0.2.0/24"), + ) + .unwrap(), + ) + .as_deref(), Some("proxy:198.51.100.20") ); @@ -700,9 +785,38 @@ mod rate_limit_tests { HeaderValue::from_static("invalid, 198.51.100.20"), ); assert_eq!( - client_key(&request, true).as_deref(), + client_key( + &request, + &private_environment::parse_proxy_config( + "production", + Some("true"), + Some("192.0.2.0/24"), + ) + .unwrap(), + ) + .as_deref(), Some("peer:192.0.2.10") ); + + let untrusted_peer: SocketAddr = "203.0.113.10:41000".parse().unwrap(); + request.extensions_mut().remove::>(); + request.extensions_mut().insert(ConnectInfo(untrusted_peer)); + request + .headers_mut() + .insert("x-forwarded-for", HeaderValue::from_static("198.51.100.20")); + assert_eq!( + client_key( + &request, + &private_environment::parse_proxy_config( + "production", + Some("true"), + Some("192.0.2.0/24"), + ) + .unwrap(), + ) + .as_deref(), + Some("peer:203.0.113.10") + ); } #[tokio::test] @@ -712,7 +826,8 @@ mod rate_limit_tests { .layer(axum::middleware::from_fn_with_state( RateLimitState { limiter: Limiter::default(), - trust_proxy: false, + proxy: private_environment::parse_proxy_config("development", None, None) + .unwrap(), }, rate_limit, )); diff --git a/src/private_environment.rs b/src/private_environment.rs new file mode 100644 index 0000000..078b6e3 --- /dev/null +++ b/src/private_environment.rs @@ -0,0 +1,289 @@ +//! Fail-closed checks for the private production-shaped runtime. +//! +//! The restriction ledger and restore snapshot are deliberately separate JSON +//! control-plane inputs. They contain opaque source keys only; they must never +//! contain addresses, coordinates, requester contact details, or source text. + +use ipnet::IpNet; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; +use sha2::{Digest, Sha256}; +use std::collections::HashSet; +use std::fs; +use std::net::IpAddr; +use std::path::Path; +use std::str::FromStr; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ProxyConfig { + pub trust_forwarded_for: bool, + pub trusted_proxy_cidrs: Vec, +} + +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq, Hash)] +pub struct RestrictionReference { + pub source_id: String, + pub source_record_key: String, + pub scope: String, + pub action: String, +} + +#[derive(Debug, Deserialize)] +struct RestrictionLedger { + schema_version: u64, + revision: String, + ledger_sha256: String, + active_restrictions: Vec, +} + +#[derive(Debug, Deserialize)] +struct RestrictionSnapshot { + ledger_revision: String, + ledger_sha256: String, + active_restrictions: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct StartupGateReport { + pub restriction_ledger: &'static str, + pub release_manifest: &'static str, +} + +fn canonical_json(value: &Value) -> String { + match value { + Value::Object(map) => { + let mut keys: Vec<_> = map.keys().collect(); + keys.sort(); + format!( + "{{{}}}", + keys.into_iter() + .map(|key| format!( + "{}:{}", + serde_json::to_string(key).expect("JSON key serialization"), + canonical_json(&map[key]) + )) + .collect::>() + .join(",") + ) + } + Value::Array(items) => format!( + "[{}]", + items + .iter() + .map(canonical_json) + .collect::>() + .join(",") + ), + _ => value.to_string(), + } +} + +fn ledger_digest(ledger: &RestrictionLedger) -> String { + let payload = json!({ + "schema_version": ledger.schema_version, + "revision": ledger.revision, + "active_restrictions": ledger.active_restrictions, + }); + format!("{:x}", Sha256::digest(canonical_json(&payload).as_bytes())) +} + +fn read_json Deserialize<'de>>(path: &Path, label: &str) -> Result { + let bytes = fs::read(path).map_err(|_| format!("{label} is unavailable"))?; + serde_json::from_slice(&bytes).map_err(|_| format!("{label} is invalid")) +} + +fn verify_ledger(ledger_path: &Path, snapshot_path: &Path) -> Result<(), String> { + if ledger_path == snapshot_path { + return Err("restriction ledger and restored snapshot must be separate files".into()); + } + let ledger: RestrictionLedger = read_json(ledger_path, "restriction ledger")?; + if ledger.schema_version != 1 || ledger.revision.trim().is_empty() { + return Err("restriction ledger schema or revision is unsupported".into()); + } + if ledger.ledger_sha256 != ledger_digest(&ledger) { + return Err("restriction ledger digest is invalid".into()); + } + if ledger.active_restrictions.iter().any(|reference| { + reference.source_id.trim().is_empty() + || reference.source_record_key.trim().is_empty() + || reference.scope.trim().is_empty() + || reference.action != "suppress" + }) { + return Err("restriction ledger references are invalid".into()); + } + let snapshot: RestrictionSnapshot = read_json(snapshot_path, "restored restriction snapshot")?; + if snapshot.ledger_revision != ledger.revision || snapshot.ledger_sha256 != ledger.ledger_sha256 + { + return Err("restored restriction state is stale".into()); + } + let expected: HashSet<_> = ledger.active_restrictions.iter().collect(); + let applied: HashSet<_> = snapshot.active_restrictions.iter().collect(); + if expected.len() != ledger.active_restrictions.len() + || applied.len() != snapshot.active_restrictions.len() + || expected != applied + || snapshot + .active_restrictions + .iter() + .any(|reference| reference.action != "suppress") + { + return Err("restored restrictions do not match current ledger".into()); + } + Ok(()) +} + +fn verify_release_manifest(path: &Path, expected_digest: &str) -> Result<(), String> { + let bytes = fs::read(path).map_err(|_| "release manifest is unavailable".to_string())?; + let manifest: Value = + serde_json::from_slice(&bytes).map_err(|_| "release manifest is invalid".to_string())?; + let actual = format!("{:x}", Sha256::digest(canonical_json(&manifest).as_bytes())); + if expected_digest != actual { + return Err("release manifest digest does not match trusted reference".into()); + } + for field in [ + "manifest_version", + "release_id", + "profile", + "ruleset_version", + ] { + if manifest + .get(field) + .and_then(Value::as_str) + .is_none_or(str::is_empty) + { + return Err(format!("release manifest field is missing: {field}")); + } + } + Ok(()) +} + +pub fn parse_proxy_config( + mode: &str, + trust_proxy: Option<&str>, + trusted_cidrs: Option<&str>, +) -> Result { + let trust_forwarded_for = match trust_proxy { + None if mode == "development" => false, + None => return Err("UEC_TRUST_PROXY must be explicitly set in production"), + Some("true") => true, + Some("false") => false, + Some(_) => return Err("UEC_TRUST_PROXY must be true or false"), + }; + let cidr_text = trusted_cidrs.unwrap_or(""); + let cidrs: Result, _> = cidr_text + .split(',') + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(IpNet::from_str) + .collect(); + let cidrs = cidrs.map_err(|_| "UEC_TRUSTED_PROXY_CIDRS contains an invalid network")?; + if trust_forwarded_for && cidrs.is_empty() { + return Err("UEC_TRUSTED_PROXY_CIDRS is required when proxy trust is enabled"); + } + if !trust_forwarded_for && !cidrs.is_empty() { + return Err("UEC_TRUSTED_PROXY_CIDRS requires UEC_TRUST_PROXY=true"); + } + Ok(ProxyConfig { + trust_forwarded_for, + trusted_proxy_cidrs: cidrs, + }) +} + +pub fn peer_is_trusted_proxy(peer: IpAddr, config: &ProxyConfig) -> bool { + config + .trusted_proxy_cidrs + .iter() + .any(|network| network.contains(&peer)) +} + +pub fn validate_startup( + mode: &str, + ledger_path: Option<&str>, + snapshot_path: Option<&str>, + manifest_path: Option<&str>, + manifest_digest: Option<&str>, +) -> Result { + if mode != "production" { + return Ok(StartupGateReport { + restriction_ledger: "not_required_development", + release_manifest: "not_required_development", + }); + } + let ledger_path = ledger_path + .filter(|value| !value.trim().is_empty()) + .ok_or("UEC_RESTRICTION_LEDGER_PATH is required in production")?; + let snapshot_path = snapshot_path + .filter(|value| !value.trim().is_empty()) + .ok_or("UEC_RESTORED_RESTRICTION_SNAPSHOT_PATH is required in production")?; + let manifest_path = manifest_path + .filter(|value| !value.trim().is_empty()) + .ok_or("UEC_RELEASE_MANIFEST_PATH is required in production")?; + let manifest_digest = manifest_digest + .filter(|value| !value.trim().is_empty()) + .ok_or("UEC_RELEASE_MANIFEST_SHA256 is required in production")?; + if manifest_digest.len() != 64 + || !manifest_digest + .bytes() + .all(|byte| byte.is_ascii_hexdigit() && !byte.is_ascii_uppercase()) + { + return Err( + "UEC_RELEASE_MANIFEST_SHA256 must be 64 lowercase hexadecimal characters".into(), + ); + } + verify_ledger(Path::new(ledger_path), Path::new(snapshot_path))?; + verify_release_manifest(Path::new(manifest_path), manifest_digest)?; + Ok(StartupGateReport { + restriction_ledger: "verified", + release_manifest: "verified", + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::fs; + use std::time::{SystemTime, UNIX_EPOCH}; + + #[test] + fn production_requires_independent_control_plane_inputs() { + assert!(validate_startup("production", None, None, None, None).is_err()); + assert!(parse_proxy_config("production", Some("true"), Some("10.0.0.0/8")).is_ok()); + assert!(parse_proxy_config("production", Some("true"), Some("10.0.0.1")).is_err()); + } + + #[test] + fn startup_accepts_matching_synthetic_ledger_and_manifest() { + let dir = std::env::temp_dir().join(format!( + "uec-private-gate-{}", + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + fs::create_dir(&dir).unwrap(); + let ledger = json!({"schema_version":1,"revision":"synthetic-r1","active_restrictions":[{"source_id":"synthetic","source_record_key":"opaque-1","scope":"whole_record","action":"suppress"}]}); + let ledger_digest = format!("{:x}", Sha256::digest(canonical_json(&ledger).as_bytes())); + let ledger = json!({"schema_version":1,"revision":"synthetic-r1","ledger_sha256":ledger_digest,"active_restrictions":ledger["active_restrictions"].clone()}); + let snapshot = json!({"ledger_revision":"synthetic-r1","ledger_sha256":ledger["ledger_sha256"].clone(),"active_restrictions":ledger["active_restrictions"].clone()}); + let manifest = json!({"manifest_version":"v1","release_id":"synthetic","profile":"official","ruleset_version":"synthetic-v1"}); + let manifest_digest = format!("{:x}", Sha256::digest(canonical_json(&manifest).as_bytes())); + let ledger_path = dir.join("ledger.json"); + let snapshot_path = dir.join("snapshot.json"); + let manifest_path = dir.join("manifest.json"); + fs::write(&ledger_path, serde_json::to_vec(&ledger).unwrap()).unwrap(); + fs::write(&snapshot_path, serde_json::to_vec(&snapshot).unwrap()).unwrap(); + fs::write(&manifest_path, canonical_json(&manifest)).unwrap(); + assert_eq!( + validate_startup( + "production", + ledger_path.to_str(), + snapshot_path.to_str(), + manifest_path.to_str(), + Some(&manifest_digest) + ) + .unwrap() + .restriction_ledger, + "verified" + ); + fs::remove_dir_all(&dir).unwrap(); + } +} From fc6529c7d5a39d7fc7573658bd3e4a7c1b796749 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Tue, 15 Sep 2026 13:59:49 -0700 Subject: [PATCH 120/311] Add private Germany and Belgium source lifecycles --- docs/country-recon-be.md | 10 +- docs/germany-source-assessment.md | 12 + docs/source-status.json | 4 +- docs/source-status.md | 4 +- pipeline/source_registry.json | 20 +- pipeline/sources/belgium/README.md | 15 + pipeline/sources/belgium/__init__.py | 5 + pipeline/sources/belgium/acquire.py | 42 +++ pipeline/sources/belgium/adapter.py | 271 ++++++++++++++++++ pipeline/sources/belgium/config.json | 12 + .../fixtures/synthetic_activity_codes.csv | 5 + .../belgium/fixtures/synthetic_operators.csv | 6 + pipeline/sources/belgium/refresh.py | 95 ++++++ pipeline/sources/belgium/test_adapter.py | 63 ++++ pipeline/sources/belgium/test_refresh.py | 30 ++ pipeline/sources/germany/README.md | 14 + pipeline/sources/germany/__init__.py | 5 + pipeline/sources/germany/adapter.py | 104 +++++++ pipeline/sources/germany/config.json | 10 + pipeline/sources/germany/refresh.py | 83 ++++++ pipeline/sources/germany/test_adapter.py | 50 ++++ pipeline/sources/germany/test_refresh.py | 31 ++ .../test_germany_belgium_candidate_import.py | 73 +++++ 23 files changed, 949 insertions(+), 15 deletions(-) create mode 100644 pipeline/sources/belgium/README.md create mode 100644 pipeline/sources/belgium/__init__.py create mode 100644 pipeline/sources/belgium/acquire.py create mode 100644 pipeline/sources/belgium/adapter.py create mode 100644 pipeline/sources/belgium/config.json create mode 100644 pipeline/sources/belgium/fixtures/synthetic_activity_codes.csv create mode 100644 pipeline/sources/belgium/fixtures/synthetic_operators.csv create mode 100644 pipeline/sources/belgium/refresh.py create mode 100644 pipeline/sources/belgium/test_adapter.py create mode 100644 pipeline/sources/belgium/test_refresh.py create mode 100644 pipeline/sources/germany/README.md create mode 100644 pipeline/sources/germany/__init__.py create mode 100644 pipeline/sources/germany/adapter.py create mode 100644 pipeline/sources/germany/config.json create mode 100644 pipeline/sources/germany/refresh.py create mode 100644 pipeline/sources/germany/test_adapter.py create mode 100644 pipeline/sources/germany/test_refresh.py create mode 100644 pipeline/tests/e2e/test_germany_belgium_candidate_import.py diff --git a/docs/country-recon-be.md b/docs/country-recon-be.md index 7c44bb6..18ad455 100644 --- a/docs/country-recon-be.md +++ b/docs/country-recon-be.md @@ -1,9 +1,17 @@ # Belgium source reconnaissance -Status: reconnaissance only. No adapter, release, publication, or row-level fixture was created. No facility rows, names, addresses, contacts, coordinates, or private artifacts are retained here. +Status: reconnaissance plus private/test-only adapter implementation. No real facility rows, names, addresses, contacts, coordinates, release, or publication are retained here. Synthetic fixtures contain no real operators. Last checked: 2026-09-15 UTC under `docs/ETHICS.md`, policy version 1.0, last reviewed 2026-09-12. This is source-status evidence, not publication approval or a runtime-health claim. +The implementation is in [`pipeline/sources/belgium/`](../pipeline/sources/belgium/). It requires two independently preserved official artifacts: the operator CSV at `https://www.static.favv.be/bo-documents/inter_actieve_actoren_EN.csv` and the LAP/PAP codebook at `https://www.static.favv.be/bo-documents/inter_PAP_omschrijving_EN.csv`. The bounded assisted command is repeatable when a browser or authorized operator supplies both files: + +```text +python -m pipeline.sources.belgium.refresh --operators --activity-codes --run-dir --retrieved-at-utc 2026-09-15T00:00:00Z +``` + +The codebook join is exact and deterministic; unresolved or ambiguous codes quarantine. The adapter preserves source activity text and distinguishes slaughter, cutting, processing, storage, animal-by-products, and export domains. It never geocodes and keeps source address/coordinate/enterprise values out of normalized/API-shaped rows. The live operator header was not obtainable in this environment, so the checked-in fixture is a schema contract and the first real capture must be reviewed for schema drift. + ## Readiness | Candidate | Evidence | Acquisition | Terms / privacy | Readiness / next action | diff --git a/docs/germany-source-assessment.md b/docs/germany-source-assessment.md index 73b2f92..df7ebf7 100644 --- a/docs/germany-source-assessment.md +++ b/docs/germany-source-assessment.md @@ -17,6 +17,8 @@ identifies BVL/BKG attribution notices for portal geodata. The portal’s export capability supports a reproducible acquisition design, but it does not by itself establish permission to redistribute the exported establishment records. +The stable entry route is [BVL BLtU](https://www.bvl.bund.de/bltu); the BVL portal supports selecting and exporting the current general or category list. Session-bound export URLs must be captured as run-specific provenance, not checked in as a source URL. + The existing V1 Germany adapter is [`static_data/de/migrate_data.py`](../static_data/de/migrate_data.py), which references the BLtU publication endpoint and expects downloaded/merged CSVs. Its current behavior also performs external geocoding and emits a wide source-shaped @@ -52,6 +54,16 @@ profile/specification, not a dataset license, and `https://bund.de` is a placeho not the BLtU resource URI. A DL-DE record for another BVL dataset cannot be inherited by BLtU without dataset-specific evidence. +## Private/test-only implementation + +[`pipeline/sources/germany/`](../pipeline/sources/germany/) now provides a typed BLtU adapter and assisted refresh. Use the stable landing page to select the current CSV export, save it in private ignored storage, and run: + +```text +python -m pipeline.sources.germany.refresh --raw --run-dir --retrieved-at-utc 2026-09-15T00:00:00Z +``` + +The adapter preserves repeated activity columns and current approval numbers, quarantines schema/identity/unmapped-code anomalies, emits shared QA and health evidence, and produces only a private candidate handoff. Address and coordinate values remain private and geocoding is disabled. Candidate import, if used, must target the disposable database guard and remains test-only; no public release is created. + ## Planned recurring acquisition after approval Once the named human reviewer records a positive terms decision, an automated runner diff --git a/docs/source-status.json b/docs/source-status.json index a0f8bb6..4e4afbb 100644 --- a/docs/source-status.json +++ b/docs/source-status.json @@ -8,7 +8,7 @@ "publication_eligibility": ["not_assessed", "blocked", "eligible_pending_release_approval"] }, "sources": [ - {"source_id":"be.locations","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-be.md","pipeline/source_registry.json"],"next_action":"Obtain an authorized bounded operator CSV response, record provenance and sanitized schema, then implement codebook/category/privacy validation before any release review."}, + {"source_id":"be.locations","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-be.md","pipeline/sources/belgium/adapter.py","pipeline/sources/belgium/refresh.py","pipeline/sources/belgium/fixtures/synthetic_operators.csv","pipeline/source_registry.json"],"next_action":"Use the assisted two-file refresh with an authorized operator capture and official activity-code CSV; compare live schema to the synthetic contract. Keep category/privacy/terms gates and publication blocked."}, {"source_id":"fr.locations","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-fr.md","pipeline/source_registry.json"],"next_action":"Acquire and snapshot an official DGAL Section I/II artifact with terms, schema, and privacy review."}, {"source_id":"it.853-2004","metadata":"verified","acquisition":"artifact_private_only","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-it.md","pipeline/source_registry.json","pipeline/sources/italy/acquire.py","pipeline/sources/italy/it_853_adapter.py","pipeline/sources/italy/README.md","pipeline/tests/e2e/test_italy_candidate_import.py"],"next_action":"Keep catalog acquisition, lifecycle, candidate import, and guarded API evidence private; resolve repeated activity identity, coordinate/address privacy, coverage, and project approval before release review."}, {"source_id":"it.1069-2009","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-it.md","pipeline/source_registry.json","pipeline/sources/italy/README.md"],"next_action":"Keep the by-products dataset separate; assess scope, schema, terms, identity links, privacy, and a dedicated adapter before any acquisition or integration."}, @@ -16,7 +16,7 @@ {"source_id":"nz.locations","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-nz.md","pipeline/source_registry.json"],"next_action":"Confirm MPI permitted acquisition route, list-specific terms, and schema before private staging."}, {"source_id":"uk.locations","metadata":"partial","acquisition":"artifact_private_only","runtime_health":"unknown","publication_eligibility":"blocked","evidence":["docs/country-recon-uk.md","docs/countries/uk/fss-approved-establishments-source-assessment.md","pipeline/sources/uk/fsa_approved/README.md","pipeline/sources/uk/fsa_approved/handoff.py","pipeline/sources/uk/fsa_approved/refresh.py","docs/architecture/disposable-candidate-import.md","pipeline/scripts/maintenance/import-candidate.py","pipeline/source_registry.json"],"next_action":"Repeatable source-local refresh dry-run passed on the retained snapshot (5,342 input, 4,300 normalized, 1,042 quarantined, expected schema fingerprint, no drift alarms); use refresh.py dry-run/handoff only in restricted staging. This does not establish production/runtime health or publication eligibility. Complete source-rights, privacy/coordinate, duplicate, coverage, and release review while keeping NI and Scotland separate."}, {"source_id":"dk.smiley","metadata":"verified","acquisition":"verified","runtime_health":"unknown","publication_eligibility":"blocked","evidence":["pipeline/sources/denmark/README.md","pipeline/contracts/README.md","pipeline/source_registry.json"],"next_action":"Keep the full privately staged artifact and candidate import validation restricted; run a bounded guarded API check with explicit test-only labeling, capture rerun exit output, and resolve coverage/effective-date/category semantics before any release review. This is not a production-health or publication claim."}, - {"source_id":"de.locations","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["pipeline/source_registry.json"],"next_action":"Replace session-bound legacy URL with a verified stable BVL endpoint and confirm terms/schema."}, + {"source_id":"de.locations","metadata":"partial","acquisition":"artifact_private_only","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/germany-source-assessment.md","pipeline/sources/germany/adapter.py","pipeline/sources/germany/refresh.py","pipeline/source_registry.json"],"next_action":"Use the stable BVL landing to select an export, then run the assisted/private refresh. The export URL, reuse terms, privacy, and project approval remain unresolved; no release or public API exposure is allowed."}, {"source_id":"ca.locations","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ca.md","pipeline/source_registry.json"],"next_action":"Keep federal/export and Ontario candidates separate; verify an authoritative permitted artifact, terms, schema, coverage, and privacy before acquisition."}, {"source_id":"es.locations","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-es.md","pipeline/source_registry.json"],"next_action":"Resolve a permitted current AESAN/MAPA artifact or query route, then verify export/schema, rights, effective-date semantics, sector coverage, privacy, and legacy/source boundaries before acquisition."}, {"source_id":"us.fsis","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-us.md","pipeline/source_registry.json"],"next_action":"Obtain authorized access to a current FSIS MPI export after 403 responses; then record URL, retrieval/effective dates, content type, byte size, SHA-256, terms, and schema before adapter or publication review."}, diff --git a/docs/source-status.md b/docs/source-status.md index 9777660..c0f00e5 100644 --- a/docs/source-status.md +++ b/docs/source-status.md @@ -15,7 +15,7 @@ No last-success timestamp is invented. Private artifacts are not proof of a publ | Source ID | Metadata | Acquisition | Runtime health | Publication | Evidence / next action | |---|---|---|---|---|---| -| `be.locations` | verified | blocked | not_run | blocked | FASFC operator/codebook reconnaissance; obtain an authorized bounded operator CSV, capture its schema and provenance, then implement category/privacy validation | +| `be.locations` | verified | blocked | not_run | blocked | Shared private adapter and assisted two-file refresh are implemented with synthetic schema coverage; obtain an authorized operator capture and compare live schema before any real run | | `fr.locations` | verified | blocked | not_run | blocked | DGAL/Alim’confiance/SIRENE/HVE/Agence Bio/Géorisques reconnaissance; acquire permitted official artifact and review terms/privacy | | `it.853-2004` | verified | artifact_private_only | not_run | blocked | Catalog acquisition, shared lifecycle, adapter, private candidate import, and guarded API checks remain review-gated; repeated activity identity, coordinate/address privacy, coverage, and project approval remain open | | `it.1069-2009` | verified | not_run | not_run | blocked | Separate by-products catalog candidate; no adapter or integration decision; assess scope, schema, terms, identity links, and privacy | @@ -23,7 +23,7 @@ No last-success timestamp is invented. Private artifacts are not proof of a publ | `nz.locations` | verified | blocked | not_run | blocked | MPI/Stats NZ reconnaissance; resolve 403/access and aggregate-vs-facility boundaries | | `uk.locations` | partial | artifact_private_only | unknown | blocked | FSA and FSS private V2 lifecycle paths and synthetic handoff tests pass; no real UK candidate has been imported or previewed; privacy/coordinate, source-rights, duplicate, coverage, and release review remain open; NI/Scotland stay separate | | `dk.smiley` | verified | verified | unknown | blocked | Shared private lifecycle and registered adapter are validated on synthetic/retained evidence; coverage/effective-date uncertainty and terms/privacy/release review remain open | -| `de.locations` | partial | not_run | not_run | blocked | Legacy session URL and current BVL endpoint/schema/terms remain unresolved | +| `de.locations` | partial | artifact_private_only | not_run | blocked | Stable BVL `/bltu` landing and portal route are verified; typed private adapter and assisted export refresh are implemented, while export-specific terms/privacy/release review remain unresolved | | `ca.locations` | verified | not_run | not_run | blocked | Federal/export and Ontario candidates are documented separately; verify permitted artifact, terms, schema, coverage, and privacy before acquisition | | `es.locations` | partial | blocked | not_run | blocked | AESAN RGSEAA and MAPA sector routes are documented, but direct acquisition was refused; rights, export/schema, effective dates, sector coverage, privacy, and legacy/source boundaries remain unresolved | | `us.fsis` | verified | blocked | not_run | blocked | Official FSIS MPI route is documented, but current CSV access returned 403; obtain authorized export access and record provenance/schema before adapter or publication review | diff --git a/pipeline/source_registry.json b/pipeline/source_registry.json index 4ed47c8..97e571c 100644 --- a/pipeline/source_registry.json +++ b/pipeline/source_registry.json @@ -7,13 +7,13 @@ "source_id": "be.locations", "jurisdiction_scope": "Belgium; FASFC-registered, approved, or authorized operators, including animal-origin food and other food-chain activities", "legacy_paths": [], - "url": "https://data.gov.be/en/datasets/favv-afsca-operators", - "access_method": "published weekly CSV download; companion PAP/LAP activity-code CSV", + "url": "https://www.static.favv.be/bo-documents/inter_actieve_actoren_EN.csv", + "access_method": "published weekly operator CSV plus companion LAP/PAP activity-code CSV; assisted capture supported", "cadence": "weekly", "attribution_licensing_notes": "CC Attribution 4.0; attribute FASFC and the last update date, do not imply FASFC affiliation/approval, and do not mislead", - "adapter_status": "not_started", - "expected_artifact_schema": "CSV operator rows with establishment/operator identity, address/geography, PAP/LAP activity codes, descriptions, and approval/authorization identifiers; exact header and field semantics require capture", - "blockers": ["Capture and verify the current operator CSV header, delimiter/encoding, identifiers, address/geography fields, status/date semantics, and slaughterhouse/cutting/processing/storage category mappings before implementation or acquisition staging."] + "adapter_status": "implemented_partial", + "expected_artifact_schema": "UTF-8/CP1252 CSV operator rows joined by LAP/PAP code to the separate FASFC activity-code CSV; live operator header remains to be captured", + "blockers": ["The official operator body was inaccessible from the current execution environment; obtain an authorized bounded capture and compare its header/field semantics with the synthetic contract before any real run. Privacy, source-terms interpretation, coverage, and project publication approval remain human gates."] }, { "source_id": "ca.locations", @@ -31,13 +31,13 @@ "source_id": "de.locations", "jurisdiction_scope": "Germany", "legacy_paths": ["static_data/de/locations.csv"], - "url": "https://bltu.bvl.bund.de/bltu/app/process/bvl-btl_p_veroeffentlichung?execution=e6s1", - "access_method": "browser-assisted export or download", + "url": "https://www.bvl.bund.de/bltu", + "access_method": "BVL portal selected CSV/XLS export or assisted capture", "cadence": "unknown", "attribution_licensing_notes": "unknown; verify BVL reuse and attribution terms", - "adapter_status": "not_started", - "expected_artifact_schema": "CSV or tabular export; establishment identifier groups are represented in the legacy file", - "blockers": ["The URL is copied from a legacy migration comment and contains a session execution parameter; confirm a stable endpoint and current schema."] + "adapter_status": "implemented_partial", + "expected_artifact_schema": "BLtU general-list semicolon CSV with repeated activity-code columns, current approval number, establishment name, state, address, and activity flags", + "blockers": ["The stable BVL landing and portal export route are verified, but the selected export URL is session/request-specific and dataset reuse terms remain pending human confirmation. Keep raw data private and publication blocked."] }, { "source_id": "dk.smiley", diff --git a/pipeline/sources/belgium/README.md b/pipeline/sources/belgium/README.md new file mode 100644 index 0000000..b4427f7 --- /dev/null +++ b/pipeline/sources/belgium/README.md @@ -0,0 +1,15 @@ +# Belgium FASFC private staging + +The official pair is the weekly FASFC operator CSV and its separate LAP/PAP +activity-code CSV. `refresh.py` accepts both as already-preserved files, records +independent hashes and retrieval metadata, joins codes exactly, and runs the +shared private lifecycle. `--fetch` is available for an operator-authored terms +review, but the current operator URL may require an assisted browser download. + +```text +python -m pipeline.sources.belgium.refresh --operators --activity-codes --run-dir --retrieved-at-utc 2026-09-15T00:00:00Z +``` + +The checked-in fixtures are synthetic only. A successful run is a private +candidate with publication blocked; it is not FASFC approval, project review, +or a public release. diff --git a/pipeline/sources/belgium/__init__.py b/pipeline/sources/belgium/__init__.py new file mode 100644 index 0000000..4f12f6c --- /dev/null +++ b/pipeline/sources/belgium/__init__.py @@ -0,0 +1,5 @@ +"""Private FASFC operator/codebook acquisition and staging.""" + +from .adapter import BelgiumOperatorsAdapter + +__all__ = ["BelgiumOperatorsAdapter"] diff --git a/pipeline/sources/belgium/acquire.py b/pipeline/sources/belgium/acquire.py new file mode 100644 index 0000000..4d48840 --- /dev/null +++ b/pipeline/sources/belgium/acquire.py @@ -0,0 +1,42 @@ +"""Bounded FASFC pair acquisition; bytes are written only to ignored storage.""" +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +from pipeline.common.acquisition import AcquisitionError, fetch_source +from pipeline.contracts.source_lifecycle import atomic_json + +from .adapter import CONFIG + + +def fetch_pair(*, output_root: str | Path, terms_review_path: str | Path, run_id: str | None = None, timeout_seconds: float = 60, max_bytes: int = 128 * 1024 * 1024) -> dict: + """Fetch operator and companion codebook with independent provenance.""" + root = Path(output_root) + operator = fetch_source(source_id=CONFIG["source_id"], url=CONFIG["operator_url"], output_root=root, artifact_name="operators.csv", run_id=run_id, timeout_seconds=timeout_seconds, max_bytes=max_bytes, terms_review_path=terms_review_path, code_version=CONFIG["adapter_version"], config_version=CONFIG["schema_version"], coverage=CONFIG["coverage"], rights_caveat=CONFIG["terms"], privacy_caveat="private staging; operator address and coordinate privacy review pending") + codebook = fetch_source(source_id="be.activity-codes", url=CONFIG["activity_code_url"], output_root=root, artifact_name="activity-codes.csv", run_id=run_id, timeout_seconds=timeout_seconds, max_bytes=max_bytes, terms_review_path=terms_review_path, code_version=CONFIG["adapter_version"], config_version=CONFIG["schema_version"], coverage="FASFC LAP/PAP activity codebook; not a facility list", rights_caveat=CONFIG["terms"], privacy_caveat="no facility rows expected") + pair = {"operator": operator, "activity_codes": codebook, "source_id": CONFIG["source_id"], "catalog_url": CONFIG["catalog_url"]} + atomic_json(root / CONFIG["source_id"] / str(run_id or operator["run_id"]) / "pair-metadata.json", pair) + return pair + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output-root", type=Path, default=Path("data/raw")) + parser.add_argument("--terms-review", type=Path, required=True) + parser.add_argument("--run-id") + parser.add_argument("--timeout-seconds", type=float, default=60) + parser.add_argument("--max-bytes", type=int, default=128 * 1024 * 1024) + args = parser.parse_args() + try: + pair = fetch_pair(output_root=args.output_root, terms_review_path=args.terms_review, run_id=args.run_id, timeout_seconds=args.timeout_seconds, max_bytes=args.max_bytes) + except (OSError, AcquisitionError) as error: + print(json.dumps({"status": "failed", "error": str(error)}, sort_keys=True)) + return 2 + print(json.dumps({"status": "archived", "operator": pair["operator"], "activity_codes": pair["activity_codes"]}, ensure_ascii=False, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/pipeline/sources/belgium/adapter.py b/pipeline/sources/belgium/adapter.py new file mode 100644 index 0000000..9709565 --- /dev/null +++ b/pipeline/sources/belgium/adapter.py @@ -0,0 +1,271 @@ +"""Private FASFC operator adapter with deterministic LAP/PAP codebook join. + +The operator feed is broader than slaughterhouses. This adapter retains every +source activity and only derives conservative, reviewable categories. Address, +coordinates, enterprise numbers, and other source values stay in private parsed +evidence; normalized rows deliberately carry no address or coordinate payload. +""" +from __future__ import annotations + +import csv +import hashlib +import json +import re +import unicodedata +from collections import Counter +from pathlib import Path +from typing import Any, Iterable + +from pipeline.contracts.adapter_contract import SourceArtifact +from pipeline.contracts.candidate_handoff import write_handoff +from pipeline.contracts.source_lifecycle import atomic_json, atomic_jsonl, private_manifest + +ROOT = Path(__file__).parent +CONFIG = json.loads((ROOT / "config.json").read_text(encoding="utf-8")) + + +class BelgiumSchemaError(ValueError): + """A supplied operator or codebook artifact is not a supported CSV schema.""" + + +_ALIASES = { + "operator_id": ("operator_id", "operator id", "operator number", "enterprise number", "enterprise id", "nummer operator", "numero operateur"), + "establishment_id": ("establishment_id", "establishment id", "establishment number", "establishment nr", "n establishment", "nummer vestiging", "numero etablissement", "n etablissement"), + "name": ("name", "operator name", "establishment name", "name operator", "naam", "nom", "name establishment"), + "address": ("address", "address line", "street", "street address", "street and number", "adres", "adresse"), + "postcode": ("postcode", "postal code", "zip", "code postal"), + "municipality": ("municipality", "city", "town", "gemeente", "commune", "municipality name"), + "region": ("region", "province", "provincie", "province"), + "latitude": ("latitude", "lat", "latitude y"), + "longitude": ("longitude", "lon", "lng", "longitude x"), + "activity_code": ("activity_code", "activity code", "activity codes", "lap code", "lap id", "pap code", "pap id", "code lap", "code pap", "activiteiten code", "code activite"), + "activity_description": ("activity_description", "activity description", "activity", "description", "omschrijving activiteit", "description activite"), + "approval_number": ("approval_number", "approval number", "approval nr", "agrément", "erkenningsnummer", "numero agrement"), + "authorization_number": ("authorization_number", "authorization number", "authorization nr", "authorisation number", "autorisatienummer", "numero autorisation"), + "status": ("status", "current status", "state", "statuut", "statut"), + "effective_date": ("effective_date", "effective date", "valid from", "start date", "geldigheid vanaf", "date debut"), +} +_CODE_ALIASES = { + "lap_code": ("lap_code", "lap code", "lap id", "pap_code", "pap code", "pap id", "activity_code", "activity code", "code lap", "code pap"), + "place_code": ("place_code", "place code", "pl code", "location code", "code lieu", "plaats code"), + "place_description": ("place_description", "place description", "location description", "lieu", "plaats"), + "activity_description": ("activity_description", "activity description", "activity", "activiteit", "activite"), + "product_description": ("product_description", "product description", "product", "produit", "productomschrijving"), + "approval_code": ("approval_code", "approval code", "approval form", "code agrement", "erkenningscode"), + "approval_description": ("approval_description", "approval description", "approval", "agrement", "erkenning"), +} +_CATEGORY_RULES: tuple[tuple[str, tuple[str, ...]], ...] = ( + ("slaughter", ("slaughter", "abattoir", "slachthuis", "killing", "abattage")), + ("cutting", ("cutting", "butchery", "deboning", "uitsnijder", "decoupe", "découpe")), + ("processing", ("processing", "meat product", "manufactur", "transformation", "verwerking", "preparation")), + ("logistics_and_storage", ("cold store", "cold-storage", "storage", "warehouse", "freezer", "refrigerat", "opslag", "entreposage")), + ("animal_by_products", ("animal by-product", "animal byproduct", "abp", "sous-produit", "dierlijke bijproduct")), + ("export", ("export", "third country trade", "handel derde landen")), +) +_PUBLIC_CATEGORIES = {"slaughter", "cutting", "processing", "logistics_and_storage"} +_RISK = re.compile(r"\b(flat|apartment|appartement|residential|home|maison|c/o|care of|caravan|woning)\b", re.I) + + +def _key(value: str) -> str: + text = unicodedata.normalize("NFKD", value).encode("ascii", "ignore").decode("ascii") + return re.sub(r"[^a-z0-9]+", " ", text.casefold()).strip() + + +def _clean(value: Any) -> str | None: + if not isinstance(value, str): + return None + value = re.sub(r"\s+", " ", value).strip() + return value or None + + +def _csv(content: bytes) -> tuple[list[str], list[list[str]], str, str]: + for encoding in ("utf-8-sig", "cp1252"): + try: + text = content.decode(encoding) + except UnicodeDecodeError: + continue + try: + sample = text[:8192] + dialect = csv.Sniffer().sniff(sample, delimiters=",;\t|") + except csv.Error: + dialect = type("FallbackDialect", (), {"delimiter": ",", "quotechar": '"'}) + try: + rows = list(csv.reader(text.splitlines(), delimiter=dialect.delimiter, quotechar=dialect.quotechar, strict=True)) + except csv.Error as exc: + raise BelgiumSchemaError("malformed CSV") from exc + if not rows or not rows[0]: + raise BelgiumSchemaError("missing CSV header") + headers = [h.strip() for h in rows[0]] + if len(set(_key(h) for h in headers)) != len(headers): + raise BelgiumSchemaError("duplicate normalized CSV headers") + return headers, rows[1:], encoding, dialect.delimiter + raise BelgiumSchemaError("unsupported CSV encoding") + + +def _header_map(headers: Iterable[str], aliases: dict[str, tuple[str, ...]], required: set[str]) -> dict[str, str]: + normalized = {_key(header): header for header in headers} + result: dict[str, str] = {} + for field, choices in aliases.items(): + for choice in choices: + if _key(choice) in normalized: + result[field] = normalized[_key(choice)] + break + missing = sorted(required - result.keys()) + if missing: + raise BelgiumSchemaError("missing supported columns: " + ", ".join(missing)) + return result + + +def _row_values(headers: list[str], values: list[str]) -> dict[str, str | None]: + return {header: (values[index] if index < len(values) else None) for index, header in enumerate(headers)} + + +def _split_codes(value: str | None) -> tuple[str, ...]: + return tuple(dict.fromkeys(part.strip() for part in re.split(r"[;,|]\s*", value or "") if part.strip())) + + +def _categories(values: Iterable[str | None]) -> tuple[str, ...]: + found: list[str] = [] + for value in values: + text = (value or "").casefold() + for category, needles in _CATEGORY_RULES: + if any(needle.casefold() in text for needle in needles) and category not in found: + found.append(category) + return tuple(found) + + +def _fingerprint(headers: list[str]) -> str: + return hashlib.sha256(json.dumps(headers, ensure_ascii=False, separators=(",", ":")).encode()).hexdigest() + + +class BelgiumOperatorsAdapter: + source_id = CONFIG["source_id"] + adapter_version = CONFIG["adapter_version"] + schema_version = CONFIG["schema_version"] + + def __init__(self, activity_codes_path: str | Path, activity_artifact: SourceArtifact | None = None): + self.activity_codes_path = Path(activity_codes_path) + self.activity_artifact = activity_artifact + + def _codebook(self) -> tuple[dict[str, dict[str, str | None]], dict[str, Any]]: + content = self.activity_codes_path.read_bytes() + if self.activity_artifact is not None: + digest = hashlib.sha256(content).hexdigest() + if digest != self.activity_artifact.sha256 or len(content) != self.activity_artifact.byte_size: + raise ValueError("activity-code artifact provenance mismatch") + headers, rows, encoding, delimiter = _csv(content) + fields = _header_map(headers, _CODE_ALIASES, {"lap_code"}) + codebook: dict[str, dict[str, str | None]] = {} + ambiguous: set[str] = set() + for values in rows: + raw = _row_values(headers, values) + code = _clean(raw.get(fields["lap_code"])) + if not code: + continue + item = {field: _clean(raw.get(header)) for field, header in fields.items()} + if code in codebook and codebook[code] != item: + ambiguous.add(code) + codebook[code] = item + metadata = {"sha256": hashlib.sha256(content).hexdigest(), "byte_size": len(content), "schema_fingerprint": _fingerprint(headers), "column_count": len(headers), "row_count": len(rows), "encoding": encoding, "delimiter": delimiter, "ambiguous_codes": sorted(ambiguous)} + for code in ambiguous: + codebook.pop(code, None) + return codebook, metadata + + def parse_bytes(self, content: bytes) -> dict[str, Any]: + codebook, codebook_meta = self._codebook() + headers, rows, encoding, delimiter = _csv(content) + fields = _header_map(headers, _ALIASES, {"establishment_id", "name", "activity_code"}) + accepted: list[dict[str, Any]] = [] + quarantined: list[dict[str, Any]] = [] + anomalies: Counter[str] = Counter() + seen: Counter[tuple[str | None, str | None]] = Counter() + prepared: list[tuple[int, dict[str, str | None], tuple[str, ...]]] = [] + for line, values in enumerate(rows, start=2): + raw = _row_values(headers, values) + codes = _split_codes(_clean(raw.get(fields["activity_code"]))) + prepared.append((line, raw, codes)) + for code in codes: + seen[(_clean(raw.get(fields["establishment_id"])), code)] += 1 + for line, raw, codes in prepared: + establishment_id = _clean(raw.get(fields["establishment_id"])) + name = _clean(raw.get(fields["name"])) + reasons: list[str] = [] + if len(raw) != len(headers) or any(value is None for value in raw.values()): + reasons.append("malformed_row") + if not establishment_id: + reasons.append("missing_establishment_id") + if not name: + reasons.append("missing_name") + if not codes: + reasons.append("missing_activity_code") + if any(code not in codebook for code in codes): + reasons.append("unresolved_activity_code") + if any(seen[(establishment_id, code)] > 1 for code in codes): + reasons.append("ambiguous_repeated_establishment_activity") + address = _clean(raw.get(fields.get("address", ""))) if "address" in fields else None + if address and _RISK.search(address): + reasons.append("address_privacy_risk") + joined = [codebook[code] for code in codes if code in codebook] + descriptions = [_clean(raw.get(fields.get("activity_description", "")))] + [item.get("activity_description") for item in joined] + categories = _categories(descriptions + [item.get("place_description") for item in joined] + [item.get("product_description") for item in joined]) + public_categories = tuple(category for category in categories if category in _PUBLIC_CATEGORIES) + record = { + "source_id": self.source_id, + "source_row": line, + "source_record_key": f"{establishment_id or 'unknown'}|{','.join(codes) or 'unknown'}|{line}", + "source_values": raw, + "normalized": { + "establishment_id": establishment_id, + "operator_id": _clean(raw.get(fields.get("operator_id", ""))) if "operator_id" in fields else None, + "name": name, + "trading_name": name, + "country_code": "BE", + "nation": "Belgium", + "municipality": _clean(raw.get(fields.get("municipality", ""))) if "municipality" in fields else None, + "city": _clean(raw.get(fields.get("municipality", ""))) if "municipality" in fields else None, + "postcode": _clean(raw.get(fields.get("postcode", ""))) if "postcode" in fields else None, + "address": None, + "address_state": "source-present-pending-privacy-review" if address else "unknown", + "coordinates": None, + "coordinate_state": "source-value-present-pending-review" if any(_clean(raw.get(fields.get(key, ""))) for key in ("latitude", "longitude") if key in fields) else "unknown", + "activity_codes": codes, + "activity_descriptions": tuple(dict.fromkeys(item.get("activity_description") for item in joined if item.get("activity_description"))), + "activity_categories": public_categories, + "source_activity_categories": categories, + "scope_flags": {"slaughterhouse": "slaughter" in categories, "cutting": "cutting" in categories, "processing": "processing" in categories, "storage": "logistics_and_storage" in categories, "animal_by_products": "animal_by_products" in categories, "export": "export" in categories}, + "approval_number": _clean(raw.get(fields.get("approval_number", ""))) if "approval_number" in fields else None, + "authorization_number": _clean(raw.get(fields.get("authorization_number", ""))) if "authorization_number" in fields else None, + "status": _clean(raw.get(fields.get("status", ""))) if "status" in fields else None, + "effective_date": _clean(raw.get(fields.get("effective_date", ""))) if "effective_date" in fields else None, + "privacy_gate": "pending-review", + "coordinate_gate": "review_required", + "publication_gate": "blocked", + }, + } + if reasons: + unique = tuple(dict.fromkeys(reasons)) + for reason in unique: + anomalies[reason] += 1 + quarantined.append({"reasons": unique, "record": record}) + else: + accepted.append(record) + return {"accepted": accepted, "quarantined": quarantined, "input_rows": len(rows), "source_sha256": hashlib.sha256(content).hexdigest(), "operator_schema_fingerprint": _fingerprint(headers), "operator_column_count": len(headers), "operator_encoding": encoding, "operator_delimiter": delimiter, "codebook": codebook_meta, "coverage_counts": dict(Counter(category for item in accepted for category in item["normalized"]["source_activity_categories"])), "anomaly_counts": dict(sorted(anomalies.items()))} + + def run(self, raw_path: str | Path, run_dir: str | Path, artifact: SourceArtifact) -> dict[str, Any]: + raw = Path(raw_path).read_bytes() + digest = hashlib.sha256(raw).hexdigest() + if artifact.sha256 != digest or artifact.byte_size != len(raw): + raise ValueError("operator artifact provenance mismatch") + result = self.parse_bytes(raw) + root = Path(run_dir) + parsed = result["accepted"] + [item["record"] for item in result["quarantined"]] + _, parsed_sha, _ = atomic_jsonl(root / "parsed" / "records.jsonl", parsed) + _, normalized_sha, _ = atomic_jsonl(root / "normalized" / "records.jsonl", result["accepted"]) + atomic_jsonl(root / "quarantined" / "records.jsonl", result["quarantined"]) + manifest = private_manifest(source_id=self.source_id, adapter_version=self.adapter_version, schema_version=self.schema_version, artifact=artifact, input_rows=result["input_rows"], normalized_rows=len(result["accepted"]), quarantined_rows=len(result["quarantined"]), normalized_sha256=normalized_sha, parsed_sha256=parsed_sha, anomaly_counts=result["anomaly_counts"]) + manifest.update({"country_code": "BE", "coverage": CONFIG["coverage"], "geocoding": "disabled", "operator_schema_fingerprint": result["operator_schema_fingerprint"], "operator_column_count": result["operator_column_count"], "operator_encoding": result["operator_encoding"], "operator_delimiter": result["operator_delimiter"], "coverage_counts": result["coverage_counts"], "activity_codebook": result["codebook"], "codebook_source_url": self.activity_artifact.source_url if self.activity_artifact else "unrecorded-companion-artifact", "codebook_retrieved_at_utc": self.activity_artifact.retrieved_at_utc if self.activity_artifact else None, "codebook_sha256": result["codebook"]["sha256"], "codebook_byte_size": result["codebook"]["byte_size"]}) + atomic_json(root / "manifest.json", manifest) + return manifest + + def write_candidate_handoff(self, run_dir: str | Path, artifact: SourceArtifact, parsed: dict[str, Any]) -> dict[str, Any]: + return write_handoff(run_dir, [item["record"] if "record" in item else item for item in parsed["accepted"]], artifact, source_id=self.source_id) diff --git a/pipeline/sources/belgium/config.json b/pipeline/sources/belgium/config.json new file mode 100644 index 0000000..e862dcf --- /dev/null +++ b/pipeline/sources/belgium/config.json @@ -0,0 +1,12 @@ +{ + "source_id": "be.locations", + "operator_url": "https://www.static.favv.be/bo-documents/inter_actieve_actoren_EN.csv", + "activity_code_url": "https://www.static.favv.be/bo-documents/inter_PAP_omschrijving_EN.csv", + "catalog_url": "https://data.gov.be/en/datasets/favv-afsca-operators", + "activity_catalog_url": "https://data.gov.be/en/datasets/fasfc-activity-codes", + "adapter_version": "be-fasfc-private-v1", + "schema_version": "be-fasfc-csv-v1", + "coverage": "Belgium FASFC operators with a current registration, approval, or authorization; activity codebook joined separately", + "terms": "CC Attribution 4.0 is indicated by the data.gov.be dataset pages; attribution and project publication review remain required", + "geocoding": "disabled" +} diff --git a/pipeline/sources/belgium/fixtures/synthetic_activity_codes.csv b/pipeline/sources/belgium/fixtures/synthetic_activity_codes.csv new file mode 100644 index 0000000..34331b9 --- /dev/null +++ b/pipeline/sources/belgium/fixtures/synthetic_activity_codes.csv @@ -0,0 +1,5 @@ +lap_code,place_code,place_description,activity_description,product_description,approval_code,approval_description +LAP-SH,PL-SH,Slaughterhouse,Slaughterhouse,Meat,APP,Approval +LAP-CS,PL-CS,Cold store,Storage,Meat,AUTH,Authorization +LAP-CP,PL-CP,Cutting plant,Cutting,Meat,APP,Approval +LAP-ABP,PL-ABP,Animal by-product facility,Animal by-products,By-products,APP,Approval diff --git a/pipeline/sources/belgium/fixtures/synthetic_operators.csv b/pipeline/sources/belgium/fixtures/synthetic_operators.csv new file mode 100644 index 0000000..c768e7d --- /dev/null +++ b/pipeline/sources/belgium/fixtures/synthetic_operators.csv @@ -0,0 +1,6 @@ +establishment_id,name,address,postcode,municipality,region,activity_code,activity_description,approval_number,status +BE-SYN-001,Synthetic Abattoir,Industrial Road 1,1000,Brussels,Brussels,LAP-SH,Slaughterhouse,BE-APP-1,current +BE-SYN-002,Synthetic Cold Store,Warehouse Road 2,2000,Antwerp,Flanders,LAP-CS,Cold storage,BE-AUT-2,current +BE-SYN-003,Synthetic Mixed Facility,Industrial Road 3,3000,Leuven,Flanders,LAP-SH;LAP-CP,Slaughter and cutting,BE-APP-3,current +BE-SYN-004,Private Residence Example,Apartment 4,4000,Liege,Wallonia,LAP-CP,Cutting plant,BE-APP-4,current +BE-SYN-005,Unknown Activity,Industrial Road 5,5000,Namur,Wallonia,LAP-UNKNOWN,Unknown,BE-APP-5,current diff --git a/pipeline/sources/belgium/refresh.py b/pipeline/sources/belgium/refresh.py new file mode 100644 index 0000000..f100ac3 --- /dev/null +++ b/pipeline/sources/belgium/refresh.py @@ -0,0 +1,95 @@ +"""Run the Belgium FASFC pair through the shared private lifecycle. + +The normal assisted mode takes two operator-provided files: the FASFC operator +CSV and the official LAP/PAP codebook CSV. ``--fetch`` is available only after +the operator records the source terms decision and uses the same bounded fetch +primitive for both files. Neither mode creates a release. +""" +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path +from typing import Any + +from pipeline.common.acquisition import AcquisitionError, fetch_source, utc_now +from pipeline.common.orchestrator import run_private_lifecycle +from pipeline.contracts.adapter_contract import SourceArtifact +from pipeline.contracts.source_lifecycle import atomic_json + +from .adapter import CONFIG, BelgiumOperatorsAdapter + + +class RefreshError(ValueError): + """A Belgium refresh cannot safely continue.""" + + +def _local_metadata(path: Path, *, source_id: str, source_url: str, retrieved_at: str, coverage: str) -> dict[str, Any]: + raw = path.read_bytes() + return {"acquisition_method": "assisted_local_capture", "source_id": source_id, "artifact": path.name, "artifact_path": str(path.resolve()), "requested_url": source_url, "final_url": source_url, "redirects": [], "response_headers": {}, "requested_at_utc": retrieved_at, "retrieved_at_utc": retrieved_at, "effective_date": "unknown", "publication_date": None, "sha256": hashlib.sha256(raw).hexdigest(), "byte_size": len(raw), "adapter_version": CONFIG["adapter_version"], "code_version": CONFIG["adapter_version"], "config_version": CONFIG["schema_version"], "coverage": coverage, "rights_caveat": CONFIG["terms"], "privacy_caveat": "private staging; address and coordinate review pending", "terms_review": "operator-assisted capture; source terms review remains a separate gate"} + + +def _artifact(metadata: dict[str, Any], *, default_url: str, default_coverage: str) -> SourceArtifact: + return SourceArtifact(source_url=str(metadata.get("final_url") or metadata.get("requested_url") or default_url), retrieved_at_utc=str(metadata.get("retrieved_at_utc") or ""), sha256=str(metadata["sha256"]), byte_size=int(metadata["byte_size"]), publication_date=metadata.get("publication_date"), effective_date=metadata.get("effective_date"), code_version=str(metadata.get("code_version") or CONFIG["adapter_version"]), config_version=str(metadata.get("config_version") or CONFIG["schema_version"]), rights_caveat=metadata.get("rights_caveat") or CONFIG["terms"], privacy_caveat=metadata.get("privacy_caveat") or "private staging; privacy review pending", coverage=metadata.get("coverage") or default_coverage, redirects=tuple(metadata.get("redirects") or ())) + + +def refresh(*, run_dir: str | Path, operators_path: str | Path | None = None, activity_codes_path: str | Path | None = None, fetch_pair: bool = False, output_root: str | Path = "data/raw", run_id: str | None = None, terms_review_path: str | Path | None = None, retrieved_at_utc: str | None = None, timeout_seconds: float = 60, max_bytes: int = 128 * 1024 * 1024, previous_normalized: str | Path | None = None) -> dict[str, Any]: + if fetch_pair == (operators_path is not None or activity_codes_path is not None): + raise RefreshError("specify --fetch or both --operators and --activity-codes") + root = Path(run_dir) + retrieved = retrieved_at_utc or utc_now() + if fetch_pair: + if terms_review_path is None: + raise RefreshError("--terms-review is required with --fetch") + try: + operator_meta = fetch_source(source_id=CONFIG["source_id"], url=CONFIG["operator_url"], output_root=Path(output_root), artifact_name="operators.csv", run_id=run_id, timeout_seconds=timeout_seconds, max_bytes=max_bytes, terms_review_path=terms_review_path, code_version=CONFIG["adapter_version"], config_version=CONFIG["schema_version"], coverage=CONFIG["coverage"], rights_caveat=CONFIG["terms"], privacy_caveat="private staging; privacy review pending") + code_meta = fetch_source(source_id="be.activity-codes", url=CONFIG["activity_code_url"], output_root=Path(output_root), artifact_name="activity-codes.csv", run_id=run_id, timeout_seconds=timeout_seconds, max_bytes=max_bytes, terms_review_path=terms_review_path, code_version=CONFIG["adapter_version"], config_version=CONFIG["schema_version"], coverage="FASFC LAP/PAP codebook; not a facility list", rights_caveat=CONFIG["terms"], privacy_caveat="no facility rows expected") + except AcquisitionError as error: + raise RefreshError(str(error)) from error + operator_path = Path(operator_meta["artifact_path"]); code_path = Path(code_meta["artifact_path"]) + else: + if operators_path is None or activity_codes_path is None: + raise RefreshError("both local artifacts are required") + operator_path, code_path = Path(operators_path).resolve(), Path(activity_codes_path).resolve() + if not operator_path.is_file() or not code_path.is_file(): + raise RefreshError("operator and activity-code artifacts must exist") + operator_meta = _local_metadata(operator_path, source_id=CONFIG["source_id"], source_url=CONFIG["operator_url"], retrieved_at=retrieved, coverage=CONFIG["coverage"]) + code_meta = _local_metadata(code_path, source_id="be.activity-codes", source_url=CONFIG["activity_code_url"], retrieved_at=retrieved, coverage="FASFC LAP/PAP codebook; not a facility list") + atomic_json(root / "acquisition-metadata.json", {"operator": operator_meta, "activity_codes": code_meta}) + operator_artifact = _artifact(operator_meta, default_url=CONFIG["operator_url"], default_coverage=CONFIG["coverage"]) + code_artifact = _artifact(code_meta, default_url=CONFIG["activity_code_url"], default_coverage="FASFC LAP/PAP codebook; not a facility list") + adapter = BelgiumOperatorsAdapter(code_path, code_artifact) + lifecycle = run_private_lifecycle(operator_path, root / "lifecycle", operator_artifact, adapter, health_as_of_utc=retrieved) + manifest = lifecycle.get("manifest") or {} + report = {"source_id": CONFIG["source_id"], "source_url": operator_artifact.source_url, "retrieved_at_utc": operator_artifact.retrieved_at_utc, "operator_sha256": operator_artifact.sha256, "activity_code_sha256": code_artifact.sha256, "input_rows": manifest.get("input_rows"), "normalized_rows": manifest.get("normalized_rows"), "quarantined_rows": manifest.get("quarantined_rows"), "operator_schema_fingerprint": manifest.get("operator_schema_fingerprint"), "activity_code_schema_fingerprint": (manifest.get("activity_codebook") or {}).get("schema_fingerprint"), "drift_alarms": [], "disappeared_not_observed_count": 0, "disappearance_semantics": "not-observed; never inferred as closure", "geocoding": "disabled", "release_state": "not-created", "publication_state": "private-candidate", "publication_eligibility": "blocked", "lifecycle_status": lifecycle.get("status"), "lifecycle_run_dir": lifecycle.get("run_dir"), "previous_normalized_supplied": previous_normalized is not None} + atomic_json(root / "refresh.json", report) + return {"report": report, "lifecycle": lifecycle} + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + source = parser.add_mutually_exclusive_group(required=True) + source.add_argument("--fetch", action="store_true") + source.add_argument("--operators", type=Path) + parser.add_argument("--activity-codes", type=Path) + parser.add_argument("--run-dir", type=Path, required=True) + parser.add_argument("--output-root", type=Path, default=Path("data/raw")) + parser.add_argument("--run-id") + parser.add_argument("--terms-review", type=Path) + parser.add_argument("--retrieved-at-utc") + parser.add_argument("--timeout-seconds", type=float, default=60) + parser.add_argument("--max-bytes", type=int, default=128 * 1024 * 1024) + parser.add_argument("--previous-normalized", type=Path) + args = parser.parse_args() + try: + result = refresh(run_dir=args.run_dir, operators_path=args.operators, activity_codes_path=args.activity_codes, fetch_pair=args.fetch, output_root=args.output_root, run_id=args.run_id, terms_review_path=args.terms_review, retrieved_at_utc=args.retrieved_at_utc, timeout_seconds=args.timeout_seconds, max_bytes=args.max_bytes, previous_normalized=args.previous_normalized) + except (OSError, RefreshError) as error: + print(json.dumps({"status": "failed", "error": str(error)}, sort_keys=True)) + return 2 + print(json.dumps(result["report"], sort_keys=True)) + return 0 if result["report"]["lifecycle_status"] == "candidate-ready" else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/pipeline/sources/belgium/test_adapter.py b/pipeline/sources/belgium/test_adapter.py new file mode 100644 index 0000000..30ae8c7 --- /dev/null +++ b/pipeline/sources/belgium/test_adapter.py @@ -0,0 +1,63 @@ +import hashlib +import json +import tempfile +import unittest +from pathlib import Path + +from pipeline.common.orchestrator import run_private_lifecycle +from pipeline.contracts.adapter_contract import SourceArtifact +from .adapter import BelgiumOperatorsAdapter + + +ROOT = Path(__file__).parent + + +class BelgiumAdapterTests(unittest.TestCase): + def _adapter(self): + return BelgiumOperatorsAdapter(ROOT / "fixtures" / "synthetic_activity_codes.csv", SourceArtifact("https://example.invalid/activity.csv", "2026-09-14T00:00:00Z", hashlib.sha256((ROOT / "fixtures" / "synthetic_activity_codes.csv").read_bytes()).hexdigest(), (ROOT / "fixtures" / "synthetic_activity_codes.csv").stat().st_size)) + + def test_join_classifies_scope_without_exposing_private_fields(self): + adapter = self._adapter() + result = adapter.parse_bytes((ROOT / "fixtures" / "synthetic_operators.csv").read_bytes()) + self.assertEqual(len(result["accepted"]), 3) + self.assertEqual(len(result["quarantined"]), 2) + mixed = next(row for row in result["accepted"] if row["source_id"] == "be.locations" and row["normalized"]["establishment_id"] == "BE-SYN-003") + self.assertEqual(mixed["normalized"]["activity_categories"], ("slaughter", "cutting")) + self.assertTrue(mixed["normalized"]["scope_flags"]["slaughterhouse"]) + self.assertIsNone(mixed["normalized"]["address"]) + self.assertIn("address", mixed["source_values"]) + + def test_unknown_activity_and_privacy_risk_quarantine(self): + result = self._adapter().parse_bytes((ROOT / "fixtures" / "synthetic_operators.csv").read_bytes()) + reasons = [set(item["reasons"]) for item in result["quarantined"]] + self.assertIn("address_privacy_risk", set().union(*reasons)) + self.assertIn("unresolved_activity_code", set().union(*reasons)) + + def test_ambiguous_codebook_key_is_not_silently_selected(self): + with tempfile.TemporaryDirectory() as directory: + codebook = Path(directory) / "codes.csv" + codebook.write_text("lap_code,place_description\nLAP-SH,Slaughterhouse\nLAP-SH,Other place\n", encoding="utf-8") + adapter = BelgiumOperatorsAdapter(codebook) + result = adapter.parse_bytes((ROOT / "fixtures" / "synthetic_operators.csv").read_bytes()) + first = next(item for item in result["quarantined"] if item["record"]["normalized"]["establishment_id"] == "BE-SYN-001") + self.assertIn("unresolved_activity_code", first["reasons"]) + + def test_run_is_deterministic_and_shared_health_is_private(self): + raw = ROOT / "fixtures" / "synthetic_operators.csv" + code = ROOT / "fixtures" / "synthetic_activity_codes.csv" + adapter = self._adapter() + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + artifact = SourceArtifact("https://example.invalid/operators.csv", "2026-09-14T00:00:00Z", hashlib.sha256(raw.read_bytes()).hexdigest(), raw.stat().st_size, publication_date="2026-09-13", code_version=adapter.adapter_version, config_version=adapter.schema_version) + status = run_private_lifecycle(raw, root / "runs", artifact, adapter) + run_dir = Path(status["run_dir"]) + self.assertEqual(status["status"], "candidate-ready") + self.assertEqual(status["manifest"]["codebook_sha256"], hashlib.sha256(code.read_bytes()).hexdigest()) + self.assertTrue((run_dir / "source-health.json").exists()) + self.assertTrue((run_dir / "release-candidate" / "records.jsonl").exists()) + self.assertFalse(json.loads((run_dir / "source-health.json").read_text())["public_exposure"]) + self.assertNotIn("source_values", (run_dir / "qa.json").read_text()) + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/sources/belgium/test_refresh.py b/pipeline/sources/belgium/test_refresh.py new file mode 100644 index 0000000..1fc2caf --- /dev/null +++ b/pipeline/sources/belgium/test_refresh.py @@ -0,0 +1,30 @@ +import tempfile +import unittest +from pathlib import Path + +from .refresh import RefreshError, refresh + + +ROOT = Path(__file__).parent + + +class BelgiumRefreshTests(unittest.TestCase): + def test_assisted_pair_is_repeatable_and_keeps_artifacts_private(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + args = {"operators_path": ROOT / "fixtures" / "synthetic_operators.csv", "activity_codes_path": ROOT / "fixtures" / "synthetic_activity_codes.csv", "retrieved_at_utc": "2026-09-14T00:00:00Z"} + first = refresh(run_dir=root / "one", **args) + second = refresh(run_dir=root / "two", **args) + self.assertEqual(first["report"]["operator_sha256"], second["report"]["operator_sha256"]) + self.assertEqual(first["report"]["activity_code_sha256"], second["report"]["activity_code_sha256"]) + self.assertEqual(first["report"]["normalized_rows"], 3) + self.assertEqual(first["report"]["publication_eligibility"], "blocked") + self.assertFalse(any((root / "one" / "lifecycle").glob("*/released/records.jsonl"))) + + def test_partial_assisted_pair_fails_closed(self): + with self.assertRaises(RefreshError): + refresh(run_dir=Path(tempfile.mkdtemp()), operators_path=ROOT / "fixtures" / "synthetic_operators.csv") + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/sources/germany/README.md b/pipeline/sources/germany/README.md new file mode 100644 index 0000000..b2666ca --- /dev/null +++ b/pipeline/sources/germany/README.md @@ -0,0 +1,14 @@ +# Germany BVL BLtU private staging + +Use the stable [BVL BLtU landing page](https://www.bvl.bund.de/bltu) and select +the current general-list CSV export in the portal. Save that export outside Git +and run the assisted refresh: + +```text +python -m pipeline.sources.germany.refresh --raw --run-dir --retrieved-at-utc 2026-09-15T00:00:00Z +``` + +The typed adapter preserves repeated activity columns and source evidence, +quarantines schema and mapping anomalies, emits shared health evidence, and +cannot create a release. Address/coordinate review, BVL reuse terms, and human +publication approval remain blocked. diff --git a/pipeline/sources/germany/__init__.py b/pipeline/sources/germany/__init__.py new file mode 100644 index 0000000..bbed509 --- /dev/null +++ b/pipeline/sources/germany/__init__.py @@ -0,0 +1,5 @@ +"""Private BVL BLtU export acquisition and staging.""" + +from .adapter import BltuAdapter + +__all__ = ["BltuAdapter"] diff --git a/pipeline/sources/germany/adapter.py b/pipeline/sources/germany/adapter.py new file mode 100644 index 0000000..b33a994 --- /dev/null +++ b/pipeline/sources/germany/adapter.py @@ -0,0 +1,104 @@ +"""Typed private adapter for the BVL BLtU general-list export.""" +from __future__ import annotations + +import csv +import hashlib +import json +from collections import Counter +from pathlib import Path +from typing import Any + +from pipeline.contracts.adapter_contract import SourceArtifact +from pipeline.contracts.candidate_handoff import write_handoff +from pipeline.contracts.source_lifecycle import atomic_json, atomic_jsonl, private_manifest +from pipeline.germany.bltu_adapter import EXPECTED_HEADERS + +CONFIG = json.loads((Path(__file__).parent / "config.json").read_text(encoding="utf-8")) +CURRENT_ID_INDEX = 5 +NAME_INDEX = 1 +STATE_INDEX = 0 +STREET_INDEX = 2 +CITY_INDEX = 3 +ACTIVITY_START = 7 +ACTIVITY_END = 41 +ACTIVITY_MAP = {"SH": "slaughter", "CP": "processing"} + + +def _source_columns(headers: list[str], values: list[str]) -> list[dict[str, str]]: + return [{"header": headers[index], "value": values[index]} for index in range(len(headers))] + + +class BltuAdapter: + source_id = CONFIG["source_id"] + adapter_version = CONFIG["adapter_version"] + schema_version = CONFIG["schema_version"] + + def parse_bytes(self, content: bytes) -> dict[str, Any]: + try: + text = content.decode("utf-8-sig") + encoding = "utf-8-sig" + except UnicodeDecodeError: + text = content.decode("cp1252") + encoding = "cp1252" + rows = list(csv.reader(text.splitlines(), delimiter=";", strict=True)) + headers = rows[0] if rows else [] + matched = headers == list(EXPECTED_HEADERS) + accepted: list[dict[str, Any]] = [] + quarantined: list[dict[str, Any]] = [] + anomalies: Counter[str] = Counter() + categories: Counter[str] = Counter() + for line, values in enumerate(rows[1:], start=2): + source = {"source_row": line, "source_headers": headers, "source_values": values} + reasons: list[str] = [] + if not matched: + reasons.append("unrecognized_header_schema") + if len(values) != len(EXPECTED_HEADERS): + reasons.append("physical_column_count_mismatch") + current_id = values[CURRENT_ID_INDEX].strip() if len(values) > CURRENT_ID_INDEX else "" + name = values[NAME_INDEX].strip() if len(values) > NAME_INDEX else "" + if not current_id: + reasons.append("missing_current_approval_id") + if not name: + reasons.append("missing_establishment_name") + activity_codes = [headers[index] for index in range(ACTIVITY_START, min(ACTIVITY_END, len(values))) if values[index].strip()] + if not activity_codes: + reasons.append("missing_activity_code") + if any(code not in ACTIVITY_MAP for code in activity_codes): + reasons.append("unmapped_activity_code") + mapped = tuple(dict.fromkeys(ACTIVITY_MAP[code] for code in activity_codes if code in ACTIVITY_MAP)) + if not mapped and "missing_activity_code" not in reasons: + reasons.append("unmapped_activity_code") + for category in mapped: + categories[category] += 1 + record = {"source_id": self.source_id, "source_row": line, "source_record_key": f"{current_id or 'unknown'}|{line}", "source_values": source, "normalized": {"establishment_id": current_id or None, "approval_number": current_id or None, "name": name or None, "trading_name": name or None, "country_code": "DE", "nation": "Germany", "state": values[STATE_INDEX].strip() if len(values) > STATE_INDEX else None, "city": values[CITY_INDEX].strip() if len(values) > CITY_INDEX else None, "address": None, "address_state": "source-present-pending-privacy-review" if len(values) > STREET_INDEX and values[STREET_INDEX].strip() else "unknown", "activity_codes": tuple(activity_codes), "activity_categories": mapped, "source_activity_categories": mapped, "classification_state": "mapped" if mapped and not any(code not in ACTIVITY_MAP for code in activity_codes) else "unresolved", "coordinates": None, "coordinate_state": "unknown", "privacy_gate": "pending-review", "coordinate_gate": "review_required", "publication_gate": "blocked"}} + if reasons: + unique = tuple(dict.fromkeys(reasons)) + for reason in unique: + anomalies[reason] += 1 + quarantined.append({"reasons": unique, "record": record}) + else: + accepted.append(record) + return {"accepted": accepted, "quarantined": quarantined, "input_rows": len(rows) - 1 if rows else 0, "source_sha256": hashlib.sha256(content).hexdigest(), "schema_status": "matched" if matched else "unrecognized", "schema_fingerprint": hashlib.sha256(json.dumps(headers, ensure_ascii=False, separators=(",", ":")).encode()).hexdigest(), "encoding": encoding, "row_length_counts": dict(Counter(str(len(row)) for row in rows[1:])), "coverage_counts": dict(categories), "anomaly_counts": dict(sorted(anomalies.items()))} + + def run(self, raw_path: str | Path, run_dir: str | Path, artifact: SourceArtifact) -> dict[str, Any]: + raw = Path(raw_path).read_bytes() + digest = hashlib.sha256(raw).hexdigest() + if artifact.sha256 != digest or artifact.byte_size != len(raw): + raise ValueError("BLtU artifact provenance mismatch") + result = self.parse_bytes(raw) + if result["schema_status"] != "matched": + # Header drift is a source-contract failure, not a row-level + # anomaly. Do not let an empty or shifted export look healthy. + raise ValueError("BLtU schema drift: unrecognized header schema") + root = Path(run_dir) + parsed = result["accepted"] + [item["record"] for item in result["quarantined"]] + _, parsed_sha, _ = atomic_jsonl(root / "parsed" / "records.jsonl", parsed) + _, normalized_sha, _ = atomic_jsonl(root / "normalized" / "records.jsonl", result["accepted"]) + atomic_jsonl(root / "quarantined" / "records.jsonl", result["quarantined"]) + manifest = private_manifest(source_id=self.source_id, adapter_version=self.adapter_version, schema_version=self.schema_version, artifact=artifact, input_rows=result["input_rows"], normalized_rows=len(result["accepted"]), quarantined_rows=len(result["quarantined"]), normalized_sha256=normalized_sha, parsed_sha256=parsed_sha, anomaly_counts=result["anomaly_counts"]) + manifest.update({"country_code": "DE", "coverage": CONFIG["coverage"], "geocoding": "disabled", "schema_status": result["schema_status"], "schema_fingerprint": result["schema_fingerprint"], "encoding": result["encoding"], "row_length_counts": result["row_length_counts"], "coverage_counts": result["coverage_counts"], "release_gate": "restricted_pending_terms"}) + atomic_json(root / "manifest.json", manifest) + return manifest + + def write_candidate_handoff(self, run_dir: str | Path, artifact: SourceArtifact, parsed: dict[str, Any]) -> dict[str, Any]: + return write_handoff(run_dir, parsed["accepted"], artifact, source_id=self.source_id) diff --git a/pipeline/sources/germany/config.json b/pipeline/sources/germany/config.json new file mode 100644 index 0000000..b96c9ae --- /dev/null +++ b/pipeline/sources/germany/config.json @@ -0,0 +1,10 @@ +{ + "source_id": "de.locations", + "source_url": "https://www.bvl.bund.de/bltu", + "portal_url": "https://gis.bvl.bund.de/datenportal/", + "adapter_version": "de-bltu-private-v1", + "schema_version": "de-bltu-csv-v1", + "coverage": "BVL BLtU list of German establishments approved under Regulation (EC) 853/2004; export selected from the BVL portal", + "terms": "Public access and export are documented by BVL, but dataset-specific reuse and redistribution terms remain pending human confirmation", + "geocoding": "disabled" +} diff --git a/pipeline/sources/germany/refresh.py b/pipeline/sources/germany/refresh.py new file mode 100644 index 0000000..08e0c2b --- /dev/null +++ b/pipeline/sources/germany/refresh.py @@ -0,0 +1,83 @@ +"""Assisted/private refresh for a BVL BLtU CSV export.""" +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path +from typing import Any + +from pipeline.common.acquisition import AcquisitionError, fetch_source, utc_now +from pipeline.common.orchestrator import run_private_lifecycle +from pipeline.contracts.adapter_contract import SourceArtifact +from pipeline.contracts.source_lifecycle import atomic_json + +from .adapter import CONFIG, BltuAdapter + + +class RefreshError(ValueError): + pass + + +def _local_metadata(path: Path, retrieved: str) -> dict[str, Any]: + raw = path.read_bytes() + return {"acquisition_method": "assisted_bvl_portal_export", "source_id": CONFIG["source_id"], "artifact": path.name, "artifact_path": str(path.resolve()), "requested_url": CONFIG["source_url"], "final_url": CONFIG["source_url"], "redirects": [], "response_headers": {}, "requested_at_utc": retrieved, "retrieved_at_utc": retrieved, "effective_date": "unknown", "publication_date": None, "sha256": hashlib.sha256(raw).hexdigest(), "byte_size": len(raw), "adapter_version": CONFIG["adapter_version"], "code_version": CONFIG["adapter_version"], "config_version": CONFIG["schema_version"], "coverage": CONFIG["coverage"], "rights_caveat": CONFIG["terms"], "privacy_caveat": "private staging; address and coordinate review pending", "terms_review": "assisted capture; recurring acquisition and redistribution remain human-gated"} + + +def refresh(*, run_dir: str | Path, raw_path: str | Path | None = None, fetch: bool = False, export_url: str | None = None, terms_review_path: str | Path | None = None, output_root: str | Path = "data/raw", run_id: str | None = None, retrieved_at_utc: str | None = None, timeout_seconds: float = 60, max_bytes: int = 128 * 1024 * 1024) -> dict[str, Any]: + if fetch == (raw_path is not None): + raise RefreshError("specify exactly one of --raw or --fetch") + retrieved = retrieved_at_utc or utc_now() + if fetch: + if not export_url: + raise RefreshError("--export-url is required: select the current CSV/XLS export in the BVL portal") + if not export_url.startswith(("https://www.bvl.bund.de/", "https://gis.bvl.bund.de/")): + raise RefreshError("BLtU export URL must be an HTTPS BVL host") + if terms_review_path is None: + raise RefreshError("--terms-review is required with --fetch") + try: + metadata = fetch_source(source_id=CONFIG["source_id"], url=export_url, output_root=Path(output_root), artifact_name="bltu-export.csv", run_id=run_id, timeout_seconds=timeout_seconds, max_bytes=max_bytes, terms_review_path=terms_review_path, code_version=CONFIG["adapter_version"], config_version=CONFIG["schema_version"], coverage=CONFIG["coverage"], rights_caveat=CONFIG["terms"], privacy_caveat="private staging; address and coordinate review pending", allowed_content_types=("text/csv", "application/csv", "application/vnd.ms-excel", "application/octet-stream")) + except AcquisitionError as error: + raise RefreshError(str(error)) from error + input_path = Path(metadata["artifact_path"]) + else: + input_path = Path(raw_path).resolve() # type: ignore[arg-type] + if not input_path.is_file(): + raise RefreshError("BLtU raw artifact does not exist") + metadata = _local_metadata(input_path, retrieved) + atomic_json(Path(run_dir) / "acquisition-metadata.json", metadata) + adapter = BltuAdapter() + raw = input_path.read_bytes() + artifact = SourceArtifact(source_url=str(metadata.get("final_url") or CONFIG["source_url"]), retrieved_at_utc=str(metadata["retrieved_at_utc"]), sha256=hashlib.sha256(raw).hexdigest(), byte_size=len(raw), publication_date=metadata.get("publication_date"), effective_date=metadata.get("effective_date"), code_version=adapter.adapter_version, config_version=adapter.schema_version, rights_caveat=CONFIG["terms"], privacy_caveat=metadata.get("privacy_caveat"), coverage=CONFIG["coverage"], redirects=tuple(metadata.get("redirects") or ())) + lifecycle = run_private_lifecycle(input_path, Path(run_dir) / "lifecycle", artifact, adapter, health_as_of_utc=str(metadata["retrieved_at_utc"])) + manifest = lifecycle.get("manifest") or {} + report = {"source_id": CONFIG["source_id"], "source_url": artifact.source_url, "portal_url": CONFIG["portal_url"], "retrieved_at_utc": artifact.retrieved_at_utc, "sha256": artifact.sha256, "byte_size": artifact.byte_size, "input_rows": manifest.get("input_rows"), "normalized_rows": manifest.get("normalized_rows"), "quarantined_rows": manifest.get("quarantined_rows"), "schema_status": manifest.get("schema_status"), "schema_fingerprint": manifest.get("schema_fingerprint"), "drift_alarms": [], "disappearance_semantics": "not-observed; never inferred as closure", "geocoding": "disabled", "release_state": "not-created", "publication_state": "private-candidate", "publication_eligibility": "blocked", "lifecycle_status": lifecycle.get("status"), "lifecycle_run_dir": lifecycle.get("run_dir")} + atomic_json(Path(run_dir) / "refresh.json", report) + return {"report": report, "lifecycle": lifecycle} + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + source = parser.add_mutually_exclusive_group(required=True) + source.add_argument("--raw", type=Path) + source.add_argument("--fetch", action="store_true") + parser.add_argument("--export-url") + parser.add_argument("--run-dir", type=Path, required=True) + parser.add_argument("--output-root", type=Path, default=Path("data/raw")) + parser.add_argument("--run-id") + parser.add_argument("--terms-review", type=Path) + parser.add_argument("--retrieved-at-utc") + parser.add_argument("--timeout-seconds", type=float, default=60) + parser.add_argument("--max-bytes", type=int, default=128 * 1024 * 1024) + args = parser.parse_args() + try: + result = refresh(run_dir=args.run_dir, raw_path=args.raw, fetch=args.fetch, export_url=args.export_url, terms_review_path=args.terms_review, output_root=args.output_root, run_id=args.run_id, retrieved_at_utc=args.retrieved_at_utc, timeout_seconds=args.timeout_seconds, max_bytes=args.max_bytes) + except (OSError, RefreshError) as error: + print(json.dumps({"status": "failed", "error": str(error)}, sort_keys=True)) + return 2 + print(json.dumps(result["report"], sort_keys=True)) + return 0 if result["report"]["lifecycle_status"] == "candidate-ready" else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/pipeline/sources/germany/test_adapter.py b/pipeline/sources/germany/test_adapter.py new file mode 100644 index 0000000..2700d80 --- /dev/null +++ b/pipeline/sources/germany/test_adapter.py @@ -0,0 +1,50 @@ +import hashlib +import tempfile +import unittest +from pathlib import Path + +from pipeline.common.orchestrator import run_private_lifecycle +from pipeline.contracts.adapter_contract import SourceArtifact +from .adapter import BltuAdapter + + +ROOT = Path(__file__).parent + + +class GermanyPrivateAdapterTests(unittest.TestCase): + def test_bltu_is_typed_and_keeps_coordinates_and_addresses_private(self): + raw = ROOT.parent.parent / "germany" / "fixtures" / "synthetic_bltu.csv" + adapter = BltuAdapter() + result = adapter.parse_bytes(raw.read_bytes()) + self.assertEqual(len(result["accepted"]), 1) + normalized = result["accepted"][0]["normalized"] + self.assertEqual(normalized["activity_categories"], ("slaughter",)) + self.assertIsNone(normalized["address"]) + self.assertIsNone(normalized["coordinates"]) + self.assertGreaterEqual(len(result["quarantined"]), 2) + + def test_shared_private_lifecycle_emits_health_and_no_release(self): + raw = ROOT.parent.parent / "germany" / "fixtures" / "synthetic_bltu.csv" + adapter = BltuAdapter(); content = raw.read_bytes() + artifact = SourceArtifact("https://example.invalid/bltu-export.csv", "2026-09-14T00:00:00Z", hashlib.sha256(content).hexdigest(), len(content), code_version=adapter.adapter_version, config_version=adapter.schema_version) + with tempfile.TemporaryDirectory() as directory: + status = run_private_lifecycle(raw, Path(directory) / "runs", artifact, adapter) + run_dir = Path(status["run_dir"]) + self.assertEqual(status["status"], "candidate-ready") + self.assertTrue((run_dir / "source-health.json").exists()) + self.assertTrue((run_dir / "release-candidate" / "records.jsonl").exists()) + self.assertFalse((run_dir / "released" / "records.jsonl").exists()) + + def test_schema_drift_fails_typed_run_before_candidate_handoff(self): + raw = ROOT.parent.parent / "germany" / "fixtures" / "synthetic_bltu.csv" + content = raw.read_bytes().replace(b"# Bundesland;", b"unexpected;", 1) + adapter = BltuAdapter() + artifact = SourceArtifact("https://example.invalid/bltu-export.csv", "2026-09-14T00:00:00Z", hashlib.sha256(content).hexdigest(), len(content), code_version=adapter.adapter_version, config_version=adapter.schema_version) + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "drift.csv"; path.write_bytes(content) + with self.assertRaisesRegex(ValueError, "schema drift"): + adapter.run(path, Path(directory) / "run", artifact) + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/sources/germany/test_refresh.py b/pipeline/sources/germany/test_refresh.py new file mode 100644 index 0000000..a247e79 --- /dev/null +++ b/pipeline/sources/germany/test_refresh.py @@ -0,0 +1,31 @@ +import tempfile +import unittest +from pathlib import Path + +from .refresh import RefreshError, refresh + + +ROOT = Path(__file__).parent + + +class GermanyRefreshTests(unittest.TestCase): + def test_assisted_refresh_is_private_and_repeatable(self): + raw = ROOT.parent.parent / "germany" / "fixtures" / "synthetic_bltu.csv" + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + first = refresh(run_dir=root / "one", raw_path=raw, retrieved_at_utc="2026-09-14T00:00:00Z") + second = refresh(run_dir=root / "two", raw_path=raw, retrieved_at_utc="2026-09-14T00:00:00Z") + self.assertEqual(first["report"]["sha256"], second["report"]["sha256"]) + self.assertEqual(first["report"]["normalized_rows"], 1) + self.assertEqual(first["report"]["publication_eligibility"], "blocked") + self.assertFalse(any((root / "one" / "lifecycle").glob("*/released/records.jsonl"))) + + def test_network_mode_requires_explicit_portal_export_url_and_terms_review(self): + with self.assertRaises(RefreshError): + refresh(run_dir=Path(tempfile.mkdtemp()), fetch=True) + with self.assertRaises(RefreshError): + refresh(run_dir=Path(tempfile.mkdtemp()), fetch=True, export_url="https://example.invalid/export.csv", terms_review_path=ROOT / "README.md") + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/tests/e2e/test_germany_belgium_candidate_import.py b/pipeline/tests/e2e/test_germany_belgium_candidate_import.py new file mode 100644 index 0000000..d9e918d --- /dev/null +++ b/pipeline/tests/e2e/test_germany_belgium_candidate_import.py @@ -0,0 +1,73 @@ +"""Guarded candidate-import/API checks for the Germany and Belgium adapters.""" +import hashlib +import json +import os +import subprocess +import sys +import tempfile +import unittest +import urllib.request +from pathlib import Path + +import psycopg + +from .fixture import E2EEnvironment +from pipeline.common.orchestrator import run_private_lifecycle +from pipeline.contracts.adapter_contract import SourceArtifact +from pipeline.sources.belgium.adapter import BelgiumOperatorsAdapter +from pipeline.sources.germany.adapter import BltuAdapter + +ROOT = Path(__file__).resolve().parents[3] +IMPORTER = ROOT / "pipeline/scripts/maintenance/import-candidate.py" + + +class GermanyBelgiumCandidateImportE2E(unittest.TestCase): + @classmethod + def setUpClass(cls): + if os.environ.get("UEC_RUN_E2E") != "1": + raise unittest.SkipTest("set UEC_RUN_E2E=1 to run Docker-backed E2E tests") + cls.env = E2EEnvironment(); cls.env.test_release_id = "candidate-de-be-e2e"; cls.env = cls.env.start() + cls.temp = tempfile.TemporaryDirectory(); root = Path(cls.temp.name) + be_raw = ROOT / "pipeline/sources/belgium/fixtures/synthetic_operators.csv" + be_codes = ROOT / "pipeline/sources/belgium/fixtures/synthetic_activity_codes.csv" + de_raw = ROOT / "pipeline/germany/fixtures/synthetic_bltu.csv" + be_adapter = BelgiumOperatorsAdapter(be_codes, SourceArtifact("https://example.invalid/be-codes.csv", "2026-09-14T00:00:00Z", hashlib.sha256(be_codes.read_bytes()).hexdigest(), be_codes.stat().st_size)) + entries = [("be", be_raw, be_adapter, "https://example.invalid/be-operators.csv"), ("de", de_raw, BltuAdapter(), "https://example.invalid/de-bltu.csv")] + cls.commands = [] + for name, raw, adapter, url in entries: + content = raw.read_bytes(); source_artifact = SourceArtifact(url, "2026-09-14T00:00:00Z", hashlib.sha256(content).hexdigest(), len(content), code_version=adapter.adapter_version, config_version=adapter.schema_version) + status = run_private_lifecycle(raw, root / "runs" / name, source_artifact, adapter) + if status["status"] != "candidate-ready": + raise RuntimeError(status) + lifecycle_dir = Path(status["run_dir"]) + cls.commands.append([sys.executable, str(IMPORTER), "--manifest", str(lifecycle_dir / "manifest.json"), "--normalized", str(lifecycle_dir / "normalized/records.jsonl"), "--raw", str(raw), "--release-id", "candidate-de-be-e2e", "--database-url", cls.env.database_url, "--disposable-db"]) + for command in cls.commands: + first = subprocess.run(command, cwd=ROOT, capture_output=True, text=True) + second = subprocess.run(command, cwd=ROOT, capture_output=True, text=True) + if first.returncode or second.returncode: + raise RuntimeError(f"candidate import failed: {first.stdout}\n{first.stderr}\n{second.stdout}\n{second.stderr}") + + @classmethod + def tearDownClass(cls): + if getattr(cls, "temp", None): cls.temp.cleanup() + if getattr(cls, "env", None): cls.env.stop() + + def test_both_sources_are_idempotent_and_candidate_only(self): + with psycopg.connect(self.env.database_url) as db: + counts = dict(db.execute("SELECT source_id, count(*) FROM uec.source_records WHERE source_id IN ('be.locations','de.locations') GROUP BY source_id").fetchall()) + self.assertEqual(counts, {"be.locations": 3, "de.locations": 1}) + self.assertTrue(db.execute("SELECT test_only FROM uec.releases WHERE release_id='candidate-de-be-e2e'").fetchone()[0]) + base = f"http://127.0.0.1:{self.env.api_port}" + with urllib.request.urlopen(base + "/api/v2/locations?profile=official") as response: + self.assertEqual(json.loads(response.read())["data"], []) + request = urllib.request.Request(base + "/api/dev/preview/test-release/locations?profile=official", headers={"X-UEC-Dev-Preview-Token": self.env.dev_preview_token}) + with urllib.request.urlopen(request) as response: + body = json.loads(response.read()) + self.assertEqual({row["country_code"] for row in body["data"]}, {"BE", "DE"}) + self.assertTrue(body["meta"]["test_only"]) + self.assertTrue(all(row["latitude"] is None for row in body["data"])) + self.assertNotIn("source_values", json.dumps(body)) + + +if __name__ == "__main__": + unittest.main() From fa1f95b22bc6bc4fcecfcaf4bc1af2b321043f56 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Tue, 15 Sep 2026 14:01:44 -0700 Subject: [PATCH 121/311] Add versioned animal scale evidence foundation --- .../animal-scale-statistics-catalog.json | 159 +++++++++ docs/statistics/aggregate-statistics-spec.md | 64 ++++ docs/story/claim-ledger.md | 4 +- docs/story/flagship-scale-evidence-spec.md | 65 ++++ .../validate-statistics-catalog.py | 30 ++ .../stages/ingest-faostat-animal-scale.py | 41 +++ pipeline/statistics/__init__.py | 5 + pipeline/statistics/catalog.py | 303 ++++++++++++++++++ pipeline/statistics/faostat.py | 156 +++++++++ .../fixtures/synthetic_catalog.json | 54 ++++ .../statistics/fixtures/synthetic_faostat.csv | 18 ++ pipeline/statistics/test_catalog.py | 84 +++++ pipeline/statistics/test_faostat.py | 44 +++ 13 files changed, 1025 insertions(+), 2 deletions(-) create mode 100644 data/manifests/animal-scale-statistics-catalog.json create mode 100644 docs/statistics/aggregate-statistics-spec.md create mode 100644 docs/story/flagship-scale-evidence-spec.md create mode 100644 pipeline/scripts/diagnostics/validate-statistics-catalog.py create mode 100644 pipeline/scripts/stages/ingest-faostat-animal-scale.py create mode 100644 pipeline/statistics/__init__.py create mode 100644 pipeline/statistics/catalog.py create mode 100644 pipeline/statistics/faostat.py create mode 100644 pipeline/statistics/fixtures/synthetic_catalog.json create mode 100644 pipeline/statistics/fixtures/synthetic_faostat.csv create mode 100644 pipeline/statistics/test_catalog.py create mode 100644 pipeline/statistics/test_faostat.py diff --git a/data/manifests/animal-scale-statistics-catalog.json b/data/manifests/animal-scale-statistics-catalog.json new file mode 100644 index 0000000..b177d5f --- /dev/null +++ b/data/manifests/animal-scale-statistics-catalog.json @@ -0,0 +1,159 @@ +{ + "schema_version": "aggregate-statistics-catalog-v1", + "catalog_id": "animal-scale-2024-edition", + "catalog_version": "1.0.0", + "created_at_utc": "2026-09-15T20:46:56Z", + "publication_state": "private-validated", + "statistics": [ + { + "statistic_id": "land-animals-slaughtered-for-meat-world-2024", + "version_id": "land-animals-slaughtered-for-meat-world-2024-faostat-qcl-2025-12-15-v1", + "label": "FAOSTAT World slaughtered animals for selected land-animal meat items, 2024", + "status": "validated-private", + "source": { + "source_id": "faostat.qcl.livestock-primary", + "publisher": "Food and Agriculture Organization of the United Nations (FAO)", + "dataset_name": "Livestock primary (Global, National - Annual)", + "dataset_url": "https://data.fao.org/catalog/iso/55375b1e-51d0-47db-ac9b-536ac8a1c738", + "origin": "intergovernmental statistical source", + "license": "CC-BY-4.0" + }, + "population_scope": { + "scope_id": "faostat-meat-leaf-items-heads-v1", + "animal_class": "land animals", + "activity": "slaughtered for meat production", + "included": [ + "asses", + "buffalo", + "camels", + "cattle", + "chickens", + "ducks", + "geese", + "goats", + "mules", + "other domestic camelids", + "other domestic rodents", + "pigs", + "pigeons and other birds n.e.c.", + "rabbits and hares", + "sheep", + "turkeys" + ], + "scope_note": "The selected scope contains the 16 non-aggregate FAOSTAT 2024 World meat-item rows that expose Producing Animals/Slaughtered. It is not a claim to count every land animal killed for food." + }, + "geography": { + "level": "global aggregate", + "area": "World (FAOSTAT Area)", + "geography_note": "FAOSTAT values describe animals slaughtered within national boundaries, irrespective of origin; the World row is an FAO aggregate." + }, + "period": { + "kind": "calendar-year", + "start": "2024-01-01", + "end": "2024-12-31", + "calendar_year": 2024 + }, + "unit": { + "kind": "count", + "name": "animals (heads)", + "numerator": "slaughtered animals", + "denominator": "calendar year", + "scale": 1 + }, + "estimate": { + "type": "point-with-qualitative-uncertainty", + "low": null, + "central": 87892277479, + "high": null + }, + "method": { + "type": "derived-from-components", + "description": "Select World rows whose item is a named Meat of ... leaf item, retain FAOSTAT's source value and flag, convert 1000 An to heads, then sum the non-overlapping rows.", + "formula": "sum(source_value * conversion_to_central_unit)" + }, + "exclusions": [ + "FAOSTAT aggregate rows such as Meat, Total; Beef and Buffalo Meat, primary; Sheep and Goat Meat; and Meat, Poultry", + "dairy and egg production, including animals culled from those systems when not represented in the selected meat rows", + "aquatic animals, fish, crustaceans, molluscs, and other aquatic populations", + "hides, fat, offal, wool, milk, eggs, and other non-meat commodities", + "species or items with no selected 2024 World leaf row", + "a numeric uncertainty interval, because this FAOSTAT release does not publish one for the selected rows" + ], + "uncertainty": { + "kind": "qualitative-data-flags-and-coverage", + "statement": "FAO says country inputs can be reported, estimated, supplemented from unofficial sources, or imputed, and flags them accordingly. The selected World rows are therefore a source-backed estimate with unknown numeric error; omitted categories and coverage gaps remain unknown." + }, + "provenance": { + "artifact_id": "faostat-qcl-2025-12-15-production-crops-livestock-normalized", + "artifact_path": "data/raw/animal-scale/faostat-qcl/Production_Crops_Livestock_E_All_Data_(Normalized).zip", + "public_artifact": false, + "sha256": "c5835418c18f9322e7decbd6800f93a216eaae3cdfa31acb08f0518c0c6d6853", + "byte_size": 33921825, + "retrieved_at_utc": "2026-09-15T20:46:56Z", + "effective_date": "2024-12-31", + "publication_date": "2025-12-15", + "retrieval_url": "https://bulks-faostat.fao.org/production/Production_Crops_Livestock_E_All_Data_(Normalized).zip", + "code_version": "animal-scale-statistics-v1", + "config_version": "faostat-qcl-leaf-meat-items-v1", + "retention_note": "The downloaded archive is ignored private research evidence; only this sanitized manifest is tracked." + }, + "revision": { + "revision_id": "faostat-qcl-2025-12-15", + "released_date": "2025-12-15", + "supersedes": null, + "change_note": "Initial private catalog entry for the 2024 FAOSTAT revision; later source revisions must create a new version and preserve this one." + }, + "citations": [ + { + "citation_id": "faostat-catalog-livestock-primary", + "title": "FAOSTAT: Livestock primary (Global, National - Annual)", + "url": "https://data.fao.org/catalog/iso/55375b1e-51d0-47db-ac9b-536ac8a1c738", + "locator": "Dataset abstract, data lineage, units, time coverage, and revision metadata", + "accessed_date": "2026-09-15" + }, + { + "citation_id": "faostat-qcl-methodology", + "title": "FAOSTAT Agricultural production — Livestock methodology", + "url": "https://files-faostat.fao.org/production/QCL/QCL_methodology_e.pdf", + "locator": "PDF page 3: meat scope; PDF page 4: reference period and totals", + "accessed_date": "2026-09-15" + }, + { + "citation_id": "faostat-qcl-bulk-archive", + "title": "FAOSTAT Crops and livestock products normalized bulk archive", + "url": "https://bulks-faostat.fao.org/production/Production_Crops_Livestock_E_All_Data_(Normalized).zip", + "locator": "2024 World rows; Element=Producing Animals/Slaughtered; selected Meat of ... items", + "accessed_date": "2026-09-15" + } + ], + "citation_ids": [ + "faostat-catalog-livestock-primary", + "faostat-qcl-methodology", + "faostat-qcl-bulk-archive" + ], + "aggregation": { + "operation": "sum", + "overlap_status": "resolved-non-overlapping", + "basis": "Each component is a distinct FAOSTAT meat leaf item; aggregate meat rows and non-meat commodity rows are excluded." + }, + "components": [ + {"component_id": "asses", "population": "asses", "source_item": "Meat of asses, fresh or chilled", "source_unit": "An", "source_value": 873452, "conversion_to_central_unit": 1, "overlap_group": "asses", "source_flag": "E"}, + {"component_id": "buffalo", "population": "buffalo", "source_item": "Meat of buffalo, fresh or chilled", "source_unit": "An", "source_value": 29040868, "conversion_to_central_unit": 1, "overlap_group": "buffalo", "source_flag": "E"}, + {"component_id": "camels", "population": "camels", "source_item": "Meat of camels, fresh or chilled", "source_unit": "An", "source_value": 3123404, "conversion_to_central_unit": 1, "overlap_group": "camels", "source_flag": "E"}, + {"component_id": "cattle", "population": "cattle", "source_item": "Meat of cattle with the bone, fresh or chilled", "source_unit": "An", "source_value": 304744668, "conversion_to_central_unit": 1, "overlap_group": "cattle", "source_flag": "E"}, + {"component_id": "chickens", "population": "chickens", "source_item": "Meat of chickens, fresh or chilled", "source_unit": "1000 An", "source_value": 78533923, "conversion_to_central_unit": 1000, "overlap_group": "chickens", "source_flag": "E"}, + {"component_id": "ducks", "population": "ducks", "source_item": "Meat of ducks, fresh or chilled", "source_unit": "1000 An", "source_value": 4227882, "conversion_to_central_unit": 1000, "overlap_group": "ducks", "source_flag": "E"}, + {"component_id": "geese", "population": "geese", "source_item": "Meat of geese, fresh or chilled", "source_unit": "1000 An", "source_value": 803201, "conversion_to_central_unit": 1000, "overlap_group": "geese", "source_flag": "E"}, + {"component_id": "goats", "population": "goats", "source_item": "Meat of goat, fresh or chilled", "source_unit": "An", "source_value": 563722557, "conversion_to_central_unit": 1, "overlap_group": "goats", "source_flag": "E"}, + {"component_id": "mules", "population": "mules", "source_item": "Meat of mules, fresh or chilled", "source_unit": "An", "source_value": 82572, "conversion_to_central_unit": 1, "overlap_group": "mules", "source_flag": "E"}, + {"component_id": "other-domestic-camelids", "population": "other domestic camelids", "source_item": "Meat of other domestic camelids, fresh or chilled", "source_unit": "An", "source_value": 972619, "conversion_to_central_unit": 1, "overlap_group": "other domestic camelids", "source_flag": "E"}, + {"component_id": "other-domestic-rodents", "population": "other domestic rodents", "source_item": "Meat of other domestic rodents, fresh or chilled", "source_unit": "1000 An", "source_value": 69257, "conversion_to_central_unit": 1000, "overlap_group": "other domestic rodents", "source_flag": "E"}, + {"component_id": "pigs", "population": "pigs", "source_item": "Meat of pig with the bone, fresh or chilled", "source_unit": "An", "source_value": 1493796985, "conversion_to_central_unit": 1, "overlap_group": "pigs", "source_flag": "A"}, + {"component_id": "pigeons-and-other-birds", "population": "pigeons and other birds n.e.c.", "source_item": "Meat of pigeons and other birds n.e.c., fresh, chilled or frozen", "source_unit": "1000 An", "source_value": 49433, "conversion_to_central_unit": 1000, "overlap_group": "pigeons and other birds n.e.c.", "source_flag": "E"}, + {"component_id": "rabbits-and-hares", "population": "rabbits and hares", "source_item": "Meat of rabbits and hares, fresh or chilled", "source_unit": "1000 An", "source_value": 604235, "conversion_to_central_unit": 1000, "overlap_group": "rabbits and hares", "source_flag": "E"}, + {"component_id": "sheep", "population": "sheep", "source_item": "Meat of sheep, fresh or chilled", "source_unit": "An", "source_value": 703861354, "conversion_to_central_unit": 1, "overlap_group": "sheep", "source_flag": "E"}, + {"component_id": "turkeys", "population": "turkeys", "source_item": "Meat of turkeys, fresh or chilled", "source_unit": "1000 An", "source_value": 504128, "conversion_to_central_unit": 1000, "overlap_group": "turkeys", "source_flag": "A"} + ] + } + ] +} diff --git a/docs/statistics/aggregate-statistics-spec.md b/docs/statistics/aggregate-statistics-spec.md new file mode 100644 index 0000000..2756673 --- /dev/null +++ b/docs/statistics/aggregate-statistics-spec.md @@ -0,0 +1,64 @@ +# Aggregate statistics evidence specification + +Status: v1 private-validated contract; not a publication approval. + +## Purpose and boundary + +Aggregate statistics are versioned evidence objects, separate from facilities and their observations. A statistic can describe a population, place, period, and unit without asserting that any facility in the map produced a particular number of animals. It is not permissible to allocate a national or global total to nearby facilities without facility-specific evidence. + +The contract is designed for a dated annual edition. It preserves source values and source item names alongside project interpretations. Unknown, unavailable, approximate, unresolved, and qualitative uncertainty remain explicit rather than being converted into a fabricated range. + +## Required object shape + +Each `statistics[]` entry in `aggregate-statistics-catalog-v1` contains: + +| Field | Required meaning | +| --- | --- | +| `statistic_id`, `version_id`, `status` | Stable identity, immutable version identity, and `validated-private` until an authorized release approves publication. | +| `source` | Publisher, dataset name, source origin, dataset URL, and license. Source origin is not review or approval. | +| `population_scope` | `scope_id`, named included populations, animal class, activity, and a scope note. | +| `geography` | Geography level and source-compatible area name. | +| `period` | Period kind and inclusive start/end dates; annual editions also carry `calendar_year`. | +| `unit` | Kind, display name, numerator, denominator, and scale. Counts are heads/animals, not mass. | +| `estimate` | `central` plus optional `low`/`high`; bounds must satisfy low ≤ central ≤ high. Missing bounds mean no numeric interval is published. | +| `method` | Direct, converted, or derived method, with a plain-language description and formula. | +| `components` / `aggregation` | Optional source rows, unit conversions, overlap assessment, and reconciled sum for derived aggregates. | +| `exclusions` | Explicit exclusions, including populations, products, periods, and rows omitted to prevent double counting. | +| `uncertainty` | Named uncertainty kind and statement. FAO flags and coverage limits are not a confidence interval. | +| `provenance` | Private artifact ID/path, checksum, byte size, retrieval timestamp, effective/publication dates, retrieval URL, code/config versions, and public-artifact flag. | +| `revision` | Revision ID, released date, superseded version when any, and change note. New source revisions create new entries. | +| `citations` / `citation_ids` | HTTPS citations with title, locator, access date, and explicit references from the entry. | + +Validation is implemented in [`pipeline/statistics/catalog.py`](../../pipeline/statistics/catalog.py) and exercised by synthetic fixtures. It rejects missing dimensions, invalid dates, non-whole converted counts, non-reconciled component totals, repeated overlap groups, missing citations, insecure URLs, missing revision metadata, numeric ranges without bounds, and ambiguous publication status. + +## Selected land-animal edition + +The first private catalog entry is `land-animals-slaughtered-for-meat-world-2024`, based on FAOSTAT's `Livestock primary (Global, National - Annual)` domain and its normalized bulk archive. The selection is: + +- Area: `World`. +- Year: 2024 calendar year. +- Element: `Producing Animals/Slaughtered`. +- Rows: the 16 named, non-aggregate `Meat of ...` item rows with head-based units. +- Conversion: rows reported as `1000 An` are multiplied by 1,000; rows reported as `An` are kept as heads. +- Result: **87,892,277,479 heads**, a source-backed selected-scope estimate. + +The result is not “all animals killed for food.” It excludes aggregate rows, aquatic animals, dairy/egg culls not represented in these meat rows, non-meat commodities, and categories without a selected 2024 World leaf row. FAO explains that source values may be reported, estimated, supplemented, or imputed and are flagged accordingly; the catalog therefore carries qualitative uncertainty and no invented low/high interval. + +The raw archive is retained in ignored local storage at `data/raw/animal-scale/faostat-qcl/`. The tracked manifest records its SHA-256 checksum, byte size, retrieval timestamp, source URLs, effective date, revision, and citations. Raw data is not a repository fixture or public artifact. + +## Aquatic animals: separate later integration + +Aquatic animals remain a separate statistic family. Do not add fish, crustaceans, molluscs, or other aquatic estimates to the land-animal head count. Candidate aquatic work must first identify: + +1. wild capture versus farmed populations; +2. species or taxonomic groups and geographic coverage; +3. year/period and whether the source reports biomass, landed weight, harvest weight, or individuals; +4. the size/weight distribution and conversion method if biomass is converted to individuals; +5. numeric ranges and sensitivity to the conversion assumptions; and +6. discard/bycatch treatment and overlap with any farmed or slaughtered category. + +Integration, if later approved, should be a derived edition with two visibly separate panels and a reconciliation table. The land and aquatic source records remain independently citable; a combined display must show both ranges, the formula, the conversion assumptions, and the fact that uncertainty is not additive in a simple precise way. If conversion assumptions are too broad, present parallel land and aquatic stories rather than a combined number. + +## Publication gate + +`validated-private` means the object passed machine validation and source review in private staging. It does not mean project-approved or project-published. Before public copy, an authorized reviewer must record a scoped approval for the named story edition, confirm the source terms, verify the exact selected rows and checksum, review the exclusions and uncertainty wording, and attach a correction/version-history path. A later source revision must never silently rewrite an earlier edition. diff --git a/docs/story/claim-ledger.md b/docs/story/claim-ledger.md index 22b2db9..3ded456 100644 --- a/docs/story/claim-ledger.md +++ b/docs/story/claim-ledger.md @@ -19,8 +19,8 @@ Status vocabulary: **candidate** means a source still needs review; **ready-for- | C-13 | “Death is much closer than you imagined.” | Interpretive/moral thesis | No empirical source as written; must be framed as the project’s interpretation. | Do not imply a tested psychological effect. | candidate editorial line | | C-14 | “The industry hides this from you.” | Intentionality claim | Evidence of intentional concealment and defined actor/scope. | Not supported by current repository evidence. | **blocked** | | C-15 | “Over 56,000 locations” | Legacy aggregate claim | Dated release manifest, country/scope, inclusion rules, suppression state. | Must distinguish facilities/records from animals and current from legacy. | blocked until release audit | -| C-16 | “FAOSTAT reports an annual number of animals slaughtered for a defined set of land-animal meat items.” | Observed/statistical-source claim | FAOSTAT **Livestock primary (Global, National - Annual)** bulk release, 2024, Area=World, Element=Producing Animals/Slaughtered. Private retrieval 2026-09-13 from [FAO bulk data](https://bulks-faostat.fao.org/production/Production_Crops_Livestock_E_All_Data_(Normalized).zip); source catalog [FAO catalog](https://data.fao.org/catalog/iso/55375b1e-51d0-47db-ac9b-536ac8a1c738); [FAO methodology](https://files-faostat.fao.org/production/QCL/QCL_methodology_e.pdf). | Selected direct meat items with head units (converting `1000 An` to heads) sum to **87,896,729,120 heads in 2024**: cattle, buffalo, sheep, goat, pig, chicken, duck, goose, pigeon/other birds, rabbit/hare, turkey, ass, horse/equine, mule, camel, other camelid, and other domestic rodents. Excludes aggregate rows (beef/buffalo, sheep/goat, poultry), hides/fat/offal duplicates, and species/items not represented in this selected head-count set. FAO states national values concern slaughter within national boundaries and aggregates may include estimates. This is a selected-scope total, not “all animals killed.” | **ready-for-prototype with scope label; not a universal total** | -| C-17 | “The selected 2024 FAOSTAT scope averages about 240.7 million heads per day, or 2,785 heads per second.” | Derived calculation | C-16 exact selected item list and 2024 value. | `daily = 87,896,729,120 / 365.2425 = 240,653,070.55`; `per_second = 87,896,729,120 / (365.2425 * 86,400) = 2,785.34`. Use “modeled average” and explain 365.2425 is a calendar-year convention; do not call it live or event-timed. | **ready-for-prototype; public release needs maintainer/source review** | +| C-16 | “FAOSTAT reports an annual number of animals slaughtered for a defined set of land-animal meat items.” | Observed/statistical-source claim | FAOSTAT **Livestock primary (Global, National - Annual)** normalized bulk release, 2024, Area=World, Element=Producing Animals/Slaughtered. Private retrieval 2026-09-15 from [FAO bulk data](https://bulks-faostat.fao.org/production/Production_Crops_Livestock_E_All_Data_(Normalized).zip); source catalog [FAO catalog](https://data.fao.org/catalog/iso/55375b1e-51d0-47db-ac9b-536ac8a1c738); [FAO methodology](https://files-faostat.fao.org/production/QCL/QCL_methodology_e.pdf). | The 16 selected non-aggregate `Meat of ...` rows (asses, buffalo, camels, cattle, chickens, ducks, geese, goats, mules, other domestic camelids, other domestic rodents, pigs, pigeons/other birds n.e.c., rabbits/hares, sheep, and turkeys), converting `1000 An` to heads, sum to **87,892,277,479 heads in 2024**. Excludes aggregate rows, dairy/egg culls not represented in these meat rows, aquatic animals, non-meat commodities, and categories without a selected 2024 World leaf row. FAO states national values concern slaughter within national boundaries and inputs may be reported, estimated, supplemented, or imputed. This is a selected-scope estimate, not “all animals killed.” | **ready-for-prototype with scope label; not a universal total** | +| C-17 | “The selected 2024 FAOSTAT scope averages about 240.1 million heads per day, or 2,779 heads per second.” | Derived calculation | C-16 exact selected item list and 2024 value. | `daily = 87,892,277,479 / 366 = 240,142,834.64`; `per_second = 87,892,277,479 / (366 * 86,400) = 2,779.43`. Use “modeled average” and explain the 366-day 2024 calendar-year convention; do not call it live or event-timed. | **ready-for-prototype; public release needs maintainer/source review** | | C-18 | “A separate global aquatic-animal annual range is [X–Y].” | Quantitative estimate | Must use a primary fish/aquatic source with explicit species, wild/captured vs farmed scope, unit, year, and conversion from biomass to individuals. | Biomass-to-individual conversion requires species/size assumptions and should be a range; fish-count methods are not interchangeable with FAOSTAT land-animal heads. | **blocked: no reviewed primary estimate yet** | | C-19 | “This is comparable to [familiar object/person/lifespan].” | Scale comparison | Comparison denominator must be independently sourced, dated, and commensurate with the same unit and scope. | Avoid comparisons that imply equivalence or certainty; show both original values and conversion. | **blocked pending independent source packet** | diff --git a/docs/story/flagship-scale-evidence-spec.md b/docs/story/flagship-scale-evidence-spec.md new file mode 100644 index 0000000..b0ed255 --- /dev/null +++ b/docs/story/flagship-scale-evidence-spec.md @@ -0,0 +1,65 @@ +# Flagship journey: individual → scale → place → evidence + +Status: evidence/narrative specification for a bounded prototype. It is not launch copy or a facility-specific death claim. + +## Narrative promise + +The visitor should leave able to answer four questions: what one individual means in this story; what population and year the annual number covers; how an annual estimate becomes an understandable rate; and how to continue into nearby documented evidence without confusing proximity with causation. + +The emotional register is calm documentary restraint. Scale and proximity do the work. No gore, invented biography, spectacle, countdown pressure, unsupported rhetoric, or claim that a map pin represents an animal count. + +## Chapters + +### 1. One animal + +Begin with a single accessible mark and text equivalent: “One mark stands for one individual animal in the selected count.” This is moral framing, not a measured biography. Do not give the animal an invented name, personality, facility, or life history. A control opens “what this number means,” the claim ledger, and source-status vocabulary. + +### 2. A group and an explicit unit change + +Show a small synthetic fixture first: one mark → ten → one thousand. Keep the multiplier visible at every transition. The visual may reuse marks or canvas bins; it must not render one DOM node per animal. Screen readers receive the same arithmetic as a table. The synthetic fixture is clearly labeled prototype-only and never mixed with the FAOSTAT edition. + +### 3. The dated annual scale + +Introduce the selected private-validated edition with a persistent scope card: + +> 2024: 87,892,277,479 heads in 16 selected FAOSTAT land-animal meat categories. This is a source-backed annual estimate for the defined scope, not a live count and not every land animal killed for food. + +The final wording remains blocked until the publication gate is completed. Keep the exact value, rounded display, included populations, exclusions, source revision, and uncertainty one interaction away and in the text equivalent. + +### 4. Elapsed time as a mathematical translation + +The 2024 calendar year has 366 days. For the selected edition: + +`87,892,277,479 / 366 = 240,142,834.64 heads/day` + +`87,892,277,479 / (366 × 86,400) = 2,779.43 heads/second` + +Display rounded values as “about 240.1 million per day” and “about 2,779 per second,” with the exact formula and year convention visible. Say: “This clock translates an annual estimate into an average rate. It does not observe events as they happen.” Pause, step, reduced-motion, keyboard, and table controls are required. The clock must calculate from elapsed time rather than drift as an independent count. + +### 5. Where is this documented? + +Offer optional town/region entry and a map-free list equivalent. Use only the selected public facility release/profile and current privacy restrictions. A nearby result may be described as “documented nearby in this dataset,” with source, observation date, location precision, review state, and correction link. Do not say the facility killed a share of the global total, that it is operating now, that a visitor’s home is near it, or that a general source proves a facility-specific practice. + +If there is no approved real nearby record, use the synthetic facility fixture and label it “synthetic evidence — demonstration only.” The synthetic record must never be counted in public facility totals or presented as a real place. + +### 6. Return to the individual and routes onward + +End by returning from the number to one mark, then offer: sources and methodology; version/correction history; local evidence search; and useful advocacy or research routes. Action routes must not imply that visiting, contacting, or targeting a facility is safe or authorized. The correction/privacy route remains visible for every record and story claim. + +## Evidence card contract + +Every numeric or nearby-evidence card includes its claim type (`observed`, `derived`, `interpretive`, or `synthetic`), source origin, source name and URL where safe, retrieval/effective/observation date, release/version, unit, scope, exclusions, uncertainty, formula if derived, and correction path. `Government-sourced`, `project-reviewed`, `project-approved`, and `project-published` are separate labels. + +## Accessibility and comprehension checks + +- A visitor can navigate chapters and controls with a keyboard. +- Reduced motion exposes the same values, multipliers, formulas, and sources. +- A text/table equivalent explains every visual unit change. +- Status is not communicated by color, hover, motion, or proximity alone. +- Readers unfamiliar with the project can state the population, year, unit, and whether the clock is live after using the prototype. +- Readers can find the source citation and distinguish the global estimate from nearby facility evidence. +- The story does not expose precise user-entered locations in URLs, analytics, shared links, logs, or error messages. + +## Claims intentionally out of scope + +The prototype does not claim an animal’s individual biography, a facility’s annual throughput, conditions at every site, a universal “all animals killed” total, an aquatic conversion, or an intentional concealment by industry actors. Those would require separate evidence packets and review. diff --git a/pipeline/scripts/diagnostics/validate-statistics-catalog.py b/pipeline/scripts/diagnostics/validate-statistics-catalog.py new file mode 100644 index 0000000..1e1fc5a --- /dev/null +++ b/pipeline/scripts/diagnostics/validate-statistics-catalog.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python3 +"""Validate a versioned aggregate-statistics catalog without publishing it.""" +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parents[3])) + +from pipeline.statistics.catalog import StatisticsCatalogError, load_catalog, validate_catalog + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("catalog", type=Path) + args = parser.parse_args() + try: + catalog = load_catalog(args.catalog) + report = validate_catalog(catalog) + except StatisticsCatalogError as error: + print(json.dumps({"status": "blocked", "error": str(error)}, indent=2)) + return 1 + print(json.dumps(report, indent=2)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/pipeline/scripts/stages/ingest-faostat-animal-scale.py b/pipeline/scripts/stages/ingest-faostat-animal-scale.py new file mode 100644 index 0000000..6a2edd8 --- /dev/null +++ b/pipeline/scripts/stages/ingest-faostat-animal-scale.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 +"""Ingest the selected FAOSTAT land-animal rows into a validated private catalog.""" +from __future__ import annotations + +import argparse +import csv +import json +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parents[3])) + +from pipeline.statistics.faostat import build_entry + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("input_csv", type=Path) + parser.add_argument("output_catalog", type=Path) + parser.add_argument("--artifact-path", required=True) + parser.add_argument("--sha256", required=True) + parser.add_argument("--byte-size", required=True, type=int) + parser.add_argument("--retrieved-at-utc", required=True) + args = parser.parse_args() + with args.input_csv.open(newline="", encoding="utf-8-sig") as handle: + entry = build_entry( + csv.DictReader(handle), + artifact_path=args.artifact_path, + sha256=args.sha256, + byte_size=args.byte_size, + retrieved_at_utc=args.retrieved_at_utc, + ) + catalog = {"schema_version": "aggregate-statistics-catalog-v1", "catalog_id": "animal-scale-2024-edition", "catalog_version": "1.0.0", "publication_state": "private-validated", "statistics": [entry]} + args.output_catalog.parent.mkdir(parents=True, exist_ok=True) + args.output_catalog.write_text(json.dumps(catalog, ensure_ascii=False, sort_keys=True, indent=2) + "\n", encoding="utf-8") + print(json.dumps({"status": "passed", "statistic_id": entry["statistic_id"], "central": entry["estimate"]["central"]}, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/pipeline/statistics/__init__.py b/pipeline/statistics/__init__.py new file mode 100644 index 0000000..f232846 --- /dev/null +++ b/pipeline/statistics/__init__.py @@ -0,0 +1,5 @@ +"""Versioned aggregate statistics contracts and validation.""" + +from .catalog import StatisticsCatalogError, load_catalog, validate_catalog + +__all__ = ["StatisticsCatalogError", "load_catalog", "validate_catalog"] diff --git a/pipeline/statistics/catalog.py b/pipeline/statistics/catalog.py new file mode 100644 index 0000000..27ce22c --- /dev/null +++ b/pipeline/statistics/catalog.py @@ -0,0 +1,303 @@ +"""Validate source-linked aggregate statistics independently of facilities. + +The catalog is deliberately conservative. It accepts a count only when its +scope, units, estimate semantics, provenance, revision, and citations are +explicit. It does not turn an annual estimate into a live observation. +""" +from __future__ import annotations + +import hashlib +import json +import re +from datetime import datetime, timezone +from pathlib import Path +from typing import Any +from urllib.parse import urlparse + + +CATALOG_SCHEMA_VERSION = "aggregate-statistics-catalog-v1" +_SHA256 = re.compile(r"^[0-9a-f]{64}$") +_DATE = re.compile(r"^\d{4}-\d{2}-\d{2}$") +_REQUIRED_ENTRY_FIELDS = { + "statistic_id", + "version_id", + "label", + "source", + "population_scope", + "geography", + "period", + "unit", + "estimate", + "method", + "exclusions", + "uncertainty", + "provenance", + "revision", + "citations", + "status", +} + + +class StatisticsCatalogError(ValueError): + """The catalog is malformed or contains an unsafe aggregate claim.""" + + +def _fail(path: str, message: str) -> None: + raise StatisticsCatalogError(f"{path}: {message}") + + +def _object(value: Any, path: str) -> dict[str, Any]: + if not isinstance(value, dict): + _fail(path, "must be an object") + return value + + +def _nonempty_string(value: Any, path: str) -> str: + if not isinstance(value, str) or not value.strip(): + _fail(path, "must be a non-empty string") + return value + + +def _nonnegative_number(value: Any, path: str) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)) or value < 0: + _fail(path, "must be a non-negative number") + return float(value) + + +def _date(value: Any, path: str) -> None: + _nonempty_string(value, path) + if not _DATE.fullmatch(value): + _fail(path, "must be YYYY-MM-DD") + + +def _timestamp(value: Any, path: str) -> None: + _nonempty_string(value, path) + try: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError as exc: + raise StatisticsCatalogError(f"{path}: must be ISO-8601") from exc + if parsed.tzinfo is None: + _fail(path, "must include a timezone") + + +def _https_url(value: Any, path: str) -> None: + _nonempty_string(value, path) + parsed = urlparse(value) + if parsed.scheme != "https" or not parsed.netloc: + _fail(path, "must be an HTTPS URL") + + +def _validate_estimate(entry: dict[str, Any]) -> None: + estimate = _object(entry["estimate"], "estimate") + if estimate.get("type") not in {"point", "range", "point-with-qualitative-uncertainty"}: + _fail("estimate.type", "must declare point, range, or point-with-qualitative-uncertainty") + central = _nonnegative_number(estimate.get("central"), "estimate.central") + low = estimate.get("low") + high = estimate.get("high") + if (low is None) != (high is None): + _fail("estimate", "low and high must be supplied together") + if low is not None and high is not None: + low_number = _nonnegative_number(low, "estimate.low") + high_number = _nonnegative_number(high, "estimate.high") + if low_number > central or central > high_number: + _fail("estimate", "must satisfy low <= central <= high") + if entry["unit"].get("kind") == "count" and not float(central).is_integer(): + _fail("estimate.central", "count estimates must be whole units") + + +def _validate_components(entry: dict[str, Any]) -> None: + components = entry.get("components") + if components is None: + return + if not isinstance(components, list) or not components: + _fail("components", "must be a non-empty list when present") + ids: set[str] = set() + overlap_groups: set[str] = set() + converted_total = 0.0 + for index, component in enumerate(components): + path = f"components[{index}]" + component = _object(component, path) + component_id = _nonempty_string(component.get("component_id"), f"{path}.component_id") + if component_id in ids: + _fail(path, "component_id must be unique") + ids.add(component_id) + _nonempty_string(component.get("population"), f"{path}.population") + _nonempty_string(component.get("source_item"), f"{path}.source_item") + _nonempty_string(component.get("source_unit"), f"{path}.source_unit") + if component.get("source_flag") is not None: + _nonempty_string(component["source_flag"], f"{path}.source_flag") + source_value = _nonnegative_number(component.get("source_value"), f"{path}.source_value") + conversion = _nonnegative_number(component.get("conversion_to_central_unit"), f"{path}.conversion_to_central_unit") + converted = source_value * conversion + if not float(converted).is_integer(): + _fail(path, "converted component must be a whole count") + converted_total += converted + group = component.get("overlap_group") + if group is not None: + group = _nonempty_string(group, f"{path}.overlap_group") + if group in overlap_groups: + _fail(path, "overlap_group repeats; overlapping components must be reconciled before summing") + overlap_groups.add(group) + aggregation = _object(entry.get("aggregation"), "aggregation") + if aggregation.get("operation") != "sum": + _fail("aggregation.operation", "component aggregates must declare sum") + if aggregation.get("overlap_status") != "resolved-non-overlapping": + _fail("aggregation.overlap_status", "must explicitly resolve overlap before summing") + _nonempty_string(aggregation.get("basis"), "aggregation.basis") + central = float(entry["estimate"]["central"]) + if abs(converted_total - central) > 0.000001: + _fail("components", "converted component total does not equal estimate.central") + + +def _validate_entry(entry: Any, index: int) -> None: + path = f"statistics[{index}]" + entry = _object(entry, path) + missing = _REQUIRED_ENTRY_FIELDS - entry.keys() + if missing: + _fail(path, "missing " + ", ".join(sorted(missing))) + _nonempty_string(entry["statistic_id"], f"{path}.statistic_id") + _nonempty_string(entry["version_id"], f"{path}.version_id") + _nonempty_string(entry["label"], f"{path}.label") + if entry["status"] != "validated-private": + _fail(f"{path}.status", "must be validated-private until an authorized release approves publication") + + source = _object(entry["source"], f"{path}.source") + _nonempty_string(source.get("source_id"), f"{path}.source.source_id") + _nonempty_string(source.get("publisher"), f"{path}.source.publisher") + _https_url(source.get("dataset_url"), f"{path}.source.dataset_url") + _nonempty_string(source.get("dataset_name"), f"{path}.source.dataset_name") + _nonempty_string(source.get("origin"), f"{path}.source.origin") + + scope = _object(entry["population_scope"], f"{path}.population_scope") + _nonempty_string(scope.get("scope_id"), f"{path}.population_scope.scope_id") + included = scope.get("included") + if not isinstance(included, list) or not included or any(not isinstance(item, str) or not item.strip() for item in included): + _fail(f"{path}.population_scope.included", "must be a non-empty list of named populations") + _nonempty_string(scope.get("activity"), f"{path}.population_scope.activity") + _nonempty_string(scope.get("animal_class"), f"{path}.population_scope.animal_class") + + geography = _object(entry["geography"], f"{path}.geography") + _nonempty_string(geography.get("level"), f"{path}.geography.level") + _nonempty_string(geography.get("area"), f"{path}.geography.area") + period = _object(entry["period"], f"{path}.period") + _nonempty_string(period.get("kind"), f"{path}.period.kind") + _date(period.get("start"), f"{path}.period.start") + _date(period.get("end"), f"{path}.period.end") + if period["start"] > period["end"]: + _fail(f"{path}.period", "start must not be after end") + if period.get("calendar_year") is not None and period["start"][:4] != str(period["calendar_year"]): + _fail(f"{path}.period.calendar_year", "must match period.start") + + unit = _object(entry["unit"], f"{path}.unit") + if unit.get("kind") not in {"count", "mass", "rate"}: + _fail(f"{path}.unit.kind", "must be count, mass, or rate") + _nonempty_string(unit.get("name"), f"{path}.unit.name") + _nonempty_string(unit.get("numerator"), f"{path}.unit.numerator") + if unit.get("scale") is not None: + _nonnegative_number(unit["scale"], f"{path}.unit.scale") + _validate_estimate(entry) + + method = _object(entry["method"], f"{path}.method") + if method.get("type") not in {"direct-source-observation", "derived-from-components", "converted-from-source"}: + _fail(f"{path}.method.type", "must declare a supported method") + _nonempty_string(method.get("description"), f"{path}.method.description") + _nonempty_string(method.get("formula"), f"{path}.method.formula") + exclusions = entry["exclusions"] + if not isinstance(exclusions, list) or any(not isinstance(item, str) or not item.strip() for item in exclusions): + _fail(f"{path}.exclusions", "must be a list of explicit strings") + uncertainty = _object(entry["uncertainty"], f"{path}.uncertainty") + _nonempty_string(uncertainty.get("kind"), f"{path}.uncertainty.kind") + _nonempty_string(uncertainty.get("statement"), f"{path}.uncertainty.statement") + if entry["estimate"]["low"] is None and uncertainty["kind"] == "numeric-range": + _fail(f"{path}.uncertainty", "numeric-range requires low and high estimate bounds") + + provenance = _object(entry["provenance"], f"{path}.provenance") + _nonempty_string(provenance.get("artifact_id"), f"{path}.provenance.artifact_id") + _nonempty_string(provenance.get("artifact_path"), f"{path}.provenance.artifact_path") + if not _SHA256.fullmatch(str(provenance.get("sha256", "")).lower()): + _fail(f"{path}.provenance.sha256", "must be a lowercase SHA-256 checksum") + byte_size = provenance.get("byte_size") + if not isinstance(byte_size, int) or byte_size <= 0: + _fail(f"{path}.provenance.byte_size", "must be a positive integer") + _timestamp(provenance.get("retrieved_at_utc"), f"{path}.provenance.retrieved_at_utc") + _date(provenance.get("effective_date"), f"{path}.provenance.effective_date") + if provenance.get("publication_date") is not None: + _date(provenance["publication_date"], f"{path}.provenance.publication_date") + if provenance.get("public_artifact") is not False: + _fail(f"{path}.provenance.public_artifact", "must be false for retained raw evidence") + + revision = _object(entry["revision"], f"{path}.revision") + _nonempty_string(revision.get("revision_id"), f"{path}.revision.revision_id") + _date(revision.get("released_date"), f"{path}.revision.released_date") + if revision.get("released_at_utc") is not None: + _timestamp(revision["released_at_utc"], f"{path}.revision.released_at_utc") + if revision.get("supersedes") is not None: + _nonempty_string(revision["supersedes"], f"{path}.revision.supersedes") + _nonempty_string(revision.get("change_note"), f"{path}.revision.change_note") + + citations = entry["citations"] + if not isinstance(citations, list) or not citations: + _fail(f"{path}.citations", "must be a non-empty list") + citation_ids: set[str] = set() + for citation_index, citation in enumerate(citations): + citation_path = f"{path}.citations[{citation_index}]" + citation = _object(citation, citation_path) + citation_id = _nonempty_string(citation.get("citation_id"), f"{citation_path}.citation_id") + if citation_id in citation_ids: + _fail(citation_path, "citation_id must be unique") + citation_ids.add(citation_id) + _nonempty_string(citation.get("title"), f"{citation_path}.title") + _https_url(citation.get("url"), f"{citation_path}.url") + _nonempty_string(citation.get("locator"), f"{citation_path}.locator") + _date(citation.get("accessed_date"), f"{citation_path}.accessed_date") + references = entry.get("citation_ids", [citation["citation_id"] for citation in citations]) + if not isinstance(references, list) or not references or any(reference not in citation_ids for reference in references): + _fail(f"{path}.citation_ids", "must reference declared citations") + _validate_components(entry) + + +def validate_catalog(catalog: dict[str, Any]) -> dict[str, Any]: + """Validate a catalog and return a row-free report suitable for CI.""" + catalog = _object(catalog, "catalog") + if catalog.get("schema_version") != CATALOG_SCHEMA_VERSION: + _fail("catalog.schema_version", f"must be {CATALOG_SCHEMA_VERSION}") + entries = catalog.get("statistics") + if not isinstance(entries, list) or not entries: + _fail("catalog.statistics", "must be a non-empty list") + ids: set[str] = set() + versions: set[str] = set() + for index, entry in enumerate(entries): + _validate_entry(entry, index) + if entry["statistic_id"] in ids: + _fail(f"statistics[{index}].statistic_id", "must be unique") + if entry["version_id"] in versions: + _fail(f"statistics[{index}].version_id", "must be unique") + ids.add(entry["statistic_id"]) + versions.add(entry["version_id"]) + return { + "schema_version": CATALOG_SCHEMA_VERSION, + "status": "passed", + "statistics_count": len(entries), + "statistic_ids": sorted(ids), + } + + +def load_catalog(path: str | Path) -> dict[str, Any]: + catalog_path = Path(path) + try: + catalog = json.loads(catalog_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise StatisticsCatalogError(f"cannot read catalog {catalog_path}: {exc}") from exc + validate_catalog(catalog) + return catalog + + +def sha256_file(path: str | Path) -> tuple[str, int]: + """Return the checksum and size used by an acquisition metadata record.""" + digest = hashlib.sha256() + size = 0 + with Path(path).open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + size += len(chunk) + return digest.hexdigest(), size diff --git a/pipeline/statistics/faostat.py b/pipeline/statistics/faostat.py new file mode 100644 index 0000000..12aa83e --- /dev/null +++ b/pipeline/statistics/faostat.py @@ -0,0 +1,156 @@ +"""Deterministically select the private FAOSTAT land-animal edition.""" +from __future__ import annotations + +from decimal import Decimal, InvalidOperation +from typing import Any, Iterable + +from .catalog import validate_catalog + + +FAOSTAT_ITEM_SCOPE = ( + ("Meat of asses, fresh or chilled", "asses", "An", 1), + ("Meat of buffalo, fresh or chilled", "buffalo", "An", 1), + ("Meat of camels, fresh or chilled", "camels", "An", 1), + ("Meat of cattle with the bone, fresh or chilled", "cattle", "An", 1), + ("Meat of chickens, fresh or chilled", "chickens", "1000 An", 1000), + ("Meat of ducks, fresh or chilled", "ducks", "1000 An", 1000), + ("Meat of geese, fresh or chilled", "geese", "1000 An", 1000), + ("Meat of goat, fresh or chilled", "goats", "An", 1), + ("Meat of mules, fresh or chilled", "mules", "An", 1), + ("Meat of other domestic camelids, fresh or chilled", "other domestic camelids", "An", 1), + ("Meat of other domestic rodents, fresh or chilled", "other domestic rodents", "1000 An", 1000), + ("Meat of pig with the bone, fresh or chilled", "pigs", "An", 1), + ("Meat of pigeons and other birds n.e.c., fresh, chilled or frozen", "pigeons and other birds n.e.c.", "1000 An", 1000), + ("Meat of rabbits and hares, fresh or chilled", "rabbits and hares", "1000 An", 1000), + ("Meat of sheep, fresh or chilled", "sheep", "An", 1), + ("Meat of turkeys, fresh or chilled", "turkeys", "1000 An", 1000), +) + + +def _whole_value(row: dict[str, Any], item: str) -> int: + try: + value = Decimal(str(row["Value"])) + except (KeyError, InvalidOperation) as exc: + raise ValueError(f"{item}: invalid Value") from exc + if value < 0 or value != value.to_integral_value(): + raise ValueError(f"{item}: source Value must be a non-negative whole number") + return int(value) + + +def build_entry( + rows: Iterable[dict[str, Any]], + *, + artifact_path: str, + sha256: str, + byte_size: int, + retrieved_at_utc: str, + effective_date: str = "2024-12-31", + publication_date: str = "2025-12-15", +) -> dict[str, Any]: + """Build and validate the 2024 entry from normalized FAOSTAT rows.""" + expected = {item: (population, unit, conversion) for item, population, unit, conversion in FAOSTAT_ITEM_SCOPE} + selected: dict[str, dict[str, Any]] = {} + for row in rows: + if row.get("Area") != "World" or row.get("Year") != "2024" or row.get("Element") != "Producing Animals/Slaughtered": + continue + item = row.get("Item") + if item not in expected: + continue + if item in selected: + raise ValueError(f"duplicate selected FAOSTAT row: {item}") + population, expected_unit, conversion = expected[item] + if row.get("Unit") != expected_unit: + raise ValueError(f"{item}: expected unit {expected_unit}, got {row.get('Unit')}") + source_value = _whole_value(row, item) + selected[item] = { + "component_id": population.replace(" ", "-"), + "population": population, + "source_item": item, + "source_unit": expected_unit, + "source_value": source_value, + "conversion_to_central_unit": conversion, + "overlap_group": population, + } + if row.get("Flag"): + selected[item]["source_flag"] = row["Flag"] + missing = [item for item, *_ in FAOSTAT_ITEM_SCOPE if item not in selected] + if missing: + raise ValueError("missing selected FAOSTAT rows: " + ", ".join(missing)) + components = [selected[item] for item, *_ in FAOSTAT_ITEM_SCOPE] + central = sum(component["source_value"] * component["conversion_to_central_unit"] for component in components) + entry = { + "statistic_id": "land-animals-slaughtered-for-meat-world-2024", + "version_id": "land-animals-slaughtered-for-meat-world-2024-faostat-qcl-2025-12-15-v1", + "label": "FAOSTAT World slaughtered animals for selected land-animal meat items, 2024", + "status": "validated-private", + "source": { + "source_id": "faostat.qcl.livestock-primary", + "publisher": "Food and Agriculture Organization of the United Nations (FAO)", + "dataset_name": "Livestock primary (Global, National - Annual)", + "dataset_url": "https://data.fao.org/catalog/iso/55375b1e-51d0-47db-ac9b-536ac8a1c738", + "origin": "intergovernmental statistical source", + "license": "CC-BY-4.0", + }, + "population_scope": { + "scope_id": "faostat-meat-leaf-items-heads-v1", + "animal_class": "land animals", + "activity": "slaughtered for meat production", + "included": [population for _, population, _, _ in FAOSTAT_ITEM_SCOPE], + "scope_note": "The selected scope contains the 16 non-aggregate FAOSTAT 2024 World meat-item rows that expose Producing Animals/Slaughtered. It is not a claim to count every land animal killed for food.", + }, + "geography": { + "level": "global aggregate", + "area": "World (FAOSTAT Area)", + "geography_note": "FAOSTAT values describe animals slaughtered within national boundaries, irrespective of origin; the World row is an FAO aggregate.", + }, + "period": {"kind": "calendar-year", "start": "2024-01-01", "end": "2024-12-31", "calendar_year": 2024}, + "unit": {"kind": "count", "name": "animals (heads)", "numerator": "slaughtered animals", "denominator": "calendar year", "scale": 1}, + "estimate": {"type": "point-with-qualitative-uncertainty", "low": None, "central": central, "high": None}, + "method": { + "type": "derived-from-components", + "description": "Select World rows whose item is a named Meat of ... leaf item, retain FAOSTAT's source value and flag, convert 1000 An to heads, then sum the non-overlapping rows.", + "formula": "sum(source_value * conversion_to_central_unit)", + }, + "exclusions": [ + "FAOSTAT aggregate rows such as Meat, Total; Beef and Buffalo Meat, primary; Sheep and Goat Meat; and Meat, Poultry", + "dairy and egg production, including animals culled from those systems when not represented in the selected meat rows", + "aquatic animals, fish, crustaceans, molluscs, and other aquatic populations", + "hides, fat, offal, wool, milk, eggs, and other non-meat commodities", + "species or items with no selected 2024 World leaf row", + "a numeric uncertainty interval, because this FAOSTAT release does not publish one for the selected rows", + ], + "uncertainty": { + "kind": "qualitative-data-flags-and-coverage", + "statement": "FAO says country inputs can be reported, estimated, supplemented from unofficial sources, or imputed, and flags them accordingly. The selected World rows are therefore a source-backed estimate with unknown numeric error; omitted categories and coverage gaps remain unknown.", + }, + "provenance": { + "artifact_id": "faostat-qcl-2025-12-15-production-crops-livestock-normalized", + "artifact_path": artifact_path, + "public_artifact": False, + "sha256": sha256, + "byte_size": byte_size, + "retrieved_at_utc": retrieved_at_utc, + "effective_date": effective_date, + "publication_date": publication_date, + "retrieval_url": "https://bulks-faostat.fao.org/production/Production_Crops_Livestock_E_All_Data_(Normalized).zip", + "code_version": "animal-scale-statistics-v1", + "config_version": "faostat-qcl-leaf-meat-items-v1", + "retention_note": "The downloaded archive is ignored private research evidence; only this sanitized manifest is tracked.", + }, + "revision": { + "revision_id": "faostat-qcl-2025-12-15", + "released_date": publication_date, + "supersedes": None, + "change_note": "Initial private catalog entry for the 2024 FAOSTAT revision; later source revisions must create a new version and preserve this one.", + }, + "citations": [ + {"citation_id": "faostat-catalog-livestock-primary", "title": "FAOSTAT: Livestock primary (Global, National - Annual)", "url": "https://data.fao.org/catalog/iso/55375b1e-51d0-47db-ac9b-536ac8a1c738", "locator": "Dataset abstract, data lineage, units, time coverage, and revision metadata", "accessed_date": "2026-09-15"}, + {"citation_id": "faostat-qcl-methodology", "title": "FAOSTAT Agricultural production — Livestock methodology", "url": "https://files-faostat.fao.org/production/QCL/QCL_methodology_e.pdf", "locator": "PDF page 3: meat scope; PDF page 4: reference period and totals", "accessed_date": "2026-09-15"}, + {"citation_id": "faostat-qcl-bulk-archive", "title": "FAOSTAT Crops and livestock products normalized bulk archive", "url": "https://bulks-faostat.fao.org/production/Production_Crops_Livestock_E_All_Data_(Normalized).zip", "locator": "2024 World rows; Element=Producing Animals/Slaughtered; selected Meat of ... items", "accessed_date": "2026-09-15"}, + ], + "citation_ids": ["faostat-catalog-livestock-primary", "faostat-qcl-methodology", "faostat-qcl-bulk-archive"], + "aggregation": {"operation": "sum", "overlap_status": "resolved-non-overlapping", "basis": "Each component is a distinct FAOSTAT meat leaf item; aggregate meat rows and non-meat commodity rows are excluded."}, + "components": components, + } + validate_catalog({"schema_version": "aggregate-statistics-catalog-v1", "statistics": [entry]}) + return entry diff --git a/pipeline/statistics/fixtures/synthetic_catalog.json b/pipeline/statistics/fixtures/synthetic_catalog.json new file mode 100644 index 0000000..a6a1237 --- /dev/null +++ b/pipeline/statistics/fixtures/synthetic_catalog.json @@ -0,0 +1,54 @@ +{ + "schema_version": "aggregate-statistics-catalog-v1", + "catalog_id": "synthetic-statistics-fixture", + "catalog_version": "1.0.0", + "statistics": [ + { + "statistic_id": "synthetic-land-count-2024", + "version_id": "synthetic-land-count-2024-v1", + "label": "Synthetic selected land animals, 2024", + "source": { + "source_id": "synthetic.source", + "publisher": "Synthetic fixture", + "dataset_name": "Synthetic annual source", + "dataset_url": "https://example.org/synthetic-source", + "origin": "synthetic test source" + }, + "population_scope": { + "scope_id": "synthetic-two-populations-v1", + "animal_class": "land animals", + "activity": "slaughtered for food", + "included": ["synthetic birds", "synthetic mammals"], + "scope_note": "Synthetic values used only to exercise validation." + }, + "geography": {"level": "global aggregate", "area": "Synthetic World"}, + "period": {"kind": "calendar-year", "start": "2024-01-01", "end": "2024-12-31", "calendar_year": 2024}, + "unit": {"kind": "count", "name": "animals (heads)", "numerator": "slaughtered animals", "denominator": "calendar year", "scale": 1}, + "estimate": {"type": "point-with-qualitative-uncertainty", "low": null, "central": 2001, "high": null}, + "method": {"type": "derived-from-components", "description": "Convert synthetic source rows to heads and sum them.", "formula": "sum(source_value * conversion_to_central_unit)"}, + "exclusions": ["Synthetic rows outside the fixture scope"], + "uncertainty": {"kind": "qualitative-data-flags-and-coverage", "statement": "Synthetic fixture has no numeric uncertainty interval."}, + "provenance": { + "artifact_id": "synthetic-artifact-v1", + "artifact_path": "pipeline/statistics/fixtures/synthetic-source.csv", + "public_artifact": false, + "sha256": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + "byte_size": 42, + "retrieved_at_utc": "2026-09-15T00:00:00Z", + "effective_date": "2024-12-31", + "publication_date": null, + "code_version": "synthetic-v1", + "config_version": "synthetic-v1" + }, + "revision": {"revision_id": "synthetic-revision-v1", "released_date": "2026-09-15", "supersedes": null, "change_note": "Initial synthetic fixture."}, + "citations": [{"citation_id": "synthetic-citation", "title": "Synthetic fixture source", "url": "https://example.org/synthetic-source", "locator": "fixture rows", "accessed_date": "2026-09-15"}], + "citation_ids": ["synthetic-citation"], + "aggregation": {"operation": "sum", "overlap_status": "resolved-non-overlapping", "basis": "The two synthetic rows are disjoint populations."}, + "components": [ + {"component_id": "birds", "population": "synthetic birds", "source_item": "Synthetic birds", "source_unit": "1000 An", "source_value": 2, "conversion_to_central_unit": 1000, "overlap_group": "synthetic birds"}, + {"component_id": "mammals", "population": "synthetic mammals", "source_item": "Synthetic mammals", "source_unit": "An", "source_value": 1, "conversion_to_central_unit": 1, "overlap_group": "synthetic mammals"} + ], + "status": "validated-private" + } + ] +} diff --git a/pipeline/statistics/fixtures/synthetic_faostat.csv b/pipeline/statistics/fixtures/synthetic_faostat.csv new file mode 100644 index 0000000..7ede58a --- /dev/null +++ b/pipeline/statistics/fixtures/synthetic_faostat.csv @@ -0,0 +1,18 @@ +Area,Item,Element,Year,Unit,Value,Flag +World,"Meat of asses, fresh or chilled",Producing Animals/Slaughtered,2024,An,1,E +World,"Meat of buffalo, fresh or chilled",Producing Animals/Slaughtered,2024,An,2,E +World,"Meat of camels, fresh or chilled",Producing Animals/Slaughtered,2024,An,3,E +World,"Meat of cattle with the bone, fresh or chilled",Producing Animals/Slaughtered,2024,An,4,E +World,"Meat of chickens, fresh or chilled",Producing Animals/Slaughtered,2024,1000 An,5,E +World,"Meat of ducks, fresh or chilled",Producing Animals/Slaughtered,2024,1000 An,6,E +World,"Meat of geese, fresh or chilled",Producing Animals/Slaughtered,2024,1000 An,7,E +World,"Meat of goat, fresh or chilled",Producing Animals/Slaughtered,2024,An,8,E +World,"Meat of mules, fresh or chilled",Producing Animals/Slaughtered,2024,An,9,E +World,"Meat of other domestic camelids, fresh or chilled",Producing Animals/Slaughtered,2024,An,10,E +World,"Meat of other domestic rodents, fresh or chilled",Producing Animals/Slaughtered,2024,1000 An,11,E +World,"Meat of pig with the bone, fresh or chilled",Producing Animals/Slaughtered,2024,An,12,A +World,"Meat of pigeons and other birds n.e.c., fresh, chilled or frozen",Producing Animals/Slaughtered,2024,1000 An,13,E +World,"Meat of rabbits and hares, fresh or chilled",Producing Animals/Slaughtered,2024,1000 An,14,E +World,"Meat of sheep, fresh or chilled",Producing Animals/Slaughtered,2024,An,15,E +World,"Meat of turkeys, fresh or chilled",Producing Animals/Slaughtered,2024,1000 An,16,A +World,"Meat, Total",Production,2024,t,999,A diff --git a/pipeline/statistics/test_catalog.py b/pipeline/statistics/test_catalog.py new file mode 100644 index 0000000..c326b29 --- /dev/null +++ b/pipeline/statistics/test_catalog.py @@ -0,0 +1,84 @@ +import copy +import json +import unittest +from pathlib import Path + +from pipeline.statistics.catalog import StatisticsCatalogError, load_catalog, validate_catalog + + +ROOT = Path(__file__).parents[2] +MANIFEST = ROOT / "data" / "manifests" / "animal-scale-statistics-catalog.json" +FIXTURE = Path(__file__).parent / "fixtures" / "synthetic_catalog.json" + + +class StatisticsCatalogTests(unittest.TestCase): + def load(self, path: Path) -> dict: + return json.loads(path.read_text(encoding="utf-8")) + + def test_private_faostat_manifest_is_valid(self): + catalog = load_catalog(MANIFEST) + self.assertEqual(catalog["statistics"][0]["estimate"]["central"], 87892277479) + self.assertEqual(validate_catalog(catalog)["status"], "passed") + + def test_synthetic_fixture_is_valid(self): + report = validate_catalog(self.load(FIXTURE)) + self.assertEqual(report["statistics_count"], 1) + + def test_missing_dimension_is_rejected(self): + catalog = self.load(FIXTURE) + del catalog["statistics"][0]["period"]["kind"] + with self.assertRaisesRegex(StatisticsCatalogError, "period.kind"): + validate_catalog(catalog) + + def test_period_order_is_rejected(self): + catalog = self.load(FIXTURE) + catalog["statistics"][0]["period"]["start"] = "2025-01-01" + with self.assertRaisesRegex(StatisticsCatalogError, "period"): + validate_catalog(catalog) + + def test_unit_conversion_must_produce_whole_count(self): + catalog = self.load(FIXTURE) + catalog["statistics"][0]["components"][0]["conversion_to_central_unit"] = 1.5 + with self.assertRaisesRegex(StatisticsCatalogError, "converted component"): + validate_catalog(catalog) + + def test_component_overlap_is_rejected(self): + catalog = self.load(FIXTURE) + catalog["statistics"][0]["components"][1]["overlap_group"] = "synthetic birds" + with self.assertRaisesRegex(StatisticsCatalogError, "overlap_group"): + validate_catalog(catalog) + + def test_aggregate_total_must_reconcile(self): + catalog = self.load(FIXTURE) + catalog["statistics"][0]["estimate"]["central"] = 2002 + with self.assertRaisesRegex(StatisticsCatalogError, "does not equal"): + validate_catalog(catalog) + + def test_numeric_range_requires_two_bounds(self): + catalog = self.load(FIXTURE) + catalog["statistics"][0]["estimate"]["type"] = "range" + catalog["statistics"][0]["uncertainty"]["kind"] = "numeric-range" + with self.assertRaisesRegex(StatisticsCatalogError, "low and high"): + validate_catalog(catalog) + + def test_citation_must_be_secure_and_declared(self): + catalog = self.load(FIXTURE) + catalog["statistics"][0]["citations"][0]["url"] = "http://example.org/source" + with self.assertRaisesRegex(StatisticsCatalogError, "HTTPS"): + validate_catalog(catalog) + + def test_revision_requires_new_version_metadata(self): + catalog = self.load(FIXTURE) + del catalog["statistics"][0]["revision"]["released_date"] + with self.assertRaisesRegex(StatisticsCatalogError, "released_date"): + validate_catalog(catalog) + + def test_copy_with_ambiguous_status_is_not_publishable(self): + catalog = copy.deepcopy(self.load(FIXTURE)) + catalog["statistics"][0]["status"] = "published" + with self.assertRaisesRegex(StatisticsCatalogError, "validated-private"): + validate_catalog(catalog) + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/statistics/test_faostat.py b/pipeline/statistics/test_faostat.py new file mode 100644 index 0000000..8007d96 --- /dev/null +++ b/pipeline/statistics/test_faostat.py @@ -0,0 +1,44 @@ +import csv +import unittest +from pathlib import Path + +from pipeline.statistics.catalog import validate_catalog +from pipeline.statistics.faostat import build_entry + + +FIXTURE = Path(__file__).parent / "fixtures" / "synthetic_faostat.csv" + + +class FaostatIngestionTests(unittest.TestCase): + def rows(self): + with FIXTURE.open(newline="", encoding="utf-8") as handle: + return list(csv.DictReader(handle)) + + def test_selects_leaf_rows_converts_thousands_and_ignores_aggregate_rows(self): + entry = build_entry( + self.rows(), + artifact_path="private/synthetic-faostat.csv", + sha256="0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + byte_size=42, + retrieved_at_utc="2026-09-15T00:00:00Z", + ) + self.assertEqual(entry["estimate"]["central"], 72064) + self.assertEqual(len(entry["components"]), 16) + self.assertEqual(validate_catalog({"schema_version": "aggregate-statistics-catalog-v1", "statistics": [entry]})["status"], "passed") + self.assertEqual(entry["components"][4]["source_flag"], "E") + + def test_missing_leaf_row_is_rejected(self): + rows = self.rows() + rows = [row for row in rows if row["Item"] != "Meat of turkeys, fresh or chilled"] + with self.assertRaisesRegex(ValueError, "missing selected FAOSTAT rows"): + build_entry(rows, artifact_path="private/synthetic.csv", sha256="0" * 64, byte_size=1, retrieved_at_utc="2026-09-15T00:00:00Z") + + def test_source_unit_must_match_selected_definition(self): + rows = self.rows() + rows[4]["Unit"] = "An" + with self.assertRaisesRegex(ValueError, "expected unit"): + build_entry(rows, artifact_path="private/synthetic.csv", sha256="0" * 64, byte_size=1, retrieved_at_utc="2026-09-15T00:00:00Z") + + +if __name__ == "__main__": + unittest.main() From ec54f4563a0fb3dd5395c8a38942f4467b6512e4 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Tue, 15 Sep 2026 14:11:18 -0700 Subject: [PATCH 122/311] Scale V2 discovery queries and local frontend --- docs/api/v2-contract.json | 6 +- docs/api/v2-contract.md | 4 +- docs/performance/v2-discovery-100k.md | 32 +++ frontend/README.md | 2 +- frontend/src/api/FilterMetadataRepository.ts | 2 +- frontend/src/api/LocalLocationRepository.ts | 2 +- frontend/src/api/wireSchema.ts | 2 +- frontend/src/app/App.svelte | 26 +-- frontend/tests/e2e/local-backend.spec.ts | 39 +++- .../unit/localLocationRepository.test.ts | 8 +- .../025_discovery_query_indexes.sql | 21 ++ pipeline/tests/benchmarks/discovery_100k.sql | 32 +++ src/lib.rs | 182 ++++++++++++++++-- 13 files changed, 323 insertions(+), 35 deletions(-) create mode 100644 docs/performance/v2-discovery-100k.md create mode 100644 pipeline/migrations/025_discovery_query_indexes.sql create mode 100644 pipeline/tests/benchmarks/discovery_100k.sql diff --git a/docs/api/v2-contract.json b/docs/api/v2-contract.json index 215d4a3..6fbb755 100644 --- a/docs/api/v2-contract.json +++ b/docs/api/v2-contract.json @@ -5,13 +5,13 @@ "GET /health/live": {"success": {"status": "ok", "service": "uec-api"}}, "GET /health/ready": {"success": {"status": "ready", "database": "ok"}, "unavailable_status": 503}, "GET /health/diagnostics": {"success": {"status": "ok", "privacy": {"diagnostic_identifiers": "excluded"}}, "payloads": "coarse status only"}, - "GET /api/v2/locations": {"success": {"api_version": "v2", "data": [], "meta": {"profile": "official", "next_cursor": null, "coverage_scope": "selected_promoted_release_public_facilities", "count_semantics": "public facility projection rows, not animal counts"}}, "no_release": 200}, + "GET /api/v2/locations": {"success": {"api_version": "v2", "data": [], "meta": {"profile": "official", "next_cursor": null, "query": {"q": null, "filters": {}}, "coverage_scope": "selected_promoted_release_public_facilities", "count_semantics": "public facility projection rows, not animal counts"}}, "query": ["q", "country_code", "region", "category", "source_type", "profile", "display_precision", "lifecycle_status", "cursor", "limit", "min_lon", "min_lat", "max_lon", "max_lat", "latitude", "longitude", "radius_km"], "spatial": "bbox or radius, never both", "no_release": 200}, "GET /api/v2/locations/{facility_id}": {"success": {"api_version": "v2", "data": {}, "meta": {"coverage_scope": "selected_promoted_release_public_facilities", "count_semantics": "public facility projection, not an animal count"}}, "not_found": 404}, "GET /api/v2/locations.csv": {"success_content_type": "text/csv", "bounded_rows": 1000, "profile_required_for_nonofficial": true, "not_found_without_promoted_manifest": 404}, - "GET /api/v2/discovery/filters": {"success": {"api_version": "v2", "contract_version": "v1", "dimensions": {"country_code": "allowlist", "category": "allowlist", "source_type": "allowlist", "profile": "allowlist", "display_precision": "allowlist", "lifecycle_status": "allowlist"}}} + "GET /api/v2/discovery/filters": {"success": {"api_version": "v2", "contract_version": "v1", "dimensions": {"country_code": "allowlist", "region": "release facets", "category": "allowlist", "source_type": "allowlist", "profile": "allowlist", "display_precision": "allowlist", "lifecycle_status": "allowlist"}, "search": {"fields": ["canonical_name", "city", "country_code", "category", "source_name"]}, "spatial": {"bbox": ["min_lon", "min_lat", "max_lon", "max_lat"], "radius": ["latitude", "longitude", "radius_km"]}}} ,"GET /api/dev/preview/candidates": {"success": {"api_version": "dev-preview-v1", "data": [], "meta": {"test_only": true, "private_preview": true, "profile": null, "coverage_scope": "candidate_release_only", "next_cursor": null}}, "auth_header": "X-UEC-Dev-Preview-Token", "production": "unavailable"} ,"GET /api/v2/discovery/facets": {"success": {"api_version": "v2", "meta": {"profile": "official", "release_id": "string", "ruleset_version": "string", "release_created_at": "timestamp", "coverage_scope": "selected_promoted_release_public_facilities", "count_semantics": "eligible public facility projection rows after current suppression; not story-wide or animal counts", "filters": {}}, "dimensions": {}}, "max_values_per_dimension": 20, "counts_are_from": "selected eligible promoted public projection"} }, - "error": {"shape": {"api_version": "v2", "error": {"code": "string", "message": "string"}}, "codes": ["invalid_filter", "invalid_profile", "invalid_limit", "invalid_offset", "invalid_cursor", "invalid_pagination", "database_not_configured", "database_pool_unavailable", "database_transaction_unavailable", "release_query_failed", "location_query_failed", "location_not_found", "rate_limited"]}, + "error": {"shape": {"api_version": "v2", "error": {"code": "string", "message": "string"}}, "codes": ["invalid_filter", "invalid_profile", "invalid_limit", "invalid_offset", "invalid_cursor", "invalid_pagination", "invalid_spatial_query", "database_not_configured", "database_pool_unavailable", "database_transaction_unavailable", "release_query_failed", "location_query_failed", "location_not_found", "rate_limited"]}, "privacy": "Public responses contain reviewed projection fields only; restricted records and raw evidence are never returned." } diff --git a/docs/api/v2-contract.md b/docs/api/v2-contract.md index 04307b7..097b5a5 100644 --- a/docs/api/v2-contract.md +++ b/docs/api/v2-contract.md @@ -10,8 +10,10 @@ Frontend clients should branch on HTTP status and `error.code`, display `message Researchers may request `GET /api/v2/locations.csv?profile=official` (or another explicit supported profile). The export is bounded to 1,000 rows, uses deterministic CSV columns and escaping, contains only the public reviewed projection, and includes `release_profile`, `release_id`, and `manifest_sha256` on every row plus matching response headers. It is unavailable when no promoted release with a manifest exists; it never exposes raw evidence or restricted records. -Clients can discover the current controlled vocabularies at `GET /api/v2/discovery/filters`. Filter values are allowlisted and versioned; clients must not invent category/source/profile/status values or send arbitrary free-text search. `country_code` is validated as an uppercase ISO alpha-2 code and returns zero rows when the project has no capability/source coverage for that country. New country adapters should register capabilities and vocabularies in the contract before becoming public. +Clients can discover the current controlled vocabularies at `GET /api/v2/discovery/filters`. Category/source/profile/precision/lifecycle values are allowlisted and versioned; `country_code` is validated as an uppercase ISO alpha-2 code and `region` is a bounded release-backed city/region value. The bounded `q` parameter searches canonical name, city, country, category, and source name server-side. Bbox (`min_lon`, `min_lat`, `max_lon`, `max_lat`) and radius (`latitude`, `longitude`, `radius_km`) are mutually exclusive and validated before release access. New country adapters should register capabilities and vocabularies in the contract before becoming public. `GET /api/v2/discovery/facets` returns deterministic value/count pairs for the same controlled dimensions, scoped to the selected promoted profile and current public projection. Its metadata includes the selected release, ruleset, release creation time, and an explicit coverage scope. Counts are eligible public facility-projection rows after current suppression; they are not story-wide totals or animal counts. It applies the supplied filters before counting, caps each dimension at 20 values, and returns no addresses, queries, raw payloads, inactive releases, or restricted records. +Clients must treat `(profile, release_id, ruleset_version, query)` as the cache key and discard or refresh cursor pages when release metadata changes. + List and detail metadata use the same coverage scope. Their source identifiers, source URL, retrieval timestamp, review state, release, and ruleset remain record-level provenance; they do not establish a story-wide denominator or an animal count. Narrative aggregate claims must come from a separately sourced, dated editorial ledger. diff --git a/docs/performance/v2-discovery-100k.md b/docs/performance/v2-discovery-100k.md new file mode 100644 index 0000000..ff4e9e2 --- /dev/null +++ b/docs/performance/v2-discovery-100k.md @@ -0,0 +1,32 @@ +# V2 discovery synthetic 100k benchmark + +This is a disposable, synthetic benchmark plan. It must never be populated with +retained source records, names, addresses, coordinates, or geocoder responses. + +## Budgets + +- First list page (50 rows): p95 <= 250 ms at the database, p95 <= 800 ms end to end. +- Cursor page (50 rows): p95 <= 200 ms at the database, p95 <= 700 ms end to end. +- Text search/filter page: p95 <= 350 ms at the database, p95 <= 1 s end to end. +- Spatial viewport/radius page: p95 <= 350 ms at the database, p95 <= 1 s end to end. +- Detail: p95 <= 200 ms at the database, p95 <= 600 ms end to end. +- Browser memory: <= 128 MB attributable to loaded discovery records at 100k rows. + +## Reproduction + +Run `pipeline/tests/benchmarks/discovery_100k.sql` against a disposable PostGIS +database after migrations through `025_discovery_query_indexes.sql` have been +applied. The script creates only `bench_discovery_100k`, fills deterministic +synthetic rows, runs `ANALYZE`, and emits `EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON)` +for the list, cursor, text, bbox, radius, and detail shapes used by the API. +Drop the benchmark table after capture. The benchmark is evidence about query +shape and budget only; it is not publication or release evidence. + +## Local disposable capture (2026-09-15) + +Against PostGIS 16 / PostGIS 3.4 with 100,000 synthetic rows, the captured +single-query timings were: first page 0.072 ms, cursor page 0.136 ms, text +search 3.206 ms, bbox 1.578 ms, radius 0.057 ms, and detail lookup 5.848 ms. +The list/cursor/search/spatial plans used the expected B-tree, trigram GIN, or +geography GiST indexes. These are cold/warm local database plan samples, not a +production load test or an end-to-end p95 claim. diff --git a/frontend/README.md b/frontend/README.md index d4765a6..258b991 100644 --- a/frontend/README.md +++ b/frontend/README.md @@ -23,7 +23,7 @@ These are documented gaps, not frontend claims or invented DTO fields. Phase 3 c Local V2 integration (opt-in only): run the backend with `cargo run` (port 8000), then run the frontend with `npm run dev` (port 5173). In development/preview, Vite proxies `/api` to `http://127.0.0.1:8000`; use `http://127.0.0.1:5173/v2-preview/?mode=local-v2#/` to opt in. The `LocalLocationRepository` still targets `/api/v2/locations?profile=...` only when explicitly invoked; fixture mode remains the default and there is no V1 fallback. The proxy is development-only configuration and production builds do not enable a backend connection. -The opt-in local view loads one API page at a time. When the response includes `next_cursor`, the UI labels its counts, search, and map as partial; text search runs only against loaded records because the current public contract intentionally has no server-side free-text query. Record context is limited to fields in the current V2 wire response and does not imply that review events, evidence hashes, or scoped approvals are available. +The opt-in local view loads one API page at a time. Search, country/region/category/source/profile/precision/lifecycle filters are evaluated by the server against the selected promoted release; cursor pages remain explicit and are never merged across release IDs. Bounded bbox and radius parameters are available to map clients. Record context is limited to fields in the current V2 wire response and does not imply that review events, evidence hashes, or scoped approvals are available. The private scale/story prototype begins with a neutral individual-animal representation and uses only bounded synthetic values. It labels model arithmetic separately from measured facility evidence; no biography, live counter, global animal total, or sourced aggregate is embedded in the production build. Candidate sourced scale figures remain outside this UI until maintainer publication approval. diff --git a/frontend/src/api/FilterMetadataRepository.ts b/frontend/src/api/FilterMetadataRepository.ts index b2c6462..26f47e6 100644 --- a/frontend/src/api/FilterMetadataRepository.ts +++ b/frontend/src/api/FilterMetadataRepository.ts @@ -2,7 +2,7 @@ import { z } from 'zod'; import type { FetchLike } from './LocalLocationRepository'; const dimension = z.object({ values: z.array(z.string()), default: z.string().optional() }); -const metadataSchema = z.object({ api_version: z.literal('v2'), contract_version: z.string(), dimensions: z.object({ country_code: dimension, category: dimension, source_type: dimension, profile: dimension, display_precision: dimension, lifecycle_status: dimension }), pagination: z.object({ limit_max: z.number().int().positive(), cursor: z.string() }), privacy: z.string().min(1) }); +const metadataSchema = z.object({ api_version: z.literal('v2'), contract_version: z.string(), dimensions: z.object({ country_code: dimension, region: dimension.optional(), category: dimension, source_type: dimension, profile: dimension, display_precision: dimension, lifecycle_status: dimension }), spatial: z.object({ bbox: z.array(z.string()), radius: z.array(z.string()) }).optional(), search: z.object({ parameter: z.string(), fields: z.array(z.string()) }).optional(), pagination: z.object({ limit_max: z.number().int().positive(), cursor: z.string() }), privacy: z.string().min(1) }); export type FilterMetadata = z.infer; export class FilterMetadataRepository { constructor(private readonly fetcher: FetchLike = globalThis.fetch, private readonly baseUrl = '') {} diff --git a/frontend/src/api/LocalLocationRepository.ts b/frontend/src/api/LocalLocationRepository.ts index b5203bd..4f54d49 100644 --- a/frontend/src/api/LocalLocationRepository.ts +++ b/frontend/src/api/LocalLocationRepository.ts @@ -4,7 +4,7 @@ import type { Location } from '../domain/location'; export type FetchLike = (input: RequestInfo | URL, init?: RequestInit) => Promise; export type LocalProfile = 'official' | 'secondary' | 'community'; -export type LocationFilters = Readonly<{ country_code?: string | undefined; category?: string | undefined; source_type?: string | undefined; display_precision?: string | undefined; lifecycle_status?: string | undefined; cursor?: string | undefined; limit?: number | undefined }>; +export type LocationFilters = Readonly<{ q?: string | undefined; country_code?: string | undefined; region?: string | undefined; category?: string | undefined; source_type?: string | undefined; display_precision?: string | undefined; lifecycle_status?: string | undefined; min_lon?: number | undefined; min_lat?: number | undefined; max_lon?: number | undefined; max_lat?: number | undefined; latitude?: number | undefined; longitude?: number | undefined; radius_km?: number | undefined; cursor?: string | undefined; limit?: number | undefined }>; export type LocalListResult = Readonly<{ locations: readonly Location[]; releaseId: string; profile: LocalProfile; coverageNote: string; coverageScope?: string; countSemantics?: string; nextCursor: string | null; ruleset?: string }>; export const localOrigin = (value: string | undefined): string | undefined => { if (!value) return undefined; const url = new URL(value); if (url.protocol !== 'http:' || !['127.0.0.1', 'localhost', '::1'].includes(url.hostname)) throw new Error('Local API origin must be loopback HTTP.'); return url.origin; }; const fail = (kind: ApiError['kind'], message: string, status?: number, code?: string): ApiError => Object.assign(new Error(message), { kind, ...(status === undefined ? {} : { status }), ...(code ? { code } : {}) }); diff --git a/frontend/src/api/wireSchema.ts b/frontend/src/api/wireSchema.ts index 88f405b..3e946ce 100644 --- a/frontend/src/api/wireSchema.ts +++ b/frontend/src/api/wireSchema.ts @@ -6,7 +6,7 @@ const locationShape={facility_id:z.string().uuid(),canonical_name:z.string().nul const coordinateRules=(row:{latitude:number|null;longitude:number|null;display_precision:string},ctx:z.RefinementCtx)=>{if((row.latitude===null)!==(row.longitude===null))ctx.addIssue({code:z.ZodIssueCode.custom,message:'coordinate pair must be complete'});if(row.display_precision==='unmapped'&&(row.latitude!==null||row.longitude!==null))ctx.addIssue({code:z.ZodIssueCode.custom,message:'unmapped record cannot have coordinates'});if(row.display_precision!=='unmapped'&&(row.latitude===null||row.longitude===null))ctx.addIssue({code:z.ZodIssueCode.custom,message:'mapped record requires coordinates'});}; export const locationSchema=z.object(locationShape).superRefine(coordinateRules); export const testReleaseLocationSchema=z.object({...locationShape,canonical_name:z.string().nullable(),publication_profile:z.enum(['official','secondary','community']).nullable(),privacy_screening_status:z.enum(['pending','passed','failed']),project_approval:z.union([z.enum(['pending','approved']),z.literal(false),z.literal('not-approved')]),release_ruleset_version:z.string().nullable()}).superRefine(coordinateRules); -export const envelopeSchema=z.object({data:z.array(locationSchema),api_version:z.literal('v2'),meta:z.object({release_id:z.string().nullable(),ruleset_version:z.string().optional(),release_created_at:z.string().optional(),profile:z.enum(['official','secondary','community']),next_cursor:z.string().nullable().optional(),coverage_note:z.string().min(1),coverage_scope:z.string().optional(),count_semantics:z.string().optional()})}); +export const envelopeSchema=z.object({data:z.array(locationSchema),api_version:z.literal('v2'),meta:z.object({release_id:z.string().nullable(),ruleset_version:z.string().optional(),release_created_at:z.string().optional(),profile:z.enum(['official','secondary','community']),next_cursor:z.string().nullable().optional(),coverage_note:z.string().min(1),coverage_scope:z.string().optional(),count_semantics:z.string().optional(),query:z.object({q:z.string().nullable().optional(),filters:z.record(z.string(),z.string().nullable())}).optional()})}); export type WireEnvelope=z.infer;export type WireLocation=z.infer; export type WireTestReleaseLocation=z.infer; export const detailEnvelopeSchema=z.object({data:locationSchema,api_version:z.literal('v2'),meta:z.object({release_id:z.string(),ruleset_version:z.string(),release_created_at:z.string(),profile:z.enum(['official','secondary','community'])})}); diff --git a/frontend/src/app/App.svelte b/frontend/src/app/App.svelte index e4df6e5..f9e39d8 100644 --- a/frontend/src/app/App.svelte +++ b/frontend/src/app/App.svelte @@ -26,6 +26,7 @@ let selected: Location | undefined = locations[0]; let search = ''; let region = 'all'; + let subregion = ''; let category = 'all'; let sourceType = 'all'; let displayPrecision = 'all'; @@ -75,12 +76,11 @@ $: filters = { search, region, category }; $: apiProfile = toApiProfile(profile); $: source = devPreviewMode ? previewRows : localMode ? loaded : (profile === 'community' ? locations.slice(0, 1) : locations); - $: visibleLocations = filterLocations(source, filters); + $: visibleLocations = localMode ? source : filterLocations(source, filters); $: exportPreview = devPreviewExportLabel(devPreviewMode) ?? previewExport(makeExportModel(visibleLocations, profile, release)); $: eligibleExport = !devPreviewMode && localMode && localStatus === 'ready' && Boolean(release); $: profileLabelText = profileLabel(profile); - // V2 deliberately has no q parameter. Search never refetches a page. - $: remoteQuery = `${apiProfile}|${region}|${category}|${sourceType}|${displayPrecision}|${lifecycleStatus}`; + $: remoteQuery = `${apiProfile}|${search.trim()}|${region}|${subregion}|${category}|${sourceType}|${displayPrecision}|${lifecycleStatus}`; $: if (localMode && (localStatus === 'ready' || localStatus === 'loading') && remoteQuery !== lastRemoteQuery) { pushFilterUrl(); void loadLocal(); @@ -96,8 +96,9 @@ const pushFilterUrl = () => { const url = new URL(window.location.href); - for (const key of ['country_code', 'category', 'source_type', 'display_precision', 'lifecycle_status', 'q']) url.searchParams.delete(key); + for (const key of ['country_code', 'region', 'category', 'source_type', 'display_precision', 'lifecycle_status', 'q']) url.searchParams.delete(key); if (region !== 'all') url.searchParams.set('country_code', region); + if (subregion.trim()) url.searchParams.set('region', subregion.trim()); if (category !== 'all') url.searchParams.set('category', category); if (sourceType !== 'all') url.searchParams.set('source_type', sourceType); if (displayPrecision !== 'all') url.searchParams.set('display_precision', displayPrecision); @@ -136,15 +137,16 @@ const loadLocal = async (cursor?: string, append = false) => { const requestProfile = toApiProfile(profile); - const queryKey = `${requestProfile}|${region}|${category}|${sourceType}|${displayPrecision}|${lifecycleStatus}`; + const queryKey = `${requestProfile}|${search.trim()}|${region}|${subregion.trim()}|${category}|${sourceType}|${displayPrecision}|${lifecycleStatus}`; if (!append && activeListKey === queryKey) return; if (!append) activeListKey = queryKey; listGeneration += 1; const generation = listGeneration; listAbort?.abort(); const controller = new AbortController(); listAbort = controller; lastRemoteQuery = queryKey; if (!append) { invalidateDetail(); localStatus = 'loading'; localFailure = 'unknown'; localError = ''; manifestSha256 = undefined; loaded = []; selected = undefined; nextCursor = null; coverageNote = ''; coverageScope = ''; countSemantics = ''; } else paging = true; try { - const result = await repo.list(requestProfile, { country_code: region === 'all' ? undefined : region, category: category === 'all' ? undefined : category, source_type: sourceType === 'all' ? undefined : sourceType, display_precision: displayPrecision === 'all' ? undefined : displayPrecision, lifecycle_status: lifecycleStatus === 'all' ? undefined : lifecycleStatus, cursor }, controller.signal); + const result = await repo.list(requestProfile, { q: search.trim() || undefined, country_code: region === 'all' ? undefined : region, region: subregion.trim() || undefined, category: category === 'all' ? undefined : category, source_type: sourceType === 'all' ? undefined : sourceType, display_precision: displayPrecision === 'all' ? undefined : displayPrecision, lifecycle_status: lifecycleStatus === 'all' ? undefined : lifecycleStatus, cursor }, controller.signal); if (generation !== listGeneration) return; + if (append && (result.releaseId !== release || result.ruleset !== ruleset)) throw Object.assign(new Error('The promoted release changed while loading this cursor page.'), { kind: 'invalid-contract' as const }); loaded = append ? [...loaded, ...result.locations] : result.locations; if (!selected) selected = result.locations[0]; release = result.releaseId; ruleset = result.ruleset; nextCursor = result.nextCursor; coverageNote = result.coverageNote; coverageScope = result.coverageScope ?? ''; countSemantics = result.countSemantics ?? ''; localStatus = 'ready'; localFailure = 'unknown'; paging = false; @@ -191,21 +193,21 @@ finally { testCsvBusy = false; } }; - const clearFilters = () => { search = ''; region = 'all'; category = 'all'; sourceType = 'all'; displayPrecision = 'all'; lifecycleStatus = 'all'; }; + const clearFilters = () => { search = ''; region = 'all'; subregion = ''; category = 'all'; sourceType = 'all'; displayPrecision = 'all'; lifecycleStatus = 'all'; }; const searchChanged = () => { const url = new URL(window.location.href); if (search.trim()) url.searchParams.set('q', search.trim()); else url.searchParams.delete('q'); history.replaceState(null, '', url); }; onMount(() => { const params = new URLSearchParams(window.location.search); localMode = params.get('mode') === 'local-v2'; testReleaseMode = params.get('preview') === 'test-release'; devPreviewMode = testReleaseMode || params.get('preview') === 'dev-candidates'; const route = parseRoute(window.location.hash); if (route.kind !== 'not-found') profile = localMode ? toApiProfile(route.profile) : route.profile; if (localMode) { - search = params.get('q') ?? ''; region = params.get('country_code') ?? 'all'; category = params.get('category') ?? 'all'; sourceType = params.get('source_type') ?? 'all'; displayPrecision = params.get('display_precision') ?? 'all'; lifecycleStatus = params.get('lifecycle_status') ?? 'all'; + search = params.get('q') ?? ''; region = params.get('country_code') ?? 'all'; subregion = params.get('region') ?? ''; category = params.get('category') ?? 'all'; sourceType = params.get('source_type') ?? 'all'; displayPrecision = params.get('display_precision') ?? 'all'; lifecycleStatus = params.get('lifecycle_status') ?? 'all'; try { const api = params.get('api') ?? undefined; repo = new LocalLocationRepository(globalThis.fetch, api); csvRepo = new LocalCsvExportRepository(globalThis.fetch, api ?? ''); metadataStatus = 'loading'; void new FilterMetadataRepository(globalThis.fetch, api ?? '').get().then((value) => { metadata = value; metadataStatus = 'ready'; }).catch(() => { metadataStatus = 'error'; }); } catch (error) { localStatus = 'error'; localFailure = 'network'; localError = error instanceof Error ? error.message : 'Local API origin was rejected safely.'; } } if (devPreviewMode) { previewStatus = import.meta.env.DEV ? 'idle' : 'blocked'; if (!import.meta.env.DEV) previewError = 'Private candidate preview is unavailable in production builds.'; } else if (localMode && localStatus !== 'error') void loadLocal(); else if (!localMode) void syncRoute(); const onHashChange = () => { const next = parseRoute(window.location.hash); if (next.kind !== 'not-found' && toApiProfile(next.profile) !== toApiProfile(profile)) { profile = localMode ? toApiProfile(next.profile) : next.profile; if (!localMode) void syncRoute(); } else void syncRoute(); }; - const onPopState = () => { const current = new URLSearchParams(window.location.search); if (localMode) { search = current.get('q') ?? ''; region = current.get('country_code') ?? 'all'; category = current.get('category') ?? 'all'; sourceType = current.get('source_type') ?? 'all'; displayPrecision = current.get('display_precision') ?? 'all'; lifecycleStatus = current.get('lifecycle_status') ?? 'all'; } onHashChange(); }; + const onPopState = () => { const current = new URLSearchParams(window.location.search); if (localMode) { search = current.get('q') ?? ''; region = current.get('country_code') ?? 'all'; subregion = current.get('region') ?? ''; category = current.get('category') ?? 'all'; sourceType = current.get('source_type') ?? 'all'; displayPrecision = current.get('display_precision') ?? 'all'; lifecycleStatus = current.get('lifecycle_status') ?? 'all'; } onHashChange(); }; window.addEventListener('hashchange', onHashChange); window.addEventListener('popstate', onPopState); return () => { listAbort?.abort(); detailAbort?.abort(); window.removeEventListener('hashchange', onHashChange); window.removeEventListener('popstate', onPopState); }; }); @@ -219,9 +221,9 @@ {#if testReleaseMode}
TEST-ONLY CSV — NOT PROJECT-APPROVED OR PUBLISHEDComplete bounded test-release rows only; this action never uses the public export route.{#if testCsvError}

{testCsvError}

{/if}
{/if} {#if !devPreviewMode || previewStatus === 'ready'} -

01 / DISCOVER

Choose the evidence lane

Profiles stay separate. A community claim is never silently promoted into a curated result.

{#if localMode && metadataStatus === 'loading'}

Loading available filter values…

{:else if localMode && metadataStatus === 'error'}

Filter metadata is unavailable; controlled filter values could not be loaded.

{/if}
-
{search || region !== 'all' || category !== 'all' || sourceType !== 'all' || displayPrecision !== 'all' || lifecycleStatus !== 'all' ? `Filters applied: ${[search && `name “${search}”`, region !== 'all' && region, category !== 'all' && category, sourceType !== 'all' && sourceType, displayPrecision !== 'all' && displayPrecision, lifecycleStatus !== 'all' && lifecycleStatus].filter(Boolean).join(' · ')}` : 'No additional filters applied'}
- {#if localMode && search.trim()}

Text search is limited to the currently loaded V2 page; the public contract does not provide a server-side free-text query. {nextCursor ? 'Later pages may contain additional matches.' : 'All rows in this response are loaded.'}

{/if} +

01 / DISCOVER

Choose the evidence lane

Profiles stay separate. A community claim is never silently promoted into a curated result.

{#if localMode && metadataStatus === 'loading'}

Loading available filter values…

{:else if localMode && metadataStatus === 'error'}

Filter metadata is unavailable; controlled filter values could not be loaded.

{/if}
+
{search || region !== 'all' || subregion.trim() || category !== 'all' || sourceType !== 'all' || displayPrecision !== 'all' || lifecycleStatus !== 'all' ? `Filters applied: ${[search && `text “${search}”`, region !== 'all' && region, subregion.trim() && subregion.trim(), category !== 'all' && category, sourceType !== 'all' && sourceType, displayPrecision !== 'all' && displayPrecision, lifecycleStatus !== 'all' && lifecycleStatus].filter(Boolean).join(' · ')}` : 'No additional filters applied'}
+ {#if localMode && (search.trim() || subregion.trim())}

Search and region filters are evaluated by the V2 server against the selected promoted release. {nextCursor ? 'Results are paginated; load the next cursor page for more matches.' : 'No additional cursor page is available.'}

{/if} {#if profile === 'community'}
Community claimsUnreviewed community claims: Not verified by Until Every Cage. Check each record’s factual review status before relying on it.
{/if} {#if localMode && localStatus === 'loading'}
Loading the {profileLabelText.toLowerCase()}…
{:else if localMode && (localStatus === 'error' || localStatus === 'no-release')}{:else if localMode && detailStatus === 'loading'}
Loading the selected local record…
{:else if localMode && detailStatus === 'error'}{:else}

02 / FILTER & COMPARE

Results {visibleLocations.length}{localMode && nextCursor ? ' on first page' : ''}

{localMode ? 'Current eligible response' : 'Fictional demonstration data'} · {profileLabelText}

diff --git a/frontend/tests/e2e/local-backend.spec.ts b/frontend/tests/e2e/local-backend.spec.ts index dfe6d16..33ac88b 100644 --- a/frontend/tests/e2e/local-backend.spec.ts +++ b/frontend/tests/e2e/local-backend.spec.ts @@ -1,8 +1,9 @@ import { test, expect } from '@playwright/test'; -test.skip(process.env.LOCAL_V2_E2E !== '1', 'Set LOCAL_V2_E2E=1 to run against the real local backend'); +const localE2eEnabled = process.env.LOCAL_V2_E2E === '1' || Boolean(process.env.UEC_E2E_API_URL); test('renders the real seeded local V2 record and opens its detail route', async ({ page }) => { + test.skip(!localE2eEnabled, 'Set LOCAL_V2_E2E=1 or UEC_E2E_API_URL to run against the real local backend'); const apiUrl = process.env.UEC_E2E_API_URL ?? process.env.LOCAL_V2_API_URL ?? 'http://127.0.0.1:8000'; let list: { data?: Array<{ facility_id: string; canonical_name: string }> } = {}; const response = await fetch(`${apiUrl}/api/v2/locations?profile=official&limit=1`); @@ -30,9 +31,43 @@ test('renders the real seeded local V2 record and opens its detail route', async await expect(page.locator('article')).toContainText('Project approval'); }); -test.skip(process.env.TEST_RELEASE_E2E !== '1', 'Set TEST_RELEASE_E2E=1 with a disposable guarded test-release backend'); +test('exercises server discovery, cursor state, map, export, and mobile basics', async ({ page }) => { + test.skip(!localE2eEnabled, 'Set LOCAL_V2_E2E=1 or UEC_E2E_API_URL to run against the real local backend'); + const apiUrl = process.env.UEC_E2E_API_URL ?? process.env.LOCAL_V2_API_URL ?? 'http://127.0.0.1:8000'; + const response = await fetch(`${apiUrl}/api/v2/locations?profile=official&limit=2`); + expect(response.ok).toBeTruthy(); + const list = await response.json() as { data?: Array<{ facility_id: string; canonical_name?: string; country_code?: string; latitude?: number | null; longitude?: number | null }>; }; + const record = list.data?.[0]; + if (!record) return; + const spatial = record.latitude !== null && record.longitude !== null + ? `&min_lon=${(record.longitude! - 1).toFixed(4)}&min_lat=${(record.latitude! - 1).toFixed(4)}&max_lon=${(record.longitude! + 1).toFixed(4)}&max_lat=${(record.latitude! + 1).toFixed(4)}` + : '&q=' + encodeURIComponent(record.canonical_name ?? record.country_code ?? 'synthetic'); + const spatialResponse = await fetch(`${apiUrl}/api/v2/locations?profile=official&limit=2${spatial}`); + expect(spatialResponse.ok).toBeTruthy(); + if (record.latitude !== null && record.longitude !== null) { + const radiusResponse = await fetch(`${apiUrl}/api/v2/locations?profile=official&limit=2&latitude=${record.latitude}&longitude=${record.longitude}&radius_km=100`); + expect(radiusResponse.ok).toBeTruthy(); + } + await page.route('**/api/v2/**', async route => { + const requestUrl = new URL(route.request().url()); + const upstream = await fetch(`${apiUrl}${requestUrl.pathname}${requestUrl.search}`); + await route.fulfill({ status: upstream.status, headers: { 'content-type': upstream.headers.get('content-type') ?? 'application/json', 'x-uec-release-id': upstream.headers.get('x-uec-release-id') ?? '' }, body: await upstream.text() }); + }); + await page.setViewportSize({ width: 390, height: 844 }); + await page.goto('./?mode=local-v2#/'); + await expect(page.getByText('LOCAL V2 API')).toBeVisible(); + const searchTerm = (record.canonical_name ?? record.country_code ?? '').slice(0, 6); + await page.getByLabel('Search locations').fill(searchTerm); + await expect(page).toHaveURL(new RegExp(`q=${encodeURIComponent(searchTerm)}`)); + await expect(page.getByRole('status').filter({ hasText: 'server' })).toBeVisible(); + await page.getByRole('button', { name: 'Show map' }).click(); + await expect(page.getByLabel(/Location map showing facility records/)).toBeVisible(); + await expect(page.getByRole('button', { name: /Download official CSV/ })).toBeVisible(); + await expect(page.locator('main')).toBeVisible(); +}); test('renders the guarded disposable test release without public fallback', async ({ page }) => { + test.skip(process.env.TEST_RELEASE_E2E !== '1', 'Set TEST_RELEASE_E2E=1 with a disposable guarded test-release backend'); const apiUrl = process.env.UEC_E2E_API_URL ?? 'http://127.0.0.1:8000'; const token = process.env.UEC_TEST_RELEASE_TOKEN ?? process.env.UEC_DEV_PREVIEW_TOKEN; expect(token, 'UEC_TEST_RELEASE_TOKEN must be supplied in memory by the test runner').toBeTruthy(); diff --git a/frontend/tests/unit/localLocationRepository.test.ts b/frontend/tests/unit/localLocationRepository.test.ts index 213cea0..9f99c16 100644 --- a/frontend/tests/unit/localLocationRepository.test.ts +++ b/frontend/tests/unit/localLocationRepository.test.ts @@ -3,16 +3,20 @@ const row={facility_id:'550e8400-e29b-41d4-a716-446655440000',canonical_name:'Lo const response=(body:unknown,status=200)=>new Response(JSON.stringify(body),{status,headers:{'content-type':'application/json'}});const envelope=(data=[row],meta={release_id:'rel-1',ruleset_version:'rules-1',profile:'official',next_cursor:null,coverage_note:'Local promoted release.'})=>({data,api_version:'v2',meta}); describe('LocalLocationRepository',()=>{it('maps a valid Rust-shaped envelope',async()=>{const result=await new LocalLocationRepository(vi.fn().mockResolvedValue(response(envelope()))).list();expect(result.locations[0]).toMatchObject({id:row.facility_id,name:'Local V2 Fixture',lat:55});});it('fails closed when no release is promoted',async()=>{await expect(new LocalLocationRepository(vi.fn().mockResolvedValue(response(envelope([],{release_id:null,profile:'official',coverage_note:'No promoted release.'})))).list()).rejects.toMatchObject({kind:'no-release'});});it('classifies server failures as unavailable',async()=>{await expect(new LocalLocationRepository(vi.fn().mockResolvedValue(response({},503))).list()).rejects.toMatchObject({kind:'unavailable',status:503});});it('rejects malformed or restricted payloads',async()=>{await expect(new LocalLocationRepository(vi.fn().mockResolvedValue(response({...envelope(),api_version:'v1'}))).list()).rejects.toMatchObject({kind:'invalid-contract'});await expect(new LocalLocationRepository(vi.fn().mockResolvedValue(response(envelope([{...row,privacy_screening_status:'failed'}])))).list()).rejects.toMatchObject({kind:'invalid-contract'});});}); describe('LocalLocationRepository query contract', () => { - it('passes supported filters and the opaque cursor without inventing search semantics', async () => { + it('passes server-side search, region, supported filters, and opaque cursor', async () => { const fetcher = vi.fn().mockResolvedValue(response(envelope([], { ...envelope().meta, next_cursor: 'cursor-2' }))); - const result = await new LocalLocationRepository(fetcher).list('official', { country_code: 'DK', category: 'dairy', source_type: 'official', display_precision: 'city', lifecycle_status: 'active_observed', cursor: 'cursor-1' }); + const result = await new LocalLocationRepository(fetcher).list('official', { q: 'North Coast', country_code: 'DK', region: 'North Coast', category: 'dairy', source_type: 'official', display_precision: 'city', lifecycle_status: 'active_observed', min_lon: 8, min_lat: 54, max_lon: 13, max_lat: 58, cursor: 'cursor-1' }); const request = String(fetcher.mock.calls[0]?.[0]); expect(request).toContain('profile=official'); + expect(request).toContain('q=North+Coast'); expect(request).toContain('country_code=DK'); + expect(request).toContain('region=North+Coast'); expect(request).toContain('category=dairy'); expect(request).toContain('source_type=official'); expect(request).toContain('display_precision=city'); expect(request).toContain('lifecycle_status=active_observed'); + expect(request).toContain('min_lon=8'); + expect(request).toContain('max_lat=58'); expect(request).toContain('cursor=cursor-1'); expect(result.nextCursor).toBe('cursor-2'); }); diff --git a/pipeline/migrations/025_discovery_query_indexes.sql b/pipeline/migrations/025_discovery_query_indexes.sql new file mode 100644 index 0000000..f4f8c42 --- /dev/null +++ b/pipeline/migrations/025_discovery_query_indexes.sql @@ -0,0 +1,21 @@ +-- Discovery read-path indexes. These are additive and safe to rerun through the +-- migration ledger. The public projection remains a view so suppression and +-- release-scoped eligibility are evaluated for every request. +CREATE EXTENSION IF NOT EXISTS pg_trgm; + +CREATE INDEX IF NOT EXISTS release_members_discovery_observation_idx + ON uec.release_members (release_id, observation_id, facility_id); + +CREATE INDEX IF NOT EXISTS facilities_discovery_country_city_idx + ON uec.facilities (country_code, city, facility_id); + +CREATE INDEX IF NOT EXISTS facilities_discovery_name_trgm_idx + ON uec.facilities USING GIN (lower(canonical_name) gin_trgm_ops); + +CREATE INDEX IF NOT EXISTS observations_discovery_category_idx + ON uec.observations (classification_category, facility_id, observation_id); + +COMMENT ON INDEX uec.release_members_discovery_observation_idx IS + 'Supports release-scoped projection joins while the primary key supplies UUID cursor order.'; +COMMENT ON INDEX uec.facilities_discovery_name_trgm_idx IS + 'Supports bounded case-insensitive substring search on canonical facility names.'; diff --git a/pipeline/tests/benchmarks/discovery_100k.sql b/pipeline/tests/benchmarks/discovery_100k.sql new file mode 100644 index 0000000..0787a28 --- /dev/null +++ b/pipeline/tests/benchmarks/discovery_100k.sql @@ -0,0 +1,32 @@ +-- Synthetic-only query-shape benchmark. This intentionally does not touch uec. +DROP TABLE IF EXISTS bench_discovery_100k; +CREATE TABLE bench_discovery_100k AS +SELECT + n AS ordinal, + format('00000000-0000-4000-8000-%s', lpad(n::text, 12, '0'))::uuid AS facility_id, + format('Synthetic facility %s', n) AS canonical_name, + CASE WHEN n % 2 = 0 THEN 'DK' ELSE 'SE' END::text AS country_code, + format('Synthetic region %s', n % 100) AS region, + CASE n % 4 WHEN 0 THEN 'slaughter' WHEN 1 THEN 'fish_processing' + WHEN 2 THEN 'logistics_and_storage' ELSE 'retail_and_prepared_food' END AS category, + CASE n % 3 WHEN 0 THEN 'official' WHEN 1 THEN 'secondary' ELSE 'user_submitted' END AS source_type, + CASE n % 3 WHEN 0 THEN 'exact' WHEN 1 THEN 'city' ELSE 'unmapped' END AS display_precision, + ST_SetSRID(ST_Point(-10 + (n % 2000) / 100.0, 45 + (n % 1000) / 100.0), 4326)::geography AS location +FROM generate_series(1, 100000) AS n; +CREATE INDEX bench_discovery_cursor ON bench_discovery_100k (country_code, facility_id); +CREATE INDEX bench_discovery_name ON bench_discovery_100k USING GIN (lower(canonical_name) gin_trgm_ops); +CREATE INDEX bench_discovery_location ON bench_discovery_100k USING GIST (location); +ANALYZE bench_discovery_100k; + +EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) +SELECT * FROM bench_discovery_100k WHERE country_code = 'DK' ORDER BY facility_id LIMIT 50; +EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) +SELECT * FROM bench_discovery_100k WHERE country_code = 'DK' AND facility_id > '00000000-0000-4000-8000-000000050000'::uuid ORDER BY facility_id LIMIT 50; +EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) +SELECT * FROM bench_discovery_100k WHERE lower(canonical_name) LIKE '%facility 999%' ORDER BY facility_id LIMIT 50; +EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) +SELECT * FROM bench_discovery_100k WHERE location && ST_MakeEnvelope(8, 54, 13, 58, 4326)::geography ORDER BY facility_id LIMIT 50; +EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) +SELECT * FROM bench_discovery_100k WHERE ST_DWithin(location, ST_SetSRID(ST_Point(10, 56), 4326)::geography, 50000) ORDER BY facility_id LIMIT 50; +EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) +SELECT * FROM bench_discovery_100k WHERE facility_id = '00000000-0000-4000-8000-000000050000'::uuid; diff --git a/src/lib.rs b/src/lib.rs index cc3368d..5cd6a9a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -900,10 +900,10 @@ const V2_LIFECYCLES: &[&str] = &[ pub async fn get_v2_filter_metadata_handler() -> impl IntoResponse { Json(json!({"api_version":"v2", "contract_version":"v1", "dimensions": { - "country_code":{"values":V2_COUNTRIES}, "category":{"values":V2_CATEGORIES}, + "country_code":{"values":V2_COUNTRIES}, "region":{"values":[],"source":"release_facets"}, "category":{"values":V2_CATEGORIES}, "source_type":{"values":V2_SOURCE_TYPES}, "profile":{"values":V2_PROFILES,"default":"official"}, "display_precision":{"values":V2_PRECISIONS}, "lifecycle_status":{"values":V2_LIFECYCLES} - }, "pagination":{"limit_max":1000,"cursor":"facility_id"}, "privacy":"Filters operate only on eligible records in the selected promoted release; filters never override suppression or publication review."})).into_response() + }, "spatial":{"bbox":["min_lon","min_lat","max_lon","max_lat"],"radius":["latitude","longitude","radius_km"]}, "search":{"parameter":"q","fields":["canonical_name","city","country_code","category","source_name"]}, "pagination":{"limit_max":1000,"cursor":"facility_id"}, "privacy":"Filters operate only on eligible records in the selected promoted release; filters never override suppression or publication review."})).into_response() } pub async fn get_v2_facets_handler( @@ -938,6 +938,10 @@ pub async fn get_v2_facets_handler( .country_code .as_deref() .is_some_and(|v| v.len() != 2 || !v.chars().all(|c| c.is_ascii_uppercase())) + || params + .region + .as_deref() + .is_some_and(|v| v.trim().is_empty() || v.len() > 120) { return v2_error( StatusCode::BAD_REQUEST, @@ -973,7 +977,7 @@ pub async fn get_v2_facets_handler( let release_id: String = release.get(0); let ruleset_version: String = release.get(1); let release_created_at: chrono::DateTime = release.get(2); - let rows = match client.query("SELECT country_code, classification_category, display_precision, lifecycle_status, provenance_origin_type FROM uec.map_facilities_display_history WHERE release_id=$1 AND ($2::text IS NULL OR country_code=$2) AND ($3::text IS NULL OR classification_category=$3) AND ($4::text IS NULL OR provenance_origin_type=$4) AND ($5::text IS NULL OR display_precision=$5) AND ($6::text IS NULL OR lifecycle_status=$6)", &[&release_id, ¶ms.country_code, ¶ms.category, ¶ms.source_type, ¶ms.display_precision, ¶ms.lifecycle_status]).await { Ok(rows) => rows, Err(_) => return v2_error(StatusCode::SERVICE_UNAVAILABLE, "facets_query_failed", "public facets unavailable") }; + let rows = match client.query("SELECT country_code, classification_category, display_precision, lifecycle_status, provenance_origin_type, city FROM uec.map_facilities_display_history WHERE release_id=$1 AND ($2::text IS NULL OR country_code=$2) AND ($3::text IS NULL OR classification_category=$3) AND ($4::text IS NULL OR provenance_origin_type=$4) AND ($5::text IS NULL OR display_precision=$5) AND ($6::text IS NULL OR lifecycle_status=$6) AND ($7::text IS NULL OR city=$7)", &[&release_id, ¶ms.country_code, ¶ms.category, ¶ms.source_type, ¶ms.display_precision, ¶ms.lifecycle_status, ¶ms.region]).await { Ok(rows) => rows, Err(_) => return v2_error(StatusCode::SERVICE_UNAVAILABLE, "facets_query_failed", "public facets unavailable") }; let mut dimensions = serde_json::Map::new(); for (name, values) in [ ( @@ -998,6 +1002,12 @@ pub async fn get_v2_facets_handler( "source_type", rows.iter().map(|r| r.get::<_, String>(4)).collect(), ), + ( + "region", + rows.iter() + .filter_map(|r| r.get::<_, Option>(5)) + .collect(), + ), ] { let mut counts = std::collections::BTreeMap::::new(); for value in values { @@ -1014,17 +1024,26 @@ pub async fn get_v2_facets_handler( ), ); } - Json(json!({"api_version":"v2", "meta":{"profile":profile,"release_id":release_id,"ruleset_version":ruleset_version,"release_created_at":release_created_at,"coverage_scope":"selected_promoted_release_public_facilities","count_semantics":"Counts are eligible public facility projection rows after current suppression; they are not story-wide or animal counts.","filters":{"country_code":params.country_code,"category":params.category,"source_type":params.source_type,"display_precision":params.display_precision,"lifecycle_status":params.lifecycle_status}}, "dimensions":dimensions})).into_response() + Json(json!({"api_version":"v2", "meta":{"profile":profile,"release_id":release_id,"ruleset_version":ruleset_version,"release_created_at":release_created_at,"coverage_scope":"selected_promoted_release_public_facilities","count_semantics":"Counts are eligible public facility projection rows after current suppression; they are not story-wide or animal counts.","filters":{"country_code":params.country_code,"region":params.region,"category":params.category,"source_type":params.source_type,"display_precision":params.display_precision,"lifecycle_status":params.lifecycle_status}}, "dimensions":dimensions})).into_response() } #[derive(Deserialize)] pub struct V2LocationParams { pub country_code: Option, + pub region: Option, pub category: Option, pub source_type: Option, pub profile: Option, pub display_precision: Option, pub lifecycle_status: Option, + pub q: Option, + pub min_lon: Option, + pub min_lat: Option, + pub max_lon: Option, + pub max_lat: Option, + pub radius_km: Option, + pub latitude: Option, + pub longitude: Option, pub limit: Option, pub offset: Option, pub cursor: Option, @@ -1112,6 +1131,116 @@ pub async fn get_v2_locations_handler( "source_type is unsupported", ); } + if params + .region + .as_deref() + .is_some_and(|v| v.trim().is_empty() || v.len() > 120) + || params + .q + .as_deref() + .is_some_and(|v| v.trim().is_empty() || v.len() > 120) + { + return v2_error( + StatusCode::BAD_REQUEST, + "invalid_filter", + "region and q must be non-empty and at most 120 characters", + ); + } + let search_text = params.q.as_deref().map(|value| { + value + .trim() + .replace('\\', "\\\\") + .replace('%', "\\%") + .replace('_', "\\_") + }); + let parse_coordinate = |value: &Option, name: &'static str, min: f64, max: f64| { + value + .as_deref() + .map(str::parse::) + .transpose() + .map_err(|_| (name, "must be a number")) + .and_then(|value| { + if value.is_some_and(|number| !number.is_finite() || number < min || number > max) { + Err((name, "is outside the supported range")) + } else { + Ok(value) + } + }) + }; + let min_lon = match parse_coordinate(¶ms.min_lon, "min_lon", -180.0, 180.0) { + Ok(v) => v, + Err((_, message)) => { + return v2_error(StatusCode::BAD_REQUEST, "invalid_spatial_query", message); + } + }; + let min_lat = match parse_coordinate(¶ms.min_lat, "min_lat", -90.0, 90.0) { + Ok(v) => v, + Err((_, message)) => { + return v2_error(StatusCode::BAD_REQUEST, "invalid_spatial_query", message); + } + }; + let max_lon = match parse_coordinate(¶ms.max_lon, "max_lon", -180.0, 180.0) { + Ok(v) => v, + Err((_, message)) => { + return v2_error(StatusCode::BAD_REQUEST, "invalid_spatial_query", message); + } + }; + let max_lat = match parse_coordinate(¶ms.max_lat, "max_lat", -90.0, 90.0) { + Ok(v) => v, + Err((_, message)) => { + return v2_error(StatusCode::BAD_REQUEST, "invalid_spatial_query", message); + } + }; + let radius_km = match parse_coordinate(¶ms.radius_km, "radius_km", 0.001, 5000.0) { + Ok(v) => v, + Err((_, message)) => { + return v2_error(StatusCode::BAD_REQUEST, "invalid_spatial_query", message); + } + }; + let latitude = match parse_coordinate(¶ms.latitude, "latitude", -90.0, 90.0) { + Ok(v) => v, + Err((_, message)) => { + return v2_error(StatusCode::BAD_REQUEST, "invalid_spatial_query", message); + } + }; + let longitude = match parse_coordinate(¶ms.longitude, "longitude", -180.0, 180.0) { + Ok(v) => v, + Err((_, message)) => { + return v2_error(StatusCode::BAD_REQUEST, "invalid_spatial_query", message); + } + }; + let bbox_values = [min_lon, min_lat, max_lon, max_lat]; + let bbox_present = bbox_values.iter().any(Option::is_some); + if bbox_present && bbox_values.iter().any(Option::is_none) { + return v2_error( + StatusCode::BAD_REQUEST, + "invalid_spatial_query", + "bounding box requires min_lon, min_lat, max_lon, and max_lat", + ); + } + if bbox_present && !(min_lon.unwrap() < max_lon.unwrap() && min_lat.unwrap() < max_lat.unwrap()) + { + return v2_error( + StatusCode::BAD_REQUEST, + "invalid_spatial_query", + "bounding box minimums must be less than maximums", + ); + } + let radius_present = radius_km.is_some() || latitude.is_some() || longitude.is_some(); + if radius_present && (radius_km.is_none() || latitude.is_none() || longitude.is_none()) { + return v2_error( + StatusCode::BAD_REQUEST, + "invalid_spatial_query", + "radius queries require radius_km, latitude, and longitude", + ); + } + if bbox_present && radius_present { + return v2_error( + StatusCode::BAD_REQUEST, + "invalid_spatial_query", + "bounding box and radius cannot be combined", + ); + } if params .profile .as_deref() @@ -1236,12 +1365,16 @@ pub async fn get_v2_locations_handler( WHERE map_facilities_display_history.release_id = $1 AND ($2::uuid IS NULL OR facility_id > $2) AND ($3::text IS NULL OR country_code = $3) - AND ($4::text IS NULL OR classification_category = $4) - AND ($5::text IS NULL OR display_precision = $5) - AND ($6::text IS NULL OR lifecycle_status = $6) - AND ($7::text IS NULL OR provenance_origin_type = $7) - ORDER BY facility_id LIMIT $8 OFFSET $9 - "#, &[&promoted_release_id, &cursor, ¶ms.country_code, ¶ms.category, ¶ms.display_precision, ¶ms.lifecycle_status, ¶ms.source_type, &query_limit, &effective_offset]).await { + AND ($4::text IS NULL OR city = $4) + AND ($5::text IS NULL OR classification_category = $5) + AND ($6::text IS NULL OR display_precision = $6) + AND ($7::text IS NULL OR lifecycle_status = $7) + AND ($8::text IS NULL OR provenance_origin_type = $8) + AND ($9::text IS NULL OR lower(coalesce(canonical_name, '') || ' ' || coalesce(city, '') || ' ' || country_code || ' ' || classification_category || ' ' || coalesce(provenance_source_name, '')) LIKE '%' || lower($9) || '%' ESCAPE '\') + AND ($10::double precision IS NULL OR (display_location && ST_MakeEnvelope($10, $11, $12, $13, 4326)::geography AND ST_Intersects(display_location::geometry, ST_MakeEnvelope($10, $11, $12, $13, 4326)))) + AND ($14::double precision IS NULL OR ST_DWithin(display_location, ST_SetSRID(ST_Point($15, $16), 4326)::geography, $14 * 1000)) + ORDER BY facility_id LIMIT $17 OFFSET $18 + "#, &[&promoted_release_id, &cursor, ¶ms.country_code, ¶ms.region, ¶ms.category, ¶ms.display_precision, ¶ms.lifecycle_status, ¶ms.source_type, &search_text, &min_lon, &min_lat, &max_lon, &max_lat, &radius_km, &longitude, &latitude, &query_limit, &effective_offset]).await { Ok(rows) => rows, Err(_) => return v2_error(StatusCode::INTERNAL_SERVER_ERROR, "location_query_failed", "V2 location query failed"), }; @@ -1297,7 +1430,8 @@ pub async fn get_v2_locations_handler( "next_cursor": next_cursor, "coverage_note": "Results are eligible public facility projection rows from the selected promoted release after current suppression; they are not story-wide or animal counts.", "coverage_scope": "selected_promoted_release_public_facilities", - "count_semantics": "Each row represents a public facility projection, not an animal count." + "count_semantics": "Each row represents a public facility projection, not an animal count.", + "query": {"q": params.q, "filters": {"country_code": params.country_code, "region": params.region, "category": params.category, "source_type": params.source_type, "display_precision": params.display_precision, "lifecycle_status": params.lifecycle_status}} }); if transaction.commit().await.is_err() { return ( @@ -1517,6 +1651,32 @@ mod v2_api_tests { assert!(!V2_COUNTRIES.contains(&"ZZ")); } + #[tokio::test] + async fn discovery_rejects_incomplete_or_conflicting_spatial_queries() { + let state = ApiState { + database: None, + dev_preview_token: None, + dev_test_release_id: None, + dev_test_release_token: None, + }; + for uri in [ + "/api/v2/locations?min_lon=8&min_lat=54&max_lon=13", + "/api/v2/locations?latitude=56&longitude=10", + "/api/v2/locations?min_lon=8&min_lat=54&max_lon=13&max_lat=58&latitude=56&longitude=10&radius_km=10", + ] { + let response = Router::new() + .route( + "/api/v2/locations", + axum::routing::get(get_v2_locations_handler), + ) + .with_state(state.clone()) + .oneshot(Request::builder().uri(uri).body(Body::empty()).unwrap()) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::BAD_REQUEST, "{uri}"); + } + } + #[tokio::test] async fn v2_response_is_json_when_database_is_configured() { let url = std::env::var("UEC_DATABASE_URL").unwrap_or_else(|_| { From b6694c8efcecbfa5dab038e308d0cdec8c193e52 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Tue, 15 Sep 2026 17:45:12 -0700 Subject: [PATCH 123/311] Align frontend safety checks with server search --- frontend/tests/e2e/local-safety.spec.ts | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/frontend/tests/e2e/local-safety.spec.ts b/frontend/tests/e2e/local-safety.spec.ts index e545cfa..90ff39e 100644 --- a/frontend/tests/e2e/local-safety.spec.ts +++ b/frontend/tests/e2e/local-safety.spec.ts @@ -44,7 +44,10 @@ test('community direct link opens in community mode with persistent record conte test('first-page-only search reports uncertainty instead of a global no-results claim', async ({ page }) => { await mockMetadata(page); - await page.route('**/api/v2/locations**', route => route.fulfill({ json: list('official', [row()], secondId) })); + await page.route('**/api/v2/locations**', route => { + const q = new URL(route.request().url()).searchParams.get('q'); + return route.fulfill({ json: q ? list('official', [], secondId) : list('official', [row()], secondId) }); + }); await page.goto('./?mode=local-v2#/'); await expect(page.getByText('Only the first page is loaded.', { exact: false })).toBeVisible(); await page.getByLabel('Search locations').fill('later record'); @@ -69,14 +72,20 @@ test('late list response cannot replace a newer community selection', async ({ p await expect(page.getByRole('note')).toContainText('Not verified'); }); -test('search filters the loaded page without refetching or claiming global completeness', async ({ page }) => { +test('search is evaluated by the server without claiming global completeness', async ({ page }) => { await mockMetadata(page); let listRequests = 0; - await page.route('**/api/v2/locations**', async route => { listRequests += 1; await route.fulfill({ json: list('official', [row(firstId, 'Current filtered result')], secondId) }); }); + let lastQuery = ''; + await page.route('**/api/v2/locations**', async route => { + listRequests += 1; + lastQuery = new URL(route.request().url()).searchParams.get('q') ?? ''; + await route.fulfill({ json: list('official', [row(firstId, 'Current filtered result')], secondId) }); + }); await page.goto('./?mode=local-v2#/'); await page.getByLabel('Search locations').fill('current'); await expect(page.getByRole('heading', { name: 'Current filtered result' })).toBeVisible(); - expect(listRequests).toBe(1); + expect(listRequests).toBe(2); + expect(lastQuery).toBe('current'); await expect(page).toHaveURL(/q=current/); }); From ef94b19ae32ebf0823eaf506ee224438fd7ed2af Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Tue, 15 Sep 2026 21:24:25 -0700 Subject: [PATCH 124/311] Decouple ledger SQL generation from psycopg --- .../maintenance/replay-restriction-ledger.py | 12 +++++++++++- pipeline/tests/test_private_environment_gate.py | 17 +++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/pipeline/scripts/maintenance/replay-restriction-ledger.py b/pipeline/scripts/maintenance/replay-restriction-ledger.py index 1de38ed..76c3e66 100644 --- a/pipeline/scripts/maintenance/replay-restriction-ledger.py +++ b/pipeline/scripts/maintenance/replay-restriction-ledger.py @@ -15,7 +15,15 @@ import tempfile from pathlib import Path -import psycopg + + +def _require_psycopg(): + """Load the database driver only for operations that contact PostgreSQL.""" + try: + import psycopg + except ModuleNotFoundError as exc: + raise RuntimeError("database replay requires the 'psycopg' package; install pipeline requirements") from exc + return psycopg def _ledger_module(): @@ -31,6 +39,7 @@ def _ledger_module(): def replay(database_url: str, ledger_path: Path, actor: str = "external-ledger-replay") -> dict[str, int | str]: verifier = _ledger_module() ledger = verifier.load_ledger(ledger_path) + psycopg = _require_psycopg() restrictions = ledger["active_restrictions"] if any(item.get("scope") != "whole_record" for item in restrictions): raise ValueError("ledger replay supports only whole_record suppression references") @@ -107,6 +116,7 @@ def write_replayed_snapshot(database_url: str, ledger_path: Path, output: Path) """Write a row-free snapshot after confirming each reference is suppressed.""" verifier = _ledger_module() ledger = verifier.load_ledger(ledger_path) + psycopg = _require_psycopg() restrictions = ledger["active_restrictions"] if any(item.get("scope") != "whole_record" for item in restrictions): raise ValueError("snapshot export supports only whole_record suppression references") diff --git a/pipeline/tests/test_private_environment_gate.py b/pipeline/tests/test_private_environment_gate.py index 0f2ae5b..68f052a 100644 --- a/pipeline/tests/test_private_environment_gate.py +++ b/pipeline/tests/test_private_environment_gate.py @@ -2,6 +2,8 @@ import importlib.util import hashlib import json +import subprocess +import sys import tempfile import unittest from pathlib import Path @@ -109,6 +111,21 @@ def test_replay_sql_is_idempotent_and_does_not_embed_sensitive_columns(self): self.assertNotIn("address", sql.lower()) self.assertNotIn("coordinate", sql.lower()) + def test_emit_sql_does_not_require_site_packages(self): + script = ROOT / "scripts" / "maintenance" / "replay-restriction-ledger.py" + replay_spec = importlib.util.spec_from_file_location("replay_restriction_ledger_no_site", script) + replay = importlib.util.module_from_spec(replay_spec) + replay_spec.loader.exec_module(replay) + with tempfile.TemporaryDirectory() as directory: + paths, _ = self.fixtures(directory) + expected = replay.replay_sql(paths["ledger.json"]) + result = subprocess.run( + [sys.executable, "-S", str(script), "--ledger", str(paths["ledger.json"]), "--emit-sql"], + capture_output=True, text=True, check=False, + ) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout, expected) + if __name__ == "__main__": unittest.main() From e5f19fbceb718b7f37f2573b0ee5a092cc903843 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Tue, 15 Sep 2026 21:47:44 -0700 Subject: [PATCH 125/311] Add private-alpha source operations --- docs/architecture/source-operations.md | 82 +++ pipeline/README.md | 13 + pipeline/common/acquisition.py | 132 +++- pipeline/common/delta.py | 7 +- pipeline/common/orchestrator.py | 26 +- pipeline/common/source_operations.py | 566 ++++++++++++++++++ pipeline/common/test_acquisition.py | 78 ++- pipeline/common/test_source_operations.py | 122 ++++ pipeline/scripts/README.md | 5 + .../build-source-operations-health.py | 35 ++ pipeline/source_operations.json | 20 + 11 files changed, 1065 insertions(+), 21 deletions(-) create mode 100644 docs/architecture/source-operations.md create mode 100644 pipeline/common/source_operations.py create mode 100644 pipeline/common/test_source_operations.py create mode 100644 pipeline/scripts/diagnostics/build-source-operations-health.py create mode 100644 pipeline/source_operations.json diff --git a/docs/architecture/source-operations.md b/docs/architecture/source-operations.md new file mode 100644 index 0000000..57cea01 --- /dev/null +++ b/docs/architecture/source-operations.md @@ -0,0 +1,82 @@ +# Private-alpha source operations + +`pipeline/source_operations.json` is the shared operational source of truth +for schedule and freshness expectations. It covers every source identity in +`pipeline/source_registry.json` without changing country adapter packages or +publication status. A `null` interval/staleness value means that cadence is +unknown; it is not permission to assume that a source is current. + +## Artifact and run layout + +The caller chooses a private operations root, normally an ignored directory +under `data/`. The shared layer uses this layout: + +```text +/ + raw/sha256/// # immutable bytes, stored once + raw/observations/.json # one provenance record per observation + history/.jsonl # append-only run ledger + notifications/.jsonl # local notification hook output + runs// # adapter-owned private stages + parsed/ normalized/ quarantined/ + manifest.json or run-manifest.json + qa.json run-status.json source-health.json + release-diff.json review-packet.json failure-report.json +``` + +Raw bytes are addressed by SHA-256 and are never overwritten. Equal bytes with +different retrieval facts receive separate observation manifests. The ledger +also preserves every failed and unchanged observation; reruns do not replace +earlier evidence. Retention is restricted research evidence subject to the +exceptional removal process in `docs/ETHICS.md`; the operational layer does not +invent an expiry or override a privacy/removal decision. + +## Run classification and failure behavior + +Every shared-orchestrator run records one of these classifications: + +- `changed`: the artifact or normalized output differs from the prior run; +- `unchanged`: both content hashes match the prior run; +- `review-required`: the adapter or QA evidence reports quarantines/drift or + explicitly requests review; +- `failed`: acquisition, adapter, evidence, or operations validation failed. + +`release_promoted` is always `false` in these records and +`release_preserved` is always `true`. A failed or partial rerun therefore +leaves any prior eligible release reference available to the separate release +process. Missing source rows are reported as `not-observed` by the aggregate +diff; they are never interpreted as closure. + +Network acquisition retries only retry bounded transport/rate-limit failures. +Terms, content-type, size-limit, malformed-input, and validation failures fail +closed. Each attempt records its category, retryability, and operator action; +the final private `failure-report.json` and optional local notification hook +contain no source rows or sensitive payloads. + +## Operator review + +`review-packet.json` and `release-diff.json` are deterministic, row-free +operator artifacts. They identify provenance, counts, drift/quarantine +signals, not-observed counts, prior release context, and required actions. They +do not approve, promote, or publish a release. Build the aggregate machine- +readable health index with: + +```powershell +python pipeline/scripts/diagnostics/build-source-operations-health.py ` + --operations-root data/restricted/source-operations ` + --output data/reports/source-operations-health.json ` + --registry pipeline/source_registry.json ` + --as-of-utc 2026-09-15T00:00:00Z +``` + +The index reports `not-run`, `private-validated`, `degraded`, +`review-required`, or `failed` per source, plus freshness and run history +facts. It always sets public exposure to false and publication eligibility to +blocked. It is operational evidence, not a production-health or publication +claim. + +The older compatibility registration helper may also retain its stable +`raw/.artifact` path and `raw/registrations/*.manifest.json` events; +those paths are still content-addressed and append-only. New integrations +should use the layout above so observation metadata and raw bytes are kept as +separate objects. diff --git a/pipeline/README.md b/pipeline/README.md index 529fea8..a05c613 100644 --- a/pipeline/README.md +++ b/pipeline/README.md @@ -81,3 +81,16 @@ counts, blocks schema changes, and never interprets source absence as closure. F or partial comparisons retain the prior eligible release reference and expose no public surface. Terms, privacy/safety, suppression, review, project approval, and publication remain separate gates. + +## Private-alpha source operations + +The shared operational layer in `common/source_operations.py` adds the +schedule/freshness inventory in `source_operations.json`, content-addressed raw +artifact deduplication, append-only run history, row-free review packets and +release diffs, bounded acquisition retry classification, and local failure +notification hooks. See [the source operations contract](../docs/architecture/source-operations.md). + +Every operational record preserves the prior eligible release reference and +keeps `release_promoted` false. A changed artifact, unchanged rerun, failed +attempt, or review-required result is recorded as a new event; no run overwrites +earlier evidence. The health index is private operational evidence only. diff --git a/pipeline/common/acquisition.py b/pipeline/common/acquisition.py index 6005bcf..77e8ac6 100644 --- a/pipeline/common/acquisition.py +++ b/pipeline/common/acquisition.py @@ -10,7 +10,9 @@ import hashlib import json import os +import socket import tempfile +import time import urllib.error import urllib.request import uuid @@ -22,6 +24,12 @@ class AcquisitionError(ValueError): """An acquisition was not bounded, authorized, valid, or complete.""" + def __init__(self, message: str, *, failure_class: str = "acquisition", retryable: bool = False, action: str = "inspect the private acquisition evidence and source terms") -> None: + super().__init__(message) + self.failure_class = failure_class + self.retryable = retryable + self.action = action + def utc_now() -> str: return datetime.now(timezone.utc).isoformat(timespec="seconds").replace("+00:00", "Z") @@ -125,34 +133,103 @@ def fetch_source( coverage: str | None = None, rights_caveat: str | None = None, privacy_caveat: str | None = None, + max_attempts: int = 1, + retry_delay_seconds: float = 0.0, + max_retry_delay_seconds: float = 30.0, + sleep_fn: Any = time.sleep, ) -> dict[str, Any]: if not source_id or not url: raise AcquisitionError("source_id and url are required") if timeout_seconds <= 0: raise AcquisitionError("timeout_seconds must be positive") + if not 1 <= max_attempts <= 10: + raise AcquisitionError("max_attempts must be between 1 and 10", failure_class="configuration") + if retry_delay_seconds < 0 or max_retry_delay_seconds < 0 or retry_delay_seconds > max_retry_delay_seconds: + raise AcquisitionError("retry delay values are invalid", failure_class="configuration") terms_review = require_terms_review(Path(terms_review_path)) run_id = run_id or default_run_id() run_dir = Path(output_root) / source_id / run_id artifact_path = run_dir / artifact_name requested_at = utc_now() - recorder = _RedirectRecorder() - try: - opener = urllib.request.build_opener(recorder) - request = urllib.request.Request(url, headers={"User-Agent": user_agent}) - with opener.open(request, timeout=timeout_seconds) as response: - if not 200 <= response.status < 300: - raise AcquisitionError(f"source returned HTTP {response.status}") - content_type = response.headers.get("Content-Type") - allowed = {item.lower() for item in allowed_content_types} - if content_type and content_type.split(";", 1)[0].strip().lower() not in allowed: - raise AcquisitionError(f"unexpected content type: {content_type}") - sha256, byte_size = archive_stream(response, artifact_path, max_bytes=max_bytes) - headers = selected_headers(response.headers) - final_url = response.geturl() - except urllib.error.HTTPError as error: - raise AcquisitionError(f"source returned HTTP {error.code}") from error - except urllib.error.URLError as error: - raise AcquisitionError(f"network error: {error.reason}") from error + # Always download to a private temporary sibling. A repeated run_id must + # never replace an existing raw artifact, even if a caller accidentally + # reuses the identifier with different bytes. + download_path = artifact_path.with_name(f".{artifact_path.name}.{uuid.uuid4().hex}.download") + artifact_state = "stored" + attempts: list[dict[str, Any]] = [] + headers: dict[str, str] = {} + final_url = url + for attempt_number in range(1, max_attempts + 1): + recorder = _RedirectRecorder() + try: + opener = urllib.request.build_opener(recorder) + request = urllib.request.Request(url, headers={"User-Agent": user_agent}) + with opener.open(request, timeout=timeout_seconds) as response: + if not 200 <= response.status < 300: + retryable = response.status == 429 or 500 <= response.status <= 599 + raise AcquisitionError( + f"source returned HTTP {response.status}", failure_class=f"http-{response.status}", + retryable=retryable, + action="retry a bounded server/rate-limit failure" if retryable else "verify URL, authorization, and terms before another run", + ) + content_type = response.headers.get("Content-Type") + allowed = {item.lower() for item in allowed_content_types} + if content_type and content_type.split(";", 1)[0].strip().lower() not in allowed: + raise AcquisitionError( + f"unexpected content type: {content_type}", failure_class="content-type", + action="inspect the source response and update the adapter contract only after review", + ) + sha256, byte_size = archive_stream(response, download_path, max_bytes=max_bytes) + headers = selected_headers(response.headers) + final_url = response.geturl() + if artifact_path.exists(): + if artifact_path.read_bytes() != download_path.read_bytes(): + raise AcquisitionError("existing run artifact differs from newly acquired bytes", failure_class="artifact-collision", action="use a new run_id and preserve both observations") + download_path.unlink(missing_ok=True) + artifact_state = "unchanged" + else: + artifact_path.parent.mkdir(parents=True, exist_ok=True) + os.replace(download_path, artifact_path) + attempts.append({"attempt": attempt_number, "outcome": "success", "redirects": recorder.redirects}) + break + except urllib.error.HTTPError as error: + retryable = error.code == 429 or 500 <= error.code <= 599 + details = {"attempt": attempt_number, "outcome": "failed", "failure_class": f"http-{error.code}", "retryable": retryable, "message": str(error)} + attempts.append(details) + if not retryable or attempt_number == max_attempts: + failure = AcquisitionError( + f"source returned HTTP {error.code}", failure_class=details["failure_class"], retryable=retryable, + action="retry a bounded server/rate-limit failure" if retryable else "verify URL, authorization, and terms before another run", + ) + _write_failure(run_dir, source_id, run_id, failure, attempts) + raise failure from error + except urllib.error.URLError as error: + details = {"attempt": attempt_number, "outcome": "failed", "failure_class": "network", "retryable": True, "message": str(error)} + attempts.append(details) + if attempt_number == max_attempts: + failure = AcquisitionError(f"network error: {error.reason}", failure_class="network", retryable=True, action="retry within the source bound; verify connectivity if it persists") + _write_failure(run_dir, source_id, run_id, failure, attempts) + raise failure from error + except (TimeoutError, socket.timeout) as error: + details = {"attempt": attempt_number, "outcome": "failed", "failure_class": "timeout", "retryable": True, "message": str(error)} + attempts.append(details) + if attempt_number == max_attempts: + failure = AcquisitionError(f"timeout: {error}", failure_class="timeout", retryable=True, action="retry within the source bound; use the manual capture route if it persists") + _write_failure(run_dir, source_id, run_id, failure, attempts) + raise failure from error + except AcquisitionError as error: + download_path.unlink(missing_ok=True) + attempts.append({"attempt": attempt_number, "outcome": "failed", "failure_class": error.failure_class, "retryable": error.retryable, "message": str(error)}) + if not error.retryable or attempt_number == max_attempts: + _write_failure(run_dir, source_id, run_id, error, attempts) + raise + if attempt_number < max_attempts: + delay = min(max_retry_delay_seconds, retry_delay_seconds * (2 ** (attempt_number - 1))) + attempts[-1]["retry_delay_seconds"] = delay + if delay: + sleep_fn(delay) + else: + raise AcquisitionError("acquisition retry loop did not complete", failure_class="runtime") metadata = { "acquisition_method": "network_fetch", "source_id": source_id, @@ -176,6 +253,25 @@ def fetch_source( "rights_caveat": rights_caveat, "privacy_caveat": privacy_caveat, "terms_review": terms_review, + "attempts": attempts, + "artifact_state": artifact_state, + "retention": {"class": "restricted-research-evidence", "public_exposure": False, "review_required": True}, } _atomic_bytes(run_dir / "acquisition-metadata.json", (json.dumps(metadata, ensure_ascii=False, sort_keys=True, indent=2) + "\n").encode()) return metadata + + +def _write_failure(run_dir: Path, source_id: str, run_id: str, error: AcquisitionError, attempts: list[dict[str, Any]]) -> None: + """Leave a private, actionable failure record without creating an artifact.""" + payload = { + "schema_version": "acquisition-failure-v1", "source_id": source_id, "run_id": run_id, + "failure_class": error.failure_class, "retryable": error.retryable, "error": str(error), + "action": error.action, "attempts": attempts, "artifact_created": False, + "public_exposure": False, + } + try: + _atomic_bytes(run_dir / "acquisition-failure.json", (json.dumps(payload, ensure_ascii=False, sort_keys=True, indent=2) + "\n").encode()) + except OSError: + # The original acquisition error is more useful than masking it with a + # best-effort diagnostic write failure. + pass diff --git a/pipeline/common/delta.py b/pipeline/common/delta.py index ec47544..fc88015 100644 --- a/pipeline/common/delta.py +++ b/pipeline/common/delta.py @@ -19,9 +19,14 @@ def _jsonl(path: Path) -> list[dict[str, Any]]: def _manifest(run_dir: Path) -> dict[str, Any]: + # Source adapters historically used both names. The shared operations + # layer treats them as the same private manifest contract so a wrapper + # rename cannot erase the release diff for an otherwise valid run. path = run_dir / "run-manifest.json" if not path.exists(): - raise ValueError(f"missing run manifest: {path}") + path = run_dir / "manifest.json" + if not path.exists(): + raise ValueError(f"missing run manifest: {run_dir / 'run-manifest.json'}") return json.loads(path.read_text(encoding="utf-8")) diff --git a/pipeline/common/orchestrator.py b/pipeline/common/orchestrator.py index 7b948de..ac26c42 100644 --- a/pipeline/common/orchestrator.py +++ b/pipeline/common/orchestrator.py @@ -13,6 +13,7 @@ from pipeline.contracts.private_run import write_private_run_report from pipeline.contracts.source_health import build_health_snapshot, write_health_snapshot from .identity import record_key +from .source_operations import classify_failure, finalize_run_operations ORCHESTRATOR_VERSION = "v2-orchestrator-3" @@ -83,8 +84,10 @@ def run_registered_input(raw_path: str | Path, runs_dir: str | Path, config: dic "suppressed_count": len(records) - len(candidate), "manifest": manifest, "prior_eligible_release": prior_eligible_release} except Exception as exc: + failure = classify_failure(exc) status = {"status": "failed", "publication_state": "unchanged", "release_promoted": False, "error_type": type(exc).__name__, "error": str(exc), + "failure_class": failure["failure_class"], "attempts": getattr(exc, "attempts", []), "prior_eligible_release": prior_eligible_release} status["run_dir"] = str(run_dir) # Health is emitted after run-status exists because it must prove that no @@ -104,12 +107,33 @@ def run_registered_input(raw_path: str | Path, runs_dir: str | Path, config: dic except Exception as exc: # A candidate with invalid evidence is not ready for any later # gate. Keep the files for diagnosis, but fail the run closed. + failure = classify_failure(exc) status = {"status": "failed", "publication_state": "unchanged", "release_promoted": False, "error_type": type(exc).__name__, - "error": f"private evidence: {exc}", + "error": f"private evidence: {exc}", "failure_class": failure["failure_class"], + "attempts": getattr(exc, "attempts", []), "prior_eligible_release": prior_eligible_release, "run_dir": str(run_dir)} status["run_dir"] = str(run_dir) + status["run_id"] = config.get("run_id") or run_dir.name + # The operations ledger is derived from the private run and is append-only. + # It records review/diff artifacts without changing the adapter contract or + # creating a release. A ledger failure closes this run rather than leaving + # an apparently complete run with missing operational evidence. + try: + status = finalize_run_operations( + runs.parent, run_dir, manifest=status.get("manifest"), status=status, + config=config, prior_eligible_release=prior_eligible_release, + ) + except Exception as exc: + failure = classify_failure(exc) + status = { + "status": "failed", "publication_state": "unchanged", "release_promoted": False, + "release_preserved": True, "error_type": type(exc).__name__, + "error": f"source operations: {exc}", "failure_class": failure["failure_class"], + "prior_eligible_release": prior_eligible_release, "run_dir": str(run_dir), + "run_id": config.get("run_id") or run_dir.name, + } _atomic(run_dir / "run-status.json", (json.dumps(status, ensure_ascii=False, sort_keys=True, indent=2) + "\n").encode()) return status diff --git a/pipeline/common/source_operations.py b/pipeline/common/source_operations.py new file mode 100644 index 0000000..a7504f2 --- /dev/null +++ b/pipeline/common/source_operations.py @@ -0,0 +1,566 @@ +"""Private operational records for repeatable source refreshes. + +This module owns the source-agnostic concerns around the adapter lifecycle: +schedule/freshness expectations, content-addressed raw evidence, append-only +run history, deterministic review packets, and bounded failure reporting. It +does not acquire, transform, import, approve, or publish records. + +All output is suitable for restricted operator use. Health and review output +is deliberately aggregate and keeps release promotion disabled. +""" +from __future__ import annotations + +import hashlib +import json +import re +import socket +import time +import urllib.error +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Callable, Iterable + +from .acquisition import AcquisitionError +from .delta import compare_runs +from pipeline.contracts.source_lifecycle import atomic_bytes, atomic_json + + +OPERATIONS_SCHEMA_VERSION = "source-operations-v1" +RUN_CLASSIFICATIONS = {"changed", "unchanged", "failed", "review-required"} +_SOURCE_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*$") + + +class SourceOperationsError(ValueError): + """A source operations contract is malformed or inconsistent.""" + + +class RetryExhaustedError(RuntimeError): + """A bounded operation failed after its permitted attempts.""" + + def __init__(self, message: str, *, attempts: list[dict[str, Any]], cause: BaseException) -> None: + super().__init__(message) + self.attempts = attempts + self.cause = cause + + +@dataclass(frozen=True) +class SourceSchedule: + """Operational expectations for one source, separate from publication.""" + + source_id: str + cadence: str + interval_hours: int | None + stale_after_hours: int | None + max_attempts: int = 1 + backoff_seconds: float = 0.0 + max_backoff_seconds: float = 30.0 + retention_class: str = "restricted-research-evidence" + manual_fallback: str = "operator review required" + + def __post_init__(self) -> None: + if not _SOURCE_ID.fullmatch(self.source_id): + raise SourceOperationsError(f"invalid source_id: {self.source_id!r}") + if not isinstance(self.cadence, str) or not self.cadence: + raise SourceOperationsError(f"{self.source_id}.cadence must be non-empty") + for name in ("interval_hours", "stale_after_hours"): + value = getattr(self, name) + if value is not None and (not isinstance(value, int) or value <= 0): + raise SourceOperationsError(f"{self.source_id}.{name} must be a positive integer or null") + if self.stale_after_hours is not None and self.interval_hours is not None and self.stale_after_hours < self.interval_hours: + raise SourceOperationsError(f"{self.source_id}.stale_after_hours must cover its interval") + if not isinstance(self.max_attempts, int) or not 1 <= self.max_attempts <= 10: + raise SourceOperationsError(f"{self.source_id}.max_attempts must be between 1 and 10") + if self.backoff_seconds < 0 or self.max_backoff_seconds < 0: + raise SourceOperationsError(f"{self.source_id} retry backoff cannot be negative") + if self.backoff_seconds > self.max_backoff_seconds: + raise SourceOperationsError(f"{self.source_id}.backoff_seconds exceeds max_backoff_seconds") + + @classmethod + def from_mapping(cls, value: object, *, index: int = 0) -> "SourceSchedule": + if not isinstance(value, dict): + raise SourceOperationsError(f"schedules[{index}] must be an object") + required = {"source_id", "cadence", "interval_hours", "stale_after_hours"} + missing = sorted(required - value.keys()) + if missing: + raise SourceOperationsError(f"schedules[{index}] missing fields: {', '.join(missing)}") + try: + return cls( + source_id=str(value["source_id"]), cadence=str(value["cadence"]), + interval_hours=value["interval_hours"], stale_after_hours=value["stale_after_hours"], + max_attempts=value.get("max_attempts", 1), + backoff_seconds=float(value.get("backoff_seconds", 0)), + max_backoff_seconds=float(value.get("max_backoff_seconds", 30)), + retention_class=str(value.get("retention_class", "restricted-research-evidence")), + manual_fallback=str(value.get("manual_fallback", "operator review required")), + ) + except (TypeError, ValueError) as error: + raise SourceOperationsError(f"schedules[{index}] has invalid values") from error + + def as_mapping(self) -> dict[str, Any]: + return { + "source_id": self.source_id, + "cadence": self.cadence, + "interval_hours": self.interval_hours, + "stale_after_hours": self.stale_after_hours, + "max_attempts": self.max_attempts, + "backoff_seconds": self.backoff_seconds, + "max_backoff_seconds": self.max_backoff_seconds, + "retention_class": self.retention_class, + "manual_fallback": self.manual_fallback, + } + + +def load_source_schedules(path: str | Path | None = None, *, registry_path: str | Path | None = None) -> dict[str, SourceSchedule]: + """Load and validate the complete operational schedule inventory.""" + config_path = Path(path) if path is not None else Path(__file__).parents[1] / "source_operations.json" + try: + payload = json.loads(config_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as error: + raise SourceOperationsError(f"cannot read source schedules: {error}") from error + if not isinstance(payload, dict) or payload.get("schema_version") != "1.0": + raise SourceOperationsError("source operations config schema_version 1.0 is required") + values = payload.get("schedules") + if not isinstance(values, list) or not values: + raise SourceOperationsError("source operations schedules must be a non-empty list") + schedules: dict[str, SourceSchedule] = {} + for index, value in enumerate(values): + schedule = SourceSchedule.from_mapping(value, index=index) + if schedule.source_id in schedules: + raise SourceOperationsError(f"duplicate schedule for {schedule.source_id}") + schedules[schedule.source_id] = schedule + if registry_path is not None: + from pipeline.source_registry import load_registry + + registry_ids = {item["source_id"] for item in load_registry(Path(registry_path))["sources"]} + missing = sorted(registry_ids - schedules.keys()) + extra = sorted(schedules.keys() - registry_ids) + if missing or extra: + raise SourceOperationsError(f"schedule/source registry mismatch; missing={missing}, extra={extra}") + return schedules + + +def _parse_time(value: str, label: str) -> datetime: + try: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + except (AttributeError, TypeError, ValueError) as error: + raise SourceOperationsError(f"{label} must be timezone-aware ISO-8601") from error + if parsed.tzinfo is None: + raise SourceOperationsError(f"{label} must include a timezone") + return parsed.astimezone(timezone.utc) + + +def freshness_for(schedule: SourceSchedule, retrieved_at_utc: str | None, *, as_of_utc: str) -> dict[str, Any]: + """Return deterministic freshness facts without calling a source.""" + as_of = _parse_time(as_of_utc, "as_of_utc") + if not retrieved_at_utc: + return {"state": "not-run", "retrieved_at_utc": None, "as_of_utc": as_of.isoformat().replace("+00:00", "Z"), "age_hours": None, "due": False} + retrieved = _parse_time(retrieved_at_utc, "retrieved_at_utc") + age_hours = round((as_of - retrieved).total_seconds() / 3600, 3) + if age_hours < 0: + state = "clock-skew" + elif schedule.stale_after_hours is None: + state = "unknown" + else: + state = "stale" if age_hours > schedule.stale_after_hours else "current" + due = bool(schedule.interval_hours is not None and age_hours >= schedule.interval_hours) + return { + "state": state, + "retrieved_at_utc": retrieved.isoformat().replace("+00:00", "Z"), + "as_of_utc": as_of.isoformat().replace("+00:00", "Z"), + "age_hours": age_hours, + "due": due, + "interval_hours": schedule.interval_hours, + "stale_after_hours": schedule.stale_after_hours, + } + + +@dataclass(frozen=True) +class RetryPolicy: + max_attempts: int = 1 + backoff_seconds: float = 0.0 + max_backoff_seconds: float = 30.0 + + def __post_init__(self) -> None: + if not 1 <= self.max_attempts <= 10: + raise SourceOperationsError("retry max_attempts must be between 1 and 10") + if self.backoff_seconds < 0 or self.max_backoff_seconds < 0 or self.backoff_seconds > self.max_backoff_seconds: + raise SourceOperationsError("retry backoff values are invalid") + + +def classify_failure(error: BaseException) -> dict[str, Any]: + """Classify failures into bounded, actionable operator categories.""" + if isinstance(error, RetryExhaustedError): + return {"failure_class": "retry-exhausted", "retryable": False, "action": "inspect attempts and use the source manual fallback", "message": str(error)} + if isinstance(error, AcquisitionError): + return { + "failure_class": getattr(error, "failure_class", "acquisition"), + "retryable": bool(getattr(error, "retryable", False)), + "action": getattr(error, "action", "inspect the private acquisition evidence and source terms"), + "message": str(error), + } + if isinstance(error, (TimeoutError, socket.timeout)): + return {"failure_class": "timeout", "retryable": True, "action": "retry within the source bound; use the manual capture route if it persists", "message": str(error)} + if isinstance(error, urllib.error.HTTPError): + retryable = error.code == 429 or 500 <= error.code <= 599 + return {"failure_class": f"http-{error.code}", "retryable": retryable, "action": "retry a bounded server/rate-limit failure" if retryable else "verify URL, authorization, and terms before another run", "message": str(error)} + if isinstance(error, urllib.error.URLError): + return {"failure_class": "network", "retryable": True, "action": "retry within the source bound; verify connectivity if it persists", "message": str(error)} + if isinstance(error, (json.JSONDecodeError, UnicodeError)): + return {"failure_class": "malformed-input", "retryable": False, "action": "retain the artifact and inspect schema/encoding before rerun", "message": str(error)} + return {"failure_class": "validation-or-runtime", "retryable": False, "action": "inspect the private run report; correct the adapter or input before rerun", "message": str(error)} + + +def run_with_bounded_retries( + operation: Callable[[], Any], + policy: RetryPolicy, + *, + sleep: Callable[[float], None] = time.sleep, +) -> tuple[Any, list[dict[str, Any]]]: + """Run an operation with retryable failures only and a hard attempt bound.""" + attempts: list[dict[str, Any]] = [] + for number in range(1, policy.max_attempts + 1): + try: + return operation(), attempts + [{"attempt": number, "outcome": "success"}] + except BaseException as error: + details = classify_failure(error) + attempt = {"attempt": number, "outcome": "failed", **details} + attempts.append(attempt) + if not details["retryable"] or number == policy.max_attempts: + raise RetryExhaustedError( + f"operation failed after {number} bounded attempt(s): {details['failure_class']}", + attempts=attempts, + cause=error, + ) from error + delay = min(policy.max_backoff_seconds, policy.backoff_seconds * (2 ** (number - 1))) + attempt["retry_delay_seconds"] = delay + if delay: + sleep(delay) + raise AssertionError("retry loop did not return or raise") + + +def register_deduplicated_artifact( + raw: bytes, + operations_root: str | Path, + metadata: dict[str, Any], + *, + artifact_name: str = "artifact", +) -> dict[str, Any]: + """Store immutable bytes once while retaining each observation manifest.""" + root = Path(operations_root) + digest = hashlib.sha256(raw).hexdigest() + artifact_path = root / "raw" / "sha256" / digest[:2] / digest / artifact_name + existed = artifact_path.exists() + if existed and artifact_path.read_bytes() != raw: + raise SourceOperationsError(f"content-addressed artifact collision: {digest}") + if not existed: + atomic_bytes(artifact_path, raw) + observation = { + **metadata, + "operations_schema_version": OPERATIONS_SCHEMA_VERSION, + "sha256": digest, + "checksum_sha256": digest, + "byte_size": len(raw), + "artifact_name": artifact_name, + "artifact_path": str(artifact_path), + "artifact_state": "deduplicated" if existed else "stored", + "retention": { + "class": metadata.get("retention_class", "restricted-research-evidence"), + "public_exposure": False, + "review_required": True, + "exceptional_removal_policy": "docs/ETHICS.md sections 2, 6, 8, and 9", + }, + } + observation_id = hashlib.sha256(json.dumps(observation, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode()).hexdigest() + observation["observation_id"] = observation_id + observation_path = root / "raw" / "observations" / f"{observation_id}.json" + if not observation_path.exists(): + atomic_json(observation_path, observation) + observation["observation_path"] = str(observation_path) + return observation + + +def _safe_source_id(source_id: str) -> str: + if not _SOURCE_ID.fullmatch(source_id): + raise SourceOperationsError(f"invalid source_id: {source_id!r}") + return source_id + + +def read_run_history(operations_root: str | Path, source_id: str) -> list[dict[str, Any]]: + path = Path(operations_root) / "history" / f"{_safe_source_id(source_id)}.jsonl" + if not path.exists(): + return [] + rows: list[dict[str, Any]] = [] + for line_number, line in enumerate(path.read_text(encoding="utf-8").splitlines(), start=1): + if not line.strip(): + continue + try: + value = json.loads(line) + except json.JSONDecodeError as error: + raise SourceOperationsError(f"invalid run history at {path}:{line_number}") from error + if not isinstance(value, dict): + raise SourceOperationsError(f"run history entry at {path}:{line_number} must be an object") + rows.append(value) + return rows + + +def append_run_history(operations_root: str | Path, source_id: str, entry: dict[str, Any]) -> Path: + path = Path(operations_root) / "history" / f"{_safe_source_id(source_id)}.jsonl" + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("a", encoding="utf-8", newline="\n") as handle: + handle.write(json.dumps(entry, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + "\n") + return path + + +def classify_run(*, failed: bool, review_required: bool, artifact_sha256: str | None, normalized_sha256: str | None, previous: dict[str, Any] | None) -> str: + if failed: + return "failed" + if review_required: + return "review-required" + if previous and artifact_sha256 == previous.get("artifact_sha256") and normalized_sha256 == previous.get("normalized_sha256"): + return "unchanged" + return "changed" + + +def _file_sha256(path: Path) -> str | None: + if not path.exists(): + return None + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def _provenance(manifest: dict[str, Any]) -> dict[str, Any]: + keys = ("source_url", "retrieved_at_utc", "publication_date", "effective_date", "sha256", "checksum_sha256", "byte_size", "code_version", "config_version", "redirects") + return {key: manifest[key] for key in keys if manifest.get(key) is not None} + + +def build_release_diff(previous_run_dir: str | Path | None, current_run_dir: str | Path, *, prior_eligible_release: dict[str, Any] | None = None) -> dict[str, Any]: + """Build a row-free, deterministic diff for a human operator.""" + current = Path(current_run_dir) + if previous_run_dir is None: + return { + "schema_version": "release-diff-v1", "status": "no-previous-validated-run", + "release_promoted": False, "prior_eligible_release": prior_eligible_release, + "counts": {"added": 0, "changed": 0, "not_observed": 0, "suppressed": 0}, + } + delta = compare_runs(Path(previous_run_dir), current, prior_eligible_release=prior_eligible_release) + return {"schema_version": "release-diff-v1", **delta, "release_promoted": False} + + +def build_review_packet( + run_dir: str | Path, + *, + manifest: dict[str, Any] | None, + status: dict[str, Any], + previous_run_dir: str | Path | None = None, + prior_eligible_release: dict[str, Any] | None = None, +) -> dict[str, Any]: + """Create a stable operator packet containing no source rows or raw fields.""" + manifest = manifest or {} + source_id = str(manifest.get("source_id") or status.get("source_id") or "unknown") + qa_path = Path(run_dir) / "qa.json" + qa: dict[str, Any] = {} + if qa_path.exists(): + value = json.loads(qa_path.read_text(encoding="utf-8")) + if isinstance(value, dict): + qa = value + failed = status.get("status") == "failed" + review_required = bool(status.get("review_required") or manifest.get("review_required") or qa.get("drift_alarms") or manifest.get("quarantined_rows", 0)) + reasons: list[str] = [] + if failed: + reasons.append("run failed; inspect failure report before rerun") + if manifest.get("quarantined_rows", 0): + reasons.append("quarantined rows require source-specific review") + if qa.get("drift_alarms"): + reasons.append("drift alarms require operator review") + if status.get("publication_state") in {"human-gate-required", "terms-gate-blocked"}: + reasons.append("publication remains behind the human/terms gate") + if not reasons: + reasons.append("confirm private evidence and release scope before any separate approval action") + diff = build_release_diff(previous_run_dir, run_dir, prior_eligible_release=prior_eligible_release) + return { + "schema_version": "review-packet-v1", + "source_id": source_id, + "run_id": status.get("run_id") or Path(run_dir).name, + "classification": status.get("run_classification", "failed" if failed else "changed"), + "review_required": review_required, + "reasons": sorted(set(reasons)), + "provenance": _provenance(manifest), + "run": { + "status": status.get("status"), + "publication_state": status.get("publication_state", "unchanged"), + "release_state": manifest.get("release_state", "not-created"), + "input_rows": manifest.get("input_rows"), + "normalized_rows": manifest.get("normalized_rows"), + "quarantined_rows": manifest.get("quarantined_rows"), + "drift_alarms": sorted(set(qa.get("drift_alarms", []))) if isinstance(qa.get("drift_alarms", []), list) else [], + }, + "release_diff": diff, + "prior_eligible_release": prior_eligible_release, + "release_promotion_allowed": False, + "public_exposure": False, + "operator_actions": [ + "inspect private acquisition metadata and QA report", + "resolve listed review reasons and source-specific blockers", + "use a separate authorized release process; this packet cannot promote a release", + ], + } + + +def finalize_run_operations( + operations_root: str | Path, + run_dir: str | Path, + *, + manifest: dict[str, Any] | None, + status: dict[str, Any], + config: dict[str, Any] | None = None, + previous_run_dir: str | Path | None = None, + prior_eligible_release: dict[str, Any] | None = None, +) -> dict[str, Any]: + """Persist the shared run ledger and deterministic operator artifacts.""" + config = config or {} + manifest = manifest or {} + source_id = str(manifest.get("source_id") or status.get("source_id") or config.get("source_id") or "unknown") + history = read_run_history(operations_root, source_id) + previous = history[-1] if history else None + if previous_run_dir is None and previous and previous.get("run_dir"): + candidate = Path(previous["run_dir"]) + if candidate.exists(): + previous_run_dir = candidate + run_path = Path(run_dir) + normalized_path = run_path / "normalized" / "records.jsonl" + artifact_sha256 = manifest.get("sha256") or manifest.get("checksum_sha256") or config.get("checksum_sha256") + normalized_sha256 = _file_sha256(normalized_path) + failed = status.get("status") == "failed" + review_required = bool(status.get("review_required") or manifest.get("review_required") or manifest.get("quarantined_rows", 0) or (run_path / "qa.json").exists() and json.loads((run_path / "qa.json").read_text(encoding="utf-8")).get("drift_alarms")) + classification = classify_run(failed=failed, review_required=review_required, artifact_sha256=artifact_sha256, normalized_sha256=normalized_sha256, previous=previous) + status.update({ + "source_id": source_id, + "run_classification": classification, + "review_required": review_required, + "artifact_state": "unchanged" if previous and artifact_sha256 == previous.get("artifact_sha256") else "changed", + "release_preserved": True, + "release_promoted": False, + }) + packet = build_review_packet(run_path, manifest=manifest, status=status, previous_run_dir=previous_run_dir, prior_eligible_release=prior_eligible_release) + packet_path = run_path / "review-packet.json" + diff_path = run_path / "release-diff.json" + atomic_json(packet_path, packet) + atomic_json(diff_path, packet["release_diff"]) + entry = { + "schema_version": "run-history-v1", + "source_id": source_id, + "run_id": status.get("run_id") or run_path.name, + "run_dir": str(run_path), + "run_status": status.get("status"), + "run_classification": classification, + "review_required": review_required, + "release_promoted": False, + "release_preserved": True, + "artifact_sha256": artifact_sha256, + "normalized_sha256": normalized_sha256, + "retrieved_at_utc": manifest.get("retrieved_at_utc") or config.get("retrieved_at_utc"), + "input_rows": manifest.get("input_rows"), + "normalized_rows": manifest.get("normalized_rows"), + "quarantined_rows": manifest.get("quarantined_rows"), + # Keep the append-only aggregate ledger row-free. The detailed error + # stays in the restricted run-status/failure report instead. + "error": "run failed; see restricted failure-report.json" if failed else None, + "failure_class": status.get("failure_class") if failed else None, + } + history_path = append_run_history(operations_root, source_id, entry) + status.update({"history_path": str(history_path), "review_packet_path": str(packet_path), "release_diff_path": str(diff_path)}) + if failed: + status["failure_report"] = failure_report(source_id=source_id, run_id=entry["run_id"], error=status.get("error", "run failed"), attempts=status.get("attempts", []), manual_fallback=config.get("manual_fallback")) + atomic_json(run_path / "failure-report.json", status["failure_report"]) + if config.get("notification_root"): + status["notification_path"] = str(write_failure_notification(status["failure_report"], config["notification_root"])) + return status + + +def failure_report(*, source_id: str, run_id: str, error: BaseException | str, attempts: Iterable[dict[str, Any]] = (), manual_fallback: str | None = None) -> dict[str, Any]: + details = classify_failure(error) if isinstance(error, BaseException) else {"failure_class": "run-failed", "retryable": False, "action": "inspect the private run and rerun after correction", "message": str(error)} + # Exception messages can accidentally echo a source value. The report is + # useful to an operator without copying that payload into an aggregate + # notification or append-only history record. + details = {key: value for key, value in details.items() if key != "message"} + safe_attempts = [ + {key: value for key, value in attempt.items() if key != "message"} + for attempt in attempts + ] + report = { + "schema_version": "failure-report-v1", "source_id": source_id, "run_id": run_id, + "failure": details, "attempts": safe_attempts, "public_exposure": False, + "release_promoted": False, "release_preserved": True, + "operator_action": manual_fallback or details["action"], + } + return report + + +def write_failure_notification(report: dict[str, Any], notifications_root: str | Path) -> Path: + """Write a local, append-only notification hook payload; no service is required.""" + source_id = _safe_source_id(str(report.get("source_id", "unknown"))) + path = Path(notifications_root) / f"{source_id}.jsonl" + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("a", encoding="utf-8", newline="\n") as handle: + handle.write(json.dumps(report, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + "\n") + return path + + +def build_source_health_index( + operations_root: str | Path, + *, + schedules_path: str | Path | None = None, + registry_path: str | Path | None = None, + as_of_utc: str, +) -> dict[str, Any]: + """Build a deterministic aggregate health index for every registered source.""" + schedules = load_source_schedules(schedules_path, registry_path=registry_path) + sources: list[dict[str, Any]] = [] + for source_id in sorted(schedules): + schedule = schedules[source_id] + history = read_run_history(operations_root, source_id) + latest = history[-1] if history else None + last_validated = next((item for item in reversed(history) if item.get("run_status") != "failed"), None) + freshness = freshness_for(schedule, last_validated.get("retrieved_at_utc") if last_validated else None, as_of_utc=as_of_utc) + if latest is None: + health_state = "not-run" + elif latest.get("run_status") == "failed": + health_state = "failed" + elif latest.get("run_classification") == "review-required": + health_state = "review-required" + elif freshness["state"] == "stale": + health_state = "degraded" + else: + health_state = "private-validated" + sources.append({ + "source_id": source_id, + "schedule": schedule.as_mapping(), + "health_state": health_state, + "private_validation": latest is not None and latest.get("run_status") != "failed", + "public_exposure": False, + "publication_eligibility": "blocked", + "freshness": freshness, + "last_run": None if latest is None else { + key: latest.get(key) for key in ("run_id", "run_dir", "run_status", "run_classification", "review_required", "artifact_sha256", "normalized_sha256", "input_rows", "normalized_rows", "quarantined_rows", "failure_class") + }, + "last_validated_run": None if last_validated is None else { + key: last_validated.get(key) for key in ("run_id", "run_dir", "run_status", "run_classification", "artifact_sha256", "normalized_sha256", "retrieved_at_utc") + }, + "run_count": len(history), + }) + return { + "schema_version": "source-health-index-v1", + "as_of_utc": _parse_time(as_of_utc, "as_of_utc").isoformat().replace("+00:00", "Z"), + "private_validation": True, + "public_exposure": False, + "publication_eligibility": "blocked", + "sources": sources, + } + + +__all__ = [ + "OPERATIONS_SCHEMA_VERSION", "RetryExhaustedError", "RetryPolicy", "SourceOperationsError", "SourceSchedule", + "append_run_history", "build_release_diff", "build_review_packet", "build_source_health_index", "classify_failure", + "classify_run", "failure_report", "finalize_run_operations", "freshness_for", "load_source_schedules", + "read_run_history", "register_deduplicated_artifact", "run_with_bounded_retries", "write_failure_notification", +] diff --git a/pipeline/common/test_acquisition.py b/pipeline/common/test_acquisition.py index f9c2ff0..528565d 100644 --- a/pipeline/common/test_acquisition.py +++ b/pipeline/common/test_acquisition.py @@ -2,8 +2,10 @@ import tempfile import unittest from pathlib import Path +from unittest.mock import patch +import urllib.error -from .acquisition import AcquisitionError, archive_stream, require_terms_review +from .acquisition import AcquisitionError, archive_stream, fetch_source, require_terms_review class AcquisitionContractTests(unittest.TestCase): @@ -26,6 +28,80 @@ def read(self, _size): self.assertFalse(target.exists()) self.assertFalse(list(Path(directory).glob("*.part"))) + def test_fetch_retries_network_failure_and_records_attempts(self): + class Response: + status = 200 + headers = {"Content-Type": "text/csv", "Content-Length": "7"} + + def __enter__(self): + return self + + def __exit__(self, *_args): + return False + + def read(self, size): + if not hasattr(self, "done"): + self.done = True + return b"a,b\n1,2\n" + return b"" + + def geturl(self): + return "https://example.test/source.csv" + + class Opener: + def __init__(self): + self.calls = 0 + + def open(self, _request, timeout): + self.calls += 1 + if self.calls == 1: + raise urllib.error.URLError("temporary") + return Response() + + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + terms = root / "terms.json" + terms.write_text(json.dumps({"reviewer": "operator", "reference": "test", "reviewed_at": "2026-09-15T00:00:00Z", "decision": "approved", "notes": "synthetic"}), encoding="utf-8") + opener = Opener() + with patch("pipeline.common.acquisition.urllib.request.build_opener", return_value=opener): + metadata = fetch_source( + source_id="test.source", url="https://example.test/source.csv", output_root=root / "raw", + artifact_name="source.csv", terms_review_path=terms, run_id="run-1", max_attempts=2, + ) + self.assertEqual(opener.calls, 2) + self.assertEqual(len(metadata["attempts"]), 2) + self.assertEqual(metadata["attempts"][0]["failure_class"], "network") + self.assertTrue(Path(metadata["artifact_path"]).exists()) + + def test_fetch_non_retryable_content_type_fails_closed_with_private_report(self): + class Response: + status = 200 + headers = {"Content-Type": "text/html"} + + def __enter__(self): + return self + + def __exit__(self, *_args): + return False + + class Opener: + def open(self, _request, timeout): + return Response() + + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + terms = root / "terms.json" + terms.write_text(json.dumps({"reviewer": "operator", "reference": "test", "reviewed_at": "2026-09-15T00:00:00Z", "decision": "approved", "notes": "synthetic"}), encoding="utf-8") + with patch("pipeline.common.acquisition.urllib.request.build_opener", return_value=Opener()): + with self.assertRaisesRegex(AcquisitionError, "unexpected content type"): + fetch_source( + source_id="test.source", url="https://example.test/source", output_root=root / "raw", + artifact_name="source", terms_review_path=terms, run_id="run-2", max_attempts=3, + ) + failure = json.loads((root / "raw/test.source/run-2/acquisition-failure.json").read_text()) + self.assertEqual(failure["failure_class"], "content-type") + self.assertFalse(failure["artifact_created"]) + if __name__ == "__main__": unittest.main() diff --git a/pipeline/common/test_source_operations.py b/pipeline/common/test_source_operations.py new file mode 100644 index 0000000..5c35138 --- /dev/null +++ b/pipeline/common/test_source_operations.py @@ -0,0 +1,122 @@ +import hashlib +import json +import tempfile +import unittest +from pathlib import Path + +from .acquisition import AcquisitionError +from .source_operations import ( + RetryPolicy, + SourceOperationsError, + SourceSchedule, + build_source_health_index, + classify_failure, + finalize_run_operations, + freshness_for, + load_source_schedules, + read_run_history, + register_deduplicated_artifact, + run_with_bounded_retries, +) + + +class SourceOperationsTests(unittest.TestCase): + def test_checked_in_schedule_inventory_matches_registry(self): + root = Path(__file__).parents[1] + schedules = load_source_schedules(registry_path=root / "source_registry.json") + self.assertEqual(len(schedules), 14) + self.assertEqual(schedules["dk.smiley"].stale_after_hours, 240) + self.assertIsNone(schedules["us.fsis"].interval_hours) + + def test_schedule_validation_rejects_missing_and_inverted_freshness(self): + with self.assertRaisesRegex(SourceOperationsError, "missing fields"): + SourceSchedule.from_mapping({"source_id": "x"}) + with self.assertRaisesRegex(SourceOperationsError, "must cover"): + SourceSchedule("x", "daily", 24, 12) + + def test_freshness_is_explicit_for_unknown_and_stale_schedules(self): + unknown = SourceSchedule("x", "unknown", None, None) + self.assertEqual(freshness_for(unknown, "2026-01-01T00:00:00Z", as_of_utc="2026-09-15T00:00:00Z")["state"], "unknown") + weekly = SourceSchedule("x", "weekly", 168, 240) + fresh = freshness_for(weekly, "2026-09-07T00:00:00Z", as_of_utc="2026-09-15T00:00:00Z") + self.assertEqual(fresh["state"], "current") + self.assertTrue(fresh["due"]) + self.assertEqual(freshness_for(weekly, "2026-09-01T00:00:00Z", as_of_utc="2026-09-15T00:00:00Z")["state"], "stale") + + def test_retries_are_bounded_and_classified(self): + calls = [] + + def operation(): + calls.append(len(calls) + 1) + raise AcquisitionError("temporary", failure_class="network", retryable=True, action="retry") + + with self.assertRaisesRegex(RuntimeError, "bounded attempt") as raised: + run_with_bounded_retries(operation, RetryPolicy(max_attempts=3), sleep=lambda _: None) + self.assertEqual(calls, [1, 2, 3]) + self.assertEqual(len(raised.exception.attempts), 3) + self.assertEqual(classify_failure(raised.exception)["failure_class"], "retry-exhausted") + + def test_deduplicated_artifacts_keep_distinct_observations(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + first = register_deduplicated_artifact(b"same", root, {"source_id": "x", "retrieved_at_utc": "one"}) + second = register_deduplicated_artifact(b"same", root, {"source_id": "x", "retrieved_at_utc": "two"}) + self.assertEqual(first["artifact_path"], second["artifact_path"]) + self.assertEqual(first["artifact_state"], "stored") + self.assertEqual(second["artifact_state"], "deduplicated") + self.assertNotEqual(first["observation_id"], second["observation_id"]) + self.assertEqual(len(list((root / "raw/sha256").glob("**/artifact"))), 1) + self.assertFalse(json.loads(Path(second["observation_path"]).read_text())["retention"]["public_exposure"]) + + def _write_run(self, root: Path, name: str, raw: bytes = b"raw") -> tuple[Path, dict]: + run = root / "runs" / name + (run / "normalized").mkdir(parents=True) + (run / "normalized/records.jsonl").write_text(json.dumps({"normalized": {"establishment_id": "A"}}) + "\n", encoding="utf-8") + manifest = { + "source_id": "dk.smiley", "source_url": "https://example.test/dk", "retrieved_at_utc": "2026-09-15T00:00:00Z", + "checksum_sha256": hashlib.sha256(raw).hexdigest(), "sha256": hashlib.sha256(raw).hexdigest(), "byte_size": len(raw), + "code_version": "test", "config_version": "test", "input_rows": 1, "normalized_rows": 1, "quarantined_rows": 0, + "schema_fingerprint": "schema-1", "release_state": "not-created", "publication_state": "private-candidate", + } + (run / "manifest.json").write_text(json.dumps(manifest, sort_keys=True), encoding="utf-8") + (run / "qa.json").write_text(json.dumps({"source_id": "dk.smiley", "input_rows": 1, "normalized_rows": 1, "quarantined_rows": 0, "drift_alarms": []}), encoding="utf-8") + return run, manifest + + def test_history_diff_packet_and_unchanged_rerun_are_deterministic(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + first, manifest = self._write_run(root, "first") + first_status = finalize_run_operations(root, first, manifest=manifest, status={"status": "candidate-ready", "publication_state": "human-gate-required"}, config={"source_id": "dk.smiley"}) + self.assertEqual(first_status["run_classification"], "changed") + self.assertTrue(first_status["release_preserved"]) + second, manifest2 = self._write_run(root, "second") + second_status = finalize_run_operations(root, second, manifest=manifest2, status={"status": "candidate-ready", "publication_state": "human-gate-required"}, config={"source_id": "dk.smiley"}) + self.assertEqual(second_status["run_classification"], "unchanged") + packet = json.loads((second / "review-packet.json").read_text()) + self.assertFalse(packet["release_promotion_allowed"]) + self.assertNotIn("establishment_id", json.dumps(packet)) + history = read_run_history(root, "dk.smiley") + self.assertEqual([item["run_classification"] for item in history], ["changed", "unchanged"]) + self.assertEqual((second / "release-diff.json").read_bytes(), (second / "release-diff.json").read_bytes()) + + def test_failed_run_keeps_prior_release_reference_and_health_is_private(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + first, manifest = self._write_run(root, "first") + finalize_run_operations(root, first, manifest=manifest, status={"status": "candidate-ready", "publication_state": "human-gate-required"}, config={"source_id": "dk.smiley"}) + failed = root / "runs/failed" + failed.mkdir(parents=True) + status = finalize_run_operations(root, failed, manifest=None, status={"status": "failed", "error": "schema drift at secret address", "publication_state": "unchanged"}, config={"source_id": "dk.smiley", "manual_fallback": "operator capture"}, prior_eligible_release={"release_id": "validated-1"}) + self.assertEqual(status["run_classification"], "failed") + self.assertTrue(status["release_preserved"]) + report = json.loads((failed / "failure-report.json").read_text()) + self.assertFalse(report["public_exposure"]) + self.assertNotIn("secret address", json.dumps(report)) + index = build_source_health_index(root, registry_path=Path(__file__).parents[1] / "source_registry.json", as_of_utc="2026-09-15T00:00:00Z") + source = next(item for item in index["sources"] if item["source_id"] == "dk.smiley") + self.assertEqual(source["health_state"], "failed") + self.assertFalse(source["public_exposure"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/scripts/README.md b/pipeline/scripts/README.md index b7ea017..29b29aa 100644 --- a/pipeline/scripts/README.md +++ b/pipeline/scripts/README.md @@ -8,6 +8,11 @@ Scripts are grouped by their role in the auditable ingestion workflow: - `diagnostics/` contains read-only inspection and sampling tools. These help evaluate a source or service and are not required for a normal full run. - `maintenance/` contains repository and migration-support utilities, such as the legacy manifest builder. +`diagnostics/build-source-operations-health.py` reads the private append-only +source run ledger and the checked-in schedule inventory to emit a deterministic, +row-free health index. It does not acquire data, update `docs/source-status`, +promote a release, or make a public health claim. + Run scripts from the repository root so their documented paths and output locations are stable. Each stage should accept explicit input and output paths (or a run identifier), preserve source timestamps and checksums, and emit useful progress logs. Generated artifacts belong under `data/`, not beside the scripts. ## Adding another country diff --git a/pipeline/scripts/diagnostics/build-source-operations-health.py b/pipeline/scripts/diagnostics/build-source-operations-health.py new file mode 100644 index 0000000..cdeec6c --- /dev/null +++ b/pipeline/scripts/diagnostics/build-source-operations-health.py @@ -0,0 +1,35 @@ +"""Build the private-alpha source operations health index.""" +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +from pipeline.common.source_operations import SourceOperationsError, build_source_health_index + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--operations-root", type=Path, required=True, help="private root containing history/") + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--as-of-utc", required=True, help="timezone-aware ISO-8601 timestamp") + parser.add_argument("--schedules", type=Path) + parser.add_argument("--registry", type=Path) + args = parser.parse_args() + try: + index = build_source_health_index( + args.operations_root, schedules_path=args.schedules, registry_path=args.registry, + as_of_utc=args.as_of_utc, + ) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(index, ensure_ascii=False, sort_keys=True, indent=2) + "\n", encoding="utf-8") + except (OSError, SourceOperationsError) as error: + print(f"source-operations-health: {error}", file=sys.stderr) + return 2 + print(f"source-operations-health: wrote {args.output}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/pipeline/source_operations.json b/pipeline/source_operations.json new file mode 100644 index 0000000..72a1b6b --- /dev/null +++ b/pipeline/source_operations.json @@ -0,0 +1,20 @@ +{ + "schema_version": "1.0", + "purpose": "Private-alpha scheduling, freshness, retention, and bounded retry expectations. This file does not authorize publication.", + "schedules": [ + {"source_id":"be.locations","cadence":"weekly","interval_hours":168,"stale_after_hours":240,"max_attempts":3,"backoff_seconds":1,"max_backoff_seconds":30,"manual_fallback":"use the authorized two-file operator capture"}, + {"source_id":"ca.locations","cadence":"unknown","interval_hours":null,"stale_after_hours":null,"max_attempts":1,"backoff_seconds":0,"max_backoff_seconds":0,"manual_fallback":"verify a permitted federal/provincial artifact before scheduling"}, + {"source_id":"de.locations","cadence":"unknown","interval_hours":null,"stale_after_hours":null,"max_attempts":1,"backoff_seconds":0,"max_backoff_seconds":0,"manual_fallback":"select and preserve the BVL export through the operator route"}, + {"source_id":"dk.smiley","cadence":"weekly","interval_hours":168,"stale_after_hours":240,"max_attempts":3,"backoff_seconds":1,"max_backoff_seconds":30,"manual_fallback":"use the reviewed Find Smiley bulk-download route"}, + {"source_id":"es.locations","cadence":"unknown","interval_hours":null,"stale_after_hours":null,"max_attempts":1,"backoff_seconds":0,"max_backoff_seconds":0,"manual_fallback":"confirm the permitted AESAN/MAPA artifact or query route"}, + {"source_id":"fr.locations","cadence":"unknown","interval_hours":null,"stale_after_hours":null,"max_attempts":1,"backoff_seconds":0,"max_backoff_seconds":0,"manual_fallback":"obtain an authorized DGAL artifact capture"}, + {"source_id":"it.1069-2009","cadence":"daily","interval_hours":24,"stale_after_hours":48,"max_attempts":3,"backoff_seconds":1,"max_backoff_seconds":30,"manual_fallback":"use a separately reviewed Ministry catalog capture"}, + {"source_id":"it.853-2004","cadence":"daily","interval_hours":24,"stale_after_hours":48,"max_attempts":3,"backoff_seconds":1,"max_backoff_seconds":30,"manual_fallback":"use the catalog-discovery capture route"}, + {"source_id":"mx.locations","cadence":"unknown","interval_hours":null,"stale_after_hours":null,"max_attempts":1,"backoff_seconds":0,"max_backoff_seconds":0,"manual_fallback":"resolve official directory access and terms before scheduling"}, + {"source_id":"nz.locations","cadence":"unknown","interval_hours":null,"stale_after_hours":null,"max_attempts":1,"backoff_seconds":0,"max_backoff_seconds":0,"manual_fallback":"use an authorized MPI register export"}, + {"source_id":"uk.locations","cadence":"monthly","interval_hours":720,"stale_after_hours":960,"max_attempts":3,"backoff_seconds":1,"max_backoff_seconds":30,"manual_fallback":"use the source-specific national operator capture"}, + {"source_id":"us.aphis","cadence":"unknown","interval_hours":null,"stale_after_hours":null,"max_attempts":1,"backoff_seconds":0,"max_backoff_seconds":0,"manual_fallback":"verify the APHIS export workflow"}, + {"source_id":"us.fsis","cadence":"unknown","interval_hours":null,"stale_after_hours":null,"max_attempts":1,"backoff_seconds":0,"max_backoff_seconds":0,"manual_fallback":"obtain authorized FSIS export access"}, + {"source_id":"us.inspections","cadence":"unknown","interval_hours":null,"stale_after_hours":null,"max_attempts":1,"backoff_seconds":0,"max_backoff_seconds":0,"manual_fallback":"verify a current inspection export"} + ] +} From 69e6a427df0ee5b0d6d574df70b244d4d3301bee Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Tue, 15 Sep 2026 21:48:55 -0700 Subject: [PATCH 126/311] Advance mature country private validation --- docs/review-packet-belgium.md | 11 ++ docs/review-packet-denmark.md | 11 ++ docs/review-packet-germany.md | 11 ++ docs/review-packet-italy.md | 11 ++ docs/review-packet-united-kingdom.md | 11 ++ docs/source-status.json | 10 +- docs/source-status.md | 10 +- pipeline/common/delta.py | 19 +++ pipeline/common/orchestrator.py | 21 +++- pipeline/common/review_packet.py | 108 ++++++++++++++++++ pipeline/common/test_review_packet.py | 34 ++++++ pipeline/sources/belgium/adapter.py | 8 +- pipeline/sources/belgium/refresh.py | 11 +- pipeline/sources/denmark/pipeline.py | 8 ++ pipeline/sources/germany/adapter.py | 5 +- pipeline/sources/germany/refresh.py | 16 ++- pipeline/sources/italy/it_853_adapter.py | 10 +- pipeline/sources/italy/refresh.py | 16 ++- pipeline/sources/italy/test_it_853_adapter.py | 8 +- pipeline/sources/uk/fsa_approved/adapter.py | 10 +- pipeline/sources/uk/fsa_approved/refresh.py | 11 +- pipeline/sources/uk/fss_approved/refresh.py | 11 +- 22 files changed, 335 insertions(+), 36 deletions(-) create mode 100644 docs/review-packet-belgium.md create mode 100644 docs/review-packet-denmark.md create mode 100644 docs/review-packet-germany.md create mode 100644 docs/review-packet-italy.md create mode 100644 docs/review-packet-united-kingdom.md create mode 100644 pipeline/common/review_packet.py create mode 100644 pipeline/common/test_review_packet.py diff --git a/docs/review-packet-belgium.md b/docs/review-packet-belgium.md new file mode 100644 index 0000000..52f40cf --- /dev/null +++ b/docs/review-packet-belgium.md @@ -0,0 +1,11 @@ +# Belgium private review packet + +As of 2026-09-15, Belgium has a private two-artifact FASFC operator/codebook lifecycle with synthetic schema fixtures. No real operator rows are retained in Git and publication is blocked. + +- Terms/licensing: CC BY 4.0 is indicated by data.gov.be; FASFC attribution, last-update labeling, non-misleading use, and project publication review remain required. +- Privacy: address, postal, enterprise, and coordinate values stay in restricted source evidence; normalized rows suppress them and geocoding is disabled. +- Completeness: the operator feed represents current FASFC registrations/approvals/authorizations, not every animal-agriculture facility or every slaughterhouse. +- Classification: the LAP/PAP codebook join is exact; slaughter, cutting, processing, storage, animal-by-products, and export remain distinct source categories. Ambiguous/repeated codes quarantine. +- Coverage/lifecycle: operator and activity-code artifacts must be captured together; the live operator header was not available in this environment. Missing rows are `not-observed`, never closure. + +Evidence: `pipeline/sources/belgium/`, `docs/country-recon-be.md`, and `pipeline/tests/e2e/test_germany_belgium_candidate_import.py`. diff --git a/docs/review-packet-denmark.md b/docs/review-packet-denmark.md new file mode 100644 index 0000000..a7335eb --- /dev/null +++ b/docs/review-packet-denmark.md @@ -0,0 +1,11 @@ +# Denmark private review packet + +As of 2026-09-15, `dk.smiley` has a private, deterministic staging path. It is not a release approval or a completeness claim. + +- Terms/licensing: the Find Smiley data page indicates attribution and currentness conditions; a named project release decision remains open. +- Privacy: addresses and source coordinates require residential/private-location screening. Geocoding is an enrichment and remains separately review-gated. +- Completeness: coverage is limited to records available in Find Smiley; the publisher supplies no dataset effective date and the result is not a census. +- Classification: source category and source key mappings must remain explicit; unknown or ambiguous values quarantine. +- Coverage/lifecycle: source disappearance is `not-observed`, never closure. Candidate import and guarded API checks must remain disposable/test-only. + +Evidence: `pipeline/sources/denmark/`, `pipeline/contracts/source_health.py`, and `docs/countries/denmark/denmark-data-flow.md`. diff --git a/docs/review-packet-germany.md b/docs/review-packet-germany.md new file mode 100644 index 0000000..e6f8d05 --- /dev/null +++ b/docs/review-packet-germany.md @@ -0,0 +1,11 @@ +# Germany private review packet + +As of 2026-09-15, Germany uses the typed BLtU adapter and assisted/private refresh. No release, geocode, or public API exposure is allowed. + +- Terms/licensing: BVL documents public access/export, but dataset-specific reuse and redistribution terms require named human confirmation. +- Privacy: facility addresses may overlap residences or identify people; addresses and coordinates stay private and geocoding is disabled. +- Completeness: BLtU covers the approved 853/2004 list, not all animal-agriculture facilities. Export effective date and currentness are run-specific. +- Classification: only pinned SH/CP mappings are accepted; unmapped activities, missing identity, duplicate approval IDs, and physical schema drift quarantine. +- Coverage/lifecycle: the portal export URL is session/request-specific and must be recorded per run. Missing rows are `not-observed`, never closure. + +Evidence: `pipeline/sources/germany/`, `pipeline/germany/bltu_adapter.py`, and `docs/germany-source-assessment.md`. diff --git a/docs/review-packet-italy.md b/docs/review-packet-italy.md new file mode 100644 index 0000000..f0ed40b --- /dev/null +++ b/docs/review-packet-italy.md @@ -0,0 +1,11 @@ +# Italy private review packet + +As of 2026-09-15, `it.853-2004` has catalog-linked private acquisition and candidate lifecycle support. `it.1069-2009` remains a separate, unimplemented candidate. Publication is blocked. + +- Terms/licensing: the Ministry catalogue indicates Italian Open Data Licence v2.0; attribution and project redistribution review remain open. +- Privacy: addresses, tax identifiers, and precise source coordinates remain in restricted evidence; normalized/API-shaped rows suppress them pending review. +- Completeness: the 853/2004 CSV is only one Ministry scope; by-products, omitted fields, and row-level effective dates are not silently included or inferred. +- Classification: recognition number plus activity code is provisional source identity; repeated pairs quarantine. Source establishment/activity categories and codes are emitted as coverage diagnostics without collapsing them. +- Coverage/lifecycle: invalid dates, missing geography, unknown status, and coordinate precision remain explicit. Source disappearance is not closure; candidate import/API checks are private and test-only. + +Evidence: `pipeline/sources/italy/`, `docs/country-recon-it.md`, and `pipeline/tests/e2e/test_italy_candidate_import.py`. diff --git a/docs/review-packet-united-kingdom.md b/docs/review-packet-united-kingdom.md new file mode 100644 index 0000000..5bf891c --- /dev/null +++ b/docs/review-packet-united-kingdom.md @@ -0,0 +1,11 @@ +# United Kingdom private review packet + +As of 2026-09-15, UK coverage is split into separate FSA England/Wales and FSS Scotland adapters. Both are private/test-only and publication remains blocked. + +- Terms/licensing: OGL v3 is indicated by the national catalogues, but attribution, national-scope terms, and project redistribution review remain separate gates. +- Privacy: FSA `AddressWithheld` rows stay suppressed; all address and coordinate precision decisions require review. FSS remarks and address-risk rows quarantine. +- Completeness: England/Wales, Northern Ireland, and Scotland are separate source scopes; disappearance is not closure and no combined UK denominator is asserted. +- Classification: source-native activities, status, authority/nation, duplicates, remarks, and privacy signals are retained or quarantined rather than silently repaired. +- Coverage/lifecycle: the inspected monthly FSA schema and FSS live header are contracts for bounded refreshes, not proof of current completeness or release eligibility. + +Evidence: `pipeline/sources/uk/`, `docs/country-recon-uk.md`, and `docs/countries/uk/fss-approved-establishments-source-assessment.md`. diff --git a/docs/source-status.json b/docs/source-status.json index 4e4afbb..bfd79a0 100644 --- a/docs/source-status.json +++ b/docs/source-status.json @@ -8,15 +8,15 @@ "publication_eligibility": ["not_assessed", "blocked", "eligible_pending_release_approval"] }, "sources": [ - {"source_id":"be.locations","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-be.md","pipeline/sources/belgium/adapter.py","pipeline/sources/belgium/refresh.py","pipeline/sources/belgium/fixtures/synthetic_operators.csv","pipeline/source_registry.json"],"next_action":"Use the assisted two-file refresh with an authorized operator capture and official activity-code CSV; compare live schema to the synthetic contract. Keep category/privacy/terms gates and publication blocked."}, + {"source_id":"be.locations","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-be.md","pipeline/sources/belgium/adapter.py","pipeline/sources/belgium/refresh.py","pipeline/sources/belgium/fixtures/synthetic_operators.csv","pipeline/sources/belgium/test_adapter.py","pipeline/common/review_packet.py","docs/review-packet-belgium.md","pipeline/source_registry.json"],"next_action":"Use the assisted two-file refresh with an authorized operator capture and official activity-code CSV; compare live schema and physical row lengths to the synthetic contract. Keep category/privacy/terms gates and publication blocked."}, {"source_id":"fr.locations","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-fr.md","pipeline/source_registry.json"],"next_action":"Acquire and snapshot an official DGAL Section I/II artifact with terms, schema, and privacy review."}, - {"source_id":"it.853-2004","metadata":"verified","acquisition":"artifact_private_only","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-it.md","pipeline/source_registry.json","pipeline/sources/italy/acquire.py","pipeline/sources/italy/it_853_adapter.py","pipeline/sources/italy/README.md","pipeline/tests/e2e/test_italy_candidate_import.py"],"next_action":"Keep catalog acquisition, lifecycle, candidate import, and guarded API evidence private; resolve repeated activity identity, coordinate/address privacy, coverage, and project approval before release review."}, + {"source_id":"it.853-2004","metadata":"verified","acquisition":"artifact_private_only","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-it.md","pipeline/source_registry.json","pipeline/sources/italy/acquire.py","pipeline/sources/italy/it_853_adapter.py","pipeline/sources/italy/README.md","pipeline/common/review_packet.py","docs/review-packet-italy.md","pipeline/tests/e2e/test_italy_candidate_import.py"],"next_action":"Keep catalog acquisition, lifecycle, candidate import, and guarded API evidence private; use source-category/activity diagnostics to resolve repeated identity, coordinate/address privacy, coverage, and project approval before release review."}, {"source_id":"it.1069-2009","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-it.md","pipeline/source_registry.json","pipeline/sources/italy/README.md"],"next_action":"Keep the by-products dataset separate; assess scope, schema, terms, identity links, privacy, and a dedicated adapter before any acquisition or integration."}, {"source_id":"mx.locations","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-mx.md","pipeline/source_registry.json"],"next_action":"Resolve DENUE token/terms and SENASICA current directory/schema before bounded acquisition."}, {"source_id":"nz.locations","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-nz.md","pipeline/source_registry.json"],"next_action":"Confirm MPI permitted acquisition route, list-specific terms, and schema before private staging."}, - {"source_id":"uk.locations","metadata":"partial","acquisition":"artifact_private_only","runtime_health":"unknown","publication_eligibility":"blocked","evidence":["docs/country-recon-uk.md","docs/countries/uk/fss-approved-establishments-source-assessment.md","pipeline/sources/uk/fsa_approved/README.md","pipeline/sources/uk/fsa_approved/handoff.py","pipeline/sources/uk/fsa_approved/refresh.py","docs/architecture/disposable-candidate-import.md","pipeline/scripts/maintenance/import-candidate.py","pipeline/source_registry.json"],"next_action":"Repeatable source-local refresh dry-run passed on the retained snapshot (5,342 input, 4,300 normalized, 1,042 quarantined, expected schema fingerprint, no drift alarms); use refresh.py dry-run/handoff only in restricted staging. This does not establish production/runtime health or publication eligibility. Complete source-rights, privacy/coordinate, duplicate, coverage, and release review while keeping NI and Scotland separate."}, - {"source_id":"dk.smiley","metadata":"verified","acquisition":"verified","runtime_health":"unknown","publication_eligibility":"blocked","evidence":["pipeline/sources/denmark/README.md","pipeline/contracts/README.md","pipeline/source_registry.json"],"next_action":"Keep the full privately staged artifact and candidate import validation restricted; run a bounded guarded API check with explicit test-only labeling, capture rerun exit output, and resolve coverage/effective-date/category semantics before any release review. This is not a production-health or publication claim."}, - {"source_id":"de.locations","metadata":"partial","acquisition":"artifact_private_only","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/germany-source-assessment.md","pipeline/sources/germany/adapter.py","pipeline/sources/germany/refresh.py","pipeline/source_registry.json"],"next_action":"Use the stable BVL landing to select an export, then run the assisted/private refresh. The export URL, reuse terms, privacy, and project approval remain unresolved; no release or public API exposure is allowed."}, + {"source_id":"uk.locations","metadata":"partial","acquisition":"artifact_private_only","runtime_health":"unknown","publication_eligibility":"blocked","evidence":["docs/country-recon-uk.md","docs/countries/uk/fss-approved-establishments-source-assessment.md","pipeline/sources/uk/fsa_approved/README.md","pipeline/sources/uk/fsa_approved/handoff.py","pipeline/sources/uk/fsa_approved/refresh.py","pipeline/sources/uk/fss_approved/refresh.py","pipeline/common/review_packet.py","docs/review-packet-united-kingdom.md","docs/architecture/disposable-candidate-import.md","pipeline/scripts/maintenance/import-candidate.py","pipeline/source_registry.json"],"next_action":"Repeatable source-local refresh dry-run passed on the retained snapshot (5,342 input, 4,300 normalized, 1,042 quarantined, expected schema fingerprint, no drift alarms); use refresh.py dry-run/handoff only in restricted staging. This does not establish production/runtime health or publication eligibility. Complete source-rights, privacy/coordinate, duplicate, coverage, and release review while keeping NI and Scotland separate."}, + {"source_id":"dk.smiley","metadata":"verified","acquisition":"verified","runtime_health":"unknown","publication_eligibility":"blocked","evidence":["pipeline/sources/denmark/README.md","pipeline/contracts/README.md","pipeline/common/review_packet.py","docs/review-packet-denmark.md","pipeline/source_registry.json"],"next_action":"Keep the full privately staged artifact and candidate import validation restricted; run a bounded guarded API check with explicit test-only labeling, capture rerun exit output, and resolve coverage/effective-date/category semantics before any release review. This is not a production-health or publication claim."}, + {"source_id":"de.locations","metadata":"partial","acquisition":"artifact_private_only","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/germany-source-assessment.md","pipeline/sources/germany/adapter.py","pipeline/sources/germany/refresh.py","pipeline/common/review_packet.py","docs/review-packet-germany.md","pipeline/source_registry.json"],"next_action":"Use the stable BVL landing to select an export, then run the assisted/private refresh with duplicate-approval and coordinate-precision checks. The export URL, reuse terms, privacy, and project approval remain unresolved; no release or public API exposure is allowed."}, {"source_id":"ca.locations","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ca.md","pipeline/source_registry.json"],"next_action":"Keep federal/export and Ontario candidates separate; verify an authoritative permitted artifact, terms, schema, coverage, and privacy before acquisition."}, {"source_id":"es.locations","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-es.md","pipeline/source_registry.json"],"next_action":"Resolve a permitted current AESAN/MAPA artifact or query route, then verify export/schema, rights, effective-date semantics, sector coverage, privacy, and legacy/source boundaries before acquisition."}, {"source_id":"us.fsis","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-us.md","pipeline/source_registry.json"],"next_action":"Obtain authorized access to a current FSIS MPI export after 403 responses; then record URL, retrieval/effective dates, content type, byte size, SHA-256, terms, and schema before adapter or publication review."}, diff --git a/docs/source-status.md b/docs/source-status.md index c0f00e5..e20e916 100644 --- a/docs/source-status.md +++ b/docs/source-status.md @@ -15,15 +15,15 @@ No last-success timestamp is invented. Private artifacts are not proof of a publ | Source ID | Metadata | Acquisition | Runtime health | Publication | Evidence / next action | |---|---|---|---|---|---| -| `be.locations` | verified | blocked | not_run | blocked | Shared private adapter and assisted two-file refresh are implemented with synthetic schema coverage; obtain an authorized operator capture and compare live schema before any real run | +| `be.locations` | verified | blocked | not_run | blocked | Shared private adapter, row-length/quarantine checks, and assisted two-file refresh are covered by synthetic fixtures; obtain an authorized operator capture and compare live schema before any real run; see `docs/review-packet-belgium.md` | | `fr.locations` | verified | blocked | not_run | blocked | DGAL/Alim’confiance/SIRENE/HVE/Agence Bio/Géorisques reconnaissance; acquire permitted official artifact and review terms/privacy | -| `it.853-2004` | verified | artifact_private_only | not_run | blocked | Catalog acquisition, shared lifecycle, adapter, private candidate import, and guarded API checks remain review-gated; repeated activity identity, coordinate/address privacy, coverage, and project approval remain open | +| `it.853-2004` | verified | artifact_private_only | not_run | blocked | Catalog acquisition, shared lifecycle, source-category/activity diagnostics, private candidate import, and guarded API checks remain review-gated; repeated activity identity, coordinate/address privacy, coverage, and project approval remain open; see `docs/review-packet-italy.md` | | `it.1069-2009` | verified | not_run | not_run | blocked | Separate by-products catalog candidate; no adapter or integration decision; assess scope, schema, terms, identity links, and privacy | | `mx.locations` | verified | blocked | not_run | blocked | DENUE/SENASICA/DGSIAP reconnaissance; resolve token, directory, terms, and schema | | `nz.locations` | verified | blocked | not_run | blocked | MPI/Stats NZ reconnaissance; resolve 403/access and aggregate-vs-facility boundaries | -| `uk.locations` | partial | artifact_private_only | unknown | blocked | FSA and FSS private V2 lifecycle paths and synthetic handoff tests pass; no real UK candidate has been imported or previewed; privacy/coordinate, source-rights, duplicate, coverage, and release review remain open; NI/Scotland stay separate | -| `dk.smiley` | verified | verified | unknown | blocked | Shared private lifecycle and registered adapter are validated on synthetic/retained evidence; coverage/effective-date uncertainty and terms/privacy/release review remain open | -| `de.locations` | partial | artifact_private_only | not_run | blocked | Stable BVL `/bltu` landing and portal route are verified; typed private adapter and assisted export refresh are implemented, while export-specific terms/privacy/release review remain unresolved | +| `uk.locations` | partial | artifact_private_only | unknown | blocked | FSA and FSS private V2 lifecycle paths, nation-qualified identity, coordinate precision states, and synthetic handoff tests pass; no real UK candidate has been imported or previewed; privacy/coordinate, source-rights, duplicate, coverage, and release review remain open; NI/Scotland stay separate; see `docs/review-packet-united-kingdom.md` | +| `dk.smiley` | verified | verified | unknown | blocked | Shared private lifecycle and registered adapter are validated on synthetic/retained evidence with explicit not-observed semantics; coverage/effective-date uncertainty and terms/privacy/release review remain open; see `docs/review-packet-denmark.md` | +| `de.locations` | partial | artifact_private_only | not_run | blocked | Stable BVL `/bltu` landing and portal route are verified; typed private adapter and assisted export refresh now include duplicate-approval and coordinate-precision checks, while export-specific terms/privacy/release review remain unresolved; see `docs/review-packet-germany.md` | | `ca.locations` | verified | not_run | not_run | blocked | Federal/export and Ontario candidates are documented separately; verify permitted artifact, terms, schema, coverage, and privacy before acquisition | | `es.locations` | partial | blocked | not_run | blocked | AESAN RGSEAA and MAPA sector routes are documented, but direct acquisition was refused; rights, export/schema, effective dates, sector coverage, privacy, and legacy/source boundaries remain unresolved | | `us.fsis` | verified | blocked | not_run | blocked | Official FSIS MPI route is documented, but current CSV access returned 403; obtain authorized export access and record provenance/schema before adapter or publication review | diff --git a/pipeline/common/delta.py b/pipeline/common/delta.py index fc88015..9e13a37 100644 --- a/pipeline/common/delta.py +++ b/pipeline/common/delta.py @@ -30,6 +30,25 @@ def _manifest(run_dir: Path) -> dict[str, Any]: return json.loads(path.read_text(encoding="utf-8")) +def compare_normalized_paths(previous_path: str | Path | None, current_path: str | Path) -> dict[str, Any]: + """Compare normalized JSONL by stable source identity without row payloads.""" + if previous_path is None: + return {"status": "not-run", "counts": {"added": 0, "changed": 0, "not_observed": 0, "suppressed": 0}, "disappearance_semantics": "not-observed; never inferred as closure"} + previous = Path(previous_path) + current = Path(current_path) + if not previous.is_file() or not current.is_file(): + return {"status": "failed", "error": "normalized state is missing", "counts": {"added": 0, "changed": 0, "not_observed": 0, "suppressed": 0}, "disappearance_semantics": "not-observed; never inferred as closure"} + try: + old = {record_key(row): row for row in _jsonl(previous)} + new = {record_key(row): row for row in _jsonl(current)} + added = sum(key not in old for key in new) + changed = sum(key in old and _fingerprint(old[key]) != _fingerprint(row) for key, row in new.items()) + not_observed = sum(key not in new for key in old) + except Exception as exc: + return {"status": "failed", "error_type": type(exc).__name__, "error": str(exc), "counts": {"added": 0, "changed": 0, "not_observed": 0, "suppressed": 0}, "disappearance_semantics": "not-observed; never inferred as closure"} + return {"status": "delta-ready", "counts": {"added": added, "changed": changed, "not_observed": not_observed, "suppressed": 0}, "disappearance_semantics": "not-observed; never inferred as closure"} + + def _fingerprint(row: dict[str, Any]) -> str: comparable = {key: value for key, value in row.items() if key not in {"provenance", "source_values", "source_columns"}} return hashlib.sha256(json.dumps(comparable, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode()).hexdigest() diff --git a/pipeline/common/orchestrator.py b/pipeline/common/orchestrator.py index ac26c42..af438ed 100644 --- a/pipeline/common/orchestrator.py +++ b/pipeline/common/orchestrator.py @@ -12,6 +12,7 @@ from pipeline.contracts.adapter_contract import SourceAdapter, SourceArtifact, source_artifact_from_mapping from pipeline.contracts.private_run import write_private_run_report from pipeline.contracts.source_health import build_health_snapshot, write_health_snapshot +from pipeline.common.review_packet import write_review_packet from .identity import record_key from .source_operations import classify_failure, finalize_run_operations @@ -53,7 +54,9 @@ def register_input(raw: bytes, staging_dir: str | Path, config: dict[str, Any]) def run_registered_input(raw_path: str | Path, runs_dir: str | Path, config: dict[str, Any], adapter_runner: Callable[..., dict[str, Any]], suppressed_ids: set[str | tuple[str, str, str]] | None = None, - prior_eligible_release: dict[str, Any] | None = None) -> dict[str, Any]: + prior_eligible_release: dict[str, Any] | None = None, + previous_normalized_path: str | Path | None = None, + review_blockers: dict[str, list[str]] | None = None) -> dict[str, Any]: """Run an adapter to a human-gated candidate, preserving prior release on failure.""" raw = Path(raw_path) runs = Path(runs_dir) @@ -114,6 +117,8 @@ def run_registered_input(raw_path: str | Path, runs_dir: str | Path, config: dic "attempts": getattr(exc, "attempts", []), "prior_eligible_release": prior_eligible_release, "run_dir": str(run_dir)} + if status.get("status") != "failed": + write_review_packet(run_dir, previous_normalized_path=previous_normalized_path, blockers=review_blockers) status["run_dir"] = str(run_dir) status["run_id"] = config.get("run_id") or run_dir.name # The operations ledger is derived from the private run and is append-only. @@ -141,7 +146,9 @@ def run_registered_input(raw_path: str | Path, runs_dir: str | Path, config: dic def run_registered_typed_input(raw_path: str | Path, runs_dir: str | Path, config: dict[str, Any], adapter: SourceAdapter, suppressed_ids: set[str | tuple[str, str, str]] | None = None, - prior_eligible_release: dict[str, Any] | None = None) -> dict[str, Any]: + prior_eligible_release: dict[str, Any] | None = None, + previous_normalized_path: str | Path | None = None, + review_blockers: dict[str, list[str]] | None = None) -> dict[str, Any]: """Run a typed adapter from registered acquisition metadata. This is the compatibility seam for adapters whose ``run`` method accepts @@ -156,14 +163,18 @@ def invoke(raw: str | Path, run_dir: str | Path, _config: dict[str, Any]) -> dic return run_registered_input(raw_path, runs_dir, config, invoke, suppressed_ids=suppressed_ids, - prior_eligible_release=prior_eligible_release) + prior_eligible_release=prior_eligible_release, + previous_normalized_path=previous_normalized_path, + review_blockers=review_blockers) def run_private_lifecycle(raw_path: str | Path, runs_dir: str | Path, artifact: SourceArtifact, adapter: SourceAdapter, *, suppressed_ids: set[str | tuple[str, str, str]] | None = None, prior_eligible_release: dict[str, Any] | None = None, - health_as_of_utc: str | None = None) -> dict[str, Any]: + health_as_of_utc: str | None = None, + previous_normalized_path: str | Path | None = None, + review_blockers: dict[str, list[str]] | None = None) -> dict[str, Any]: """Run the canonical typed lifecycle from a preserved ``SourceArtifact``. This is the source-local integration seam for new countries. Acquisition @@ -191,4 +202,6 @@ def run_private_lifecycle(raw_path: str | Path, runs_dir: str | Path, raw_path, runs_dir, config, adapter, suppressed_ids=suppressed_ids, prior_eligible_release=prior_eligible_release, + previous_normalized_path=previous_normalized_path, + review_blockers=review_blockers, ) diff --git a/pipeline/common/review_packet.py b/pipeline/common/review_packet.py new file mode 100644 index 0000000..92faf55 --- /dev/null +++ b/pipeline/common/review_packet.py @@ -0,0 +1,108 @@ +"""Deterministic, row-free private review packets for source runs. + +The packet is an operator aid, not a release approval. It contains only +aggregate counts, schema/provenance facts, and explicit gate state; source +values and addresses remain in the restricted run artifacts. +""" +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +from typing import Any + +from .delta import compare_normalized_paths +from pipeline.contracts.source_lifecycle import atomic_json + + +REVIEW_PACKET_VERSION = "private-review-packet-v1" + + +def _read_json(path: Path) -> dict[str, Any]: + if not path.is_file(): + raise ValueError(f"missing review input: {path.name}") + value = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(value, dict): + raise ValueError(f"review input must be an object: {path.name}") + return value + + +def _digest(path: Path) -> str | None: + if not path.is_file(): + return None + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def _counts(manifest: dict[str, Any], qa: dict[str, Any]) -> dict[str, Any]: + values = {key: manifest.get(key) for key in ("input_rows", "normalized_rows", "quarantined_rows")} + qa_values = {key: qa.get(key) for key in values} + valid = all(isinstance(value, int) and value >= 0 for value in values.values()) + return { + **values, + "reconciles": valid and values["input_rows"] == values["normalized_rows"] + values["quarantined_rows"], + "qa_matches_manifest": values == qa_values, + } + + +def build_review_packet( + run_dir: str | Path, + *, + previous_normalized_path: str | Path | None = None, + blockers: dict[str, list[str]] | None = None, +) -> dict[str, Any]: + root = Path(run_dir) + manifest = _read_json(root / "manifest.json") + qa = _read_json(root / "qa.json") + status = _read_json(root / "run-status.json") + counts = _counts(manifest, qa) + normalized = root / "normalized" / "records.jsonl" + packet: dict[str, Any] = { + "schema_version": REVIEW_PACKET_VERSION, + "source_id": manifest.get("source_id"), + "run_dir_digest": _digest(root / "manifest.json"), + "provenance": { + key: manifest.get(key) + for key in ("source_url", "retrieved_at_utc", "publication_date", "effective_date", "sha256", "byte_size", "code_version", "config_version") + if manifest.get(key) is not None + }, + "schema": { + "adapter_version": manifest.get("adapter_version"), + "schema_version": manifest.get("schema_version"), + "schema_fingerprint": manifest.get("schema_fingerprint"), + "schema_status": manifest.get("schema_status", "not-reported"), + }, + "counts": counts, + "quarantine": { + "rows": manifest.get("quarantined_rows"), + "reasons": manifest.get("anomaly_counts", {}), + }, + "release_diff": compare_normalized_paths(previous_normalized_path, normalized) if previous_normalized_path else { + "status": "not-run", + "counts": {"added": None, "changed": None, "not_observed": None, "suppressed": 0}, + "disappearance_semantics": "not-observed; never inferred as closure", + }, + "gates": { + "release_state": manifest.get("release_state"), + "publication_state": manifest.get("publication_state"), + "release_promoted": status.get("release_promoted"), + "public_surfaces": status.get("public_surfaces", {surface: False for surface in ("api", "map", "export", "cache", "history")}), + "geocoding": manifest.get("geocoding", "disabled"), + }, + "blockers": blockers or {}, + } + if not counts["reconciles"] or not counts["qa_matches_manifest"]: + packet["blockers"].setdefault("validation", []).append("manifest and QA row counts must reconcile") + if packet["gates"]["release_state"] != "not-created" or packet["gates"]["release_promoted"] is not False: + packet["blockers"].setdefault("release", []).append("private review requires release_state=not-created and release_promoted=false") + return packet + + +def write_review_packet( + run_dir: str | Path, + *, + previous_normalized_path: str | Path | None = None, + blockers: dict[str, list[str]] | None = None, +) -> dict[str, Any]: + packet = build_review_packet(run_dir, previous_normalized_path=previous_normalized_path, blockers=blockers) + atomic_json(Path(run_dir) / "review-packet.json", packet) + return packet diff --git a/pipeline/common/test_review_packet.py b/pipeline/common/test_review_packet.py new file mode 100644 index 0000000..f87843a --- /dev/null +++ b/pipeline/common/test_review_packet.py @@ -0,0 +1,34 @@ +import hashlib +import json +import tempfile +import unittest +from pathlib import Path + +from pipeline.common.orchestrator import run_private_lifecycle +from pipeline.contracts.adapter_contract import SourceArtifact +from pipeline.sources.denmark.adapter import DenmarkSmileyAdapter + + +class ReviewPacketTests(unittest.TestCase): + def test_packet_is_row_free_and_delta_is_explicitly_not_observed(self): + raw = b"oneSynthetic" + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + source = root / "source.xml" + source.write_bytes(raw) + artifact = SourceArtifact("https://example.invalid/smiley.xml", "2026-09-15T00:00:00Z", hashlib.sha256(raw).hexdigest(), len(raw), code_version="test", config_version="test") + first = run_private_lifecycle(source, root / "one", artifact, DenmarkSmileyAdapter()) + second = run_private_lifecycle(source, root / "two", artifact, DenmarkSmileyAdapter(), previous_normalized_path=Path(first["run_dir"]) / "normalized" / "records.jsonl") + packet = json.loads((Path(second["run_dir"]) / "review-packet.json").read_text(encoding="utf-8")) + self.assertEqual(packet["schema_version"], "private-review-packet-v1") + self.assertTrue(packet["counts"]["reconciles"]) + self.assertTrue(packet["counts"]["qa_matches_manifest"]) + self.assertEqual(packet["release_diff"]["status"], "delta-ready") + self.assertEqual(packet["release_diff"]["counts"]["not_observed"], 0) + self.assertEqual(packet["gates"]["release_state"], "not-created") + self.assertFalse(packet["gates"]["release_promoted"]) + self.assertNotIn("source_values", json.dumps(packet)) + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/sources/belgium/adapter.py b/pipeline/sources/belgium/adapter.py index 9709565..224670e 100644 --- a/pipeline/sources/belgium/adapter.py +++ b/pipeline/sources/belgium/adapter.py @@ -180,8 +180,10 @@ def parse_bytes(self, content: bytes) -> dict[str, Any]: anomalies: Counter[str] = Counter() seen: Counter[tuple[str | None, str | None]] = Counter() prepared: list[tuple[int, dict[str, str | None], tuple[str, ...]]] = [] + row_lengths: Counter[str] = Counter() for line, values in enumerate(rows, start=2): raw = _row_values(headers, values) + row_lengths[str(len(values))] += 1 codes = _split_codes(_clean(raw.get(fields["activity_code"]))) prepared.append((line, raw, codes)) for code in codes: @@ -190,7 +192,7 @@ def parse_bytes(self, content: bytes) -> dict[str, Any]: establishment_id = _clean(raw.get(fields["establishment_id"])) name = _clean(raw.get(fields["name"])) reasons: list[str] = [] - if len(raw) != len(headers) or any(value is None for value in raw.values()): + if len(values) != len(headers) or any(value is None for value in raw.values()): reasons.append("malformed_row") if not establishment_id: reasons.append("missing_establishment_id") @@ -249,7 +251,7 @@ def parse_bytes(self, content: bytes) -> dict[str, Any]: quarantined.append({"reasons": unique, "record": record}) else: accepted.append(record) - return {"accepted": accepted, "quarantined": quarantined, "input_rows": len(rows), "source_sha256": hashlib.sha256(content).hexdigest(), "operator_schema_fingerprint": _fingerprint(headers), "operator_column_count": len(headers), "operator_encoding": encoding, "operator_delimiter": delimiter, "codebook": codebook_meta, "coverage_counts": dict(Counter(category for item in accepted for category in item["normalized"]["source_activity_categories"])), "anomaly_counts": dict(sorted(anomalies.items()))} + return {"accepted": accepted, "quarantined": quarantined, "input_rows": len(rows), "source_sha256": hashlib.sha256(content).hexdigest(), "operator_schema_fingerprint": _fingerprint(headers), "operator_column_count": len(headers), "operator_encoding": encoding, "operator_delimiter": delimiter, "row_length_counts": dict(sorted(row_lengths.items())), "codebook": codebook_meta, "coverage_counts": dict(Counter(category for item in accepted for category in item["normalized"]["source_activity_categories"])), "anomaly_counts": dict(sorted(anomalies.items()))} def run(self, raw_path: str | Path, run_dir: str | Path, artifact: SourceArtifact) -> dict[str, Any]: raw = Path(raw_path).read_bytes() @@ -263,7 +265,7 @@ def run(self, raw_path: str | Path, run_dir: str | Path, artifact: SourceArtifac _, normalized_sha, _ = atomic_jsonl(root / "normalized" / "records.jsonl", result["accepted"]) atomic_jsonl(root / "quarantined" / "records.jsonl", result["quarantined"]) manifest = private_manifest(source_id=self.source_id, adapter_version=self.adapter_version, schema_version=self.schema_version, artifact=artifact, input_rows=result["input_rows"], normalized_rows=len(result["accepted"]), quarantined_rows=len(result["quarantined"]), normalized_sha256=normalized_sha, parsed_sha256=parsed_sha, anomaly_counts=result["anomaly_counts"]) - manifest.update({"country_code": "BE", "coverage": CONFIG["coverage"], "geocoding": "disabled", "operator_schema_fingerprint": result["operator_schema_fingerprint"], "operator_column_count": result["operator_column_count"], "operator_encoding": result["operator_encoding"], "operator_delimiter": result["operator_delimiter"], "coverage_counts": result["coverage_counts"], "activity_codebook": result["codebook"], "codebook_source_url": self.activity_artifact.source_url if self.activity_artifact else "unrecorded-companion-artifact", "codebook_retrieved_at_utc": self.activity_artifact.retrieved_at_utc if self.activity_artifact else None, "codebook_sha256": result["codebook"]["sha256"], "codebook_byte_size": result["codebook"]["byte_size"]}) + manifest.update({"country_code": "BE", "coverage": CONFIG["coverage"], "geocoding": "disabled", "operator_schema_fingerprint": result["operator_schema_fingerprint"], "operator_column_count": result["operator_column_count"], "operator_encoding": result["operator_encoding"], "operator_delimiter": result["operator_delimiter"], "row_length_counts": result["row_length_counts"], "coverage_counts": result["coverage_counts"], "activity_codebook": result["codebook"], "codebook_source_url": self.activity_artifact.source_url if self.activity_artifact else "unrecorded-companion-artifact", "codebook_retrieved_at_utc": self.activity_artifact.retrieved_at_utc if self.activity_artifact else None, "codebook_sha256": result["codebook"]["sha256"], "codebook_byte_size": result["codebook"]["byte_size"]}) atomic_json(root / "manifest.json", manifest) return manifest diff --git a/pipeline/sources/belgium/refresh.py b/pipeline/sources/belgium/refresh.py index f100ac3..ef9ad77 100644 --- a/pipeline/sources/belgium/refresh.py +++ b/pipeline/sources/belgium/refresh.py @@ -25,6 +25,15 @@ class RefreshError(ValueError): """A Belgium refresh cannot safely continue.""" +REVIEW_BLOCKERS = { + "terms": ["CC BY 4.0 attribution and non-misleading-use review remain human gates; the operator feed has no checked-in licence grant beyond the catalogue evidence."], + "privacy": ["Address, postal, enterprise, and coordinate fields require field-level privacy review; normalized rows intentionally suppress them and geocoding is disabled."], + "completeness": ["The feed covers current FASFC registrations/approvals/authorizations, not every animal-agriculture facility; live operator schema and currentness semantics still require capture review."], + "classification": ["LAP/PAP codebook joins are explicit, but slaughter, cutting, processing, storage, animal-by-products, and export remain separate source categories."], + "coverage": ["Operator CSV and activity-code CSV must be captured together; the live operator header was not available in this environment."], +} + + def _local_metadata(path: Path, *, source_id: str, source_url: str, retrieved_at: str, coverage: str) -> dict[str, Any]: raw = path.read_bytes() return {"acquisition_method": "assisted_local_capture", "source_id": source_id, "artifact": path.name, "artifact_path": str(path.resolve()), "requested_url": source_url, "final_url": source_url, "redirects": [], "response_headers": {}, "requested_at_utc": retrieved_at, "retrieved_at_utc": retrieved_at, "effective_date": "unknown", "publication_date": None, "sha256": hashlib.sha256(raw).hexdigest(), "byte_size": len(raw), "adapter_version": CONFIG["adapter_version"], "code_version": CONFIG["adapter_version"], "config_version": CONFIG["schema_version"], "coverage": coverage, "rights_caveat": CONFIG["terms"], "privacy_caveat": "private staging; address and coordinate review pending", "terms_review": "operator-assisted capture; source terms review remains a separate gate"} @@ -60,7 +69,7 @@ def refresh(*, run_dir: str | Path, operators_path: str | Path | None = None, ac operator_artifact = _artifact(operator_meta, default_url=CONFIG["operator_url"], default_coverage=CONFIG["coverage"]) code_artifact = _artifact(code_meta, default_url=CONFIG["activity_code_url"], default_coverage="FASFC LAP/PAP codebook; not a facility list") adapter = BelgiumOperatorsAdapter(code_path, code_artifact) - lifecycle = run_private_lifecycle(operator_path, root / "lifecycle", operator_artifact, adapter, health_as_of_utc=retrieved) + lifecycle = run_private_lifecycle(operator_path, root / "lifecycle", operator_artifact, adapter, health_as_of_utc=retrieved, previous_normalized_path=previous_normalized, review_blockers=REVIEW_BLOCKERS) manifest = lifecycle.get("manifest") or {} report = {"source_id": CONFIG["source_id"], "source_url": operator_artifact.source_url, "retrieved_at_utc": operator_artifact.retrieved_at_utc, "operator_sha256": operator_artifact.sha256, "activity_code_sha256": code_artifact.sha256, "input_rows": manifest.get("input_rows"), "normalized_rows": manifest.get("normalized_rows"), "quarantined_rows": manifest.get("quarantined_rows"), "operator_schema_fingerprint": manifest.get("operator_schema_fingerprint"), "activity_code_schema_fingerprint": (manifest.get("activity_codebook") or {}).get("schema_fingerprint"), "drift_alarms": [], "disappeared_not_observed_count": 0, "disappearance_semantics": "not-observed; never inferred as closure", "geocoding": "disabled", "release_state": "not-created", "publication_state": "private-candidate", "publication_eligibility": "blocked", "lifecycle_status": lifecycle.get("status"), "lifecycle_run_dir": lifecycle.get("run_dir"), "previous_normalized_supplied": previous_normalized is not None} atomic_json(root / "refresh.json", report) diff --git a/pipeline/sources/denmark/pipeline.py b/pipeline/sources/denmark/pipeline.py index 4138044..8134e7d 100644 --- a/pipeline/sources/denmark/pipeline.py +++ b/pipeline/sources/denmark/pipeline.py @@ -15,6 +15,7 @@ from pipeline.contracts.private_run import write_private_run_report from pipeline.contracts.source_health import build_health_snapshot, write_health_snapshot from pipeline.contracts.source_lifecycle import atomic_json, validate_private_manifest +from pipeline.common.review_packet import write_review_packet LOGGER = logging.getLogger("uec.denmark.pipeline") @@ -120,6 +121,13 @@ def _canonical_evidence(run_dir: Path, input_path: Path, metadata: dict, if retrieved: snapshot = build_health_snapshot(run_dir, as_of_utc=retrieved) write_health_snapshot(run_dir / "source-health.json", snapshot) + write_review_packet(run_dir, blockers={ + "terms": ["Find Smiley attribution/currentness conditions are recorded; named project release approval remains open."], + "privacy": ["Address and source-coordinate residential/private-location screening remains required; geocoding is separately review-gated."], + "completeness": ["Find Smiley coverage is not a census and has no supplied dataset effective date."], + "classification": ["Source category and stable-key mappings remain explicit; unknown or ambiguous values quarantine."], + "coverage": ["Source disappearance is not-observed, never closure; candidate import and API checks remain disposable/test-only."], + }) LOGGER.info("private evidence source=dk.smiley rows=%d findings=%d qa=%s", report["normalized_rows"], validation.get("finding_records", 0), run_dir / "qa.json") diff --git a/pipeline/sources/germany/adapter.py b/pipeline/sources/germany/adapter.py index b33a994..6b3de76 100644 --- a/pipeline/sources/germany/adapter.py +++ b/pipeline/sources/germany/adapter.py @@ -43,6 +43,7 @@ def parse_bytes(self, content: bytes) -> dict[str, Any]: rows = list(csv.reader(text.splitlines(), delimiter=";", strict=True)) headers = rows[0] if rows else [] matched = headers == list(EXPECTED_HEADERS) + id_counts = Counter(values[CURRENT_ID_INDEX].strip() for values in rows[1:] if len(values) > CURRENT_ID_INDEX and values[CURRENT_ID_INDEX].strip()) accepted: list[dict[str, Any]] = [] quarantined: list[dict[str, Any]] = [] anomalies: Counter[str] = Counter() @@ -58,6 +59,8 @@ def parse_bytes(self, content: bytes) -> dict[str, Any]: name = values[NAME_INDEX].strip() if len(values) > NAME_INDEX else "" if not current_id: reasons.append("missing_current_approval_id") + elif id_counts[current_id] > 1: + reasons.append("duplicate_current_approval_id") if not name: reasons.append("missing_establishment_name") activity_codes = [headers[index] for index in range(ACTIVITY_START, min(ACTIVITY_END, len(values))) if values[index].strip()] @@ -70,7 +73,7 @@ def parse_bytes(self, content: bytes) -> dict[str, Any]: reasons.append("unmapped_activity_code") for category in mapped: categories[category] += 1 - record = {"source_id": self.source_id, "source_row": line, "source_record_key": f"{current_id or 'unknown'}|{line}", "source_values": source, "normalized": {"establishment_id": current_id or None, "approval_number": current_id or None, "name": name or None, "trading_name": name or None, "country_code": "DE", "nation": "Germany", "state": values[STATE_INDEX].strip() if len(values) > STATE_INDEX else None, "city": values[CITY_INDEX].strip() if len(values) > CITY_INDEX else None, "address": None, "address_state": "source-present-pending-privacy-review" if len(values) > STREET_INDEX and values[STREET_INDEX].strip() else "unknown", "activity_codes": tuple(activity_codes), "activity_categories": mapped, "source_activity_categories": mapped, "classification_state": "mapped" if mapped and not any(code not in ACTIVITY_MAP for code in activity_codes) else "unresolved", "coordinates": None, "coordinate_state": "unknown", "privacy_gate": "pending-review", "coordinate_gate": "review_required", "publication_gate": "blocked"}} + record = {"source_id": self.source_id, "source_row": line, "source_record_key": f"{current_id or 'unknown'}|{line}", "source_values": source, "normalized": {"establishment_id": current_id or None, "approval_number": current_id or None, "name": name or None, "trading_name": name or None, "country_code": "DE", "nation": "Germany", "state": values[STATE_INDEX].strip() if len(values) > STATE_INDEX else None, "city": values[CITY_INDEX].strip() if len(values) > CITY_INDEX else None, "address": None, "address_state": "source-present-pending-privacy-review" if len(values) > STREET_INDEX and values[STREET_INDEX].strip() else "unknown", "activity_codes": tuple(activity_codes), "activity_categories": mapped, "source_activity_categories": mapped, "classification_state": "mapped" if mapped and not any(code not in ACTIVITY_MAP for code in activity_codes) else "unresolved", "coordinates": None, "coordinate_state": "unknown", "coordinate_precision": "not-supplied", "privacy_gate": "pending-review", "coordinate_gate": "review_required", "publication_gate": "blocked"}} if reasons: unique = tuple(dict.fromkeys(reasons)) for reason in unique: diff --git a/pipeline/sources/germany/refresh.py b/pipeline/sources/germany/refresh.py index 08e0c2b..1db41ca 100644 --- a/pipeline/sources/germany/refresh.py +++ b/pipeline/sources/germany/refresh.py @@ -19,12 +19,21 @@ class RefreshError(ValueError): pass +REVIEW_BLOCKERS = { + "terms": ["BVL documents public access/export but dataset-specific reuse and redistribution terms remain pending named human confirmation."], + "privacy": ["Facility addresses may overlap residences or identify people; address and coordinate publication review is pending and geocoding remains disabled."], + "completeness": ["BLtU is the continuously updated 853/2004 approved-establishment list, not a census of all animal-agriculture facilities; export effective date is unknown until captured."], + "classification": ["Only the pinned SH/CP activity mapping is accepted; unmapped activity flags quarantine and species/activity interpretation remains source-scoped."], + "coverage": ["The portal export URL is session/request-specific; a selected current general-list export and its schema must be evidenced per run."], +} + + def _local_metadata(path: Path, retrieved: str) -> dict[str, Any]: raw = path.read_bytes() return {"acquisition_method": "assisted_bvl_portal_export", "source_id": CONFIG["source_id"], "artifact": path.name, "artifact_path": str(path.resolve()), "requested_url": CONFIG["source_url"], "final_url": CONFIG["source_url"], "redirects": [], "response_headers": {}, "requested_at_utc": retrieved, "retrieved_at_utc": retrieved, "effective_date": "unknown", "publication_date": None, "sha256": hashlib.sha256(raw).hexdigest(), "byte_size": len(raw), "adapter_version": CONFIG["adapter_version"], "code_version": CONFIG["adapter_version"], "config_version": CONFIG["schema_version"], "coverage": CONFIG["coverage"], "rights_caveat": CONFIG["terms"], "privacy_caveat": "private staging; address and coordinate review pending", "terms_review": "assisted capture; recurring acquisition and redistribution remain human-gated"} -def refresh(*, run_dir: str | Path, raw_path: str | Path | None = None, fetch: bool = False, export_url: str | None = None, terms_review_path: str | Path | None = None, output_root: str | Path = "data/raw", run_id: str | None = None, retrieved_at_utc: str | None = None, timeout_seconds: float = 60, max_bytes: int = 128 * 1024 * 1024) -> dict[str, Any]: +def refresh(*, run_dir: str | Path, raw_path: str | Path | None = None, fetch: bool = False, export_url: str | None = None, terms_review_path: str | Path | None = None, output_root: str | Path = "data/raw", run_id: str | None = None, retrieved_at_utc: str | None = None, timeout_seconds: float = 60, max_bytes: int = 128 * 1024 * 1024, previous_normalized: str | Path | None = None) -> dict[str, Any]: if fetch == (raw_path is not None): raise RefreshError("specify exactly one of --raw or --fetch") retrieved = retrieved_at_utc or utc_now() @@ -49,7 +58,7 @@ def refresh(*, run_dir: str | Path, raw_path: str | Path | None = None, fetch: b adapter = BltuAdapter() raw = input_path.read_bytes() artifact = SourceArtifact(source_url=str(metadata.get("final_url") or CONFIG["source_url"]), retrieved_at_utc=str(metadata["retrieved_at_utc"]), sha256=hashlib.sha256(raw).hexdigest(), byte_size=len(raw), publication_date=metadata.get("publication_date"), effective_date=metadata.get("effective_date"), code_version=adapter.adapter_version, config_version=adapter.schema_version, rights_caveat=CONFIG["terms"], privacy_caveat=metadata.get("privacy_caveat"), coverage=CONFIG["coverage"], redirects=tuple(metadata.get("redirects") or ())) - lifecycle = run_private_lifecycle(input_path, Path(run_dir) / "lifecycle", artifact, adapter, health_as_of_utc=str(metadata["retrieved_at_utc"])) + lifecycle = run_private_lifecycle(input_path, Path(run_dir) / "lifecycle", artifact, adapter, health_as_of_utc=str(metadata["retrieved_at_utc"]), previous_normalized_path=previous_normalized, review_blockers=REVIEW_BLOCKERS) manifest = lifecycle.get("manifest") or {} report = {"source_id": CONFIG["source_id"], "source_url": artifact.source_url, "portal_url": CONFIG["portal_url"], "retrieved_at_utc": artifact.retrieved_at_utc, "sha256": artifact.sha256, "byte_size": artifact.byte_size, "input_rows": manifest.get("input_rows"), "normalized_rows": manifest.get("normalized_rows"), "quarantined_rows": manifest.get("quarantined_rows"), "schema_status": manifest.get("schema_status"), "schema_fingerprint": manifest.get("schema_fingerprint"), "drift_alarms": [], "disappearance_semantics": "not-observed; never inferred as closure", "geocoding": "disabled", "release_state": "not-created", "publication_state": "private-candidate", "publication_eligibility": "blocked", "lifecycle_status": lifecycle.get("status"), "lifecycle_run_dir": lifecycle.get("run_dir")} atomic_json(Path(run_dir) / "refresh.json", report) @@ -69,9 +78,10 @@ def main() -> int: parser.add_argument("--retrieved-at-utc") parser.add_argument("--timeout-seconds", type=float, default=60) parser.add_argument("--max-bytes", type=int, default=128 * 1024 * 1024) + parser.add_argument("--previous-normalized", type=Path) args = parser.parse_args() try: - result = refresh(run_dir=args.run_dir, raw_path=args.raw, fetch=args.fetch, export_url=args.export_url, terms_review_path=args.terms_review, output_root=args.output_root, run_id=args.run_id, retrieved_at_utc=args.retrieved_at_utc, timeout_seconds=args.timeout_seconds, max_bytes=args.max_bytes) + result = refresh(run_dir=args.run_dir, raw_path=args.raw, fetch=args.fetch, export_url=args.export_url, terms_review_path=args.terms_review, output_root=args.output_root, run_id=args.run_id, retrieved_at_utc=args.retrieved_at_utc, timeout_seconds=args.timeout_seconds, max_bytes=args.max_bytes, previous_normalized=args.previous_normalized) except (OSError, RefreshError) as error: print(json.dumps({"status": "failed", "error": str(error)}, sort_keys=True)) return 2 diff --git a/pipeline/sources/italy/it_853_adapter.py b/pipeline/sources/italy/it_853_adapter.py index 4d854c9..4c482be 100644 --- a/pipeline/sources/italy/it_853_adapter.py +++ b/pipeline/sources/italy/it_853_adapter.py @@ -71,6 +71,8 @@ def parse_bytes(self, content: bytes) -> dict[str, Any]: raise ValueError("schema drift") occurrences: Counter[tuple[str | None, str | None]] = Counter() + category_counts: Counter[str] = Counter() + activity_counts: Counter[str] = Counter() accepted: list[dict[str, Any]] = [] quarantined: list[dict[str, Any]] = [] for line, row in enumerate(rows, 2): @@ -81,6 +83,8 @@ def parse_bytes(self, content: bytes) -> dict[str, Any]: raise ValueError("schema drift: row has extra columns") rec = clean(row.get(REQUIRED[1])) activity = clean(row.get("codice_impianto_attivita")) + category_counts[clean(row.get("classificazione_stabilimento")) or "unknown"] += 1 + activity_counts[activity or "unknown"] += 1 key = (rec, activity) occurrences[key] += 1 reasons: list[str] = [] @@ -119,6 +123,7 @@ def parse_bytes(self, content: bytes) -> dict[str, Any]: "geography_precision": geography_precision, "geography_state": "source-municipality-code" if geography_precision != "unknown" else "unknown", "classification": clean(row.get("classificazione_stabilimento")), + "classification_state": "source-category-preserved" if clean(row.get("classificazione_stabilimento")) else "unknown", "activity_code": activity, "activity_description": clean(row.get("descrizione_impianto_attivita")), "products": clean(row.get("prodotti_abilitati")), @@ -144,7 +149,7 @@ def parse_bytes(self, content: bytes) -> dict[str, Any]: quarantined.append({"reasons": tuple(dict.fromkeys(reasons)), "record": record}) else: accepted.append(record) - return {"accepted": accepted, "quarantined": quarantined, "source_sha256": digest, "input_rows": len(rows)} + return {"accepted": accepted, "quarantined": quarantined, "source_sha256": digest, "input_rows": len(rows), "schema_fingerprint": hashlib.sha256(json.dumps(headers, ensure_ascii=False, separators=(",", ":")).encode()).hexdigest(), "source_category_counts": dict(sorted(category_counts.items())), "source_activity_counts": dict(sorted(activity_counts.items()))} def parse_file(self, path: str | Path) -> dict[str, Any]: return self.parse_bytes(Path(path).read_bytes()) @@ -175,7 +180,6 @@ def run(self, raw_path: str | Path, run_dir: str | Path, artifact: SourceArtifac parsed_sha256=parsed_sha256, anomaly_counts=dict(sorted(anomaly_counts.items())), ) - manifest["coverage"] = "Italian Ministry 853/2004 CSV; one source row per establishment/activity; 1069/2009 excluded" - manifest["geocoding"] = "disabled" + manifest.update({"coverage": "Italian Ministry 853/2004 CSV; one source row per establishment/activity; 1069/2009 excluded", "geocoding": "disabled", "schema_fingerprint": result["schema_fingerprint"], "source_category_counts": result["source_category_counts"], "source_activity_counts": result["source_activity_counts"]}) atomic_json(root / "manifest.json", manifest) return manifest diff --git a/pipeline/sources/italy/refresh.py b/pipeline/sources/italy/refresh.py index 0804ad2..3277971 100644 --- a/pipeline/sources/italy/refresh.py +++ b/pipeline/sources/italy/refresh.py @@ -17,6 +17,15 @@ from .it_853_adapter import Italy853Adapter +REVIEW_BLOCKERS = { + "terms": ["Italian Open Data Licence v2.0 is indicated by the Ministry catalogue; licence/attribution and project redistribution review remain open."], + "privacy": ["Address, tax identifiers, and precise source coordinates stay in restricted source evidence; privacy classification and coordinate precision review are pending."], + "completeness": ["The run covers the 853/2004 Ministry CSV only; the separate 1069/2009 by-products dataset is intentionally excluded and no national completeness claim is made."], + "classification": ["Recognition number plus activity code is the provisional identity; repeated recognition/activity pairs quarantine, and source classification/activity text is preserved without collapsing categories."], + "coverage": ["Catalog filename/publication date are recorded when supplied, but row-level effective dates and coded category coverage require review."], +} + + def _metadata_for_local(raw_path: Path, metadata: dict, *, url: str, retrieved_at: str | None, adapter: Italy853Adapter) -> dict: import hashlib @@ -49,6 +58,7 @@ def refresh( retrieved_at_utc: str | None = None, timeout_seconds: float = 60.0, max_bytes: int = DEFAULT_MAX_BYTES, + previous_normalized: str | Path | None = None, ) -> dict: """Run the shared private lifecycle from a preserved or acquired artifact.""" if fetch_source == (raw_path is not None): @@ -77,7 +87,7 @@ def refresh( code_version=str(facts["code_version"]), config_version=str(facts["config_version"]), rights_caveat=facts["rights_caveat"], privacy_caveat=facts["privacy_caveat"], coverage=facts["coverage"], ) - status = run_private_lifecycle(input_path, run_dir, artifact, adapter) + status = run_private_lifecycle(input_path, run_dir, artifact, adapter, previous_normalized_path=previous_normalized, review_blockers=REVIEW_BLOCKERS) # Keep catalog/response/terms evidence beside the lifecycle run without # copying row payloads into QA, health, or API-shaped artifacts. if metadata: @@ -99,12 +109,14 @@ def main() -> int: parser.add_argument("--retrieved-at-utc") parser.add_argument("--timeout-seconds", type=float, default=60.0) parser.add_argument("--max-bytes", type=int, default=DEFAULT_MAX_BYTES) + parser.add_argument("--previous-normalized", type=Path) args = parser.parse_args() try: status = refresh(run_dir=args.run_dir, raw_path=args.raw, fetch_source=args.fetch, catalog_url=args.catalog_url, output_root=args.output_root, run_id=args.run_id, terms_review=args.terms_review, retrieved_at_utc=args.retrieved_at_utc, - timeout_seconds=args.timeout_seconds, max_bytes=args.max_bytes) + timeout_seconds=args.timeout_seconds, max_bytes=args.max_bytes, + previous_normalized=args.previous_normalized) except (OSError, ValueError) as error: print(json.dumps({"status": "failed", "error": str(error)}, sort_keys=True)) return 2 diff --git a/pipeline/sources/italy/test_it_853_adapter.py b/pipeline/sources/italy/test_it_853_adapter.py index 150de8e..1b1d27a 100644 --- a/pipeline/sources/italy/test_it_853_adapter.py +++ b/pipeline/sources/italy/test_it_853_adapter.py @@ -14,9 +14,11 @@ def test_sensitive_and_deterministic_identity(self): content=(H+"\n"+row()).encode(); a=Italy853Adapter(); x=a.parse_bytes(content)["accepted"][0]; y=a.parse_bytes(content)["accepted"][0]; self.assertEqual(x["source_row_id"],y["source_row_id"]); self.assertNotIn("p_iva",x["normalized"]); self.assertIsNone(x["normalized"]["coordinates"]) def test_shape_drift_quarantine(self): content=(H+"\n"+row().replace("Name","Name;extra")).encode(); self.assertRaises(ValueError,Italy853Adapter().parse_bytes,content) - def test_missing_date_and_geography_are_explicit(self): - r=Italy853Adapter().parse_bytes((H+"\n"+row().replace("001001","001").replace("2026-09-13","")).encode())["accepted"][0] - self.assertEqual(r["normalized"]["date_state"]["data_inizio_attivita"],"unknown"); self.assertEqual(r["normalized"]["geography_precision"],"unknown"); self.assertEqual(r["normalized"]["coordinate_state"],"source-value-present-pending-review") + def test_missing_date_and_geography_are_explicit(self): + r=Italy853Adapter().parse_bytes((H+"\n"+row().replace("001001","001").replace("2026-09-13","")).encode())["accepted"][0] + self.assertEqual(r["normalized"]["date_state"]["data_inizio_attivita"],"unknown"); self.assertEqual(r["normalized"]["geography_precision"],"unknown"); self.assertEqual(r["normalized"]["coordinate_state"],"source-value-present-pending-review") + def test_source_category_and_activity_coverage_are_explicit(self): + result=Italy853Adapter().parse_bytes((H+"\n"+row()).encode()); self.assertEqual(result["source_category_counts"], {"X": 1}); self.assertEqual(result["source_activity_counts"], {"10": 1}); self.assertTrue(result["schema_fingerprint"]) def test_repeated_activity_quarantines_collision_without_merge(self): result=Italy853Adapter().parse_bytes((H+"\n"+row()+row()).encode()); self.assertEqual(len(result["accepted"]),1); self.assertEqual(len(result["quarantined"]),1); self.assertIn("ambiguous_repeated_recognition_activity",result["quarantined"][0]["reasons"]) def test_run_writes_contract_manifest_and_row_quarantine(self): diff --git a/pipeline/sources/uk/fsa_approved/adapter.py b/pipeline/sources/uk/fsa_approved/adapter.py index 82c436f..5869bc6 100644 --- a/pipeline/sources/uk/fsa_approved/adapter.py +++ b/pipeline/sources/uk/fsa_approved/adapter.py @@ -73,7 +73,7 @@ def _monthly_record(row,line): acts=tuple(x for x in (_clean(row.get("All_Activities")),_clean(row.get("Part_A__All_sections_")),_clean(row.get("Part B All sections "))) if x) privacy_gate="restricted-withheld-address" if withheld else "privacy-review-required" coordinate_gate="restricted-withheld-address" if withheld else "privacy-review-required" - return {"source_id":CONFIG["source_id"],"source_row":line,"source_values":dict(row),"normalized":{"establishment_id":_clean(row.get("AppNo")),"trading_name":_clean(row.get("TradingName")),"address_lines":None if withheld else tuple(_clean(row.get(k)) for k in ("Address1","Address2","Address3")),"postcode":_clean(row.get("Postcode")),"activities":acts,"activity_categories":classify_activities(acts),"species":_clean(row.get("Species")),"competent_authority":_clean(row.get("CompetentAuthority")),"nation":_clean(row.get("Country")),"authority_nation_key":_clean(row.get("Country")),"status":None,"remarks":_clean(row.get("Remarks")),"published_date":None,"coordinates":None,"coordinate_gate":coordinate_gate,"privacy_gate":privacy_gate,"publication_gate":"blocked"}} + return {"source_id":CONFIG["source_id"],"source_row":line,"source_values":dict(row),"normalized":{"establishment_id":_clean(row.get("AppNo")),"trading_name":_clean(row.get("TradingName")),"address_lines":None if withheld else tuple(_clean(row.get(k)) for k in ("Address1","Address2","Address3")),"postcode":_clean(row.get("Postcode")),"activities":acts,"activity_categories":classify_activities(acts),"species":_clean(row.get("Species")),"competent_authority":_clean(row.get("CompetentAuthority")),"nation":_clean(row.get("Country")),"authority_nation_key":_clean(row.get("Country")),"status":None,"remarks":_clean(row.get("Remarks")),"published_date":None,"coordinates":None,"coordinate_state":status,"coordinate_precision":"withheld" if withheld else "source-precision-unspecified","coordinate_gate":coordinate_gate,"privacy_gate":privacy_gate,"publication_gate":"blocked"}} class FsaApprovedEstablishmentsAdapter: source_id=CONFIG["source_id"];schema_version=CONFIG["contract_version"];adapter_version=CONFIG["adapter_version"] @@ -82,13 +82,13 @@ def parse_bytes(self,content): if not synthetic and not monthly:raise FsaContractError("schema drift: unsupported FSA header profile") return self._synthetic(headers,rows,digest,fp) if synthetic else self._monthly(headers,rows,digest,fp) def _synthetic(self,headers,rows,digest,fp): - accepted=[];quarantined=[];parsed=[];keys=[] + accepted=[];quarantined=[];parsed=[];keys=[];coverage={};anomalies={} for line,row in enumerate(rows,2): v={h:row[i] if i1} for line,v in parsed: reasons=[];nation,ident=_clean(v.get("nation")),_clean(v.get("establishment_id")) - if len(v)!=len(headers) or any(x is None for x in v.values()):reasons.append("malformed_row") + if len(row)!=len(headers) or any(x is None for x in v.values()):reasons.append("malformed_row") if not ident:reasons.append("missing_establishment_id") if (nation,ident) in duplicates:reasons.append("duplicate_id_within_nation") if nation not in CONFIG["covered_nations"]:reasons.append("unknown_nation") @@ -101,8 +101,10 @@ def _synthetic(self,headers,rows,digest,fp): if status and status.lower() not in ALLOWED_STATUSES:reasons.append("unknown_status") if _clean(v.get("remarks")):reasons.append("remarks_present") if ADDRESS_RISK.search(" ".join(_clean(v.get(f"address_line_{n}")) or "" for n in range(1,4))):reasons.append("address_privacy_risk") + coverage[nation or ""] = coverage.get(nation or "", 0) + 1 + for reason in dict.fromkeys(reasons): anomalies[reason] = anomalies.get(reason, 0) + 1 record=_synthetic_record(v,line);(quarantined if reasons else accepted).append({"reasons":tuple(dict.fromkeys(reasons)),"record":record} if reasons else record) - return ValidationResult(tuple(accepted),tuple(quarantined),digest,profile="synthetic",schema_fingerprint=fp) + return ValidationResult(tuple(accepted),tuple(quarantined),digest,profile="synthetic",schema_fingerprint=fp,coverage_counts=coverage,anomaly_counts=anomalies) def _monthly(self,headers,rows,digest,fp): accepted=[];quarantined=[];parsed=[];keys=[];coverage={};anomalies={} for line,row in enumerate(rows,2): diff --git a/pipeline/sources/uk/fsa_approved/refresh.py b/pipeline/sources/uk/fsa_approved/refresh.py index 7dd07bc..0e6396d 100644 --- a/pipeline/sources/uk/fsa_approved/refresh.py +++ b/pipeline/sources/uk/fsa_approved/refresh.py @@ -15,6 +15,15 @@ from .handoff import write_private_monthly_handoff +REVIEW_BLOCKERS = { + "terms": ["UK OGL v3 is indicated by the catalogue; attribution, national-scope terms, and project redistribution review remain separate gates."], + "privacy": ["AddressWithheld rows and precise X/Y coordinates require privacy classification; withheld addresses remain suppressed and geocoding is disabled."], + "completeness": ["The monthly feed covers England and Wales in this adapter; Northern Ireland remains a separate authority/source scope and disappearance is not closure."], + "classification": ["Activity values are source-native and mapped conservatively; unknown activity, status, authority/nation mismatch, duplicates, and remarks quarantine."], + "coverage": ["The inspected monthly schema/fingerprint and baseline are evidence for the retained snapshot only; live drift and source effective-date semantics require repeatable refresh review."], +} + + class RefreshError(ValueError): """The source refresh cannot safely continue.""" @@ -153,7 +162,7 @@ def refresh_monthly( handoff = None if mode == "handoff": handoff = write_private_monthly_handoff(input_path, root / "handoff", artifact) - lifecycle = run_private_lifecycle(input_path, root / "lifecycle", artifact, adapter, health_as_of_utc=retrieved_at_utc) + lifecycle = run_private_lifecycle(input_path, root / "lifecycle", artifact, adapter, health_as_of_utc=retrieved_at_utc, previous_normalized_path=previous_normalized, review_blockers=REVIEW_BLOCKERS) report = { "source_url": source_url, "retrieved_at_utc": retrieved_at_utc, diff --git a/pipeline/sources/uk/fss_approved/refresh.py b/pipeline/sources/uk/fss_approved/refresh.py index 819eb02..5e75749 100644 --- a/pipeline/sources/uk/fss_approved/refresh.py +++ b/pipeline/sources/uk/fss_approved/refresh.py @@ -16,6 +16,15 @@ from .handoff import write_private_handoff +REVIEW_BLOCKERS = { + "terms": ["FSS indicates OGL v3; exact CSV terms, attribution, and project redistribution review remain human gates."], + "privacy": ["Address fields and remarks require privacy/safety review; no coordinates are published or geocoded by this adapter."], + "completeness": ["This lane is Scotland only; FSA England/Wales/Northern Ireland are separate feeds, and a missing source row is not closure."], + "classification": ["Live activity columns are preserved and classified conservatively; remarks, duplicate approval IDs, unknown activity/status, and privacy-risk addresses quarantine."], + "coverage": ["The retained live header contract is inspected but current source effective-date and full category coverage remain subject to each bounded refresh."], +} + + def _write_json(path: Path, value: dict[str, Any]) -> None: path.parent.mkdir(parents=True, exist_ok=True) path.write_text(json.dumps(value, ensure_ascii=False, sort_keys=True, indent=2, default=list) + "\n", encoding="utf-8") @@ -128,7 +137,7 @@ def refresh_scotland( current_ids.discard(None) previous_ids = _read_previous_ids(Path(previous_normalized) if previous_normalized else None) disappeared = len(previous_ids - current_ids) if previous_ids else 0 - lifecycle = run_private_lifecycle(input_path, root / "lifecycle", artifact, adapter, health_as_of_utc=retrieved_at_utc) + lifecycle = run_private_lifecycle(input_path, root / "lifecycle", artifact, adapter, health_as_of_utc=retrieved_at_utc, previous_normalized_path=previous_normalized, review_blockers=REVIEW_BLOCKERS) handoff = None if mode == "handoff": handoff = write_private_handoff(input_path, root / "handoff", artifact) From 43bfe2e2a635fab83958f5650202991f88a30d85 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Tue, 15 Sep 2026 21:48:27 -0700 Subject: [PATCH 127/311] Add private France and Canada source pipelines --- docs/countries/canada/meat-plants-pipeline.md | 30 ++++ docs/countries/france/dgal-853-pipeline.md | 30 ++++ docs/source-status.json | 6 +- docs/source-status.md | 8 +- pipeline/adapter-capabilities.json | 4 + pipeline/common/review.py | 46 ++++++ pipeline/common/tabular.py | 94 +++++++++++++ pipeline/common/test_registry.py | 2 +- pipeline/common/test_source_operations.py | 3 +- pipeline/source_operations.json | 6 +- pipeline/source_registry.json | 60 +++++--- pipeline/sources/canada/__init__.py | 1 + pipeline/sources/canada/acquire.py | 17 +++ pipeline/sources/canada/adapter.py | 110 +++++++++++++++ pipeline/sources/canada/fixtures/cfia.csv | 4 + pipeline/sources/canada/fixtures/ontario.csv | 4 + pipeline/sources/canada/refresh.py | 47 +++++++ pipeline/sources/canada/test_adapter.py | 33 +++++ pipeline/sources/canada/test_refresh.py | 21 +++ pipeline/sources/france/__init__.py | 1 + pipeline/sources/france/acquire.py | 17 +++ pipeline/sources/france/adapter.py | 132 ++++++++++++++++++ .../sources/france/fixtures/section_i.csv | 4 + .../sources/france/fixtures/section_ii.csv | 3 + pipeline/sources/france/refresh.py | 54 +++++++ pipeline/sources/france/test_adapter.py | 36 +++++ pipeline/sources/france/test_refresh.py | 18 +++ pipeline/tests/test_source_registry.py | 4 +- 28 files changed, 766 insertions(+), 29 deletions(-) create mode 100644 docs/countries/canada/meat-plants-pipeline.md create mode 100644 docs/countries/france/dgal-853-pipeline.md create mode 100644 pipeline/common/review.py create mode 100644 pipeline/common/tabular.py create mode 100644 pipeline/sources/canada/__init__.py create mode 100644 pipeline/sources/canada/acquire.py create mode 100644 pipeline/sources/canada/adapter.py create mode 100644 pipeline/sources/canada/fixtures/cfia.csv create mode 100644 pipeline/sources/canada/fixtures/ontario.csv create mode 100644 pipeline/sources/canada/refresh.py create mode 100644 pipeline/sources/canada/test_adapter.py create mode 100644 pipeline/sources/canada/test_refresh.py create mode 100644 pipeline/sources/france/__init__.py create mode 100644 pipeline/sources/france/acquire.py create mode 100644 pipeline/sources/france/adapter.py create mode 100644 pipeline/sources/france/fixtures/section_i.csv create mode 100644 pipeline/sources/france/fixtures/section_ii.csv create mode 100644 pipeline/sources/france/refresh.py create mode 100644 pipeline/sources/france/test_adapter.py create mode 100644 pipeline/sources/france/test_refresh.py diff --git a/docs/countries/canada/meat-plants-pipeline.md b/docs/countries/canada/meat-plants-pipeline.md new file mode 100644 index 0000000..17349fa --- /dev/null +++ b/docs/countries/canada/meat-plants-pipeline.md @@ -0,0 +1,30 @@ +# Canada federal/provincial private meat-plant pipelines + +Status: implemented private candidate pipelines; no release or public API exposure. + +The source registry keeps Ontario and CFIA as separate identities. Ontario is +the verified Government of Ontario provincially licensed meat-plant dataset and +is Ontario-only. CFIA is the federal registry download for federally registered +meat establishments and licensed operators. Neither source is a complete +Canada-wide facility register, and the CFIA export-eligibility lists are not +substituted for the federal registry. + +Both sources use the same compact adapter contract while retaining jurisdiction +level, jurisdiction name, plant/registration number, source activity/function +codes, animal class, names, contact fields, addresses, and coordinates in +restricted source values. Normalized candidate fields suppress street address, +phone, and coordinates behind privacy/review gates. Ontario plant-type labels +derive conservative categories; CFIA function codes derive slaughter, cutting, +processing, and storage only for recognized codes. Unknown federal codes, +missing identifiers/names, duplicate source rows, inconsistent columns, and +schema drift fail closed or quarantine without silent merging. + +Lifecycle: + +`bounded fetch or assisted capture -> immutable provenance -> parse/normalize -> +validate/quarantine -> private QA/health -> operator review packet -> candidate +handoff`. Reruns are deterministic. Missing observations are not closure. No +geocoding is performed. Current licence, attribution, redistribution, privacy, +function-code semantics, freshness, and project approval remain human gates. + +Run with `python -m pipeline.sources.canada.refresh --source ontario --raw --run-dir ` or `--source cfia --fetch --terms-review `. Keep federal and provincial candidate releases separate. diff --git a/docs/countries/france/dgal-853-pipeline.md b/docs/countries/france/dgal-853-pipeline.md new file mode 100644 index 0000000..d30f62e --- /dev/null +++ b/docs/countries/france/dgal-853-pipeline.md @@ -0,0 +1,30 @@ +# France DGAL Section I/II private pipeline + +Status: implemented private candidate pipeline; no release or public API exposure. + +The source registry keeps the Ministry's daily Regulation (EC) 853/2004 lists +as two identities: Section I (`SSA1_VIAN_ONG_DOM.txt`, domestic ungulates) and +Section II (`SSA1_VIAN_COL_LAGO.txt`, poultry and lagomorphs). The Ministry's +current 853/2004 page and the file host are the authoritative route evidence. +The adapter preserves every source cell in restricted parsed evidence and +derives only conservative category labels from the source category/activity +strings. Approval number is a source identifier, not permission to merge rows +or evidence that the site is operating. + +Lifecycle: + +`bounded fetch or assisted capture -> immutable hash/size/URL metadata -> +encoding/delimiter/schema validation -> parsed JSONL -> normalized JSONL with +street address/coordinates suppressed -> quarantine JSONL -> row-free QA, +health, and operator review packet -> private candidate handoff`. + +The adapter supports UTF-8/CP1252, semicolon/tab/comma/pipe-delimited inputs, +stable schema fingerprints, duplicate detection, missing identity/name/commune +quarantine, explicit unclassified activity quarantine, deterministic row IDs, +and reruns. A missing row in a later snapshot is `not observed`, never closure. +Geocoding is disabled. Address, SIRET, names, and any future coordinates remain +restricted pending privacy review. Terms/attribution evidence is not treated as +publication approval; the current site-wide Etalab indication still needs +file-specific confirmation. + +Run with `python -m pipeline.sources.france.refresh --section I --raw --run-dir ` or `--fetch --terms-review `. Raw artifacts belong in ignored private storage only. diff --git a/docs/source-status.json b/docs/source-status.json index bfd79a0..d6cbea8 100644 --- a/docs/source-status.json +++ b/docs/source-status.json @@ -9,7 +9,8 @@ }, "sources": [ {"source_id":"be.locations","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-be.md","pipeline/sources/belgium/adapter.py","pipeline/sources/belgium/refresh.py","pipeline/sources/belgium/fixtures/synthetic_operators.csv","pipeline/sources/belgium/test_adapter.py","pipeline/common/review_packet.py","docs/review-packet-belgium.md","pipeline/source_registry.json"],"next_action":"Use the assisted two-file refresh with an authorized operator capture and official activity-code CSV; compare live schema and physical row lengths to the synthetic contract. Keep category/privacy/terms gates and publication blocked."}, - {"source_id":"fr.locations","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-fr.md","pipeline/source_registry.json"],"next_action":"Acquire and snapshot an official DGAL Section I/II artifact with terms, schema, and privacy review."}, + {"source_id":"fr.dgal.section-i","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-fr.md","docs/countries/france/dgal-853-pipeline.md","pipeline/sources/france/adapter.py","pipeline/sources/france/acquire.py","pipeline/sources/france/refresh.py","pipeline/sources/france/fixtures/section_i.csv","pipeline/common/review_packet.py","pipeline/source_registry.json"],"next_action":"Run the bounded private Section I refresh with an approved terms record or authorized capture; review category semantics, address privacy, schema drift, and release approval."}, + {"source_id":"fr.dgal.section-ii","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-fr.md","docs/countries/france/dgal-853-pipeline.md","pipeline/sources/france/adapter.py","pipeline/sources/france/acquire.py","pipeline/sources/france/refresh.py","pipeline/sources/france/fixtures/section_ii.csv","pipeline/common/review_packet.py","pipeline/source_registry.json"],"next_action":"Run the bounded private Section II refresh separately from Section I; review category/species semantics, address privacy, schema drift, and release approval."}, {"source_id":"it.853-2004","metadata":"verified","acquisition":"artifact_private_only","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-it.md","pipeline/source_registry.json","pipeline/sources/italy/acquire.py","pipeline/sources/italy/it_853_adapter.py","pipeline/sources/italy/README.md","pipeline/common/review_packet.py","docs/review-packet-italy.md","pipeline/tests/e2e/test_italy_candidate_import.py"],"next_action":"Keep catalog acquisition, lifecycle, candidate import, and guarded API evidence private; use source-category/activity diagnostics to resolve repeated identity, coordinate/address privacy, coverage, and project approval before release review."}, {"source_id":"it.1069-2009","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-it.md","pipeline/source_registry.json","pipeline/sources/italy/README.md"],"next_action":"Keep the by-products dataset separate; assess scope, schema, terms, identity links, privacy, and a dedicated adapter before any acquisition or integration."}, {"source_id":"mx.locations","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-mx.md","pipeline/source_registry.json"],"next_action":"Resolve DENUE token/terms and SENASICA current directory/schema before bounded acquisition."}, @@ -17,7 +18,8 @@ {"source_id":"uk.locations","metadata":"partial","acquisition":"artifact_private_only","runtime_health":"unknown","publication_eligibility":"blocked","evidence":["docs/country-recon-uk.md","docs/countries/uk/fss-approved-establishments-source-assessment.md","pipeline/sources/uk/fsa_approved/README.md","pipeline/sources/uk/fsa_approved/handoff.py","pipeline/sources/uk/fsa_approved/refresh.py","pipeline/sources/uk/fss_approved/refresh.py","pipeline/common/review_packet.py","docs/review-packet-united-kingdom.md","docs/architecture/disposable-candidate-import.md","pipeline/scripts/maintenance/import-candidate.py","pipeline/source_registry.json"],"next_action":"Repeatable source-local refresh dry-run passed on the retained snapshot (5,342 input, 4,300 normalized, 1,042 quarantined, expected schema fingerprint, no drift alarms); use refresh.py dry-run/handoff only in restricted staging. This does not establish production/runtime health or publication eligibility. Complete source-rights, privacy/coordinate, duplicate, coverage, and release review while keeping NI and Scotland separate."}, {"source_id":"dk.smiley","metadata":"verified","acquisition":"verified","runtime_health":"unknown","publication_eligibility":"blocked","evidence":["pipeline/sources/denmark/README.md","pipeline/contracts/README.md","pipeline/common/review_packet.py","docs/review-packet-denmark.md","pipeline/source_registry.json"],"next_action":"Keep the full privately staged artifact and candidate import validation restricted; run a bounded guarded API check with explicit test-only labeling, capture rerun exit output, and resolve coverage/effective-date/category semantics before any release review. This is not a production-health or publication claim."}, {"source_id":"de.locations","metadata":"partial","acquisition":"artifact_private_only","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/germany-source-assessment.md","pipeline/sources/germany/adapter.py","pipeline/sources/germany/refresh.py","pipeline/common/review_packet.py","docs/review-packet-germany.md","pipeline/source_registry.json"],"next_action":"Use the stable BVL landing to select an export, then run the assisted/private refresh with duplicate-approval and coordinate-precision checks. The export URL, reuse terms, privacy, and project approval remain unresolved; no release or public API exposure is allowed."}, - {"source_id":"ca.locations","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ca.md","pipeline/source_registry.json"],"next_action":"Keep federal/export and Ontario candidates separate; verify an authoritative permitted artifact, terms, schema, coverage, and privacy before acquisition."}, + {"source_id":"ca.ontario.meat-plants","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ca.md","docs/countries/canada/meat-plants-pipeline.md","pipeline/sources/canada/adapter.py","pipeline/sources/canada/acquire.py","pipeline/sources/canada/refresh.py","pipeline/sources/canada/fixtures/ontario.csv","pipeline/common/review_packet.py","pipeline/source_registry.json"],"next_action":"Run the bounded private Ontario refresh; keep plant/contact/coordinate fields restricted pending privacy and licence review, and do not generalize Ontario coverage nationally."}, + {"source_id":"ca.cfia.federal-meat","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ca.md","docs/countries/canada/meat-plants-pipeline.md","pipeline/sources/canada/adapter.py","pipeline/sources/canada/acquire.py","pipeline/sources/canada/refresh.py","pipeline/sources/canada/fixtures/cfia.csv","pipeline/common/review_packet.py","pipeline/source_registry.json"],"next_action":"Run the bounded private CFIA registry refresh; validate the live export schema/function codes, keep federal scope separate from provincial scope, and retain publication gates."}, {"source_id":"es.locations","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-es.md","pipeline/source_registry.json"],"next_action":"Resolve a permitted current AESAN/MAPA artifact or query route, then verify export/schema, rights, effective-date semantics, sector coverage, privacy, and legacy/source boundaries before acquisition."}, {"source_id":"us.fsis","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-us.md","pipeline/source_registry.json"],"next_action":"Obtain authorized access to a current FSIS MPI export after 403 responses; then record URL, retrieval/effective dates, content type, byte size, SHA-256, terms, and schema before adapter or publication review."}, {"source_id":"us.aphis","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-us.md","pipeline/source_registry.json"],"next_action":"Verify the current APHIS export workflow and keep licences, registrants, annual reports, and exception reports separately attributed before acquisition."}, diff --git a/docs/source-status.md b/docs/source-status.md index e20e916..bf9e148 100644 --- a/docs/source-status.md +++ b/docs/source-status.md @@ -16,7 +16,8 @@ No last-success timestamp is invented. Private artifacts are not proof of a publ | Source ID | Metadata | Acquisition | Runtime health | Publication | Evidence / next action | |---|---|---|---|---|---| | `be.locations` | verified | blocked | not_run | blocked | Shared private adapter, row-length/quarantine checks, and assisted two-file refresh are covered by synthetic fixtures; obtain an authorized operator capture and compare live schema before any real run; see `docs/review-packet-belgium.md` | -| `fr.locations` | verified | blocked | not_run | blocked | DGAL/Alim’confiance/SIRENE/HVE/Agence Bio/Géorisques reconnaissance; acquire permitted official artifact and review terms/privacy | +| `fr.dgal.section-i` | verified | not_run | not_run | blocked | DGAL Section I private adapter/refresh is implemented; run only with approved terms or authorized capture, then review category semantics, address privacy, schema drift, and release approval | +| `fr.dgal.section-ii` | verified | not_run | not_run | blocked | DGAL Section II remains a separate private adapter/refresh scope; review species/category semantics, address privacy, schema drift, and release approval | | `it.853-2004` | verified | artifact_private_only | not_run | blocked | Catalog acquisition, shared lifecycle, source-category/activity diagnostics, private candidate import, and guarded API checks remain review-gated; repeated activity identity, coordinate/address privacy, coverage, and project approval remain open; see `docs/review-packet-italy.md` | | `it.1069-2009` | verified | not_run | not_run | blocked | Separate by-products catalog candidate; no adapter or integration decision; assess scope, schema, terms, identity links, and privacy | | `mx.locations` | verified | blocked | not_run | blocked | DENUE/SENASICA/DGSIAP reconnaissance; resolve token, directory, terms, and schema | @@ -24,10 +25,11 @@ No last-success timestamp is invented. Private artifacts are not proof of a publ | `uk.locations` | partial | artifact_private_only | unknown | blocked | FSA and FSS private V2 lifecycle paths, nation-qualified identity, coordinate precision states, and synthetic handoff tests pass; no real UK candidate has been imported or previewed; privacy/coordinate, source-rights, duplicate, coverage, and release review remain open; NI/Scotland stay separate; see `docs/review-packet-united-kingdom.md` | | `dk.smiley` | verified | verified | unknown | blocked | Shared private lifecycle and registered adapter are validated on synthetic/retained evidence with explicit not-observed semantics; coverage/effective-date uncertainty and terms/privacy/release review remain open; see `docs/review-packet-denmark.md` | | `de.locations` | partial | artifact_private_only | not_run | blocked | Stable BVL `/bltu` landing and portal route are verified; typed private adapter and assisted export refresh now include duplicate-approval and coordinate-precision checks, while export-specific terms/privacy/release review remain unresolved; see `docs/review-packet-germany.md` | -| `ca.locations` | verified | not_run | not_run | blocked | Federal/export and Ontario candidates are documented separately; verify permitted artifact, terms, schema, coverage, and privacy before acquisition | +| `ca.ontario.meat-plants` | verified | not_run | not_run | blocked | Ontario private adapter/refresh is implemented; keep plant/contact/coordinate fields restricted pending privacy and licence review, and do not generalize Ontario coverage nationally | +| `ca.cfia.federal-meat` | verified | not_run | not_run | blocked | CFIA federal private adapter/refresh is implemented; validate live export schema/function codes, keep federal scope separate from provincial scope, and retain publication gates | | `es.locations` | partial | blocked | not_run | blocked | AESAN RGSEAA and MAPA sector routes are documented, but direct acquisition was refused; rights, export/schema, effective dates, sector coverage, privacy, and legacy/source boundaries remain unresolved | | `us.fsis` | verified | blocked | not_run | blocked | Official FSIS MPI route is documented, but current CSV access returned 403; obtain authorized export access and record provenance/schema before adapter or publication review | | `us.aphis` | partial | not_run | not_run | blocked | APHIS export workflow and separate report/license provenance require review | | `us.inspections` | partial | not_run | not_run | blocked | Inspection observations require a current export and explicit identity matching | -The machine-readable file is the source of truth for these statuses. Legacy `.locations` paths may represent composite coverage, but the Italy Ministry candidates are now split into explicit 853/2004 and 1069/2009 source IDs. Candidate feeds mentioned in the France, Mexico, and New Zealand reconnaissance documents are not silently conflated into a single healthy source. Country reconnaissance documents provide evidence and next actions; they do not override this status vocabulary or authorize publication. +The machine-readable file is the source of truth for these statuses. Legacy `.locations` paths may represent composite coverage, but source identities are split where the evidence establishes separate feeds: France Section I/II, Canada Ontario/CFIA, and Italy 853/2004/1069/2009. Candidate feeds mentioned in the Mexico and New Zealand reconnaissance documents are not silently conflated into a single healthy source. Country reconnaissance documents provide evidence and next actions; they do not override this status vocabulary or authorize publication. diff --git a/pipeline/adapter-capabilities.json b/pipeline/adapter-capabilities.json index f413f79..7354050 100644 --- a/pipeline/adapter-capabilities.json +++ b/pipeline/adapter-capabilities.json @@ -2,6 +2,10 @@ "schema_version": "adapter-capabilities-v1", "adapters": [ {"country_code": "de", "source_id": "de-bvl-bltu", "adapter_version": "de-v2-foundation-1", "schema_version": "location-v2-foundation-1", "adapter_path": "pipeline/germany/adapter.py", "acquisition": "restricted_pending_terms", "geocoding": "disabled", "publication": "human_gate_required"}, + {"country_code": "fr", "source_id": "fr.dgal.section-i", "adapter_version": "fr-dgal-853-v1", "schema_version": "fr-dgal-853-txt-v1", "adapter_path": "pipeline/sources/france/adapter.py", "acquisition": "bounded_private_fetch", "geocoding": "disabled", "publication": "human_gate_required"}, + {"country_code": "fr", "source_id": "fr.dgal.section-ii", "adapter_version": "fr-dgal-853-v1", "schema_version": "fr-dgal-853-txt-v1", "adapter_path": "pipeline/sources/france/adapter.py", "acquisition": "bounded_private_fetch", "geocoding": "disabled", "publication": "human_gate_required"}, + {"country_code": "ca", "source_id": "ca.ontario.meat-plants", "adapter_version": "ca-meat-v1", "schema_version": "ca-meat-delimited-v1", "adapter_path": "pipeline/sources/canada/adapter.py", "acquisition": "bounded_private_fetch", "geocoding": "disabled", "publication": "human_gate_required"}, + {"country_code": "ca", "source_id": "ca.cfia.federal-meat", "adapter_version": "ca-meat-v1", "schema_version": "ca-meat-delimited-v1", "adapter_path": "pipeline/sources/canada/adapter.py", "acquisition": "bounded_private_fetch", "geocoding": "disabled", "publication": "human_gate_required"}, {"country_code": "gb", "source_id": "fss_approved_establishments", "adapter_version": "fss-scotland-v2-1", "schema_version": "fss-scotland-approved-v1", "adapter_path": "pipeline/sources/uk/fss_approved/adapter.py", "acquisition": "bounded_private_fetch", "geocoding": "disabled", "publication": "human_gate_required"}, {"country_code": "gb", "source_id": "fsa_approved_establishments", "adapter_version": "fsa-uk-v2-1", "schema_version": "fsa-uk-approved-v1", "adapter_path": "pipeline/sources/uk/fsa_approved/adapter.py", "acquisition": "bounded_private_fetch", "geocoding": "disabled", "publication": "human_gate_required"} ] diff --git a/pipeline/common/review.py b/pipeline/common/review.py new file mode 100644 index 0000000..75875d4 --- /dev/null +++ b/pipeline/common/review.py @@ -0,0 +1,46 @@ +"""Row-free operator review packets for private source runs.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any, Iterable + +from pipeline.contracts.source_lifecycle import atomic_json + + +def write_operator_review_packet( + run_dir: str | Path, + manifest: dict[str, Any], + *, + source_scope: str, + checks: Iterable[str], + blockers: Iterable[str], +) -> dict[str, Any]: + """Write deterministic, row-free review instructions beside a run.""" + packet = { + "packet_version": "operator-review-v1", + "source_id": manifest["source_id"], + "source_scope": source_scope, + "source_url": manifest.get("source_url"), + "retrieved_at_utc": manifest.get("retrieved_at_utc"), + "effective_date": manifest.get("effective_date"), + "checksum_sha256": manifest.get("checksum_sha256"), + "schema_version": manifest.get("schema_version"), + "schema_fingerprint": manifest.get("schema_fingerprint"), + "adapter_version": manifest.get("adapter_version"), + "counts": { + "input_rows": manifest.get("input_rows", 0), + "normalized_rows": manifest.get("normalized_rows", 0), + "quarantined_rows": manifest.get("quarantined_rows", 0), + }, + "anomaly_counts": manifest.get("anomaly_counts", {}), + "review_state": "review_required", + "publication_state": "private-candidate", + "release_state": "not-created", + "geocoding": "disabled", + "checks": sorted(set(checks)), + "blockers": sorted(set(blockers)), + "row_payloads_included": False, + } + atomic_json(Path(run_dir) / "operator-review-packet.json", packet) + return packet diff --git a/pipeline/common/tabular.py b/pipeline/common/tabular.py new file mode 100644 index 0000000..03ace9a --- /dev/null +++ b/pipeline/common/tabular.py @@ -0,0 +1,94 @@ +"""Small deterministic helpers for delimited government source artifacts.""" + +from __future__ import annotations + +import csv +import hashlib +import json +import unicodedata +from collections import Counter +from typing import Any, Iterable + + +class TabularSchemaError(ValueError): + """The artifact does not match the source-local tabular contract.""" + + +def canonical_header(value: str) -> str: + text = unicodedata.normalize("NFKD", value.lstrip("\ufeff")) + text = "".join(char for char in text if not unicodedata.combining(char)) + return "".join(char.lower() for char in text if char.isalnum()) + + +def decode_text(content: bytes) -> str: + for encoding in ("utf-8-sig", "cp1252"): + try: + return content.decode(encoding) + except UnicodeDecodeError: + continue + raise TabularSchemaError("artifact is not valid UTF-8 or CP1252 text") + + +def detect_delimiter(text: str) -> str: + line = next((line for line in text.splitlines() if line.strip()), "") + scores = {delimiter: line.count(delimiter) for delimiter in (";", "\t", ",", "|")} + delimiter, score = max(scores.items(), key=lambda item: (item[1], {";": 4, "\t": 3, ",": 2, "|": 1}[item[0]])) + if score == 0: + raise TabularSchemaError("could not detect a delimited header") + return delimiter + + +def read_rows(content: bytes, aliases: dict[str, Iterable[str]], *, required: Iterable[str]) -> tuple[tuple[str, ...], list[dict[str, str]], str, str]: + text = decode_text(content) + delimiter = detect_delimiter(text) + try: + reader = csv.DictReader(text.splitlines(), delimiter=delimiter, strict=True) + headers = tuple(reader.fieldnames or ()) + if not headers or len(set(headers)) != len(headers): + raise TabularSchemaError("missing or duplicate header columns") + resolved = resolve_mapping(headers, aliases) + missing = sorted(set(required) - resolved.keys()) + if missing: + raise TabularSchemaError("schema drift; missing columns: " + ", ".join(missing)) + rows: list[dict[str, str]] = [] + for row in reader: + if None in row or any(value is None for value in row.values()): + raise TabularSchemaError("schema drift; row has an inconsistent column count") + rows.append({str(key): str(value) for key, value in row.items()}) + except csv.Error as error: + raise TabularSchemaError("malformed delimited artifact") from error + fingerprint = hashlib.sha256(json.dumps(tuple(canonical_header(header) for header in headers), separators=(",", ":")).encode()).hexdigest() + return headers, rows, delimiter, fingerprint + + +def resolve_mapping(headers: Iterable[str], aliases: dict[str, Iterable[str]]) -> dict[str, str]: + canonical = {canonical_header(header): header for header in headers} + resolved: dict[str, str] = {} + alias_map = {field: {canonical_header(alias) for alias in names} for field, names in aliases.items()} + for field, names in alias_map.items(): + match = next((canonical_name for canonical_name in canonical if canonical_name in names), None) + if match: + resolved[field] = canonical[match] + return resolved + + +def value(row: dict[str, str], mapping: dict[str, str], field: str) -> str | None: + source_key = mapping.get(field) + if source_key is None: + return None + raw = row.get(source_key, "") + return raw.strip() or None + + +def row_identity(row: dict[str, str], occurrence: int) -> str: + payload = json.dumps(row, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + return hashlib.sha256(f"{payload}|{occurrence}".encode()).hexdigest() + + +def occurrence_key(row: dict[str, str], mapping: dict[str, str], fields: Iterable[str]) -> tuple[str | None, ...]: + return tuple(value(row, mapping, field) for field in fields) + + +def count_values(rows: Iterable[dict[str, Any]], field: str) -> dict[str, int]: + counts = Counter(str(row.get("normalized", {}).get(field) or "unknown") for row in rows) + return dict(sorted(counts.items())) diff --git a/pipeline/common/test_registry.py b/pipeline/common/test_registry.py index c4ed561..233fdb4 100644 --- a/pipeline/common/test_registry.py +++ b/pipeline/common/test_registry.py @@ -8,7 +8,7 @@ class RegistryTests(unittest.TestCase): def test_registered_adapters_have_versioned_capabilities(self): registry = load(Path(__file__).parents[1] / "adapter-capabilities.json") - self.assertEqual({entry["country_code"] for entry in registry["adapters"]}, {"de", "gb"}) + self.assertEqual({entry["country_code"] for entry in registry["adapters"]}, {"ca", "de", "fr", "gb"}) self.assertTrue(all(entry["geocoding"] == "disabled" for entry in registry["adapters"])) self.assertTrue(all(entry["publication"] == "human_gate_required" for entry in registry["adapters"])) diff --git a/pipeline/common/test_source_operations.py b/pipeline/common/test_source_operations.py index 5c35138..eda5c79 100644 --- a/pipeline/common/test_source_operations.py +++ b/pipeline/common/test_source_operations.py @@ -24,8 +24,9 @@ class SourceOperationsTests(unittest.TestCase): def test_checked_in_schedule_inventory_matches_registry(self): root = Path(__file__).parents[1] schedules = load_source_schedules(registry_path=root / "source_registry.json") - self.assertEqual(len(schedules), 14) + self.assertEqual(len(schedules), 16) self.assertEqual(schedules["dk.smiley"].stale_after_hours, 240) + self.assertEqual(schedules["fr.dgal.section-i"].interval_hours, 24) self.assertIsNone(schedules["us.fsis"].interval_hours) def test_schedule_validation_rejects_missing_and_inverted_freshness(self): diff --git a/pipeline/source_operations.json b/pipeline/source_operations.json index 72a1b6b..612c21f 100644 --- a/pipeline/source_operations.json +++ b/pipeline/source_operations.json @@ -3,11 +3,13 @@ "purpose": "Private-alpha scheduling, freshness, retention, and bounded retry expectations. This file does not authorize publication.", "schedules": [ {"source_id":"be.locations","cadence":"weekly","interval_hours":168,"stale_after_hours":240,"max_attempts":3,"backoff_seconds":1,"max_backoff_seconds":30,"manual_fallback":"use the authorized two-file operator capture"}, - {"source_id":"ca.locations","cadence":"unknown","interval_hours":null,"stale_after_hours":null,"max_attempts":1,"backoff_seconds":0,"max_backoff_seconds":0,"manual_fallback":"verify a permitted federal/provincial artifact before scheduling"}, + {"source_id":"ca.cfia.federal-meat","cadence":"unknown","interval_hours":null,"stale_after_hours":null,"max_attempts":1,"backoff_seconds":0,"max_backoff_seconds":0,"manual_fallback":"verify the CFIA federal registry artifact before scheduling"}, + {"source_id":"ca.ontario.meat-plants","cadence":"unknown","interval_hours":null,"stale_after_hours":null,"max_attempts":1,"backoff_seconds":0,"max_backoff_seconds":0,"manual_fallback":"verify the permitted Ontario artifact before scheduling"}, {"source_id":"de.locations","cadence":"unknown","interval_hours":null,"stale_after_hours":null,"max_attempts":1,"backoff_seconds":0,"max_backoff_seconds":0,"manual_fallback":"select and preserve the BVL export through the operator route"}, {"source_id":"dk.smiley","cadence":"weekly","interval_hours":168,"stale_after_hours":240,"max_attempts":3,"backoff_seconds":1,"max_backoff_seconds":30,"manual_fallback":"use the reviewed Find Smiley bulk-download route"}, {"source_id":"es.locations","cadence":"unknown","interval_hours":null,"stale_after_hours":null,"max_attempts":1,"backoff_seconds":0,"max_backoff_seconds":0,"manual_fallback":"confirm the permitted AESAN/MAPA artifact or query route"}, - {"source_id":"fr.locations","cadence":"unknown","interval_hours":null,"stale_after_hours":null,"max_attempts":1,"backoff_seconds":0,"max_backoff_seconds":0,"manual_fallback":"obtain an authorized DGAL artifact capture"}, + {"source_id":"fr.dgal.section-i","cadence":"daily","interval_hours":24,"stale_after_hours":48,"max_attempts":3,"backoff_seconds":1,"max_backoff_seconds":30,"manual_fallback":"obtain an authorized DGAL Section I artifact capture"}, + {"source_id":"fr.dgal.section-ii","cadence":"daily","interval_hours":24,"stale_after_hours":48,"max_attempts":3,"backoff_seconds":1,"max_backoff_seconds":30,"manual_fallback":"obtain an authorized DGAL Section II artifact capture"}, {"source_id":"it.1069-2009","cadence":"daily","interval_hours":24,"stale_after_hours":48,"max_attempts":3,"backoff_seconds":1,"max_backoff_seconds":30,"manual_fallback":"use a separately reviewed Ministry catalog capture"}, {"source_id":"it.853-2004","cadence":"daily","interval_hours":24,"stale_after_hours":48,"max_attempts":3,"backoff_seconds":1,"max_backoff_seconds":30,"manual_fallback":"use the catalog-discovery capture route"}, {"source_id":"mx.locations","cadence":"unknown","interval_hours":null,"stale_after_hours":null,"max_attempts":1,"backoff_seconds":0,"max_backoff_seconds":0,"manual_fallback":"resolve official directory access and terms before scheduling"}, diff --git a/pipeline/source_registry.json b/pipeline/source_registry.json index 97e571c..ceecd4e 100644 --- a/pipeline/source_registry.json +++ b/pipeline/source_registry.json @@ -16,16 +16,28 @@ "blockers": ["The official operator body was inaccessible from the current execution environment; obtain an authorized bounded capture and compare its header/field semantics with the synthetic contract before any real run. Privacy, source-terms interpretation, coverage, and project publication approval remain human gates."] }, { - "source_id": "ca.locations", - "jurisdiction_scope": "Canada; federal and Ontario legacy coverage", + "source_id": "ca.ontario.meat-plants", + "jurisdiction_scope": "Canada; Ontario provincial meat plants only", "legacy_paths": ["static_data/ca/locations.csv"], - "url": "unknown", - "access_method": "download or assisted export", - "cadence": "unknown", - "attribution_licensing_notes": "unknown; confirm source terms and attribution before acquisition", - "adapter_status": "not_started", - "expected_artifact_schema": "CSV; legacy normalized location rows; exact upstream schema unknown", - "blockers": ["Separate federal and Ontario upstream identities and verify current URLs, terms, and fields."] + "url": "https://data.ontario.ca/dataset/a763088c-018d-48b7-bf47-3027a8c725b8/resource/ee6d559a-78de-40e6-b2ba-ad3c4a674b96/download/1._all_meat_plants.csv", + "access_method": "bounded CSV fetch or assisted local capture", + "cadence": "unknown; dataset metadata checked 2026-09-14", + "attribution_licensing_notes": "Government of Ontario dataset; current licence, attribution, privacy, and redistribution review remain explicit gates", + "adapter_status": "implemented_partial", + "expected_artifact_schema": "Delimited CSV; plant number/name, contact, coordinates, and animal class; schema fingerprint is captured per run", + "blockers": ["Ontario coverage is not national; privacy/coordinate review and project publication approval remain pending."] + }, + { + "source_id": "ca.cfia.federal-meat", + "jurisdiction_scope": "Canada; CFIA federally registered meat establishments and licensed operators", + "legacy_paths": [], + "url": "https://active.inspection.gc.ca/scripts/meavia/reglist/download.asp?lang=e", + "access_method": "bounded registry download or assisted local capture", + "cadence": "unknown; registry page states list update 2023-12-04", + "attribution_licensing_notes": "CFIA government source; registry page calls the consolidation a convenience reference and current reuse/privacy review remains required", + "adapter_status": "implemented_partial", + "expected_artifact_schema": "Delimited export; registration number, operator/DBA, location, province, function codes, and contact fields", + "blockers": ["Federal registry is not complete provincial coverage; function-code schema, currency, privacy, and project publication approval remain pending."] }, { "source_id": "de.locations", @@ -64,16 +76,28 @@ "blockers": ["Identify the exact competent-authority publication and distinguish source values from legacy transformations."] }, { - "source_id": "fr.locations", - "jurisdiction_scope": "France", + "source_id": "fr.dgal.section-i", + "jurisdiction_scope": "France; DGAL Regulation (EC) 853/2004 Section I domestic ungulate establishments", "legacy_paths": ["static_data/fr/locations.csv", "france-data.kml"], - "url": "unknown", - "access_method": "downloaded spreadsheet/text/KML or assisted export", - "cadence": "unknown", - "attribution_licensing_notes": "unknown; verify DGAL source identity, licence, and attribution", - "adapter_status": "not_started", - "expected_artifact_schema": "CSV/KML; legacy locations and geometry; exact upstream schema unknown", - "blockers": ["Confirm current DGAL publication and whether the KML is an original source or a derived artifact."] + "url": "https://fichiers-publics.agriculture.gouv.fr/dgal/ListesOfficielles/SSA1_VIAN_ONG_DOM.txt", + "access_method": "bounded TXT fetch or assisted local capture", + "cadence": "daily; Ministry page checked 2026-09-15", + "attribution_licensing_notes": "Ministry page indicates Etalab 2.0 for site content; file-specific terms and attribution confirmation remain pending", + "adapter_status": "implemented_partial", + "expected_artifact_schema": "Delimited TXT; department, approval number, SIRET, name, address, commune, category, activities, and species", + "blockers": ["Privacy/coordinate review, file-specific terms, category codebook, and project publication approval remain pending."] + }, + { + "source_id": "fr.dgal.section-ii", + "jurisdiction_scope": "France; DGAL Regulation (EC) 853/2004 Section II poultry and lagomorph establishments", + "legacy_paths": [], + "url": "https://fichiers-publics.agriculture.gouv.fr/dgal/ListesOfficielles/SSA1_VIAN_COL_LAGO.txt", + "access_method": "bounded TXT fetch or assisted local capture", + "cadence": "daily; Ministry page checked 2026-09-15", + "attribution_licensing_notes": "Ministry page indicates Etalab 2.0 for site content; file-specific terms and attribution confirmation remain pending", + "adapter_status": "implemented_partial", + "expected_artifact_schema": "Delimited TXT; same source fields as Section I, with Section II scope retained", + "blockers": ["Section II is separate from Section I; privacy/coordinate review, terms, codebook, and project publication approval remain pending."] }, { "source_id": "it.853-2004", diff --git a/pipeline/sources/canada/__init__.py b/pipeline/sources/canada/__init__.py new file mode 100644 index 0000000..0262c2a --- /dev/null +++ b/pipeline/sources/canada/__init__.py @@ -0,0 +1 @@ +"""Private federal and provincial Canadian source adapters.""" diff --git a/pipeline/sources/canada/acquire.py b/pipeline/sources/canada/acquire.py new file mode 100644 index 0000000..88fd77f --- /dev/null +++ b/pipeline/sources/canada/acquire.py @@ -0,0 +1,17 @@ +"""Bounded private acquisition for Ontario and the CFIA federal registry.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +from pipeline.common.acquisition import fetch_source + +from .adapter import CfiaFederalMeatAdapter, OntarioMeatPlantsAdapter + +ADAPTERS = {"ontario": OntarioMeatPlantsAdapter, "cfia": CfiaFederalMeatAdapter} + + +def fetch_source_artifact(*, source: str, output_root: str | Path, terms_review_path: str | Path, run_id: str | None = None, timeout_seconds: float = 60.0, max_bytes: int = 64 * 1024 * 1024) -> dict[str, Any]: + adapter = ADAPTERS[source]() + return fetch_source(source_id=adapter.source_id, url=adapter.source_url, output_root=output_root, artifact_name="source.csv", terms_review_path=terms_review_path, run_id=run_id, timeout_seconds=timeout_seconds, max_bytes=max_bytes, allowed_content_types=("text/csv", "application/csv", "application/octet-stream", "text/plain"), code_version=adapter.adapter_version, config_version=adapter.schema_version, coverage=adapter.coverage, rights_caveat="Government source; current licence, attribution, and redistribution review remain explicit gates.", privacy_caveat="Private staging; names, addresses, phones, and coordinates require review.") diff --git a/pipeline/sources/canada/adapter.py b/pipeline/sources/canada/adapter.py new file mode 100644 index 0000000..d53724e --- /dev/null +++ b/pipeline/sources/canada/adapter.py @@ -0,0 +1,110 @@ +"""Private adapters for the verified Ontario and CFIA meat-plant routes.""" + +from __future__ import annotations + +import hashlib +import json +import re +from collections import Counter +from pathlib import Path +from typing import Any + +from pipeline.common.review import write_operator_review_packet +from pipeline.common.tabular import occurrence_key, read_rows, resolve_mapping, row_identity, value +from pipeline.contracts.adapter_contract import SourceArtifact +from pipeline.contracts.source_lifecycle import atomic_json, atomic_jsonl, private_manifest + + +ALIASES = { + "plant_number": ("plant number", "registration number", "establishment number", "establishment id", "plant id", "registration no"), + "name": ("plant name", "operator name", "operators name", "name of operator", "establishment name", "operator", "name"), + "doing_business_as": ("doing business as", "dba name", "also doing business as name", "trade name"), + "address": ("address", "location address", "street address", "location"), + "city": ("city", "location city", "municipality", "town"), + "province": ("province", "location province", "prov", "state"), + "postal_code": ("postal code", "postcode", "zip"), + "phone": ("phone", "telephone", "telephone numbers", "contact phone"), + "latitude": ("latitude", "lat", "y"), + "longitude": ("longitude", "lon", "lng", "x"), + "animal_class": ("animal class", "animal classes", "species", "species processed"), + "plant_type": ("plant type", "type", "dataset", "facility type"), + "function_codes": ("function codes", "function code", "activities", "activity codes", "activity"), + "status": ("status", "current status", "state"), + "effective_date": ("effective date", "date updated", "last updated", "updated"), +} + + +def _clean(raw: str | None) -> str | None: + return raw.strip() if isinstance(raw, str) and raw.strip() else None + + +def _categories(*values_: str | None) -> tuple[str, ...]: + text = " ".join(item for item in values_ if item).lower() + categories: list[str] = [] + if any(token in text for token in ("abattoir", "slaughter", "slaughterhouse")) or re.search(r"(?:^|[ ,;/])1[a-h]?(?:$|[ ,;/])", text): categories.append("slaughter") + if any(token in text for token in ("cutting", "boning", "further processing")) or re.search(r"(?:^|[ ,;/])3x?(?:$|[ ,;/])", text): categories.append("cutting") + if any(token in text for token in ("processing", "canning", "rendering")) or re.search(r"(?:^|[ ,;/])(?:6|7)(?:$|[ ,;/])", text): categories.append("processing") + if any(token in text for token in ("storage", "cold store", "dry store")) or re.search(r"(?:^|[ ,;/])10(?:$|[ ,;/])", text): categories.append("logistics_and_storage") + return tuple(dict.fromkeys(categories)) + + +class CanadaMeatAdapter: + def __init__(self, source_id: str, jurisdiction_level: str, jurisdiction: str, source_url: str, coverage: str, require_categories: bool = False) -> None: + self.source_id, self.jurisdiction_level, self.jurisdiction, self.source_url, self.coverage = source_id, jurisdiction_level, jurisdiction, source_url, coverage + self.require_categories = require_categories + self.adapter_version, self.schema_version = "ca-meat-v1", "ca-meat-delimited-v1" + + def parse_bytes(self, content: bytes) -> dict[str, Any]: + required = ("plant_number", "name") + headers, rows, delimiter, schema_fingerprint = read_rows(content, ALIASES, required=required) + mapping = resolve_mapping(headers, ALIASES) + occurrences: Counter[tuple[str | None, ...]] = Counter(); accepted: list[dict[str, Any]] = []; quarantined: list[dict[str, Any]] = [] + for line, row in enumerate(rows, 2): + plant_number, name = _clean(value(row, mapping, "plant_number")), _clean(value(row, mapping, "name")) + key = occurrence_key(row, mapping, ("plant_number", "name", "city", "province", "function_codes", "animal_class")); occurrences[key] += 1 + plant_type, functions, animal_class = _clean(value(row, mapping, "plant_type")), _clean(value(row, mapping, "function_codes")), _clean(value(row, mapping, "animal_class")) + categories = _categories(plant_type, functions, animal_class) + reasons: list[str] = [] + if not plant_number: reasons.append("missing_plant_number") + if not name: reasons.append("missing_operator_or_plant_name") + if self.require_categories and not categories: reasons.append("unknown_function_code") + if occurrences[key] > 1: reasons.append("duplicate_source_row") + normalized = { + "establishment_id": plant_number, "recognition_number": plant_number, "facility_grouping": f"provisional-{self.jurisdiction_level}-plant-number", "identity_review": "required-before-merge", + "name": name, "trading_name": _clean(value(row, mapping, "doing_business_as")) or name, "address": None, + "address_state": "source-value-present-pending-review" if _clean(value(row, mapping, "address")) else "unknown", "city": _clean(value(row, mapping, "city")), "postal_code": _clean(value(row, mapping, "postal_code")), "province": _clean(value(row, mapping, "province")), + "country_code": "CA", "nation": "Canada", "jurisdiction_level": self.jurisdiction_level, "jurisdiction": self.jurisdiction, + "source_plant_type": plant_type, "source_function_codes": functions, "animal_class": animal_class, "activity_categories": categories, + "classification_state": "derived-from-source-label" if categories else "unclassified", "observation_state": "listed-at-retrieval", "disappearance_semantics": "not-observed; never inferred as closure", + "coordinates": None, "coordinate_state": "source-value-present-pending-review" if _clean(value(row, mapping, "latitude")) or _clean(value(row, mapping, "longitude")) else "not-supplied-by-source", "privacy_gate": "pending-review", "coordinate_gate": "review_required", "publication_gate": "blocked", + } + record = {"source_id": self.source_id, "source_row": line, "source_row_id": row_identity(row, occurrences[key]), "source_record_key": f"{plant_number or 'unknown'}|{occurrences[key]}", "source_values": row, "normalized": normalized} + if reasons: quarantined.append({"reasons": tuple(dict.fromkeys(reasons)), "record": record}) + else: accepted.append(record) + return {"accepted": accepted, "quarantined": quarantined, "input_rows": len(rows), "delimiter": delimiter, "headers": headers, "schema_fingerprint": schema_fingerprint, "source_sha256": hashlib.sha256(content).hexdigest()} + + def parse_file(self, path: str | Path) -> dict[str, Any]: return self.parse_bytes(Path(path).read_bytes()) + + def run(self, raw_path: str | Path, run_dir: str | Path, artifact: SourceArtifact) -> dict[str, Any]: + raw = Path(raw_path).read_bytes() + if artifact.sha256 != hashlib.sha256(raw).hexdigest() or artifact.byte_size != len(raw): raise ValueError("artifact provenance mismatch") + result = self.parse_bytes(raw); root = Path(run_dir); accepted, quarantined = result["accepted"], result["quarantined"]; parsed = accepted + [item["record"] for item in quarantined] + _, parsed_sha256, _ = atomic_jsonl(root / "parsed" / "records.jsonl", parsed); _, normalized_sha256, _ = atomic_jsonl(root / "normalized" / "records.jsonl", accepted); atomic_jsonl(root / "quarantined" / "records.jsonl", quarantined) + anomaly_counts = Counter(reason for item in quarantined for reason in item["reasons"]) + manifest = private_manifest(source_id=self.source_id, adapter_version=self.adapter_version, schema_version=self.schema_version, artifact=artifact, input_rows=result["input_rows"], normalized_rows=len(accepted), quarantined_rows=len(quarantined), normalized_sha256=normalized_sha256, parsed_sha256=parsed_sha256, anomaly_counts=dict(sorted(anomaly_counts.items()))) + manifest.update({"country_code": "CA", "jurisdiction_level": self.jurisdiction_level, "jurisdiction": self.jurisdiction, "delimiter": result["delimiter"], "schema_fingerprint": result["schema_fingerprint"], "coverage": self.coverage, "geocoding": "disabled"}) + atomic_json(root / "manifest.json", manifest) + write_operator_review_packet(root, manifest, source_scope=self.coverage, checks=("keep federal and provincial identities separate", "review address, phone, and coordinate privacy", "review duplicate plant-number rows without silent merge", "confirm function-code or plant-type mapping", "approve any project release separately"), blockers=("no national completeness claim", "publication and privacy approval pending", "source disappearance means not observed, not closure")) + return manifest + + +ONTARIO_URL = "https://data.ontario.ca/dataset/a763088c-018d-48b7-bf47-3027a8c725b8/resource/ee6d559a-78de-40e6-b2ba-ad3c4a674b96/download/1._all_meat_plants.csv" +CFIA_URL = "https://active.inspection.gc.ca/scripts/meavia/reglist/download.asp?lang=e" + + +class OntarioMeatPlantsAdapter(CanadaMeatAdapter): + def __init__(self) -> None: super().__init__("ca.ontario.meat-plants", "provincial", "Ontario", ONTARIO_URL, "Government of Ontario provincially licensed meat plants; Ontario only; plant name/number, contact, coordinates, and animal-class source fields; no national completeness claim") + + +class CfiaFederalMeatAdapter(CanadaMeatAdapter): + def __init__(self) -> None: super().__init__("ca.cfia.federal-meat", "federal", "Canada", CFIA_URL, "CFIA federally registered meat establishments and licensed operators; federal registry only; provincial establishments are excluded", require_categories=True) diff --git a/pipeline/sources/canada/fixtures/cfia.csv b/pipeline/sources/canada/fixtures/cfia.csv new file mode 100644 index 0000000..63a3850 --- /dev/null +++ b/pipeline/sources/canada/fixtures/cfia.csv @@ -0,0 +1,4 @@ +Registration Number,Operator's Name,Also Doing Business As Name,Location Address,Location City,Location Province,Function Codes,Telephone Numbers +001,Synthetic Federal Meats,Synthetic Federal Brand,3 Federal Way,Test City,ON,1a 3x,(555) 010-0001 +002,Synthetic Federal Storage,,4 Federal Way,Test City,QC,10,(555) 010-0002 +003,Synthetic Unknown Functions,,5 Federal Way,Test City,AB,99,(555) 010-0003 diff --git a/pipeline/sources/canada/fixtures/ontario.csv b/pipeline/sources/canada/fixtures/ontario.csv new file mode 100644 index 0000000..13f05aa --- /dev/null +++ b/pipeline/sources/canada/fixtures/ontario.csv @@ -0,0 +1,4 @@ +Plant Number,Plant Name,Address,City,Province,Postal Code,Phone,Latitude,Longitude,Animal Class,Plant Type +ON-001,Synthetic Ontario Abattoir,1 Private Road,Testville,ON,A1A 1A1,555-0100,45.0000,-75.0000,Cattle,Abattoir +ON-002,Synthetic Ontario Processor,2 Example Road,Testburgh,ON,B2B 2B2,555-0101,44.0000,-76.0000,Pigs,Freestanding Meat Plant +ON-002,Synthetic Ontario Processor,2 Example Road,Testburgh,ON,B2B 2B2,555-0101,44.0000,-76.0000,Pigs,Freestanding Meat Plant diff --git a/pipeline/sources/canada/refresh.py b/pipeline/sources/canada/refresh.py new file mode 100644 index 0000000..3749b57 --- /dev/null +++ b/pipeline/sources/canada/refresh.py @@ -0,0 +1,47 @@ +"""Run Ontario or CFIA through private acquisition or assisted capture.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path +from typing import Any + +from pipeline.common.acquisition import utc_now +from pipeline.common.orchestrator import run_private_lifecycle +from pipeline.common.review import write_operator_review_packet +from pipeline.contracts.adapter_contract import SourceArtifact +from pipeline.contracts.candidate_handoff import write_handoff +from pipeline.contracts.source_lifecycle import atomic_json + +from .acquire import ADAPTERS, fetch_source_artifact + + +def refresh(*, source: str, run_dir: str | Path, raw_path: str | Path | None = None, fetch: bool = False, terms_review_path: str | Path | None = None, output_root: str | Path = "data/raw", run_id: str | None = None, retrieved_at_utc: str | None = None, timeout_seconds: float = 60.0, max_bytes: int = 64 * 1024 * 1024) -> dict[str, Any]: + if fetch == (raw_path is not None): raise ValueError("specify exactly one of --fetch or --raw") + adapter = ADAPTERS[source](); retrieved = retrieved_at_utc or utc_now() + if fetch: + if terms_review_path is None: raise ValueError("--terms-review is required with --fetch") + metadata = fetch_source_artifact(source=source, output_root=output_root, terms_review_path=terms_review_path, run_id=run_id, timeout_seconds=timeout_seconds, max_bytes=max_bytes); raw = Path(metadata["artifact_path"]) + artifact = SourceArtifact(metadata["final_url"], metadata["retrieved_at_utc"], metadata["sha256"], metadata["byte_size"], effective_date=metadata.get("effective_date"), publication_date=metadata.get("publication_date"), code_version=adapter.adapter_version, config_version=adapter.schema_version, rights_caveat=metadata.get("rights_caveat"), privacy_caveat=metadata.get("privacy_caveat"), coverage=metadata.get("coverage"), redirects=tuple(metadata.get("redirects") or ())) + else: + raw = Path(raw_path).resolve() + if not raw.is_file(): raise ValueError("--raw artifact must exist") + data = raw.read_bytes(); artifact = SourceArtifact(adapter.source_url, retrieved, hashlib.sha256(data).hexdigest(), len(data), code_version=adapter.adapter_version, config_version=adapter.schema_version, rights_caveat="assisted capture; current terms remain pending", privacy_caveat="private staging; privacy review pending", coverage=adapter.coverage) + metadata = {"acquisition_method": "assisted_local_capture", "source_id": adapter.source_id, "artifact_path": str(raw), "sha256": artifact.sha256, "byte_size": artifact.byte_size, "retrieved_at_utc": retrieved, "requested_url": adapter.source_url, "final_url": adapter.source_url} + root = Path(run_dir); atomic_json(root / "acquisition-metadata.json", metadata); lifecycle = run_private_lifecycle(raw, root / "lifecycle", artifact, adapter, health_as_of_utc=retrieved) + if lifecycle.get("status") == "candidate-ready": + run_root = Path(lifecycle["run_dir"]); rows = [json.loads(line) for line in (run_root / "normalized" / "records.jsonl").read_text(encoding="utf-8").splitlines() if line]; write_handoff(run_root / "candidate-handoff", rows, artifact, source_id=adapter.source_id) + write_operator_review_packet(run_root, lifecycle["manifest"], source_scope=lifecycle["manifest"]["coverage"], checks=("review candidate-handoff/records.jsonl in restricted staging", "keep Ontario and CFIA candidates separate", "confirm no public promotion"), blockers=("candidate is private and human-gated", "privacy and attribution review pending")) + report = {"source_id": adapter.source_id, "jurisdiction": adapter.jurisdiction, "source_url": artifact.source_url, "retrieved_at_utc": artifact.retrieved_at_utc, "checksum_sha256": artifact.sha256, "byte_size": artifact.byte_size, "lifecycle_status": lifecycle.get("status"), "run_dir": lifecycle.get("run_dir"), "publication_state": lifecycle.get("publication_state", "unchanged"), "release_state": "not-created", "geocoding": "disabled"}; atomic_json(root / "refresh.json", report); return {"report": report, "lifecycle": lifecycle} + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__); parser.add_argument("--source", choices=sorted(ADAPTERS), required=True); group = parser.add_mutually_exclusive_group(required=True); group.add_argument("--fetch", action="store_true"); group.add_argument("--raw", type=Path); parser.add_argument("--run-dir", type=Path, required=True); parser.add_argument("--terms-review", type=Path); parser.add_argument("--output-root", type=Path, default=Path("data/raw")); parser.add_argument("--run-id"); parser.add_argument("--retrieved-at-utc"); parser.add_argument("--timeout-seconds", type=float, default=60.0); parser.add_argument("--max-bytes", type=int, default=64 * 1024 * 1024); args = parser.parse_args() + try: result = refresh(source=args.source, run_dir=args.run_dir, raw_path=args.raw, fetch=args.fetch, terms_review_path=args.terms_review, output_root=args.output_root, run_id=args.run_id, retrieved_at_utc=args.retrieved_at_utc, timeout_seconds=args.timeout_seconds, max_bytes=args.max_bytes) + except (OSError, ValueError) as error: print(json.dumps({"status": "failed", "error": str(error)}, sort_keys=True)); return 2 + print(json.dumps(result["report"], sort_keys=True)); return 0 if result["report"]["lifecycle_status"] == "candidate-ready" else 1 + + +if __name__ == "__main__": raise SystemExit(main()) diff --git a/pipeline/sources/canada/test_adapter.py b/pipeline/sources/canada/test_adapter.py new file mode 100644 index 0000000..b4d6405 --- /dev/null +++ b/pipeline/sources/canada/test_adapter.py @@ -0,0 +1,33 @@ +import hashlib +import tempfile +import unittest +from pathlib import Path + +from pipeline.common.orchestrator import run_private_lifecycle +from pipeline.contracts.adapter_contract import SourceArtifact +from .adapter import CfiaFederalMeatAdapter, OntarioMeatPlantsAdapter + +FIXTURES = Path(__file__).parent / "fixtures" + + +class CanadaAdapterTests(unittest.TestCase): + def test_ontario_is_provincial_and_privacy_safe(self): + adapter = OntarioMeatPlantsAdapter(); result = adapter.parse_file(FIXTURES / "ontario.csv") + self.assertEqual(len(result["accepted"]), 2); self.assertEqual(len(result["quarantined"]), 1) + row = result["accepted"][0] + self.assertEqual(row["normalized"]["jurisdiction_level"], "provincial"); self.assertEqual(row["normalized"]["jurisdiction"], "Ontario"); self.assertEqual(row["normalized"]["activity_categories"], ("slaughter",)); self.assertIsNone(row["normalized"]["coordinates"]) + self.assertEqual(row["source_values"]["Phone"], "555-0100") + + def test_cfia_function_codes_and_unknown_code_quarantine(self): + adapter = CfiaFederalMeatAdapter(); result = adapter.parse_file(FIXTURES / "cfia.csv") + self.assertEqual(len(result["accepted"]), 2); self.assertEqual(len(result["quarantined"]), 1); self.assertEqual(result["accepted"][0]["normalized"]["activity_categories"], ("slaughter", "cutting")); self.assertEqual(result["accepted"][1]["normalized"]["activity_categories"], ("logistics_and_storage",)); self.assertEqual(result["quarantined"][0]["reasons"], ("unknown_function_code",)) + + def test_federal_and_provincial_lifecycles_are_separate(self): + with tempfile.TemporaryDirectory() as d: + for adapter, fixture in ((OntarioMeatPlantsAdapter(), "ontario.csv"), (CfiaFederalMeatAdapter(), "cfia.csv")): + raw = (FIXTURES / fixture).read_bytes(); artifact = SourceArtifact(adapter.source_url, "2026-09-15T00:00:00Z", hashlib.sha256(raw).hexdigest(), len(raw), code_version=adapter.adapter_version, config_version=adapter.schema_version) + status = run_private_lifecycle(FIXTURES / fixture, Path(d) / adapter.source_id, artifact, adapter) + self.assertEqual(status["status"], "candidate-ready"); self.assertEqual(status["manifest"]["jurisdiction_level"], adapter.jurisdiction_level); self.assertTrue((Path(status["run_dir"]) / "release-candidate" / "records.jsonl").exists()) + + +if __name__ == "__main__": unittest.main() diff --git a/pipeline/sources/canada/test_refresh.py b/pipeline/sources/canada/test_refresh.py new file mode 100644 index 0000000..0c1edf5 --- /dev/null +++ b/pipeline/sources/canada/test_refresh.py @@ -0,0 +1,21 @@ +import json +import tempfile +import unittest +from pathlib import Path + +from .refresh import refresh + + +class CanadaRefreshTests(unittest.TestCase): + def test_assisted_refresh_emits_separate_private_candidates(self): + with tempfile.TemporaryDirectory() as d: + root = Path(d); results = [] + for source, fixture in (("ontario", "ontario.csv"), ("cfia", "cfia.csv")): + result = refresh(source=source, raw_path=Path(__file__).parent / "fixtures" / fixture, run_dir=root / source, retrieved_at_utc="2026-09-15T00:00:00Z"); results.append(result) + self.assertEqual(result["report"]["lifecycle_status"], "candidate-ready") + run = Path(result["report"]["run_dir"]); self.assertTrue((run / "candidate-handoff/manifest.json").exists()); self.assertTrue((run / "operator-review-packet.json").exists()) + self.assertEqual({item["report"]["source_id"] for item in results}, {"ca.ontario.meat-plants", "ca.cfia.federal-meat"}) + self.assertFalse(json.loads((Path(results[0]["report"]["run_dir"]) / "operator-review-packet.json").read_text())["row_payloads_included"]) + + +if __name__ == "__main__": unittest.main() diff --git a/pipeline/sources/france/__init__.py b/pipeline/sources/france/__init__.py new file mode 100644 index 0000000..db04447 --- /dev/null +++ b/pipeline/sources/france/__init__.py @@ -0,0 +1 @@ +"""Private DGAL Section I and II adapters.""" diff --git a/pipeline/sources/france/acquire.py b/pipeline/sources/france/acquire.py new file mode 100644 index 0000000..e775ece --- /dev/null +++ b/pipeline/sources/france/acquire.py @@ -0,0 +1,17 @@ +"""Bounded private acquisition for the two current DGAL 853/2004 TXT routes.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +from pipeline.common.acquisition import fetch_source + +from .adapter import FranceDgalSectionIAdapter, FranceDgalSectionIIAdapter + +ADAPTERS = {"I": FranceDgalSectionIAdapter, "II": FranceDgalSectionIIAdapter} + + +def fetch_section(*, section: str, output_root: str | Path, terms_review_path: str | Path, run_id: str | None = None, timeout_seconds: float = 60.0, max_bytes: int = 32 * 1024 * 1024) -> dict[str, Any]: + adapter = ADAPTERS[section]() + return fetch_source(source_id=adapter.source_id, url=adapter.source_url, output_root=output_root, artifact_name="source.txt", terms_review_path=terms_review_path, run_id=run_id, timeout_seconds=timeout_seconds, max_bytes=max_bytes, allowed_content_types=("text/plain", "text/csv", "application/octet-stream"), code_version=adapter.adapter_version, config_version=adapter.schema_version, coverage=f"France DGAL Regulation (EC) 853/2004 Section {section}; source rows only", rights_caveat="Ministry page indicates Etalab 2.0 for site content; file-specific reuse confirmation remains a human gate.", privacy_caveat="Private staging; names, addresses, SIRET, and any location enrichment require review.") diff --git a/pipeline/sources/france/adapter.py b/pipeline/sources/france/adapter.py new file mode 100644 index 0000000..18880a0 --- /dev/null +++ b/pipeline/sources/france/adapter.py @@ -0,0 +1,132 @@ +"""Private, provenance-preserving adapters for France's DGAL 853/2004 lists.""" + +from __future__ import annotations + +import hashlib +import json +from collections import Counter +from pathlib import Path +from typing import Any + +from pipeline.common.review import write_operator_review_packet +from pipeline.common.tabular import TabularSchemaError, occurrence_key, read_rows, resolve_mapping, row_identity, value +from pipeline.contracts.adapter_contract import SourceArtifact +from pipeline.contracts.candidate_handoff import write_handoff +from pipeline.contracts.source_lifecycle import atomic_json, atomic_jsonl, private_manifest + + +ALIASES = { + "department_number": ("department number", "department", "n departement", "numero departement", "code departement"), + "approval_number": ("approval number", "approval no", "n dagrement", "numero dagrement", "num dagrement", "agrément", "agrement"), + "siret": ("siret", "siret number"), + "legal_name": ("legal name", "company name", "raison sociale", "nom de letablissement", "nom de l'etablissement", "establishment name"), + "address": ("address", "adresse", "location address"), + "postal_code": ("postal code", "code postal", "postcode"), + "commune": ("commune", "municipality", "city", "town"), + "category": ("category", "categorie", "catégorie", "establishment category"), + "associated_activities": ("associated activities", "activites associees", "activités associées", "activities", "activity"), + "species": ("species", "especes", "espèces", "animal species"), +} +REQUIRED = ("approval_number", "legal_name", "commune", "category") + + +def _clean(raw: str | None) -> str | None: + return raw.strip() if isinstance(raw, str) and raw.strip() else None + + +def _categories(category: str | None, activities: str | None) -> tuple[tuple[str, ...], bool]: + text = " ".join(item for item in (category, activities) if item).upper() + categories: list[str] = [] + if any(token in text for token in ("SH", "ABAT", "SLAUGHT", "ABATTAGE")): + categories.append("slaughter") + if any(token in text for token in ("CP", "CUT", "DECOUPE", "DÉCOUPE")): + categories.append("cutting") + if any(token in text for token in ("TRANSFORM", "PROCESS", "PREPAR", "PRÉPAR")): + categories.append("processing") + if any(token in text for token in ("ENTREP", "STOCK", "STORAGE")): + categories.append("logistics_and_storage") + return tuple(dict.fromkeys(categories)), bool(categories) + + +class FranceDgalAdapter: + """One adapter instance represents exactly one DGAL section/source.""" + + def __init__(self, source_id: str, section: str, source_url: str) -> None: + if section not in {"I", "II"}: + raise ValueError("DGAL section must be I or II") + self.source_id, self.section, self.source_url = source_id, section, source_url + self.adapter_version = "fr-dgal-853-v1" + self.schema_version = "fr-dgal-853-txt-v1" + + def parse_bytes(self, content: bytes) -> dict[str, Any]: + headers, rows, delimiter, schema_fingerprint = read_rows(content, ALIASES, required=REQUIRED) + mapping = resolve_mapping(headers, ALIASES) + occurrences: Counter[tuple[str | None, ...]] = Counter() + accepted: list[dict[str, Any]] = [] + quarantined: list[dict[str, Any]] = [] + for line, row in enumerate(rows, 2): + approval, category = _clean(value(row, mapping, "approval_number")), _clean(value(row, mapping, "category")) + activities = _clean(value(row, mapping, "associated_activities")) + key = occurrence_key(row, mapping, ("approval_number", "category", "associated_activities", "species")) + occurrences[key] += 1 + reasons: list[str] = [] + if not approval: reasons.append("missing_approval_number") + if not _clean(value(row, mapping, "legal_name")): reasons.append("missing_establishment_name") + if not _clean(value(row, mapping, "commune")): reasons.append("missing_commune") + if not category: reasons.append("missing_category") + categories, recognized = _categories(category, activities) + if not recognized: reasons.append("unknown_category_code") + if occurrences[key] > 1: reasons.append("duplicate_source_row") + record = { + "source_id": self.source_id, "source_row": line, + "source_row_id": row_identity(row, occurrences[key]), + "source_record_key": f"{approval or 'unknown'}|{category or 'unknown'}|{activities or 'unknown'}|{occurrences[key]}", + "source_values": row, + "normalized": { + "establishment_id": approval, "recognition_number": approval, + "facility_grouping": "provisional-dgal-approval-number", "identity_review": "required-before-merge", + "name": _clean(value(row, mapping, "legal_name")), "trading_name": _clean(value(row, mapping, "legal_name")), + "address": None, "address_state": "source-value-present-pending-review" if _clean(value(row, mapping, "address")) else "unknown", + "postal_code": _clean(value(row, mapping, "postal_code")), "municipality": _clean(value(row, mapping, "commune")), + "city": _clean(value(row, mapping, "commune")), "department_number": _clean(value(row, mapping, "department_number")), + "country_code": "FR", "nation": "France", "jurisdiction_level": "national", "source_section": self.section, + "source_category": category, "source_activity": activities, "species": _clean(value(row, mapping, "species")), + "activity_categories": categories, "classification_state": "derived-from-source-label" if recognized else "unclassified", + "observation_state": "listed-at-retrieval", "disappearance_semantics": "not-observed; never inferred as closure", + "coordinates": None, "coordinate_state": "not-supplied-by-source", "privacy_gate": "pending-review", + "coordinate_gate": "review_required", "publication_gate": "blocked", + }, + } + if reasons: quarantined.append({"reasons": tuple(dict.fromkeys(reasons)), "record": record}) + else: accepted.append(record) + return {"accepted": accepted, "quarantined": quarantined, "input_rows": len(rows), "delimiter": delimiter, "headers": headers, "schema_fingerprint": schema_fingerprint, "source_sha256": hashlib.sha256(content).hexdigest()} + + def parse_file(self, path: str | Path) -> dict[str, Any]: + return self.parse_bytes(Path(path).read_bytes()) + + def run(self, raw_path: str | Path, run_dir: str | Path, artifact: SourceArtifact) -> dict[str, Any]: + raw = Path(raw_path).read_bytes() + if artifact.sha256 != hashlib.sha256(raw).hexdigest() or artifact.byte_size != len(raw): raise ValueError("artifact provenance mismatch") + result = self.parse_bytes(raw); root = Path(run_dir); accepted, quarantined = result["accepted"], result["quarantined"] + parsed = accepted + [item["record"] for item in quarantined] + _, parsed_sha256, _ = atomic_jsonl(root / "parsed" / "records.jsonl", parsed) + _, normalized_sha256, _ = atomic_jsonl(root / "normalized" / "records.jsonl", accepted) + atomic_jsonl(root / "quarantined" / "records.jsonl", quarantined) + anomaly_counts = Counter(reason for item in quarantined for reason in item["reasons"]) + manifest = private_manifest(source_id=self.source_id, adapter_version=self.adapter_version, schema_version=self.schema_version, artifact=artifact, input_rows=result["input_rows"], normalized_rows=len(accepted), quarantined_rows=len(quarantined), normalized_sha256=normalized_sha256, parsed_sha256=parsed_sha256, anomaly_counts=dict(sorted(anomaly_counts.items()))) + manifest.update({"country_code": "FR", "section": self.section, "delimiter": result["delimiter"], "schema_fingerprint": result["schema_fingerprint"], "coverage": f"France DGAL Regulation (EC) 853/2004 Section {self.section}; source rows only; no completeness claim", "geocoding": "disabled"}) + atomic_json(root / "manifest.json", manifest) + write_operator_review_packet(root, manifest, source_scope=manifest["coverage"], checks=("confirm DGAL file terms and attribution", "review residential or mixed-use addresses", "review duplicate approval/activity identities", "confirm category codebook and current-list semantics", "approve any project release separately"), blockers=("publication approval not granted", "privacy and coordinate review pending", "source disappearance means not observed, not closure")) + return manifest + + +SECTION_I_URL = "https://fichiers-publics.agriculture.gouv.fr/dgal/ListesOfficielles/SSA1_VIAN_ONG_DOM.txt" +SECTION_II_URL = "https://fichiers-publics.agriculture.gouv.fr/dgal/ListesOfficielles/SSA1_VIAN_COL_LAGO.txt" + + +class FranceDgalSectionIAdapter(FranceDgalAdapter): + def __init__(self) -> None: super().__init__("fr.dgal.section-i", "I", SECTION_I_URL) + + +class FranceDgalSectionIIAdapter(FranceDgalAdapter): + def __init__(self) -> None: super().__init__("fr.dgal.section-ii", "II", SECTION_II_URL) diff --git a/pipeline/sources/france/fixtures/section_i.csv b/pipeline/sources/france/fixtures/section_i.csv new file mode 100644 index 0000000..faa281a --- /dev/null +++ b/pipeline/sources/france/fixtures/section_i.csv @@ -0,0 +1,4 @@ +N° département;N° d'agrément;SIRET;Raison sociale;Adresse;Code postal;Commune;Catégorie;Activités associées;Espèces +01;FR 01.001.001 CE;12345678901234;Synthetic Ungulate Foods;12 Rue Exemple;01000;Bourg-Test;SH;Abattage;BOVINS +75;FR 75.002.002 CE;98765432109876;Synthetic Cutting Foods;45 Avenue Test;75000;Paris-Test;CP;Découpe;PORCINS +75;FR 75.002.002 CE;98765432109876;Synthetic Cutting Foods;45 Avenue Test;75000;Paris-Test;CP;Découpe;PORCINS diff --git a/pipeline/sources/france/fixtures/section_ii.csv b/pipeline/sources/france/fixtures/section_ii.csv new file mode 100644 index 0000000..d554d6b --- /dev/null +++ b/pipeline/sources/france/fixtures/section_ii.csv @@ -0,0 +1,3 @@ +N° département;N° d'agrément;SIRET;Raison sociale;Adresse;Code postal;Commune;Catégorie;Activités associées;Espèces +69;FR 69.003.003 CE;11111111111111;Synthetic Poultry Foods;1 Chemin Test;69000;Lyon-Test;SH;Abattage;VOLAILLES +69;;22222222222222;Missing Approval;2 Chemin Test;69000;Lyon-Test;SH;Abattage;VOLAILLES diff --git a/pipeline/sources/france/refresh.py b/pipeline/sources/france/refresh.py new file mode 100644 index 0000000..6542c7f --- /dev/null +++ b/pipeline/sources/france/refresh.py @@ -0,0 +1,54 @@ +"""Run one DGAL section through private acquisition or assisted capture.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path +from typing import Any + +from pipeline.common.acquisition import utc_now +from pipeline.common.orchestrator import run_private_lifecycle +from pipeline.common.review import write_operator_review_packet +from pipeline.contracts.adapter_contract import SourceArtifact +from pipeline.contracts.candidate_handoff import write_handoff +from pipeline.contracts.source_lifecycle import atomic_json + +from .acquire import ADAPTERS, fetch_section + + +def refresh(*, section: str, run_dir: str | Path, raw_path: str | Path | None = None, fetch: bool = False, terms_review_path: str | Path | None = None, output_root: str | Path = "data/raw", run_id: str | None = None, retrieved_at_utc: str | None = None, timeout_seconds: float = 60.0, max_bytes: int = 32 * 1024 * 1024) -> dict[str, Any]: + if fetch == (raw_path is not None): raise ValueError("specify exactly one of --fetch or --raw") + adapter = ADAPTERS[section](); retrieved = retrieved_at_utc or utc_now() + if fetch: + if terms_review_path is None: raise ValueError("--terms-review is required with --fetch") + metadata = fetch_section(section=section, output_root=output_root, terms_review_path=terms_review_path, run_id=run_id, timeout_seconds=timeout_seconds, max_bytes=max_bytes) + source = Path(metadata["artifact_path"]) + artifact = SourceArtifact(metadata["final_url"], metadata["retrieved_at_utc"], metadata["sha256"], metadata["byte_size"], effective_date=metadata.get("effective_date"), publication_date=metadata.get("publication_date"), code_version=adapter.adapter_version, config_version=adapter.schema_version, rights_caveat=metadata.get("rights_caveat"), privacy_caveat=metadata.get("privacy_caveat"), coverage=metadata.get("coverage"), redirects=tuple(metadata.get("redirects") or ())) + else: + source = Path(raw_path).resolve() + if not source.is_file(): raise ValueError("--raw artifact must exist") + raw = source.read_bytes(); artifact = SourceArtifact(adapter.source_url, retrieved, hashlib.sha256(raw).hexdigest(), len(raw), code_version=adapter.adapter_version, config_version=adapter.schema_version, rights_caveat="assisted capture; file-specific terms remain pending", privacy_caveat="private staging; privacy review pending", coverage=f"France DGAL Regulation (EC) 853/2004 Section {section}; source rows only") + metadata = {"acquisition_method": "assisted_local_capture", "source_id": adapter.source_id, "artifact_path": str(source), "sha256": artifact.sha256, "byte_size": artifact.byte_size, "retrieved_at_utc": retrieved, "requested_url": adapter.source_url, "final_url": adapter.source_url} + root = Path(run_dir); atomic_json(root / "acquisition-metadata.json", metadata) + lifecycle = run_private_lifecycle(source, root / "lifecycle", artifact, adapter, health_as_of_utc=retrieved) + if lifecycle.get("status") == "candidate-ready": + run_root = Path(lifecycle["run_dir"]); rows = [json.loads(line) for line in (run_root / "normalized" / "records.jsonl").read_text(encoding="utf-8").splitlines() if line] + write_handoff(run_root / "candidate-handoff", rows, artifact, source_id=adapter.source_id) + write_operator_review_packet(run_root, lifecycle["manifest"], source_scope=lifecycle["manifest"]["coverage"], checks=("review candidate-handoff/records.jsonl in restricted staging", "confirm no public release or API promotion", "review source terms and privacy before approval"), blockers=("candidate is private and human-gated", "address and coordinate publication blocked")) + report = {"source_id": adapter.source_id, "section": section, "source_url": artifact.source_url, "retrieved_at_utc": artifact.retrieved_at_utc, "checksum_sha256": artifact.sha256, "byte_size": artifact.byte_size, "lifecycle_status": lifecycle.get("status"), "run_dir": lifecycle.get("run_dir"), "publication_state": lifecycle.get("publication_state", "unchanged"), "release_state": "not-created", "geocoding": "disabled"} + atomic_json(root / "refresh.json", report); return {"report": report, "lifecycle": lifecycle} + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__); parser.add_argument("--section", choices=sorted(ADAPTERS), required=True) + source = parser.add_mutually_exclusive_group(required=True); source.add_argument("--fetch", action="store_true"); source.add_argument("--raw", type=Path) + parser.add_argument("--run-dir", type=Path, required=True); parser.add_argument("--terms-review", type=Path); parser.add_argument("--output-root", type=Path, default=Path("data/raw")); parser.add_argument("--run-id"); parser.add_argument("--retrieved-at-utc"); parser.add_argument("--timeout-seconds", type=float, default=60.0); parser.add_argument("--max-bytes", type=int, default=32 * 1024 * 1024) + args = parser.parse_args() + try: result = refresh(section=args.section, run_dir=args.run_dir, raw_path=args.raw, fetch=args.fetch, terms_review_path=args.terms_review, output_root=args.output_root, run_id=args.run_id, retrieved_at_utc=args.retrieved_at_utc, timeout_seconds=args.timeout_seconds, max_bytes=args.max_bytes) + except (OSError, ValueError) as error: print(json.dumps({"status": "failed", "error": str(error)}, sort_keys=True)); return 2 + print(json.dumps(result["report"], sort_keys=True)); return 0 if result["report"]["lifecycle_status"] == "candidate-ready" else 1 + + +if __name__ == "__main__": raise SystemExit(main()) diff --git a/pipeline/sources/france/test_adapter.py b/pipeline/sources/france/test_adapter.py new file mode 100644 index 0000000..47c3616 --- /dev/null +++ b/pipeline/sources/france/test_adapter.py @@ -0,0 +1,36 @@ +import hashlib +import json +import tempfile +import unittest +from pathlib import Path + +from pipeline.common.orchestrator import run_private_lifecycle +from pipeline.contracts.adapter_contract import SourceArtifact +from .adapter import FranceDgalSectionIAdapter, FranceDgalSectionIIAdapter + +FIXTURES = Path(__file__).parent / "fixtures" + + +class FranceAdapterTests(unittest.TestCase): + def test_section_i_preserves_source_and_quarantines_duplicate(self): + adapter = FranceDgalSectionIAdapter(); result = adapter.parse_file(FIXTURES / "section_i.csv") + self.assertEqual(len(result["accepted"]), 2); self.assertEqual(len(result["quarantined"]), 1) + row = result["accepted"][0] + self.assertEqual(row["normalized"]["source_section"], "I"); self.assertEqual(row["normalized"]["activity_categories"], ("slaughter",)) + self.assertEqual(row["source_values"]["SIRET"], "12345678901234"); self.assertIsNone(row["normalized"]["address"]); self.assertIsNone(row["normalized"]["coordinates"]) + self.assertEqual(result["quarantined"][0]["reasons"], ("duplicate_source_row",)) + + def test_section_ii_missing_identity_is_quarantined(self): + result = FranceDgalSectionIIAdapter().parse_file(FIXTURES / "section_ii.csv") + self.assertEqual(len(result["accepted"]), 1); self.assertIn("missing_approval_number", result["quarantined"][0]["reasons"]) + + def test_schema_drift_and_lifecycle_are_closed(self): + adapter = FranceDgalSectionIAdapter(); raw = (FIXTURES / "section_i.csv").read_bytes(); artifact = SourceArtifact(adapter.source_url, "2026-09-15T00:00:00Z", hashlib.sha256(raw).hexdigest(), len(raw), code_version=adapter.adapter_version, config_version=adapter.schema_version) + with self.assertRaises(ValueError): adapter.parse_bytes(raw.replace("N° d'agrément".encode(), b"wrong")) + with tempfile.TemporaryDirectory() as d: + status = run_private_lifecycle(FIXTURES / "section_i.csv", Path(d) / "runs", artifact, adapter); root = Path(status["run_dir"]) + self.assertEqual(status["status"], "candidate-ready"); self.assertTrue((root / "operator-review-packet.json").exists()); self.assertFalse((root / "released").exists()) + self.assertFalse(json.loads((root / "operator-review-packet.json").read_text())["row_payloads_included"]) + + +if __name__ == "__main__": unittest.main() diff --git a/pipeline/sources/france/test_refresh.py b/pipeline/sources/france/test_refresh.py new file mode 100644 index 0000000..02e9609 --- /dev/null +++ b/pipeline/sources/france/test_refresh.py @@ -0,0 +1,18 @@ +import json +import tempfile +import unittest +from pathlib import Path + +from .refresh import refresh + + +class FranceRefreshTests(unittest.TestCase): + def test_assisted_refresh_emits_candidate_handoff_and_review_packet(self): + with tempfile.TemporaryDirectory() as d: + root = Path(d); result = refresh(section="I", raw_path=Path(__file__).parent / "fixtures/section_i.csv", run_dir=root / "refresh", retrieved_at_utc="2026-09-15T00:00:00Z") + self.assertEqual(result["report"]["lifecycle_status"], "candidate-ready") + run = Path(result["report"]["run_dir"]); self.assertTrue((run / "candidate-handoff/manifest.json").exists()); self.assertTrue((run / "operator-review-packet.json").exists()) + packet = json.loads((run / "operator-review-packet.json").read_text()); self.assertFalse(packet["row_payloads_included"]) + + +if __name__ == "__main__": unittest.main() diff --git a/pipeline/tests/test_source_registry.py b/pipeline/tests/test_source_registry.py index 6856582..549aacd 100644 --- a/pipeline/tests/test_source_registry.py +++ b/pipeline/tests/test_source_registry.py @@ -9,8 +9,8 @@ class SourceRegistryTests(unittest.TestCase): def test_repository_registry_loads_and_references_existing_legacy_paths(self): registry = load_registry() - self.assertEqual(len(registry["sources"]), 14) - self.assertEqual(len({source["source_id"] for source in registry["sources"]}), 14) + self.assertEqual(len(registry["sources"]), 16) + self.assertEqual(len({source["source_id"] for source in registry["sources"]}), 16) def test_unknowns_are_explicit(self): registry = json.loads(REGISTRY_PATH.read_text(encoding="utf-8")) From e4fbe493e0bfa5cc049a8d314ae0ffe72fa15a0c Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Tue, 15 Sep 2026 21:48:33 -0700 Subject: [PATCH 128/311] Add private US source recovery pipeline slice --- docs/countries/us/README.md | 32 +++++ docs/countries/us/v1-field-crosswalk.json | 19 +++ docs/country-recon-us.md | 8 ++ docs/source-status.json | 6 +- docs/source-status.md | 6 +- pipeline/source-inventory.csv | 6 +- pipeline/source_registry.json | 38 +++--- pipeline/sources/us/__init__.py | 1 + pipeline/sources/us/aphis/__init__.py | 1 + pipeline/sources/us/aphis/adapter.py | 105 ++++++++++++++++ pipeline/sources/us/aphis/config.json | 13 ++ .../us/aphis/fixtures/annual_reports.csv | 2 + .../sources/us/aphis/fixtures/inspections.csv | 2 + .../us/aphis/fixtures/registrations.csv | 2 + pipeline/sources/us/aphis/refresh.py | 30 +++++ pipeline/sources/us/aphis/test_adapter.py | 34 +++++ pipeline/sources/us/fsis/__init__.py | 1 + pipeline/sources/us/fsis/adapter.py | 117 ++++++++++++++++++ pipeline/sources/us/fsis/config.json | 13 ++ .../sources/us/fsis/fixtures/malformed.csv | 4 + pipeline/sources/us/fsis/fixtures/valid.csv | 3 + pipeline/sources/us/fsis/refresh.py | 51 ++++++++ pipeline/sources/us/fsis/test_adapter.py | 37 ++++++ 23 files changed, 503 insertions(+), 28 deletions(-) create mode 100644 docs/countries/us/README.md create mode 100644 docs/countries/us/v1-field-crosswalk.json create mode 100644 pipeline/sources/us/__init__.py create mode 100644 pipeline/sources/us/aphis/__init__.py create mode 100644 pipeline/sources/us/aphis/adapter.py create mode 100644 pipeline/sources/us/aphis/config.json create mode 100644 pipeline/sources/us/aphis/fixtures/annual_reports.csv create mode 100644 pipeline/sources/us/aphis/fixtures/inspections.csv create mode 100644 pipeline/sources/us/aphis/fixtures/registrations.csv create mode 100644 pipeline/sources/us/aphis/refresh.py create mode 100644 pipeline/sources/us/aphis/test_adapter.py create mode 100644 pipeline/sources/us/fsis/__init__.py create mode 100644 pipeline/sources/us/fsis/adapter.py create mode 100644 pipeline/sources/us/fsis/config.json create mode 100644 pipeline/sources/us/fsis/fixtures/malformed.csv create mode 100644 pipeline/sources/us/fsis/fixtures/valid.csv create mode 100644 pipeline/sources/us/fsis/refresh.py create mode 100644 pipeline/sources/us/fsis/test_adapter.py diff --git a/docs/countries/us/README.md b/docs/countries/us/README.md new file mode 100644 index 0000000..e65d28c --- /dev/null +++ b/docs/countries/us/README.md @@ -0,0 +1,32 @@ +# United States recovery packet + +This packet is private pipeline documentation, not publication approval. + +## Source boundaries + +The facility-master candidate is USDA FSIS's [Meat, Poultry and Egg Product Inspection Directory](https://www.fsis.usda.gov/inspection/establishments/meat-poultry-and-egg-product-inspection-directory) and its supplemental establishment-demographic CSV. FSIS describes the directory as a listing of FSIS-regulated meat, poultry, and egg establishments, with a weekly replacement edition and generalized activity categories. State meat-and-poultry inspection programs are not silently included. + +APHIS is a separate evidence family. The [Animal Care Public Search Tool](https://www.aphis.usda.gov/animal-care/awa-services/usda-animal-care-public-search-tool) exposes licensed/registered persons, inspection reports, and research facility annual reports. The [annual usage summary](https://direct.aphis.usda.gov/awa/research-facility-report/annual-summary) notes that annual reports can be amended. The adapters require an explicit `registrations`, `annual_reports`, or `inspections` profile, and never turn an APHIS row into an FSIS facility or laboratory master record. + +## Acquisition boundary + +FSIS direct links returned HTTP 403 during the prior reconnaissance. This sprint does not bypass that control. `pipeline.sources.us.fsis.refresh` provides a reproducible operator-assisted capture contract: an authorized operator saves the current CSV export shown on the official page, records the edition/date and final URL, and runs the private adapter. Direct fetch is available only with a terms-review JSON and the shared bounded acquisition primitive; HTML, login, 403, content-type, and schema failures remain fail-closed. + +APHIS is UI-mediated. `pipeline.sources.us.aphis.refresh` records the selected profile, official route, query/export context, retrieval time, hash, byte size, and separate evidence type. No hidden endpoint automation is required. + +## Validation and handoff + +Both adapters preserve source values only in restricted staging and emit parsed, normalized, quarantined, QA, run-status, health, and private candidate-handoff artifacts. They disable geocoding, mark address/coordinate review as pending, quarantine duplicate or missing identities, and set `release_state=not-created`, `publication_state=private-candidate`, and `publication_gate=blocked`. Candidate import remains limited to the disposable loopback database and existing test-only API path; no public promotion is performed by these commands. + +## V1 reconciliation + +[`v1-field-crosswalk.json`](v1-field-crosswalk.json) is the row-free inventory and field/category map. The checked-in FSIS V1 snapshot has 7,101 rows and 269 columns. Its slaughter and processing flags overlap, so the counts are field-presence observations rather than totals. Until an authorized current artifact exists, V1 rows are not claimed current and a missing current observation is `not-observed`, never closure. + +## Review checklist + +- authority, edition/effective date, URL, terms/attribution, and retention are recorded; +- source schema fingerprint and count changes are reviewed; +- FSIS, APHIS registrations, APHIS inspections, annual reports, laboratories, and aggregates remain separate; +- phone, DUNS, names, addresses, and precise coordinates receive privacy review; +- current coverage is compared with V1 only by exact source keys and aggregate reports; +- no raw artifact or row-level report is committed or exposed. diff --git a/docs/countries/us/v1-field-crosswalk.json b/docs/countries/us/v1-field-crosswalk.json new file mode 100644 index 0000000..4ef3f54 --- /dev/null +++ b/docs/countries/us/v1-field-crosswalk.json @@ -0,0 +1,19 @@ +{ + "schema_version": "us-v1-field-crosswalk-1", + "legacy_snapshot": "static_data/us/locations.csv", + "legacy_rows": 7101, + "legacy_columns": 269, + "status": "inventory-only; no current FSIS artifact was safely acquired", + "groups": { + "identity": {"fields": ["establishment_id", "establishment_number", "establishment_name", "duns_number"], "current_evidence": "FSIS MPI directory identity fields are the intended source boundary; exact current header mapping pending authorized export", "state": "pending_current_artifact"}, + "location": {"fields": ["street", "city", "state", "zip", "county", "fips_code", "latitude", "longitude"], "current_evidence": "FSIS directory/demographic source describes physical establishments and geographic fields; source values remain private and coordinates require review", "state": "pending_privacy_and_current_artifact"}, + "contact": {"fields": ["phone"], "current_evidence": "May exist in legacy or current directory; not copied to normalized output", "state": "unresolved"}, + "administrative": {"fields": ["grant_date", "dbas", "district", "circuit", "size", "type"], "current_evidence": "FSIS demographic documentation describes district, HACCP size, and production activity categories; exact current field mapping pending export", "state": "partial"}, + "slaughter_categories": {"fields": ["slaughter", "meat_slaughter", "beef_cow_slaughter", "steer_slaughter", "heifer_slaughter", "bull_stag_slaughter", "dairy_cow_slaughter", "heavy_calf_slaughter", "bob_veal_slaughter", "formula_fed_veal_slaughter", "non_formula_fed_veal_slaughter", "market_swine_slaughter", "sow_slaughter", "roaster_swine_slaughter", "boar_stag_swine_slaughter", "stag_swine_slaughter", "feral_swine_slaughter", "goat_slaughter", "young_goat_slaughter", "adult_goat_slaughter", "sheep_slaughter", "lamb_slaughter", "deer_reindeer_slaughter", "antelope_slaughter", "elk_slaughter", "bison_slaughter", "buffalo_slaughter", "water_buffalo_slaughter", "cattalo_slaughter", "yak_slaughter", "other_voluntary_livestock_slaughter", "rabbit_slaughter", "poultry_slaughter", "young_chicken_slaughter", "light_fowl_slaughter", "heavy_fowl_slaughter", "capon_slaughter", "young_turkey_slaughter", "young_breeder_turkey_slaughter", "old_breeder_turkey_slaughter", "fryer_roaster_turkey_slaughter", "duck_slaughter", "goose_slaughter", "pheasant_slaughter", "quail_slaughter", "guinea_slaughter", "ostrich_slaughter", "emu_slaughter", "rhea_slaughter", "squab_slaughter", "other_voluntary_poultry_slaughter"], "current_evidence": "FSIS supplemental demographic dataset documents species slaughter subclasses; adapter preserves source activity vocabulary and does not infer absent species", "state": "partial_pending_current_artifact"}, + "processing_categories": {"fields": ["processing", "meat_processing", "poultry_processing", "egg_processing", "*_processing"], "current_evidence": "FSIS supplemental demographic dataset documents categorical processing, RTE/NRTE/raw-intact/raw-non-intact, species, cell-cultured, and exemption distinctions", "state": "partial_pending_current_artifact"}, + "inspection_systems": {"fields": ["inspection_system_*"], "current_evidence": "FSIS source documentation describes inspection activities; retain each source flag separately", "state": "partial_pending_current_artifact"}, + "derived_legacy_categories": {"fields": ["slaughter_or_processing_only", "slaughter_only_class", "slaughter_only_species", "meat_slaughter_only_species", "poultry_slaughter_only_species", "slaughter_volume_category", "processing_volume_category"], "current_evidence": "Project-derived from legacy FSIS columns; must be recomputed only from a reviewed current artifact and never treated as source facts", "state": "unresolved_current_reproduction"} + }, + "legacy_aggregate_observations": {"slaughter_flag_presence": {"slaughter": 1340, "meat_slaughter": 1061, "poultry_slaughter": 320}, "processing_flag_presence": {"processing": 5755, "meat_processing": 4981, "poultry_processing": 3142, "egg_processing": 96}, "grant_flag_presence": {"active_meat_grant": 5550, "active_poultry_grant": 4462}, "warning": "presence counts overlap and are not mutually exclusive facility totals"}, + "other_v1_surfaces": {"static_data/us/aphis_data_final.csv": "APHIS annual-use observations; not FSIS and not a laboratory census", "static_data/us/inspection_reports.csv": "APHIS registration/license-shaped legacy surface; current inspection exports remain separate observations", "policy": "No cross-source identity merge without an explicit reviewed identity/link event"} +} diff --git a/docs/country-recon-us.md b/docs/country-recon-us.md index 95f728c..d13195c 100644 --- a/docs/country-recon-us.md +++ b/docs/country-recon-us.md @@ -38,3 +38,11 @@ publication decision. ## Blockers and recommendation No safe bounded private fetch was performed, so current hashes/bytes and deterministic reproduction are intentionally unavailable. FSIS is the strongest automation candidate because recurring CSV downloads and source descriptions are available. APHIS is secondary/manual/UI-mediated and should be an explicitly versioned, human-reviewed annual-report adapter or restricted manual input. Do not build a laboratory-supplier layer from APHIS records without a separately identified, licensed source. Existing Selenium/compiler code is not production-grade: obsolete selectors, no provenance manifest, quarantine, terms/schema/privacy gates, and unsafe duplicate handling. + +## 2026-09-15 recovery slice + +The private implementation is in `pipeline/sources/us/`. FSIS now has a profile-aware adapter and refresh command with a sanctioned operator-assisted capture contract. APHIS now has one adapter with explicit `registrations`, `annual_reports`, and `inspections` profiles. All three APHIS populations remain observations, not a laboratory or facility master, and no identity merge with FSIS is performed. + +The row-free V1 inventory and field/category crosswalk is [`docs/countries/us/v1-field-crosswalk.json`](countries/us/v1-field-crosswalk.json). It records 7,101 rows and 269 columns, maps identity/location/contact/administrative/slaughter/processing/inspection-system/derived fields, and records overlapping legacy field-presence counts. Since no authorized current FSIS artifact was available, current-versus-V1 reconciliation remains blocked; the existing exact-key crosswalk reports `not_observed`, never closure. + +Focused adapter, lifecycle, registry, status, and contract tests pass. No raw artifact, current source hash, or publication candidate from a real US source was created. Publication remains blocked pending authorized capture, terms, schema, privacy, coverage, review, and test-only import checks. diff --git a/docs/source-status.json b/docs/source-status.json index d6cbea8..3aa3488 100644 --- a/docs/source-status.json +++ b/docs/source-status.json @@ -21,8 +21,8 @@ {"source_id":"ca.ontario.meat-plants","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ca.md","docs/countries/canada/meat-plants-pipeline.md","pipeline/sources/canada/adapter.py","pipeline/sources/canada/acquire.py","pipeline/sources/canada/refresh.py","pipeline/sources/canada/fixtures/ontario.csv","pipeline/common/review_packet.py","pipeline/source_registry.json"],"next_action":"Run the bounded private Ontario refresh; keep plant/contact/coordinate fields restricted pending privacy and licence review, and do not generalize Ontario coverage nationally."}, {"source_id":"ca.cfia.federal-meat","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ca.md","docs/countries/canada/meat-plants-pipeline.md","pipeline/sources/canada/adapter.py","pipeline/sources/canada/acquire.py","pipeline/sources/canada/refresh.py","pipeline/sources/canada/fixtures/cfia.csv","pipeline/common/review_packet.py","pipeline/source_registry.json"],"next_action":"Run the bounded private CFIA registry refresh; validate the live export schema/function codes, keep federal scope separate from provincial scope, and retain publication gates."}, {"source_id":"es.locations","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-es.md","pipeline/source_registry.json"],"next_action":"Resolve a permitted current AESAN/MAPA artifact or query route, then verify export/schema, rights, effective-date semantics, sector coverage, privacy, and legacy/source boundaries before acquisition."}, - {"source_id":"us.fsis","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-us.md","pipeline/source_registry.json"],"next_action":"Obtain authorized access to a current FSIS MPI export after 403 responses; then record URL, retrieval/effective dates, content type, byte size, SHA-256, terms, and schema before adapter or publication review."}, - {"source_id":"us.aphis","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-us.md","pipeline/source_registry.json"],"next_action":"Verify the current APHIS export workflow and keep licences, registrants, annual reports, and exception reports separately attributed before acquisition."}, - {"source_id":"us.inspections","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-us.md","pipeline/source_registry.json"],"next_action":"Verify a current inspection export and use explicit, reviewable identity matching rather than treating observations as a facility master."} + {"source_id":"us.fsis","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-us.md","docs/countries/us/README.md","docs/countries/us/v1-field-crosswalk.json","pipeline/sources/us/fsis/config.json","pipeline/sources/us/fsis/adapter.py","pipeline/sources/us/fsis/refresh.py","pipeline/source_registry.json"],"next_action":"Use the assisted official export contract after the 403 blocker; record URL, retrieval/effective dates, content type, byte size, SHA-256, terms, schema fingerprint, privacy review, and reconciliation before any test-only handoff."}, + {"source_id":"us.aphis","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-us.md","docs/countries/us/README.md","pipeline/sources/us/aphis/config.json","pipeline/sources/us/aphis/adapter.py","pipeline/sources/us/aphis/refresh.py","pipeline/source_registry.json"],"next_action":"Use the explicit profile-based assisted export for registrations, annual reports, or inspections; preserve each evidence type separately and complete terms, privacy, schema, and review gates."}, + {"source_id":"us.inspections","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-us.md","docs/countries/us/README.md","pipeline/sources/us/aphis/config.json","pipeline/sources/us/aphis/adapter.py","pipeline/sources/us/aphis/refresh.py","pipeline/source_registry.json"],"next_action":"Capture the APHIS inspections profile through the documented public-search route; treat rows as observations, not a facility master, and use explicit reviewable identity matching only."} ] } diff --git a/docs/source-status.md b/docs/source-status.md index bf9e148..dbe981c 100644 --- a/docs/source-status.md +++ b/docs/source-status.md @@ -28,8 +28,8 @@ No last-success timestamp is invented. Private artifacts are not proof of a publ | `ca.ontario.meat-plants` | verified | not_run | not_run | blocked | Ontario private adapter/refresh is implemented; keep plant/contact/coordinate fields restricted pending privacy and licence review, and do not generalize Ontario coverage nationally | | `ca.cfia.federal-meat` | verified | not_run | not_run | blocked | CFIA federal private adapter/refresh is implemented; validate live export schema/function codes, keep federal scope separate from provincial scope, and retain publication gates | | `es.locations` | partial | blocked | not_run | blocked | AESAN RGSEAA and MAPA sector routes are documented, but direct acquisition was refused; rights, export/schema, effective dates, sector coverage, privacy, and legacy/source boundaries remain unresolved | -| `us.fsis` | verified | blocked | not_run | blocked | Official FSIS MPI route is documented, but current CSV access returned 403; obtain authorized export access and record provenance/schema before adapter or publication review | -| `us.aphis` | partial | not_run | not_run | blocked | APHIS export workflow and separate report/license provenance require review | -| `us.inspections` | partial | not_run | not_run | blocked | Inspection observations require a current export and explicit identity matching | +| `us.fsis` | verified | blocked | not_run | blocked | Private adapter and assisted-capture contract are implemented; current CSV access returned 403, so obtain an authorized export and record provenance/schema/privacy/reconciliation before test-only handoff | +| `us.aphis` | verified | not_run | not_run | blocked | Profile-explicit private adapter and assisted-capture contract cover registrations, annual reports, and inspections; capture current exports and review terms/schema/privacy | +| `us.inspections` | verified | not_run | not_run | blocked | APHIS inspections profile is implemented as observation evidence; capture current export and use explicit reviewable identity links only | The machine-readable file is the source of truth for these statuses. Legacy `.locations` paths may represent composite coverage, but source identities are split where the evidence establishes separate feeds: France Section I/II, Canada Ontario/CFIA, and Italy 853/2004/1069/2009. Candidate feeds mentioned in the Mexico and New Zealand reconnaissance documents are not silently conflated into a single healthy source. Country reconnaissance documents provide evidence and next actions; they do not override this status vocabulary or authorize publication. diff --git a/pipeline/source-inventory.csv b/pipeline/source-inventory.csv index 192b32e..6ebf25f 100644 --- a/pipeline/source-inventory.csv +++ b/pipeline/source-inventory.csv @@ -9,6 +9,6 @@ it.1069-2009,IT,,Italian Ministry of Health 1069/2009 animal by-products establi mx.locations,MX,static_data/mx/locations.csv;Old CSVs/mexico, Mexican official registers and INEGI-derived work,download/API or assisted export,source_candidate,Separate official source records from derived aquaculture research files. nz.locations,NZ,static_data/nz/locations.csv,New Zealand MPI approved premises/registers,download/API,source_candidate,MPI country listings expose identifier, address, processes, species, and expiry fields. uk.locations,GB,static_data/uk/locations.csv;static_data/uk/locations.csv.backup,Food Standards Agency approved food establishments,monthly CSV download,source_candidate,Current FSA publication is split across England/Wales, Northern Ireland, and Scotland. -us.fsis,US,static_data/us/locations.csv,USDA FSIS MPI directory and establishment demographic data,download or official directory export,source_candidate,Confirm which legacy rows came from FSIS versus other USDA files. -us.aphis,US,static_data/us/aphis_data_final.csv,USDA APHIS Animal Care Public Search Tool,interactive export/browser-assisted,source_candidate,Preserve license/registrant data and annual-report/exception-report provenance separately. -us.inspections,US,static_data/us/inspection_reports.csv,USDA APHIS inspection-reports public search,interactive export/browser-assisted,source_candidate,Inspection reports are observations, not facility master records. +us.fsis,US,static_data/us/locations.csv,USDA FSIS MPI directory and supplemental establishment-demographic data,operator-assisted official CSV export; bounded direct fetch with terms review,implemented_partial,Current route verified; direct links returned 403, so acquire an authorized edition and preserve provenance/schema/privacy review before test-only handoff. +us.aphis,US,static_data/us/aphis_data_final.csv,USDA APHIS Animal Care Public Search Tool,operator-assisted profile-explicit export,implemented_partial,Registrations, annual reports, inspections, laboratories, and aggregate summaries remain separate evidence types. +us.inspections,US,static_data/us/inspection_reports.csv,USDA APHIS inspection-reports public search,operator-assisted inspection export,implemented_partial,Inspection reports are observations, not facility master records; use explicit reviewable identity links only. diff --git a/pipeline/source_registry.json b/pipeline/source_registry.json index ceecd4e..76ab644 100644 --- a/pipeline/source_registry.json +++ b/pipeline/source_registry.json @@ -163,37 +163,37 @@ "source_id": "us.fsis", "jurisdiction_scope": "United States; USDA FSIS meat and poultry establishment coverage", "legacy_paths": ["static_data/us/locations.csv"], - "url": "unknown", - "access_method": "download or official directory export", - "cadence": "unknown", - "attribution_licensing_notes": "unknown; determine whether legacy rows came from FSIS, another USDA file, or a combination", - "adapter_status": "not_started", - "expected_artifact_schema": "CSV/directory export; establishment identity, address, and activity fields expected; exact schema unknown", - "blockers": ["Trace legacy rows to a specific FSIS artifact and separate any other USDA inputs."] + "url": "https://www.fsis.usda.gov/inspection/establishments/meat-poultry-and-egg-product-inspection-directory", + "access_method": "operator-assisted official CSV export; bounded direct fetch only with terms review", + "cadence": "weekly replacement", + "attribution_licensing_notes": "FSIS authority and directory scope verified; terms/attribution and current export URL must be recorded per run before publication", + "adapter_status": "implemented_partial", + "expected_artifact_schema": "FSIS MPI directory CSV; source values preserved privately; supplemental demographic activity flags remain distinct", + "blockers": ["Current direct links returned HTTP 403 during reconnaissance; obtain an authorized current export and complete terms, schema, privacy, and project review."] }, { "source_id": "us.aphis", "jurisdiction_scope": "United States; USDA APHIS Animal Care public search and annual-report data", "legacy_paths": ["static_data/us/aphis_data_final.csv"], "url": "https://aphis.my.site.com/PublicSearchTool/s/annual-reports", - "access_method": "interactive export/browser-assisted retrieval", - "cadence": "unknown", - "attribution_licensing_notes": "Legacy scripts identify USDA APHIS; confirm export terms, attribution, and handling of license/registrant information.", - "adapter_status": "reference_only", - "expected_artifact_schema": "CSV export; license, registrant, annual-report, and exception-report fields must remain separately attributed", - "blockers": ["Confirm current export workflow and schema; do not merge annual reports or exception reports into a facility master without explicit provenance."] + "access_method": "operator-assisted public-search export with explicit registrations/annual_reports/inspections profile", + "cadence": "unknown; record selected report/search date", + "attribution_licensing_notes": "APHIS authority and public-search scope verified; export terms, attribution, and privacy handling remain review gates", + "adapter_status": "implemented_partial", + "expected_artifact_schema": "Profile-explicit CSV; registrations, annual reports, inspections, and aggregate summaries are separate evidence types", + "blockers": ["Capture and verify the current export schema and terms; no facility/laboratory merge or public release until review."] }, { "source_id": "us.inspections", "jurisdiction_scope": "United States; USDA APHIS inspection-report observations", "legacy_paths": ["static_data/us/inspection_reports.csv"], "url": "https://efile.aphis.usda.gov/PublicSearchTool/s/inspection-reports", - "access_method": "interactive export/browser-assisted retrieval", - "cadence": "unknown", - "attribution_licensing_notes": "Legacy script identifies USDA APHIS; verify current export terms and attribution.", - "adapter_status": "reference_only", - "expected_artifact_schema": "CSV inspection observations; not a facility master record", - "blockers": ["Confirm current export schema and link observations to facilities only through explicit, reviewable identity matching."] + "access_method": "operator-assisted APHIS public-search inspection export", + "cadence": "unknown; record selected search date", + "attribution_licensing_notes": "APHIS inspection evidence; export terms, attribution, redaction/privacy handling, and review outcome remain required", + "adapter_status": "implemented_partial", + "expected_artifact_schema": "CSV inspection observations; not a facility master record and not silently joined to registrations or FSIS", + "blockers": ["Verify current inspection export schema and use explicit, reviewable identity matching only; absence is not closure."] } ] } diff --git a/pipeline/sources/us/__init__.py b/pipeline/sources/us/__init__.py new file mode 100644 index 0000000..ad0de91 --- /dev/null +++ b/pipeline/sources/us/__init__.py @@ -0,0 +1 @@ +"""Private US source adapters.""" diff --git a/pipeline/sources/us/aphis/__init__.py b/pipeline/sources/us/aphis/__init__.py new file mode 100644 index 0000000..6876f2d --- /dev/null +++ b/pipeline/sources/us/aphis/__init__.py @@ -0,0 +1 @@ +"""USDA APHIS Animal Care observation adapters.""" diff --git a/pipeline/sources/us/aphis/adapter.py b/pipeline/sources/us/aphis/adapter.py new file mode 100644 index 0000000..4f35ef0 --- /dev/null +++ b/pipeline/sources/us/aphis/adapter.py @@ -0,0 +1,105 @@ +"""Private, profile-explicit APHIS Public Search Tool CSV adapter.""" +from __future__ import annotations +import csv, hashlib, json +from collections import Counter +from pathlib import Path +from typing import Any +from pipeline.contracts.adapter_contract import SourceArtifact +from pipeline.contracts.source_lifecycle import atomic_json, atomic_jsonl, private_manifest + +ROOT = Path(__file__).parent +CONFIG = json.loads((ROOT / "config.json").read_text(encoding="utf-8")) +PROFILES = { + "registrations": ("Account Name", "Customer Number", "Certificate Number", "License Type", "Certificate Status", "Status Date"), + "annual_reports": ("Account Name", "Customer Number_y", "Certificate Number", "Registration Type", "Certificate Status", "Year"), + "inspections": ("Account Name", "Customer Number", "Certificate Number", "Certificate Status", "Status Date"), +} + +class AphisContractError(ValueError): + pass + +def _clean(value: Any) -> str | None: + if value is None: return None + value = str(value).strip() + return value or None + +def _read(content: bytes) -> tuple[tuple[str, ...], list[dict[str, Any]]]: + try: + reader = csv.DictReader(content.decode("utf-8-sig").splitlines(), strict=True) + headers = tuple(reader.fieldnames or ()); rows = list(reader) + except (UnicodeDecodeError, csv.Error) as exc: + raise AphisContractError("malformed or unsupported APHIS CSV") from exc + if not headers or None in headers or len(headers) != len(set(headers)) or any(None in row for row in rows): + raise AphisContractError("APHIS schema drift or malformed row") + return headers, rows + +def _profile(headers: tuple[str, ...]) -> str: + if "License Type" in headers and set(PROFILES["registrations"]).issubset(headers): + return "registrations" + if "Registration Type" in headers and set(PROFILES["annual_reports"]).issubset(headers): + return "annual_reports" + if set(PROFILES["inspections"]).issubset(headers): + return "inspections" + raise AphisContractError("profile is unsupported") + +def _observation_key(profile: str, row: dict[str, Any]) -> str | None: + key = _clean(row.get("Certificate Number")) or _clean(row.get("Customer Number")) or _clean(row.get("Customer Number_y")) + if key and profile == "annual_reports": + return f"{key}:{_clean(row.get('Year'))}" + return key + +def _record(profile: str, row: dict[str, Any], line: int) -> dict[str, Any]: + certificate = _clean(row.get("Certificate Number")) + customer = _clean(row.get("Customer Number")) or _clean(row.get("Customer Number_y")) + key = certificate or customer + observation_key = _observation_key(profile, row) + normalized = { + "establishment_id": None, + "source_observation_key": observation_key, + "country_code": "US", + "evidence_type": profile, + "account_name": _clean(row.get("Account Name")), + "certificate_number": certificate, + "customer_number": customer, + "registration_or_license_type": _clean(row.get("Registration Type")) or _clean(row.get("License Type")), + "status": _clean(row.get("Certificate Status")), + "status_date": _clean(row.get("Status Date")), + "report_year": _clean(row.get("Year")), + "animal_use_fields_present": tuple(sorted(key for key, value in row.items() if key not in {"Account Name", "Customer Number", "Customer Number_y", "Certificate Number", "Registration Type", "License Type", "Certificate Status", "Status Date", "Year", "Address Line 1", "Address Line 2", "City-State-Zip", "County", "City", "State", "Zip", "latitude", "longitude", "Geocodio Latitude", "Geocodio Longitude", "Exception Report"} and _clean(value))), + "coordinates": None, + "address_state": "source-address-retained-private-pending-review", + "privacy_gate": "pending-review", + "coordinate_gate": "review_required", + "publication_gate": "blocked", + } + return {"source_id": CONFIG["source_id"], "source_row": line, "source_record_key": f"{profile}:{observation_key or 'unknown'}", "source_values": {str(k): v for k, v in row.items()}, "normalized": normalized} + +class AphisPublicSearchAdapter: + source_id = CONFIG["source_id"] + adapter_version = CONFIG["adapter_version"] + schema_version = CONFIG["contract_version"] + + def parse_bytes(self, content: bytes) -> dict[str, Any]: + digest = hashlib.sha256(content).hexdigest(); headers, rows = _read(content); profile = _profile(headers) + keys = [_observation_key(profile, row) for row in rows] + duplicates = {key for key, count in Counter(key for key in keys if key).items() if count > 1} + accepted=[]; quarantined=[] + for line, row in enumerate(rows, 2): + key = (_clean(row.get("Certificate Number")) or _clean(row.get("Customer Number")) or _clean(row.get("Customer Number_y"))) + duplicate_key = _observation_key(profile, row) + reasons=[] + if not key: reasons.append("missing_certificate_or_customer_id") + if duplicate_key is not None and duplicate_key in duplicates: reasons.append("duplicate_observation_id") + if profile == "annual_reports" and not _clean(row.get("Year")): reasons.append("missing_report_year") + record = _record(profile, row, line); (quarantined if reasons else accepted).append({"reasons": tuple(dict.fromkeys(reasons)), "record": record} if reasons else record) + return {"accepted": accepted, "quarantined": quarantined, "profile": profile, "headers": headers, "schema_fingerprint": hashlib.sha256(json.dumps(headers, separators=(",", ":")).encode()).hexdigest(), "source_sha256": digest, "input_rows": len(rows)} + + def run(self, raw_path: str | Path, run_dir: str | Path, artifact: SourceArtifact) -> dict[str, Any]: + raw=Path(raw_path).read_bytes(); digest=hashlib.sha256(raw).hexdigest() + if artifact.sha256 != digest or artifact.byte_size != len(raw): raise ValueError("artifact provenance mismatch") + result=self.parse_bytes(raw); accepted=result["accepted"]; quarantined=result["quarantined"]; parsed=accepted+[item["record"] for item in quarantined] + _, parsed_sha, _=atomic_jsonl(Path(run_dir)/"parsed/records.jsonl", parsed); _, normalized_sha, _=atomic_jsonl(Path(run_dir)/"normalized/records.jsonl", accepted); atomic_jsonl(Path(run_dir)/"quarantined/records.jsonl", quarantined) + anomalies=Counter(reason for item in quarantined for reason in item["reasons"]) + manifest=private_manifest(source_id=self.source_id, adapter_version=self.adapter_version, schema_version=self.schema_version, artifact=artifact, input_rows=result["input_rows"], normalized_rows=len(accepted), quarantined_rows=len(quarantined), normalized_sha256=normalized_sha, parsed_sha256=parsed_sha, anomaly_counts=dict(sorted(anomalies.items()))) + manifest.update({"source_profile": result["profile"], "schema_fingerprint": result["schema_fingerprint"], "entity_policy": CONFIG["entity_policy"], "geocoding": "disabled", "coverage": f"APHIS Public Search Tool {result['profile']} observations only; other APHIS profiles excluded"}) + atomic_json(Path(run_dir)/"manifest.json", manifest); return manifest diff --git a/pipeline/sources/us/aphis/config.json b/pipeline/sources/us/aphis/config.json new file mode 100644 index 0000000..491673b --- /dev/null +++ b/pipeline/sources/us/aphis/config.json @@ -0,0 +1,13 @@ +{ + "source_id": "us.aphis", + "contract_version": "us-aphis-public-search-v1", + "adapter_version": "us-aphis-candidate-v1", + "authority": "USDA Animal and Plant Health Inspection Service, Animal Care", + "public_search_url": "https://www.aphis.usda.gov/animal-care/awa-services/usda-animal-care-public-search-tool", + "annual_reports_url": "https://direct.aphis.usda.gov/awa/research-facility-report/annual-summary", + "inspection_reports_url": "https://direct.aphis.usda.gov/awa/annual-inspection-reports", + "profiles": ["registrations", "annual_reports", "inspections"], + "acquisition": "operator-assisted-public-search-export", + "release_allowed_by_default": false, + "entity_policy": "registrations, annual reports, inspections, laboratories, and aggregates remain separate evidence types; no facility merge" +} diff --git a/pipeline/sources/us/aphis/fixtures/annual_reports.csv b/pipeline/sources/us/aphis/fixtures/annual_reports.csv new file mode 100644 index 0000000..444e87c --- /dev/null +++ b/pipeline/sources/us/aphis/fixtures/annual_reports.csv @@ -0,0 +1,2 @@ +Customer Number_x,Account Name,Certificate Number,Registration Type,Certificate Status,Status Date,Address Line 1,Address Line 2,City-State-Zip,County,Customer Number_y,Year,Dogs,Cats,Guinea Pigs,Hamsters,Rabbits,Non-Human Primates,Sheep,Pigs,Other Farm Animals,All Other Animals,latitude,longitude,Exception Report +Synthetic Laboratory,2,00-R-0002,Class R - Research Facility,Active,2026-01-01,Research Office,,Testville,Example,2,2025,,1,2,,,,3,4,5,,32.1,-96.7,False diff --git a/pipeline/sources/us/aphis/fixtures/inspections.csv b/pipeline/sources/us/aphis/fixtures/inspections.csv new file mode 100644 index 0000000..84de78d --- /dev/null +++ b/pipeline/sources/us/aphis/fixtures/inspections.csv @@ -0,0 +1,2 @@ +Account Name,Customer Number,Certificate Number,Certificate Status,Status Date,Address Line 1,Address Line 2,City-State-Zip,County,City,State,Zip,Geocodio Latitude,Geocodio Longitude +Synthetic Exhibitor,3,00-C-0003,Active,2026-02-01,Exhibit Office,,Testville,Example,Testville,TX,75001,32.1,-96.7 diff --git a/pipeline/sources/us/aphis/fixtures/registrations.csv b/pipeline/sources/us/aphis/fixtures/registrations.csv new file mode 100644 index 0000000..4175e17 --- /dev/null +++ b/pipeline/sources/us/aphis/fixtures/registrations.csv @@ -0,0 +1,2 @@ +Account Name,Customer Number,Certificate Number,License Type,Certificate Status,Status Date,Address Line 1,Address Line 2,City-State-Zip,County,City,State,Zip,Geocodio Latitude,Geocodio Longitude +Synthetic Registrant,1,00-B-0001,Class B - Dealer,Active,2026-01-01,Office,,Testville,Example,Testville,TX,75001,32.1,-96.7 diff --git a/pipeline/sources/us/aphis/refresh.py b/pipeline/sources/us/aphis/refresh.py new file mode 100644 index 0000000..ce3c5b6 --- /dev/null +++ b/pipeline/sources/us/aphis/refresh.py @@ -0,0 +1,30 @@ +"""Assisted private refresh for APHIS public-search exports.""" +from __future__ import annotations +import argparse, hashlib, json +from pathlib import Path +from pipeline.common.orchestrator import run_private_lifecycle +from pipeline.common.acquisition import utc_now +from pipeline.contracts.adapter_contract import SourceArtifact +from pipeline.contracts.source_lifecycle import atomic_json +from .adapter import CONFIG, AphisPublicSearchAdapter + +def assisted_capture_contract(profile: str) -> dict: + if profile not in CONFIG["profiles"]: raise ValueError("unsupported APHIS profile") + urls={"registrations":CONFIG["public_search_url"],"annual_reports":CONFIG["annual_reports_url"],"inspections":CONFIG["inspection_reports_url"]} + return {"source_id": CONFIG["source_id"], "profile": profile, "method": "operator-assisted-public-search-export", "source_url": urls[profile], "steps": ["Open the APHIS Animal Care Public Search Tool or the documented annual-summary page.", f"Select the {profile.replace('_',' ')} view and use only its documented export/download control.", "Save the export without editing it; retain the query parameters, displayed date/year, and final URL in the run metadata.", "Run dry-run and review profile, schema, duplicate IDs, missing dates, privacy, and coverage before private test-only import."], "boundaries": ["No automation against hidden endpoints or access-control bypass", "Registrations/licenses, annual reports, inspections, laboratories, and aggregate summaries are separate evidence types", "Absence is not closure and an inspection is not a facility-master assertion"]} + +def refresh(*, run_dir: str | Path, raw_path: str | Path, profile: str, source_url: str | None = None, retrieved_at_utc: str | None = None, effective_date: str | None = None) -> dict: + path=Path(raw_path); raw=path.read_bytes(); adapter=AphisPublicSearchAdapter(); metadata={"acquisition_method":"preserved_local_artifact","source_id":CONFIG["source_id"],"profile":profile,"artifact":path.name,"artifact_path":str(path),"requested_url":source_url or CONFIG["public_search_url"],"final_url":source_url or CONFIG["public_search_url"],"retrieved_at_utc":retrieved_at_utc or utc_now(),"effective_date":effective_date or "unknown","sha256":hashlib.sha256(raw).hexdigest(),"byte_size":len(raw),"code_version":adapter.adapter_version,"config_version":adapter.schema_version,"rights_caveat":"APHIS export terms and attribution require operator review","privacy_caveat":"restricted private staging; address and coordinate review pending","coverage":f"APHIS {profile} export only; no facility merge","terms_review":"required before publication"} + result=adapter.parse_bytes(raw) + if result["profile"] != profile: raise ValueError(f"captured APHIS profile is {result['profile']}, expected {profile}") + root=Path(run_dir); atomic_json(root/"acquisition-metadata.json",metadata) + artifact=SourceArtifact(source_url=metadata["final_url"],retrieved_at_utc=metadata["retrieved_at_utc"],sha256=metadata["sha256"],byte_size=metadata["byte_size"],effective_date=effective_date or "unknown",code_version=adapter.adapter_version,config_version=adapter.schema_version,rights_caveat=metadata["rights_caveat"],privacy_caveat=metadata["privacy_caveat"],coverage=metadata["coverage"]) + status=run_private_lifecycle(path,root,artifact,adapter,health_as_of_utc=metadata["retrieved_at_utc"]); contract=assisted_capture_contract(profile); status["assisted_capture_contract"]=contract; atomic_json(root/"assisted-capture-contract.json",contract); return status + +def main() -> int: + parser=argparse.ArgumentParser(description=__doc__); parser.add_argument("--raw",type=Path,required=True); parser.add_argument("--run-dir",type=Path,required=True); parser.add_argument("--profile",choices=CONFIG["profiles"],required=True); parser.add_argument("--source-url"); parser.add_argument("--retrieved-at-utc"); parser.add_argument("--effective-date"); args=parser.parse_args() + try: result=refresh(run_dir=args.run_dir,raw_path=args.raw,profile=args.profile,source_url=args.source_url,retrieved_at_utc=args.retrieved_at_utc,effective_date=args.effective_date) + except (OSError,ValueError) as exc: print(json.dumps({"status":"failed","error":str(exc)})); return 2 + print(json.dumps({"status":result.get("status"),"run_dir":result.get("run_dir")},sort_keys=True)); return 0 + +if __name__ == "__main__": raise SystemExit(main()) diff --git a/pipeline/sources/us/aphis/test_adapter.py b/pipeline/sources/us/aphis/test_adapter.py new file mode 100644 index 0000000..0baf565 --- /dev/null +++ b/pipeline/sources/us/aphis/test_adapter.py @@ -0,0 +1,34 @@ +import unittest +from pathlib import Path +from .adapter import AphisContractError, AphisPublicSearchAdapter + +ROOT=Path(__file__).parent + +class AphisAdapterTests(unittest.TestCase): + def test_profiles_remain_distinct(self): + adapter=AphisPublicSearchAdapter() + for profile in ("registrations","annual_reports","inspections"): + with self.subTest(profile=profile): + result=adapter.parse_bytes((ROOT/f"fixtures/{profile}.csv").read_bytes()) + self.assertEqual(result["profile"],profile); self.assertEqual(len(result["accepted"]),1) + row=result["accepted"][0] + self.assertEqual(row["normalized"]["evidence_type"],profile) + self.assertIsNone(row["normalized"]["establishment_id"]) + self.assertEqual(row["normalized"]["publication_gate"],"blocked") + + def test_annual_report_requires_year_and_duplicate_ids_quarantine(self): + raw=(ROOT/"fixtures/annual_reports.csv").read_text(encoding="utf-8").replace(",2025,", ",,") + result=AphisPublicSearchAdapter().parse_bytes(raw.encode()) + self.assertEqual(len(result["quarantined"]),1); self.assertIn("missing_report_year",result["quarantined"][0]["reasons"]) + + def test_annual_reports_use_year_in_observation_identity(self): + raw=(ROOT/"fixtures/annual_reports.csv").read_text(encoding="utf-8") + second=raw.splitlines()[1].replace(",2025,", ",2024,") + result=AphisPublicSearchAdapter().parse_bytes((raw + second + "\n").encode()) + self.assertEqual(len(result["accepted"]),2) + self.assertNotEqual(result["accepted"][0]["source_record_key"], result["accepted"][1]["source_record_key"]) + + def test_unsupported_profile_fails_closed(self): + with self.assertRaises(AphisContractError): AphisPublicSearchAdapter().parse_bytes(b"Name,Value\nA,B\n") + +if __name__ == "__main__": unittest.main() diff --git a/pipeline/sources/us/fsis/__init__.py b/pipeline/sources/us/fsis/__init__.py new file mode 100644 index 0000000..07a909e --- /dev/null +++ b/pipeline/sources/us/fsis/__init__.py @@ -0,0 +1 @@ +"""USDA FSIS MPI directory adapter.""" diff --git a/pipeline/sources/us/fsis/adapter.py b/pipeline/sources/us/fsis/adapter.py new file mode 100644 index 0000000..5969083 --- /dev/null +++ b/pipeline/sources/us/fsis/adapter.py @@ -0,0 +1,117 @@ +"""Fail-closed private adapter for the FSIS MPI directory CSV.""" +from __future__ import annotations + +import csv +import hashlib +import json +from collections import Counter +from pathlib import Path +from typing import Any + +from pipeline.contracts.adapter_contract import SourceArtifact +from pipeline.contracts.candidate_handoff import write_handoff +from pipeline.contracts.source_lifecycle import atomic_json, atomic_jsonl, private_manifest + +ROOT = Path(__file__).parent +CONFIG = json.loads((ROOT / "config.json").read_text(encoding="utf-8")) +V1_HEADER = Path(__file__).resolve().parents[4] / "static_data/us/locations.csv" +V1_COLUMNS = tuple(next(csv.reader([V1_HEADER.read_text(encoding="utf-8-sig").splitlines()[0]]))) +CORE_COLUMNS = ("establishment_id", "establishment_number", "establishment_name", "street", "city", "state", "zip", "phone", "grant_date", "type", "dbas", "district", "circuit", "size", "latitude", "longitude", "county", "fips_code") +ACTIVITY_COLUMNS = frozenset(column for column in V1_COLUMNS if column.endswith("_slaughter") or column.endswith("_processing") or column in {"slaughter", "processing", "egg_processing", "ratite_processing", "siluriformes_processing"}) +ALLOWED_STATES = frozenset("AL AK AZ AR CA CO CT DE FL GA HI ID IL IN IA KS KY LA ME MD MA MI MN MS MO MT NE NV NH NJ NM NY NC ND OH OK OR PA RI SC SD TN TX UT VT VA WA WV WI WY DC PR VI GU AS MP".split()) + + +class FsisContractError(ValueError): + """The captured artifact is not a supported FSIS profile.""" + + +def _clean(value: Any) -> str | None: + if value is None: + return None + value = str(value).strip() + return value or None + + +def _schema_fingerprint(headers: tuple[str, ...]) -> str: + return hashlib.sha256(json.dumps(headers, ensure_ascii=False, separators=(",", ":")).encode()).hexdigest() + + +def _csv(content: bytes) -> tuple[tuple[str, ...], list[dict[str, Any]]]: + try: + reader = csv.DictReader(content.decode("utf-8-sig").splitlines(), strict=True) + headers = tuple(reader.fieldnames or ()) + rows = list(reader) + except (UnicodeDecodeError, csv.Error) as exc: + raise FsisContractError("malformed or unsupported UTF-8 CSV") from exc + if not headers: + raise FsisContractError("missing FSIS header") + if None in headers or len(set(headers)) != len(headers): + raise FsisContractError("duplicate or unnamed FSIS columns") + if any(None in row for row in rows): + raise FsisContractError("schema drift: row has extra columns") + if not {"establishment_id", "establishment_name", "state"}.issubset(headers): + raise FsisContractError("unsupported FSIS profile: missing facility identity fields") + return headers, rows + + +def _record(row: dict[str, Any], line: int) -> dict[str, Any]: + source_values = {str(key): value for key, value in row.items()} + activities = tuple(key for key in sorted(ACTIVITY_COLUMNS) if _clean(row.get(key))) + normalized = { + "establishment_id": _clean(row.get("establishment_id")), + "establishment_number": _clean(row.get("establishment_number")), + "canonical_name": _clean(row.get("establishment_name")), + "country_code": "US", + "city": _clean(row.get("city")), "state": _clean(row.get("state")), "postal_code": _clean(row.get("zip")), + "county": _clean(row.get("county")), "district": _clean(row.get("district")), "circuit": _clean(row.get("circuit")), + "size": _clean(row.get("size")), "source_type": _clean(row.get("type")), "activities": activities, + "activity_categories": tuple(sorted({"slaughter" if key.endswith("_slaughter") or key == "slaughter" else "processing" for key in activities})), + "grant_date": _clean(row.get("grant_date")), "coordinates": None, + "coordinate_state": "source-value-present-pending-review" if _clean(row.get("latitude")) or _clean(row.get("longitude")) else "unknown", + "address_state": "source-address-retained-private-pending-review" if _clean(row.get("street")) else "unknown", + "privacy_gate": "pending-review", "coordinate_gate": "review_required", "publication_gate": "blocked", + } + return {"source_id": CONFIG["source_id"], "source_row": line, "source_record_key": normalized["establishment_id"], "source_values": source_values, "normalized": normalized} + + +class FsisMpiAdapter: + source_id = CONFIG["source_id"] + adapter_version = CONFIG["adapter_version"] + schema_version = CONFIG["contract_version"] + + def parse_bytes(self, content: bytes) -> dict[str, Any]: + digest = hashlib.sha256(content).hexdigest() + headers, rows = _csv(content) + accepted: list[dict[str, Any]] = [] + quarantined: list[dict[str, Any]] = [] + keys = [_clean(row.get("establishment_id")) for row in rows] + duplicates = {key for key, count in Counter(key for key in keys if key).items() if count > 1} + for line, row in enumerate(rows, 2): + reasons: list[str] = [] + identifier = _clean(row.get("establishment_id")); state = (_clean(row.get("state")) or "").upper() + if not identifier: reasons.append("missing_establishment_id") + if identifier in duplicates: reasons.append("duplicate_establishment_id") + if state and state not in ALLOWED_STATES: reasons.append("unknown_state") + if not _clean(row.get("establishment_name")): reasons.append("missing_establishment_name") + record = _record(row, line) + (quarantined if reasons else accepted).append({"reasons": tuple(dict.fromkeys(reasons)), "record": record} if reasons else record) + return {"accepted": accepted, "quarantined": quarantined, "source_sha256": digest, "schema_fingerprint": _schema_fingerprint(headers), "headers": headers, "input_rows": len(rows)} + + def run(self, raw_path: str | Path, run_dir: str | Path, artifact: SourceArtifact) -> dict[str, Any]: + raw = Path(raw_path).read_bytes(); digest = hashlib.sha256(raw).hexdigest() + if artifact.sha256 != digest or artifact.byte_size != len(raw): raise ValueError("artifact provenance mismatch") + result = self.parse_bytes(raw); accepted = result["accepted"]; quarantined = result["quarantined"] + parsed = accepted + [item["record"] for item in quarantined] + _, parsed_sha, _ = atomic_jsonl(Path(run_dir) / "parsed/records.jsonl", parsed) + _, normalized_sha, _ = atomic_jsonl(Path(run_dir) / "normalized/records.jsonl", accepted) + atomic_jsonl(Path(run_dir) / "quarantined/records.jsonl", quarantined) + anomalies = Counter(reason for item in quarantined for reason in item["reasons"]) + manifest = private_manifest(source_id=self.source_id, adapter_version=self.adapter_version, schema_version=self.schema_version, artifact=artifact, input_rows=result["input_rows"], normalized_rows=len(accepted), quarantined_rows=len(quarantined), normalized_sha256=normalized_sha, parsed_sha256=parsed_sha, anomaly_counts=dict(sorted(anomalies.items()))) + manifest.update({"schema_fingerprint": result["schema_fingerprint"], "source_profile": "fsis-mpi-directory-plus-demographics", "geocoding": "disabled", "coverage": "FSIS-regulated meat, poultry, and egg establishments in the captured edition; state-inspection programs and non-FSIS populations excluded"}) + atomic_json(Path(run_dir) / "manifest.json", manifest) + return manifest + + def write_candidate_handoff(self, run_dir: str | Path, artifact: SourceArtifact, *, output_dir: str | Path | None = None) -> dict[str, Any]: + root = Path(run_dir) + rows = [json.loads(line) for line in (root / "normalized/records.jsonl").read_text(encoding="utf-8").splitlines() if line] + return write_handoff(output_dir or root, rows, artifact, source_id=self.source_id, profile="us-fsis-test-only") diff --git a/pipeline/sources/us/fsis/config.json b/pipeline/sources/us/fsis/config.json new file mode 100644 index 0000000..daeb12e --- /dev/null +++ b/pipeline/sources/us/fsis/config.json @@ -0,0 +1,13 @@ +{ + "source_id": "us.fsis", + "contract_version": "us-fsis-mpi-v1", + "adapter_version": "us-fsis-candidate-v1", + "authority": "USDA Food Safety and Inspection Service", + "directory_url": "https://www.fsis.usda.gov/inspection/establishments/meat-poultry-and-egg-product-inspection-directory", + "data_url": "https://www.fsis.usda.gov/inspection/establishments/meat-poultry-and-egg-product-inspection-directory", + "cadence": "weekly replacement; verify current edition before each run", + "format": "CSV directory and supplemental establishment-demographic CSV", + "release_allowed_by_default": false, + "privacy_default": "address-and-coordinate-review-required", + "category_policy": "preserve source activity vocabulary; do not collapse species, inspection systems, exemptions, or volume categories" +} diff --git a/pipeline/sources/us/fsis/fixtures/malformed.csv b/pipeline/sources/us/fsis/fixtures/malformed.csv new file mode 100644 index 0000000..b1a3e76 --- /dev/null +++ b/pipeline/sources/us/fsis/fixtures/malformed.csv @@ -0,0 +1,4 @@ +establishment_id,establishment_number,establishment_name,street,city,state,zip,phone,grant_date,type +FSIS-001,M001,Example Meat Plant,100 Industrial Way,Testville,TX,75001,,2026-01-01,Meat Slaughter +FSIS-001,M002,Duplicate,200 Industrial Way,Testville,TX,75002,,2026-01-01,Meat Processing +FSIS-003,M003,,300 Industrial Way,Testville,ZZ,75003,,2026-01-01,Meat Processing diff --git a/pipeline/sources/us/fsis/fixtures/valid.csv b/pipeline/sources/us/fsis/fixtures/valid.csv new file mode 100644 index 0000000..4102dc3 --- /dev/null +++ b/pipeline/sources/us/fsis/fixtures/valid.csv @@ -0,0 +1,3 @@ +establishment_id,establishment_number,establishment_name,street,city,state,zip,phone,grant_date,type,dbas,district,circuit,size,latitude,longitude,county,fips_code,meat_slaughter,poultry_slaughter,meat_processing,poultry_processing,processing_volume_category +FSIS-001,M001,Example Meat Plant,100 Industrial Way,Testville,TX,75001,555-0100,2026-01-01,Meat Slaughter; Meat Processing,Example Brand,5,501,Small,32.1,-96.7,Example County,48001,Yes,,Yes,,2.0 +FSIS-002,P002,Example Poultry Plant,200 Industrial Way,Testville,GA,30001,,2026-01-02,Poultry Slaughter; Poultry Processing,,4,402,Very Small,,,Example County,13001,,Yes,,Yes,1.0 diff --git a/pipeline/sources/us/fsis/refresh.py b/pipeline/sources/us/fsis/refresh.py new file mode 100644 index 0000000..772868c --- /dev/null +++ b/pipeline/sources/us/fsis/refresh.py @@ -0,0 +1,51 @@ +"""Private FSIS refresh with a sanctioned assisted-acquisition boundary.""" +from __future__ import annotations +import argparse, hashlib, json +from pathlib import Path +from pipeline.common.acquisition import AcquisitionError, fetch_source, utc_now +from pipeline.common.orchestrator import run_private_lifecycle +from pipeline.contracts.adapter_contract import SourceArtifact +from pipeline.contracts.source_lifecycle import atomic_json +from .adapter import CONFIG, FsisMpiAdapter + +def assisted_capture_contract(*, source_url: str = CONFIG["directory_url"]) -> dict: + return {"source_id": CONFIG["source_id"], "method": "operator-assisted-official-export", "steps": ["Open the official FSIS MPI Directory page in an authorized browser session.", "Select the current downloadable MPI Directory and Establishment Demographic CSV files shown on that page.", "Save the files without editing them; record the displayed edition/publication date and the final download URLs.", "Place the selected directory CSV at the --raw path or use --fetch only when a terms-reviewed direct URL is authorized.", "Run dry-run first and review schema, count, duplicate, privacy, and category results before any test-only handoff."], "controls": ["No credential or access-control bypass", "HTML, login, 403, and schema-drift responses fail closed", "No raw artifact in Git", "FSIS facility evidence is not joined to APHIS rows"], "source_url": source_url} + +def _local_facts(path: Path, *, retrieved_at_utc: str, effective_date: str | None) -> dict: + raw = path.read_bytes() + return {"acquisition_method": "preserved_local_artifact", "source_id": CONFIG["source_id"], "artifact": path.name, "artifact_path": str(path), "requested_url": CONFIG["directory_url"], "final_url": CONFIG["directory_url"], "retrieved_at_utc": retrieved_at_utc, "effective_date": effective_date or "unknown", "publication_date": None, "sha256": hashlib.sha256(raw).hexdigest(), "byte_size": len(raw), "code_version": CONFIG["adapter_version"], "config_version": CONFIG["contract_version"], "rights_caveat": "FSIS source terms and attribution require operator review before publication", "privacy_caveat": "private staging; address, phone, DUNS, and coordinate review pending", "coverage": "FSIS MPI edition only; state-inspection and APHIS populations excluded", "terms_review": "required before network acquisition or handoff"} + +def refresh(*, run_dir: str | Path, raw_path: str | Path | None = None, fetch: bool = False, source_url: str = CONFIG["directory_url"], terms_review_path: str | Path | None = None, retrieved_at_utc: str | None = None, effective_date: str | None = None, mode: str = "dry-run", max_bytes: int = 128 * 1024 * 1024) -> dict: + if fetch == (raw_path is not None): raise ValueError("specify exactly one of raw_path or fetch") + if mode not in {"dry-run", "handoff"}: raise ValueError("mode must be dry-run or handoff") + root = Path(run_dir) + if fetch: + if terms_review_path is None: raise ValueError("terms_review_path is required for network acquisition") + try: acquisition = fetch_source(source_id=CONFIG["source_id"], url=source_url, output_root=root / "acquisition", artifact_name="source.csv", terms_review_path=terms_review_path, max_bytes=max_bytes, allowed_content_types=("text/csv", "application/csv", "application/octet-stream"), code_version=CONFIG["adapter_version"], config_version=CONFIG["contract_version"], coverage="FSIS MPI edition only; state-inspection and APHIS populations excluded", rights_caveat="terms review retained with run", privacy_caveat="private staging; privacy review pending", effective_date=effective_date) + except AcquisitionError as exc: raise ValueError(str(exc)) from exc + path = Path(acquisition["artifact_path"]) + else: + path = Path(raw_path) # type: ignore[arg-type] + if not path.is_file(): raise ValueError(f"raw artifact does not exist: {path}") + acquisition = _local_facts(path, retrieved_at_utc=retrieved_at_utc or utc_now(), effective_date=effective_date) + raw = path.read_bytes(); acquired_at = acquisition.get("retrieved_at_utc") or retrieved_at_utc or utc_now() + artifact = SourceArtifact(source_url=str(acquisition.get("final_url") or source_url), retrieved_at_utc=str(acquired_at), sha256=hashlib.sha256(raw).hexdigest(), byte_size=len(raw), publication_date=acquisition.get("publication_date"), effective_date=acquisition.get("effective_date") or effective_date, code_version=CONFIG["adapter_version"], config_version=CONFIG["contract_version"], rights_caveat=acquisition.get("rights_caveat"), privacy_caveat=acquisition.get("privacy_caveat"), coverage=acquisition.get("coverage"), redirects=tuple(acquisition.get("redirects") or ())) + atomic_json(root / "acquisition-metadata.json", acquisition) + status = run_private_lifecycle(path, root, artifact, FsisMpiAdapter(), health_as_of_utc=str(acquired_at)) + if mode == "handoff" and status.get("status") in {"candidate-ready", "success"}: + lifecycle_run = Path(status["run_dir"]) + status["handoff"] = FsisMpiAdapter().write_candidate_handoff(lifecycle_run, artifact, output_dir=lifecycle_run / "handoff") + status["mode"] = mode + if status.get("run_dir"): + atomic_json(Path(status["run_dir"]) / "run-status.json", status) + status["assisted_capture_contract"] = assisted_capture_contract(source_url=source_url); atomic_json(root / "assisted-capture-contract.json", status["assisted_capture_contract"]) + return status + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__); source = parser.add_mutually_exclusive_group(required=True); source.add_argument("--raw", type=Path); source.add_argument("--fetch", action="store_true") + parser.add_argument("--run-dir", type=Path, required=True); parser.add_argument("--source-url", default=CONFIG["directory_url"]); parser.add_argument("--terms-review", type=Path); parser.add_argument("--retrieved-at-utc"); parser.add_argument("--effective-date"); parser.add_argument("--mode", choices=("dry-run", "handoff"), default="dry-run"); parser.add_argument("--max-bytes", type=int, default=128 * 1024 * 1024); args = parser.parse_args() + try: result = refresh(run_dir=args.run_dir, raw_path=args.raw, fetch=args.fetch, source_url=args.source_url, terms_review_path=args.terms_review, retrieved_at_utc=args.retrieved_at_utc, effective_date=args.effective_date, mode=args.mode, max_bytes=args.max_bytes) + except (OSError, ValueError) as exc: print(json.dumps({"status": "failed", "error": str(exc)})); return 2 + print(json.dumps({"status": result.get("status"), "run_dir": result.get("run_dir"), "manifest": result.get("manifest", {}).get("source_id")}, sort_keys=True)); return 0 + +if __name__ == "__main__": raise SystemExit(main()) diff --git a/pipeline/sources/us/fsis/test_adapter.py b/pipeline/sources/us/fsis/test_adapter.py new file mode 100644 index 0000000..b57b202 --- /dev/null +++ b/pipeline/sources/us/fsis/test_adapter.py @@ -0,0 +1,37 @@ +import hashlib +import json +import tempfile +import unittest +from pathlib import Path + +from pipeline.contracts.adapter_contract import SourceArtifact +from .adapter import FsisContractError, FsisMpiAdapter + +ROOT = Path(__file__).parent + +class FsisAdapterTests(unittest.TestCase): + def test_valid_fixture_preserves_source_values_and_blocks_coordinates(self): + raw = (ROOT / "fixtures/valid.csv").read_bytes(); adapter = FsisMpiAdapter(); result = adapter.parse_bytes(raw) + self.assertEqual(result["input_rows"], 2); self.assertEqual(len(result["accepted"]), 2); self.assertFalse(result["quarantined"]) + row = result["accepted"][0]; self.assertEqual(row["source_values"]["phone"], "555-0100") + self.assertIsNone(row["normalized"]["coordinates"]); self.assertEqual(row["normalized"]["country_code"], "US") + + def test_duplicates_missing_name_and_unknown_state_quarantine(self): + result = FsisMpiAdapter().parse_bytes((ROOT / "fixtures/malformed.csv").read_bytes()) + self.assertEqual(len(result["accepted"]), 0); self.assertEqual(len(result["quarantined"]), 3) + reasons = [set(item["reasons"]) for item in result["quarantined"]] + self.assertTrue(all("duplicate_establishment_id" in reason for reason in reasons[:2])) + self.assertIn("unknown_state", reasons[2]); self.assertIn("missing_establishment_name", reasons[2]) + + def test_schema_drift_fails_closed(self): + with self.assertRaises(FsisContractError): FsisMpiAdapter().parse_bytes(b"wrong,header\n1,2\n") + + def test_run_is_deterministic_and_private(self): + raw=(ROOT/"fixtures/valid.csv").read_bytes(); artifact=SourceArtifact("https://example.invalid/fsis.csv","2026-09-15T00:00:00Z",hashlib.sha256(raw).hexdigest(),len(raw),effective_date="2026-09-01",code_version="test",config_version="test") + with tempfile.TemporaryDirectory() as directory: + manifest=FsisMpiAdapter().run(ROOT/"fixtures/valid.csv",directory,artifact) + self.assertEqual(manifest["release_state"],"not-created"); self.assertEqual(manifest["publication_state"],"private-candidate") + self.assertEqual(manifest["input_rows"],manifest["normalized_rows"]+manifest["quarantined_rows"]) + self.assertTrue((Path(directory)/"parsed/records.jsonl").exists()); self.assertEqual(json.loads((Path(directory)/"normalized/records.jsonl").read_text().splitlines()[0])["normalized"]["publication_gate"],"blocked") + +if __name__ == "__main__": unittest.main() From b945086bb75f340da6d00aff7d843fca0bdebdbf Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Tue, 15 Sep 2026 21:59:09 -0700 Subject: [PATCH 129/311] Reconcile shared review packet contracts --- pipeline/common/orchestrator.py | 7 ++--- pipeline/common/source_operations.py | 42 ++++++++++++++++++++++++---- 2 files changed, 40 insertions(+), 9 deletions(-) diff --git a/pipeline/common/orchestrator.py b/pipeline/common/orchestrator.py index af438ed..c116d8c 100644 --- a/pipeline/common/orchestrator.py +++ b/pipeline/common/orchestrator.py @@ -12,7 +12,6 @@ from pipeline.contracts.adapter_contract import SourceAdapter, SourceArtifact, source_artifact_from_mapping from pipeline.contracts.private_run import write_private_run_report from pipeline.contracts.source_health import build_health_snapshot, write_health_snapshot -from pipeline.common.review_packet import write_review_packet from .identity import record_key from .source_operations import classify_failure, finalize_run_operations @@ -117,8 +116,6 @@ def run_registered_input(raw_path: str | Path, runs_dir: str | Path, config: dic "attempts": getattr(exc, "attempts", []), "prior_eligible_release": prior_eligible_release, "run_dir": str(run_dir)} - if status.get("status") != "failed": - write_review_packet(run_dir, previous_normalized_path=previous_normalized_path, blockers=review_blockers) status["run_dir"] = str(run_dir) status["run_id"] = config.get("run_id") or run_dir.name # The operations ledger is derived from the private run and is append-only. @@ -128,7 +125,9 @@ def run_registered_input(raw_path: str | Path, runs_dir: str | Path, config: dic try: status = finalize_run_operations( runs.parent, run_dir, manifest=status.get("manifest"), status=status, - config=config, prior_eligible_release=prior_eligible_release, + config={**config, "review_blockers": review_blockers or {}}, + previous_normalized_path=previous_normalized_path, + prior_eligible_release=prior_eligible_release, ) except Exception as exc: failure = classify_failure(exc) diff --git a/pipeline/common/source_operations.py b/pipeline/common/source_operations.py index a7504f2..84666aa 100644 --- a/pipeline/common/source_operations.py +++ b/pipeline/common/source_operations.py @@ -22,7 +22,7 @@ from typing import Any, Callable, Iterable from .acquisition import AcquisitionError -from .delta import compare_runs +from .delta import compare_normalized_paths, compare_runs from pipeline.contracts.source_lifecycle import atomic_bytes, atomic_json @@ -333,9 +333,12 @@ def _provenance(manifest: dict[str, Any]) -> dict[str, Any]: return {key: manifest[key] for key in keys if manifest.get(key) is not None} -def build_release_diff(previous_run_dir: str | Path | None, current_run_dir: str | Path, *, prior_eligible_release: dict[str, Any] | None = None) -> dict[str, Any]: +def build_release_diff(previous_run_dir: str | Path | None, current_run_dir: str | Path, *, previous_normalized_path: str | Path | None = None, prior_eligible_release: dict[str, Any] | None = None) -> dict[str, Any]: """Build a row-free, deterministic diff for a human operator.""" current = Path(current_run_dir) + if previous_run_dir is None and previous_normalized_path is not None: + normalized_delta = compare_normalized_paths(previous_normalized_path, current / "normalized" / "records.jsonl") + return {"schema_version": "release-diff-v1", **normalized_delta, "release_promoted": False, "prior_eligible_release": prior_eligible_release} if previous_run_dir is None: return { "schema_version": "release-diff-v1", "status": "no-previous-validated-run", @@ -352,6 +355,7 @@ def build_review_packet( manifest: dict[str, Any] | None, status: dict[str, Any], previous_run_dir: str | Path | None = None, + previous_normalized_path: str | Path | None = None, prior_eligible_release: dict[str, Any] | None = None, ) -> dict[str, Any]: """Create a stable operator packet containing no source rows or raw fields.""" @@ -376,15 +380,33 @@ def build_review_packet( reasons.append("publication remains behind the human/terms gate") if not reasons: reasons.append("confirm private evidence and release scope before any separate approval action") - diff = build_release_diff(previous_run_dir, run_dir, prior_eligible_release=prior_eligible_release) + diff = build_release_diff(previous_run_dir, run_dir, previous_normalized_path=previous_normalized_path, prior_eligible_release=prior_eligible_release) + counts = { + key: manifest.get(key) for key in ("input_rows", "normalized_rows", "quarantined_rows") + } + qa_counts = {key: qa.get(key) for key in counts} + blockers = status.get("review_blockers", {}) return { - "schema_version": "review-packet-v1", + "schema_version": "private-review-packet-v1", "source_id": source_id, + "run_dir_digest": _file_sha256(Path(run_dir) / "manifest.json") or _file_sha256(Path(run_dir) / "run-manifest.json"), "run_id": status.get("run_id") or Path(run_dir).name, "classification": status.get("run_classification", "failed" if failed else "changed"), "review_required": review_required, "reasons": sorted(set(reasons)), "provenance": _provenance(manifest), + "schema": { + "adapter_version": manifest.get("adapter_version"), + "schema_version": manifest.get("schema_version"), + "schema_fingerprint": manifest.get("schema_fingerprint"), + "schema_status": manifest.get("schema_status", "not-reported"), + }, + "counts": { + **counts, + "reconciles": all(isinstance(value, int) and value >= 0 for value in counts.values()) and counts["input_rows"] == counts["normalized_rows"] + counts["quarantined_rows"], + "qa_matches_manifest": counts == qa_counts, + }, + "quarantine": {"rows": manifest.get("quarantined_rows"), "reasons": manifest.get("anomaly_counts", {})}, "run": { "status": status.get("status"), "publication_state": status.get("publication_state", "unchanged"), @@ -395,6 +417,14 @@ def build_review_packet( "drift_alarms": sorted(set(qa.get("drift_alarms", []))) if isinstance(qa.get("drift_alarms", []), list) else [], }, "release_diff": diff, + "gates": { + "release_state": manifest.get("release_state", "not-created"), + "publication_state": status.get("publication_state"), + "release_promoted": status.get("release_promoted"), + "public_surfaces": status.get("public_surfaces", {surface: False for surface in ("api", "map", "export", "cache", "history")}), + "geocoding": manifest.get("geocoding", "disabled"), + }, + "blockers": blockers, "prior_eligible_release": prior_eligible_release, "release_promotion_allowed": False, "public_exposure": False, @@ -414,6 +444,7 @@ def finalize_run_operations( status: dict[str, Any], config: dict[str, Any] | None = None, previous_run_dir: str | Path | None = None, + previous_normalized_path: str | Path | None = None, prior_eligible_release: dict[str, Any] | None = None, ) -> dict[str, Any]: """Persist the shared run ledger and deterministic operator artifacts.""" @@ -441,7 +472,8 @@ def finalize_run_operations( "release_preserved": True, "release_promoted": False, }) - packet = build_review_packet(run_path, manifest=manifest, status=status, previous_run_dir=previous_run_dir, prior_eligible_release=prior_eligible_release) + status["review_blockers"] = config.get("review_blockers", {}) + packet = build_review_packet(run_path, manifest=manifest, status=status, previous_run_dir=previous_run_dir, previous_normalized_path=previous_normalized_path, prior_eligible_release=prior_eligible_release) packet_path = run_path / "review-packet.json" diff_path = run_path / "release-diff.json" atomic_json(packet_path, packet) From 87fde1136f09be22e6d65ef1e741298544746a1f Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Tue, 15 Sep 2026 22:23:49 -0700 Subject: [PATCH 130/311] Remove committed Geocodio credential literals --- dirty-datasets/it/geocode_italy.py | 11 +++- dirty-datasets/it/geocode_italy_test.py | 11 +++- dirty-datasets/it/geocode_italy_v2.py | 11 +++- docs/security/geocodio-credentials.md | 19 +++++++ pipeline/tests/test_credential_scan.py | 75 +++++++++++++++++++++++++ 5 files changed, 121 insertions(+), 6 deletions(-) create mode 100644 docs/security/geocodio-credentials.md create mode 100644 pipeline/tests/test_credential_scan.py diff --git a/dirty-datasets/it/geocode_italy.py b/dirty-datasets/it/geocode_italy.py index 67c388f..8eb083e 100644 --- a/dirty-datasets/it/geocode_italy.py +++ b/dirty-datasets/it/geocode_italy.py @@ -5,8 +5,15 @@ import json import time import sys +import os + + +def get_geocodio_api_key(): + key = os.environ.get("GEOCODIO_API_KEY", "").strip() + if not key: + raise RuntimeError("GEOCODIO_API_KEY is required for geocoding; set it in the process environment") + return key -GEOCODIO_API_KEY = "e1cd921cddddc5dddbd6bc965b1cd5c6666c229" GEOCODIO_URL = "https://api.geocod.io/v1.7/geocode" def build_address(row): @@ -23,7 +30,7 @@ def geocode_address(address): try: params = urllib.parse.urlencode({ 'q': address, - 'api_key': GEOCODIO_API_KEY + 'api_key': get_geocodio_api_key() }) url = f"{GEOCODIO_URL}?{params}" diff --git a/dirty-datasets/it/geocode_italy_test.py b/dirty-datasets/it/geocode_italy_test.py index 4fbb30b..c637bbe 100644 --- a/dirty-datasets/it/geocode_italy_test.py +++ b/dirty-datasets/it/geocode_italy_test.py @@ -5,8 +5,15 @@ import json import time import re +import os + + +def get_geocodio_api_key(): + key = os.environ.get("GEOCODIO_API_KEY", "").strip() + if not key: + raise RuntimeError("GEOCODIO_API_KEY is required for geocoding; set it in the process environment") + return key -GEOCODIO_API_KEY = "e1cd921cddddc5dddbd6bc965b1cd5c6666c229" GEOCODIO_URL = "https://api.geocod.io/v1.7/geocode" def extract_province_code(city_state_str): @@ -32,7 +39,7 @@ def geocode_address(address): try: params = urllib.parse.urlencode({ 'q': address, - 'api_key': GEOCODIO_API_KEY + 'api_key': get_geocodio_api_key() }) url = f"{GEOCODIO_URL}?{params}" diff --git a/dirty-datasets/it/geocode_italy_v2.py b/dirty-datasets/it/geocode_italy_v2.py index 6b2000b..52492a2 100644 --- a/dirty-datasets/it/geocode_italy_v2.py +++ b/dirty-datasets/it/geocode_italy_v2.py @@ -5,8 +5,15 @@ import json import time import re +import os + + +def get_geocodio_api_key(): + key = os.environ.get("GEOCODIO_API_KEY", "").strip() + if not key: + raise RuntimeError("GEOCODIO_API_KEY is required for geocoding; set it in the process environment") + return key -GEOCODIO_API_KEY = "e1cd921cddddc5dddbd6bc965b1cd5c6666c229" GEOCODIO_URL = "https://api.geocod.io/v1.7/geocode" ITALY_PROVINCES = { @@ -42,7 +49,7 @@ def geocode_address(address): params = urllib.parse.urlencode({ 'q': address, 'country': 'IT', - 'api_key': GEOCODIO_API_KEY + 'api_key': get_geocodio_api_key() }) url = f"{GEOCODIO_URL}?{params}" diff --git a/docs/security/geocodio-credentials.md b/docs/security/geocodio-credentials.md new file mode 100644 index 0000000..4c06606 --- /dev/null +++ b/docs/security/geocodio-credentials.md @@ -0,0 +1,19 @@ +# Geocodio credential handling + +The historical Italy geocoding scripts read `GEOCODIO_API_KEY` from the +process environment only when a geocoding request is invoked. Importing the +scripts and running repository tests do not contact Geocodio. If the variable +is absent, the scripts fail with an actionable configuration error. + +For a private local run, configure the variable through the process environment +or a secret manager, for example: + +```powershell +$env:GEOCODIO_API_KEY = '' +python dirty-datasets/it/geocode_italy_v2.py +``` + +Do not put a real key in Python, CSV, documentation, shell history, or a +committed environment file. The key previously present in the repository must +be treated as compromised and revoked/rotated with Geocodio outside the +repository. Removing the literal from source does not revoke or rotate it. diff --git a/pipeline/tests/test_credential_scan.py b/pipeline/tests/test_credential_scan.py new file mode 100644 index 0000000..f872309 --- /dev/null +++ b/pipeline/tests/test_credential_scan.py @@ -0,0 +1,75 @@ +"""Regression checks preventing committed non-placeholder credentials.""" + +from __future__ import annotations + +import re +import subprocess +import unittest +import importlib.util +import os +from pathlib import Path +from unittest.mock import patch + + +ROOT = Path(__file__).resolve().parents[2] +NAMED_SCRIPTS = ( + ROOT / "dirty-datasets/it/geocode_italy.py", + ROOT / "dirty-datasets/it/geocode_italy_v2.py", + ROOT / "dirty-datasets/it/geocode_italy_test.py", +) +ASSIGNMENT = re.compile(r"(?i)\b(?:api[_-]?key|access[_-]?token|client[_-]?secret|password)\b\s*[:=]\s*(['\"])([^'\"]+)\1") +PLACEHOLDER_MARKERS = ("example", "placeholder", "redacted", "dummy", "changeme", "your_", "replace_me", "test-token", "not-a-real") + + +def tracked_texts() -> list[tuple[str, str]]: + result = subprocess.run(["git", "ls-files", "-z"], cwd=ROOT, check=True, capture_output=True) + paths = [Path(raw.decode()) for raw in result.stdout.split(b"\0") if raw] + texts: list[tuple[str, str]] = [] + for relative in paths: + path = ROOT / relative + try: + texts.append((relative.as_posix(), path.read_text(encoding="utf-8"))) + except (UnicodeDecodeError, OSError): + continue + return texts + + +def embedded_non_placeholder_credentials() -> list[tuple[str, int]]: + findings: list[tuple[str, int]] = [] + for filename, text in tracked_texts(): + for line_number, line in enumerate(text.splitlines(), 1): + match = ASSIGNMENT.search(line) + if not match: + continue + candidate = match.group(2).lower() + if len(candidate) >= 12 and not any(marker in candidate for marker in PLACEHOLDER_MARKERS): + findings.append((filename, line_number)) + return findings + + +class CredentialRemediationTests(unittest.TestCase): + def test_named_geocodio_scripts_use_runtime_environment_only(self): + assignment = re.compile(r"(?im)^\s*GEOCODIO_API_KEY\s*=") + for path in NAMED_SCRIPTS: + text = path.read_text(encoding="utf-8") + self.assertIsNone(assignment.search(text), path.as_posix()) + self.assertIn("os.environ.get(\"GEOCODIO_API_KEY\", \"\")", text) + self.assertIn("get_geocodio_api_key()", text) + + def test_imports_are_network_free_and_missing_key_fails_clearly(self): + for index, path in enumerate(NAMED_SCRIPTS): + with self.subTest(path=path.name): + spec = importlib.util.spec_from_file_location(f"credential_scan_target_{index}", path) + module = importlib.util.module_from_spec(spec) + with patch("urllib.request.urlopen", side_effect=AssertionError("network access during import")): + spec.loader.exec_module(module) + with patch.dict(os.environ, {}, clear=True): + with self.assertRaisesRegex(RuntimeError, "GEOCODIO_API_KEY is required"): + module.get_geocodio_api_key() + + def test_tracked_files_have_no_obvious_non_placeholder_credentials(self): + self.assertEqual(embedded_non_placeholder_credentials(), []) + + +if __name__ == "__main__": + unittest.main() From 3b69c8ee5b98b6c039c0552e84d85fe2a3c19812 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Wed, 16 Sep 2026 00:33:32 -0700 Subject: [PATCH 131/311] Consolidate Sprint 2 integration --- docs/api/v2-contract.json | 6 +- docs/api/v2-contract.md | 2 + docs/architecture/adr-graph-foundation.md | 58 +++ docs/architecture/graph-data-dictionary.md | 31 ++ .../release-manifest-verification.md | 4 +- docs/countries/br/v1-field-crosswalk.json | 149 +++++++ docs/countries/us/README.md | 13 + docs/country-recon-br.md | 117 +++++ docs/country-rehearsal-2026-09-15.json | 39 ++ docs/country-rehearsal-2026-09-15.md | 39 ++ docs/data-dictionary.json | 34 ++ docs/data-product.md | 46 ++ docs/review-packet-us-accountability.md | 70 +++ docs/source-status.json | 5 + docs/source-status.md | 8 + pipeline/common/data_product.py | 410 ++++++++++++++++++ pipeline/common/test_data_product.py | 164 +++++++ pipeline/common/test_source_operations.py | 2 +- pipeline/contracts/GRAPH-CANDIDATE-HANDOFF.md | 17 + .../fixtures/synthetic_graph_candidate.json | 13 + pipeline/contracts/graph_candidate_handoff.py | 190 ++++++++ .../contracts/test_graph_candidate_handoff.py | 78 ++++ .../026_graph_entities_crosswalks.sql | 121 ++++++ .../027_graph_relationship_observations.sql | 89 ++++ .../migrations/028_graph_claims_support.sql | 107 +++++ .../029_graph_publication_projections.sql | 108 +++++ .../maintenance/private-environment-gate.py | 13 + .../maintenance/verify-data-product.py | 33 ++ pipeline/scripts/stages/export-release.py | 169 ++++++++ pipeline/scripts/stages/promote-release.py | 52 ++- pipeline/source_operations.json | 4 + pipeline/source_registry.json | 48 ++ pipeline/sources/belgium/adapter.py | 8 +- pipeline/sources/belgium/refresh.py | 13 + pipeline/sources/belgium/test_adapter.py | 8 + pipeline/sources/canada/adapter.py | 16 +- pipeline/sources/canada/refresh.py | 16 +- pipeline/sources/canada/test_adapter.py | 12 + pipeline/sources/denmark/pipeline.py | 23 + pipeline/sources/france/adapter.py | 16 +- pipeline/sources/france/refresh.py | 17 +- pipeline/sources/france/test_adapter.py | 11 + pipeline/sources/germany/refresh.py | 13 + pipeline/sources/italy/it_853_adapter.py | 36 +- pipeline/sources/italy/refresh.py | 13 + pipeline/sources/italy/test_it_853_adapter.py | 8 +- pipeline/sources/us/accountability/README.md | 69 +++ .../sources/us/accountability/__init__.py | 5 + pipeline/sources/us/accountability/adapter.py | 327 ++++++++++++++ .../sources/us/accountability/config.json | 11 + .../fixtures/synthetic_link_ledger.csv | 13 + .../sources/us/accountability/reconcile.py | 39 ++ pipeline/sources/us/accountability/refresh.py | 148 +++++++ .../sources/us/accountability/test_adapter.py | 127 ++++++ .../us/accountability/test_reconcile.py | 28 ++ pipeline/tests/test_export_release.py | 26 ++ .../tests/test_graph_database_contract.py | 127 ++++++ pipeline/tests/test_graph_migrations.py | 51 +++ .../tests/test_private_environment_gate.py | 14 + pipeline/tests/test_source_registry.py | 4 +- src/lib.rs | 122 +++--- 61 files changed, 3471 insertions(+), 89 deletions(-) create mode 100644 docs/architecture/adr-graph-foundation.md create mode 100644 docs/architecture/graph-data-dictionary.md create mode 100644 docs/countries/br/v1-field-crosswalk.json create mode 100644 docs/country-recon-br.md create mode 100644 docs/country-rehearsal-2026-09-15.json create mode 100644 docs/country-rehearsal-2026-09-15.md create mode 100644 docs/data-dictionary.json create mode 100644 docs/data-product.md create mode 100644 docs/review-packet-us-accountability.md create mode 100644 pipeline/common/data_product.py create mode 100644 pipeline/common/test_data_product.py create mode 100644 pipeline/contracts/GRAPH-CANDIDATE-HANDOFF.md create mode 100644 pipeline/contracts/fixtures/synthetic_graph_candidate.json create mode 100644 pipeline/contracts/graph_candidate_handoff.py create mode 100644 pipeline/contracts/test_graph_candidate_handoff.py create mode 100644 pipeline/migrations/026_graph_entities_crosswalks.sql create mode 100644 pipeline/migrations/027_graph_relationship_observations.sql create mode 100644 pipeline/migrations/028_graph_claims_support.sql create mode 100644 pipeline/migrations/029_graph_publication_projections.sql create mode 100644 pipeline/scripts/maintenance/verify-data-product.py create mode 100644 pipeline/scripts/stages/export-release.py create mode 100644 pipeline/sources/us/accountability/README.md create mode 100644 pipeline/sources/us/accountability/__init__.py create mode 100644 pipeline/sources/us/accountability/adapter.py create mode 100644 pipeline/sources/us/accountability/config.json create mode 100644 pipeline/sources/us/accountability/fixtures/synthetic_link_ledger.csv create mode 100644 pipeline/sources/us/accountability/reconcile.py create mode 100644 pipeline/sources/us/accountability/refresh.py create mode 100644 pipeline/sources/us/accountability/test_adapter.py create mode 100644 pipeline/sources/us/accountability/test_reconcile.py create mode 100644 pipeline/tests/test_export_release.py create mode 100644 pipeline/tests/test_graph_database_contract.py create mode 100644 pipeline/tests/test_graph_migrations.py diff --git a/docs/api/v2-contract.json b/docs/api/v2-contract.json index 6fbb755..fa240a1 100644 --- a/docs/api/v2-contract.json +++ b/docs/api/v2-contract.json @@ -5,9 +5,11 @@ "GET /health/live": {"success": {"status": "ok", "service": "uec-api"}}, "GET /health/ready": {"success": {"status": "ready", "database": "ok"}, "unavailable_status": 503}, "GET /health/diagnostics": {"success": {"status": "ok", "privacy": {"diagnostic_identifiers": "excluded"}}, "payloads": "coarse status only"}, - "GET /api/v2/locations": {"success": {"api_version": "v2", "data": [], "meta": {"profile": "official", "next_cursor": null, "query": {"q": null, "filters": {}}, "coverage_scope": "selected_promoted_release_public_facilities", "count_semantics": "public facility projection rows, not animal counts"}}, "query": ["q", "country_code", "region", "category", "source_type", "profile", "display_precision", "lifecycle_status", "cursor", "limit", "min_lon", "min_lat", "max_lon", "max_lat", "latitude", "longitude", "radius_km"], "spatial": "bbox or radius, never both", "no_release": 200}, + "GET /api/v2/locations": {"success": {"api_version": "v2", "data": [], "meta": {"profile": "official", "next_cursor": null, "query": {"q": null, "filters": {}}, "data_product_version": "uec-public-data-product-v1", "schema_version": "uec-location-projection-v1", "coverage_scope": "selected_promoted_release_public_facilities", "count_semantics": "public facility projection rows, not animal counts"}}, "query": ["q", "country_code", "region", "category", "source_type", "profile", "display_precision", "lifecycle_status", "cursor", "limit", "min_lon", "min_lat", "max_lon", "max_lat", "latitude", "longitude", "radius_km"], "spatial": "bbox or radius, never both", "no_release": 200}, "GET /api/v2/locations/{facility_id}": {"success": {"api_version": "v2", "data": {}, "meta": {"coverage_scope": "selected_promoted_release_public_facilities", "count_semantics": "public facility projection, not an animal count"}}, "not_found": 404}, - "GET /api/v2/locations.csv": {"success_content_type": "text/csv", "bounded_rows": 1000, "profile_required_for_nonofficial": true, "not_found_without_promoted_manifest": 404}, + "GET /api/v2/locations.csv": {"success_content_type": "text/csv", "bounded_rows": 1000, "profile_required_for_nonofficial": true, "not_found_without_promoted_manifest": 404, "row_contract": "docs/data-dictionary.json", "rights_field": "source_rights_status"}, + "bulk_snapshot_cli": {"command": "pipeline/scripts/stages/export-release.py RELEASE_ID --profile PROFILE --output-dir DIR", "formats": ["csv", "geojson"], "artifacts": ["manifest.json", "SHA256SUMS.json", "data-dictionary.json"], "gate": "explicitly promoted non-test release/profile and release-scoped public projection"}, + "GET /api/v2/releases/manifest": {"success": {"api_version": "v2", "data": {"release_id": "string", "profile": "string", "manifest": "versioned release metadata", "manifest_sha256": "sha256"}}, "not_found_without_promoted_manifest": 404}, "GET /api/v2/discovery/filters": {"success": {"api_version": "v2", "contract_version": "v1", "dimensions": {"country_code": "allowlist", "region": "release facets", "category": "allowlist", "source_type": "allowlist", "profile": "allowlist", "display_precision": "allowlist", "lifecycle_status": "allowlist"}, "search": {"fields": ["canonical_name", "city", "country_code", "category", "source_name"]}, "spatial": {"bbox": ["min_lon", "min_lat", "max_lon", "max_lat"], "radius": ["latitude", "longitude", "radius_km"]}}} ,"GET /api/dev/preview/candidates": {"success": {"api_version": "dev-preview-v1", "data": [], "meta": {"test_only": true, "private_preview": true, "profile": null, "coverage_scope": "candidate_release_only", "next_cursor": null}}, "auth_header": "X-UEC-Dev-Preview-Token", "production": "unavailable"} ,"GET /api/v2/discovery/facets": {"success": {"api_version": "v2", "meta": {"profile": "official", "release_id": "string", "ruleset_version": "string", "release_created_at": "timestamp", "coverage_scope": "selected_promoted_release_public_facilities", "count_semantics": "eligible public facility projection rows after current suppression; not story-wide or animal counts", "filters": {}}, "dimensions": {}}, "max_values_per_dimension": 20, "counts_are_from": "selected eligible promoted public projection"} diff --git a/docs/api/v2-contract.md b/docs/api/v2-contract.md index 097b5a5..84c24ee 100644 --- a/docs/api/v2-contract.md +++ b/docs/api/v2-contract.md @@ -10,6 +10,8 @@ Frontend clients should branch on HTTP status and `error.code`, display `message Researchers may request `GET /api/v2/locations.csv?profile=official` (or another explicit supported profile). The export is bounded to 1,000 rows, uses deterministic CSV columns and escaping, contains only the public reviewed projection, and includes `release_profile`, `release_id`, and `manifest_sha256` on every row plus matching response headers. It is unavailable when no promoted release with a manifest exists; it never exposes raw evidence or restricted records. +For reproducible bulk snapshots, use `pipeline/scripts/stages/export-release.py` with an explicit release ID and profile. It emits deterministic CSV and GeoJSON, a schema/data dictionary, `manifest.json`, and `SHA256SUMS.json`; it fails closed on test-only, unapproved, privacy-failed, suppressed, profile-mismatched, or unclear-rights rows. See [the data-product contract](../data-product.md) and [the machine-readable dictionary](../data-dictionary.json). The package is the citation artifact; the bounded API export is a convenience view of the same public projection. + Clients can discover the current controlled vocabularies at `GET /api/v2/discovery/filters`. Category/source/profile/precision/lifecycle values are allowlisted and versioned; `country_code` is validated as an uppercase ISO alpha-2 code and `region` is a bounded release-backed city/region value. The bounded `q` parameter searches canonical name, city, country, category, and source name server-side. Bbox (`min_lon`, `min_lat`, `max_lon`, `max_lat`) and radius (`latitude`, `longitude`, `radius_km`) are mutually exclusive and validated before release access. New country adapters should register capabilities and vocabularies in the contract before becoming public. `GET /api/v2/discovery/facets` returns deterministic value/count pairs for the same controlled dimensions, scoped to the selected promoted profile and current public projection. Its metadata includes the selected release, ruleset, release creation time, and an explicit coverage scope. Counts are eligible public facility-projection rows after current suppression; they are not story-wide totals or animal counts. It applies the supplied filters before counting, caps each dimension at 20 values, and returns no addresses, queries, raw payloads, inactive releases, or restricted records. diff --git a/docs/architecture/adr-graph-foundation.md b/docs/architecture/adr-graph-foundation.md new file mode 100644 index 0000000..08fe783 --- /dev/null +++ b/docs/architecture/adr-graph-foundation.md @@ -0,0 +1,58 @@ +# ADR: Accountability Graph Foundation + +Status: accepted foundation; no production graph import or publication + +## Decision + +Use PostgreSQL tables and read-only views as a compact evidence graph. A +facility is a place/entity and an organization is a legal/operating entity; +neither is represented as the other. Source-native identifiers are retained in +`uec.source_entity_identifiers`. A crosswalk links two such identifiers with a +source-scoped decision (`candidate`, `accepted`, `disputed`, or `rejected`), +but never merges rows or creates a universal identity. + +Relationship rows are dated observations, not mutable edges. They support +operator, owner, parent, brand, supplier, and customer assertions, including +explicit unknown observations. Conflicting observations remain queryable and +are not resolved by a latest-write overwrite. `uec.organization_relationship_current` +is only a convenience projection over retained observations. + +Claims are append-only, source-backed facts or unknowns targeting one facility +or organization. `claim_support` links them to source records and/or preserved +artifacts and labels corroborating, contradicting, primary, and contextual +support. Inspection, violation, commitment, investigation, public funding, +and animal-count domains are reserved attachment points. Domain-specific +schemas, scoring, and workflow are deferred until evidence and governance +requirements are known. + +All graph evidence carries storage, review, privacy, and publication state. +Released rows must be accepted, privacy-passed, tied to a release, and marked +released; public views additionally require a promoted release and exclude +active `uec.public_access_restricted` source records. This is compatibility +with the existing release/suppression controls, not a claim that a release is +automatically approved. + +## Threats and mitigations + +* Residential identities, doxxing, and mixed-use addresses: source values stay + private; graph rows default to pending privacy and public projections require + an explicit passed decision. Facility suppression propagates through the + existing payload-free restriction views. +* Uncertain ownership, stale operator edges, and source disappearance: validity + dates and observation time are separate; no disappearance implies closure; + contradictions and rejected findings remain append-only. +* Community allegations and contested evidence: source origin/review/approval + remain separate, and claim support records can be marked contradicting. +* Cross-source false matches: identifiers remain source-qualified and + crosswalks are source-scoped, reviewable, and never universal IDs. +* Suppression reimports/restores: public graph views resolve current + `public_access_restricted` state at query time; raw/private evidence is not + exposed by these projections. + +## Consequences + +Country adapters can hand off a deterministic private candidate without knowing +the final canonical entity. A later importer may create a facility or +organization projection and append observations/claims. No graph database, +traversal language, automatic identity merge, relationship inference, or UI is +introduced in this sprint. diff --git a/docs/architecture/graph-data-dictionary.md b/docs/architecture/graph-data-dictionary.md new file mode 100644 index 0000000..ce38de9 --- /dev/null +++ b/docs/architecture/graph-data-dictionary.md @@ -0,0 +1,31 @@ +# Graph foundation data dictionary + +| Object | Meaning | Important invariants | +| --- | --- | --- | +| `organizations` | Canonical organization projection | Separate from `facilities`; names are not evidence by themselves. | +| `source_entity_identifiers` | Native identifier observed in one source record | Source-qualified; exactly one facility or organization target; append-only. | +| `source_entity_crosswalks` | A reviewed or candidate mapping between two native identifiers | `identity_scope = source_scoped`; does not merge or assert universal identity. | +| `organization_relationship_observations` | Dated organization-to-facility or organization-to-organization assertion | Validity dates, observation time, source, confidence, review state, and explicit unknowns are distinct. | +| `organization_relationship_current` | Latest observation per scoped endpoint/type | Projection only; different targets remain visible for contradictions. | +| `claims` | Source-backed typed value/unknown for one facility or organization | Multiple values can coexist; unknown requires a reason. | +| `claim_support` | Link from a claim to a source record/artifact | Role is explicit: primary, corroborating, contradicting, or context. | +| `graph_public_claims` / `graph_public_relationships` | Release-scoped public projections | Promoted release + accepted review + passed privacy + released storage + no current suppression. | + +Graph row state fields use four independent axes: + +* `storage_state`: `raw`, `private`, `reviewed`, or `released`. +* `review_state`: factual review/decision state; it is not source origin. +* `privacy_status`: exposure screening state; it is not factual review. +* `publication_status`: release eligibility state; it is not actual publication. + +`claim_domain` values `inspection`, `violation`, `commitment`, +`investigation`, `public_funding`, and `animal_count` are reserved attachment +points. This migration does not implement inspection systems, violation +adjudication, commitment tracking, investigation case management, funding +accounting, or animal-count methodology. + +Source records and artifacts remain the provenance boundary. The graph layer +does not copy raw source payloads into public projections. Facility-level +privacy/suppression decisions are expected to propagate through the existing +source-record restriction references and must be checked again on every public +query. diff --git a/docs/architecture/release-manifest-verification.md b/docs/architecture/release-manifest-verification.md index a87d120..f75d064 100644 --- a/docs/architecture/release-manifest-verification.md +++ b/docs/architecture/release-manifest-verification.md @@ -1,8 +1,8 @@ # Release manifest verification -Promotion stores an immutable machine-readable manifest with the release ID, profile, ruleset version, source coverage, database creation time, and an inventory of declared distributed files. Supply each file with a repeated `--artifact ` option; promotion hashes the file bytes and records its basename, size, and SHA-256. If there really are no distributed files, the operator must explicitly pass `--no-distributed-artifacts`. `--manifest ` exports the canonical JSON whose SHA-256 is stored in `uec.release_manifests`; the CLI result printed to stdout is a separate operation receipt. Obtain the manifest from a trusted project channel, verify its SHA-256 against the stored digest, then hash each listed artifact locally. +Promotion stores an immutable machine-readable `uec-release-manifest-v2` with the release/profile IDs, ruleset and projection schema versions, generated/retrieved timestamps, source coverage, eligible row counts, independent review/publication state, limitations, supersession, and an inventory of declared distributed files. Supply each file with a repeated `--artifact ` option; promotion hashes the file bytes and records its basename, size, and SHA-256. If there really are no distributed files, the operator must explicitly pass `--no-distributed-artifacts`. `--manifest ` exports the canonical JSON whose SHA-256 is stored in `uec.release_manifests`; the CLI result printed to stdout is a separate operation receipt. Obtain the manifest from a trusted project channel, verify its SHA-256 against the stored digest, then hash each listed artifact locally. -The workflow does not discover distributed files or prove that the operator supplied a complete inventory. An empty declared inventory is not evidence that no files were distributed. The manifest records the ruleset version, but other configuration and code versions are not yet part of the release schema; the source ID list is not complete record-level provenance. Hashing at promotion does not freeze later distribution bytes. The database promotion and writing `--manifest` to disk are separate operations; if the file write fails, the database manifest remains stored and an operator must recover and verify it before distribution. These limits require operational review before making a full release-integrity claim. +The workflow does not discover distributed files or prove that the operator supplied a complete inventory. An empty declared inventory is not evidence that no files were distributed. `source_coverage` is aggregate coverage metadata; record-level provenance remains in the public projection rows. Source `rights_status` is recorded as `cleared`, `attribution_required`, or `unknown`; the bulk packager refuses unknown/restricted rows. Hashing at promotion does not freeze later distribution bytes. The database promotion and writing `--manifest` to disk are separate operations; if the file write fails, the database manifest remains stored and an operator must recover and verify it before distribution. These limits require operational review before making a full release-integrity claim. Migration 022 binds publication review events to releases. After importing the candidate, use the actual `release_id` recorded in `uec.releases` and `uec.release_members` when an authorized maintainer records a reviewed decision. For example, a maintainer can parameterize the following SQL with a source record ID from the candidate and the candidate's real release ID; the values and decision must be chosen by that maintainer: diff --git a/docs/countries/br/v1-field-crosswalk.json b/docs/countries/br/v1-field-crosswalk.json new file mode 100644 index 0000000..ecc2259 --- /dev/null +++ b/docs/countries/br/v1-field-crosswalk.json @@ -0,0 +1,149 @@ +{ + "schema_version": "br-v1-recon-crosswalk-1", + "status": "reconnaissance_only_private_artifacts_no_adapter", + "checked_at_utc": "2026-09-16T06:36:11.5283352Z", + "country_code": "BR", + "policy": "docs/ETHICS.md", + "source_boundaries": { + "br.sif.registered": { + "origin": "government_sourced", + "route": "https://dados.agricultura.gov.br/dataset/062166e3-b515-4274-8e7d-68aadd64b820/resource/97277e92-264a-4dc0-9aea-f87b8ea93798/download/sigsifestabelecimentosregistradosnosif.csv", + "format_observed": "UTF-8 semicolon-delimited CSV", + "retrieval_utc": "2026-09-16T06:27:27.9394921Z", + "http": 200, + "bytes": 11374764, + "sha256": "6a9b944ac34f9872b00b32338d5b5f4969c724167d6e2006ec2f71b372538c5a", + "catalog_update_utc": "2026-09-02T06:26:00Z", + "observed_rows": 24174, + "observed_unique_sif": 3147, + "identity": { + "primary_source_key": "NR_SIF", + "supporting_keys": ["CPF_CNPJ", "NUMERO_PROCESSO"], + "warning": "Rows repeat by source activity/category/occurrence; row count is not facility count." + }, + "fields": { + "CPF_CNPJ": "source_cnpj", + "RAZAO_SOCIAL": "source_legal_name", + "NOME_FANTASIA": "source_trade_name", + "NR_SIF": "source_inspection_number", + "DATA_RESERVA": "source_reservation_date", + "DT_REGISTRO": "source_registration_date", + "NUMERO_PROCESSO": "source_process_number", + "SITUACAO": "source_status_code", + "LOGRADOURO": "source_address", + "BAIRRO": "source_neighborhood", + "CEP": "source_postal_code", + "MUNICIPIO": "source_municipality", + "UF": "source_state", + "TELEFONE": "restricted_source_phone", + "EMAIL": "restricted_source_email", + "AREA_CATEGORIA": "source_area_category", + "CATEGORIA_CLASSE": "source_category_class", + "DATA_OCORRENCIA": "source_occurrence_date", + "DESCRICAO_OCORRENCIA": "source_occurrence_description" + }, + "coordinates": "not present; no geocoding performed", + "publication": "blocked_pending_terms_privacy_review_project_approval" + }, + "br.sif.export": { + "origin": "government_sourced", + "route": "https://dados.agricultura.gov.br/dataset/062166e3-b515-4274-8e7d-68aadd64b820/resource/fcb7f87d-0092-4a52-a44b-b3550747b4c2/download/sigsifestabelecimentosnacionais.csv", + "format_observed": "UTF-8 semicolon-delimited CSV", + "retrieval_utc": "2026-09-16T06:27:31.7523210Z", + "http": 200, + "bytes": 4864067, + "sha256": "7b0540b3f9df244d562495eba07e3be20245219caaebd5d701beba0de636d113", + "observed_rows": 37508, + "observed_unique_sif": 2912, + "observed_unique_countries": 55, + "identity": { + "source_establishment_key": "SIF", + "observation_key": ["PAIS", "SIF", "PRODUTO", "DT_OCORRENCIA", "DT_SUSPENSAO"], + "warning": "One establishment has many country/product authorization rows." + }, + "fields": { + "PAIS": "source_export_country", + "AREA": "source_area", + "ESTABELECIMENTO": "source_establishment_name", + "SIF": "source_inspection_number", + "UF": "source_state", + "MUNICIPIO": "source_municipality", + "PRODUTO": "source_authorized_product", + "DT_VALIDADE": "source_valid_until", + "DT_OCORRENCIA": "source_authorization_event_date", + "DT_SUSPENSAO": "source_suspension_date" + }, + "publication": "blocked_pending_terms_privacy_review_project_approval" + }, + "br.sisbi.public": { + "origin": "government_sourced", + "routes": { + "establishments": "https://sistemasweb.agricultura.gov.br/sisbi_api/estabelecimentos-sisbi", + "products": "https://sistemasweb.agricultura.gov.br/sisbi_api/produtos-sisbi", + "capacities": "https://sistemasweb.agricultura.gov.br/sisbi_api/estabs-capacidades", + "inspection_services": "https://sistemasweb.agricultura.gov.br/sisbi_api/servicos-inspecao", + "gis": "https://sistemasweb.agricultura.gov.br/gis_api/gis/genderecos/{idEndereco}" + }, + "access_observed": { + "get": "HTTP 200, application/json; charset=UTF-8", + "default_list_items": 10, + "establishment_count": 10541, + "head": "HTTP 403 on establishments, capacities, and products list routes; GET must be used/tested", + "cache_control": "no-store, must-revalidate, no-cache, max-age=0" + }, + "identity": { + "establishment_primary": "idEstabSisbi", + "supporting_keys": ["idServicoInspecao", "nrRegistro", "nrCnpj", "idEndereco"], + "warning": "Lifecycle/stability and cross-service uniqueness remain unverified." + }, + "establishment_fields": [ + "idEstabSisbi", "nrProcesso", "nrRegistro", "csEstabelecimento", "csSituacaoEstabelecimento", + "situacaoExclusao", "dtRegistro", "idEndereco", "sgUf", "nmMunicipio", "nome", + "tipoEstabelecimento", "servicoInspecao", "pessoa" + ], + "nested_fields": { + "pessoa.pessoaJuridica": ["nrCnpj", "nmRazaoSocial", "nmFantasia"], + "servicoInspecao": ["idServicoInspecao", "ufs", "nmServico", "csSituacao", "stSuspenso", "enderecos"], + "capacity": ["idEstabCapacidade", "categEstabEspecie", "estabSisbiClassificacao", "tipoCapacProducao", "qtCapacidade"], + "product": ["idProdutoSisbi", "nmVenda", "csComercio", "csSisbi", "nrRegistro", "estabelecimentoSisbi", "csAtivoSisbi", "csSuspensaoSisbi", "seloSisbi", "situacaoEscopo"] + }, + "coordinates": "GIS lookup by idEndereco returns longitude/latitude; provider is MAPA GIS, not a project geocoder", + "publication": "blocked_pending_api_contract_terms_privacy_review_project_approval" + }, + "br.trase.facilities": { + "origin": "secondary_project_compilation", + "route": "https://trase.earth/open-data/datasets/brazil-facilities", + "artifact_route": "https://resources.trase.earth/data/facilities-data/2026-05-07-br_beef_logistics_map_v6.geo.json", + "retrieval_utc": "2026-09-16T06:28:09.2156309Z", + "http": 200, + "bytes": 15888949, + "sha256": "11eb5131e0a9379c0bbbff731425726d76db74d8ac21b0356773293c75be32c5", + "dataset_year": 2025, + "page_updated": "2026-01-01", + "observed_features": 18090, + "observed_unique_facility_id": 15119, + "observed_unique_id": 18077, + "sources_observed": ["SISBI", "SIF database", "SIE database"], + "fields": [ + "approved_export_countries", "capacity_units_english", "capacity_units_portuguese", "capacity_value_num", + "capacity_value_text", "cnpj", "commodity", "company", "facility_id", "inspection_level", "inspection_num", + "lat", "long", "municipality", "source", "state", "status", "type_detailed_english", + "type_detailed_portuguese", "type_english", "type_portuguese", "unique_id" + ], + "identity": { + "source_local_key": "unique_id", + "warning": "Trase constructs IDs and disclaims lifetime uniqueness; rows can repeat by commodity/activity. No silent merge with MAPA." + }, + "coordinates": "Trase-derived; methodology says SISBI GIS API or Google Maps fallback; retain as secondary evidence", + "publication": "blocked_pending_trase_terms_privacy_review_project_approval" + } + }, + "implementation_gates": [ + "confirm e-SISBI pagination/query contract and code lists with authorized operator", + "confirm dataset-specific MAPA reuse and attribution terms", + "preserve raw/parsed/normalized/quarantine/review layers", + "screen full addresses, contacts, CNPJ/legal-person fields, and precise coordinates", + "use explicit reviewed cross-source identity links and separate counts by inspection authority", + "keep publication blocked until human privacy/terms/release approval" + ] +} diff --git a/docs/countries/us/README.md b/docs/countries/us/README.md index e65d28c..c748d58 100644 --- a/docs/countries/us/README.md +++ b/docs/countries/us/README.md @@ -22,6 +22,19 @@ Both adapters preserve source values only in restricted staging and emit parsed, [`v1-field-crosswalk.json`](v1-field-crosswalk.json) is the row-free inventory and field/category map. The checked-in FSIS V1 snapshot has 7,101 rows and 269 columns. Its slaughter and processing flags overlap, so the counts are field-presence observations rather than totals. Until an authorized current artifact exists, V1 rows are not claimed current and a missing current observation is `not-observed`, never closure. +## Accountability pilot + +The private accountability pilot in +[`pipeline/sources/us/accountability`](../../pipeline/sources/us/accountability/README.md) +adds a deterministic, graph-foundation-compatible link ledger. It starts from +FSIS establishment/approval IDs and the modeled APHIS registration/inspection +IDs, while keeping operators, legal entities, parents, brands, inspections, +violations, enforcements, laboratories, and aggregate observations distinct. +It accepts only exact source IDs or explicit reviewed link events. Ambiguous, +stale, conflicting, overlapping-ownership, and suppressed relationships are +quarantined. The checked-in fixture is synthetic/sanitized, private/test-only, +and does not add a graph migration or public release. + ## Review checklist - authority, edition/effective date, URL, terms/attribution, and retention are recorded; diff --git a/docs/country-recon-br.md b/docs/country-recon-br.md new file mode 100644 index 0000000..bd36b39 --- /dev/null +++ b/docs/country-recon-br.md @@ -0,0 +1,117 @@ +# Brazil source reconnaissance + +Status: reconnaissance and private artifact/schema validation only. No adapter, candidate import, release, publication, or public API exposure was created. Raw files and row-level samples are retained only under the ignored local path `data/raw/brazil/`. + +Last checked: 2026-09-16 UTC, against `docs/ETHICS.md` version 1.0 (last reviewed 2026-09-12), on baseline `87fde1136f09be22e6d65ef1e741298544746a1f`. This report is source-status evidence, not a publication approval or healthy-pipeline claim. + +## Decision summary + +| Source family | Current finding | Integration decision | +| --- | --- | --- | +| MAPA SIF registered establishments | Official CKAN CSV is live and retrievable. The capture has 24,174 rows but only 3,147 distinct `NR_SIF`; 1,025 SIF numbers repeat, so rows are not facility counts. It contains CNPJ, names, SIF, registration dates, situation, full address, contacts, area/category/class, and occurrence text, but no coordinates. | Conditional go for restricted acquisition and schema validation. Keep establishment identity separate from export/product observations; privacy, terms, and release gates remain blocked. | +| MAPA SIF export-authorized establishments | Official CKAN CSV is live and retrievable. The capture has 37,508 country/product rows and 2,912 distinct SIF numbers across 55 countries. | Conditional go for a separate capability-observation feed. Never use row count as facility count or infer current operation from export presence alone. | +| MAPA SIF establishment report | Official CKAN route and current data dictionary are documented. The dictionary describes area, category, class, SIF, legal name, address, municipality, and UF; it is a supplemental classification report, not a replacement for the registered-establishment file. | Keep separate until a current CSV snapshot and relation semantics are verified. | +| e-SISBI public | MAPA documents public access to service, establishment, and product registrations. The live client exposes JSON routes for establishments, products, capacities, and a GIS address route. A bounded GET returned HTTP 200; the establishment count route returned 10,541, while the list returned 10 records by default. | Conditional go for bounded private capture. Confirm pagination, route parameters, code lists, effective-date/status semantics, address linkage, and update cadence with an authorized operator before recurring retrieval. | +| Trase Brazil facilities | Current 2026 dataset page says it covers 2025 and compiles federal, state, and municipal inspection registries. The private GeoJSON capture has 18,090 feature rows from `SIF database`, `SISBI`, and `SIE database`, with 15,119 source-local facility IDs and 13 repeated `unique_id` groups. It is a high-quality secondary compilation, not MAPA-origin data. | Use as a separately labeled corroboration/coverage benchmark. Do not silently merge it into MAPA or treat its geocodes, status, categories, or constructed IDs as government source facts. | + +## Official routes and observed schemas + +### MAPA / SIF + +The [MAPA SIF page](https://www.gov.br/agricultura/pt-br/assuntos/inspecao/produtos-animal/sif) links the public SIF establishment, export, establishment-report, and statistical routes. The [MAPA open-data catalog](https://dados.agricultura.gov.br/dataset/servico-de-inspecao-federal-sif) identifies the responsible unit as DIPOA/SDA, reports monthly update frequency, UTF-8 encoding, a last-update timestamp of 2 September 2026 03:26 BRT, and displays a Creative Commons Attribution link. That catalog-level indication is recorded as source evidence, not a blanket conclusion about every field, downstream use, personal-data handling, or project publication permission. + +The current identity-relevant CSV resources were: + +- [Estabelecimentos Registrados no SIF](https://dados.agricultura.gov.br/dataset/062166e3-b515-4274-8e7d-68aadd64b820/resource/97277e92-264a-4dc0-9aea-f87b8ea93798/download/sigsifestabelecimentosregistradosnosif.csv) +- [Estabelecimentos Nacionais Habilitados](https://dados.agricultura.gov.br/dataset/062166e3-b515-4274-8e7d-68aadd64b820/resource/fcb7f87d-0092-4a52-a44b-b3550747b4c2/download/sigsifestabelecimentosnacionais.csv) +- [Relatório de Estabelecimentos](https://dados.agricultura.gov.br/dataset/062166e3-b515-4274-8e7d-68aadd64b820/resource/7d02af92-e3cf-4ae4-af8a-0dad334ffdfa/download/sigsifrelatorioestabelecimentos.csv) + +The [registered-establishment dictionary](https://dados.agricultura.gov.br/dataset/062166e3-b515-4274-8e7d-68aadd64b820/resource/c01a51a6-033e-4470-8c31-63998c0eaa38/download/estabelecimentosregistradosnosif.pdf) names the observed fields: CNPJ, legal/trade name, SIF, reservation/registration dates, process number, situation, address, neighborhood, CEP, municipality, UF, telephone, email, product name, category/class, occurrence date, and occurrence description. The observed file is semicolon-delimited UTF-8 and preserves source values as strings. Its `SITUACAO` was `A` for all 24,174 captured rows; that is an observation of this snapshot, not a claim that every SIF facility is active outside it. + +The [export-authorized dictionary](https://dados.agricultura.gov.br/dataset/062166e3-b515-4274-8e7d-68aadd64b820/resource/36ba4d24-d828-4f58-a01c-dd8f1dede1e1/download/listasdeestab.nacionaishabilitados.pdf) names country, area, establishment, SIF, UF, municipality, product, validity date, habilitation-occurrence date, and suspension date. It is naturally one-to-many by SIF, country, and product. It should be modeled as dated authorization evidence, not joined into a single establishment row by lossy aggregation. + +The catalog and live download behavior observed from this environment was HTTP 200 for GET. The two CSV responses had `Content-Type: text/csv`, ETags, and `Last-Modified: 2 Sep 2026` headers. The raw downloads are ignored and their hashes/sizes are listed below. + +### e-SISBI / SISBI-POA + +MAPA's [e-SISBI documentation](https://www.gov.br/agricultura/pt-br/assuntos/defesa-agropecuaria/suasa/manuais-e-tutoriais-do-e-sisbi/e-sisbi) says the public access function exposes information about inspection services, establishments, and products registered in services that are integrated or not integrated into SISBI-POA. The [SISBI-POA page](https://www.gov.br/agricultura/pt-br/assuntos/defesa-agropecuaria/suasa/sisbi-1) explicitly distinguishes an active service `Situação SISBI` from establishment and product status, so those states must not be collapsed. + +The public application [SGSI establishment route](https://sistemasweb.agricultura.gov.br/sgsi/app/estabelecimentos) is a JavaScript client. Its public bundle exposed these service bases and route families: + +| Service | Observed route | Role | +| --- | --- | --- | +| SISBI API | `https://sistemasweb.agricultura.gov.br/sisbi_api/estabelecimentos-sisbi` | Establishment records; separate `/count` route returned `10541`. | +| SISBI API | `https://sistemasweb.agricultura.gov.br/sisbi_api/produtos-sisbi` | Product registrations; `/count` route exists in the client route map. | +| SISBI API | `https://sistemasweb.agricultura.gov.br/sisbi_api/estabs-capacidades` | Capacity/species/category records. | +| SISBI API | `https://sistemasweb.agricultura.gov.br/sisbi_api/servicos-inspecao` | Inspection-service records and status/scope context. | +| GIS API | `https://sistemasweb.agricultura.gov.br/gis_api/gis/genderecos/{idEndereco}` | Coordinate lookup by address identifier. | + +Bounded GETs returned `application/json; charset=UTF-8` and HTTP 200. The establishment, capacity, and product list routes returned 10-item arrays by default in this observation; the route did not return a documented page object. Repeated GET responses advertised `Cache-Control: no-store, must-revalidate, no-cache, max-age=0` and no ETag/content length. HEAD requests to the three SISBI list routes returned HTTP 403 while GET succeeded, so acquisition must test the actual method used by the public client. The GIS route returned HTTP 200 with a 106-byte JSON object containing `idEndereco`, `dsTipo`, `nrLongitude`, and `nrLatitude`; the establishment payload carries `idEndereco` but not a complete flat address. + +Observed establishment keys include `idEstabSisbi`, `nrProcesso`, `nrRegistro`, `csEstabelecimento`, `csSituacaoEstabelecimento`, `situacaoExclusao`, `dtRegistro`, `idEndereco`, `sgUf`, `nmMunicipio`, `nome`, `tipoEstabelecimento`, and nested `servicoInspecao` and `pessoa`. The nested legal-person object exposes `nrCnpj`, `nmRazaoSocial`, and `nmFantasia`; the service object exposes service ID, UFs, name, status/suspension fields, and address references. Capacity records link `estabSisbiClassificacao` to an establishment and expose category, area, species, capacity type, and `qtCapacidade`. Product records expose product registration/name, commerce and SISBI codes, establishment, standardized product references, active/suspension fields, seal, and scope status. + +The public route proves current access and shape, not a stable API contract. Before a recurring adapter, obtain an operator-confirmed pagination/query contract, current code lists, status/effective-date meaning, and source terms. Do not assume that an `idEstabSisbi`, `nrRegistro`, status code, or GIS point is lifetime-stable or publication-eligible without that validation. + +MAPA's [service description for SISBI-POA](https://www.gov.br/agricultura/pt-br/assuntos/defesa-agropecuaria/suasa/perguntas-e-respostas-decreto-12-408-2025-e-o-sisbi/perguntas-e-respostas-sisbi) distinguishes SIM, SIE, SIF, and SISBI-POA: SISBI-POA is a standardisation/equivalence system, not a fourth inspection level. The [Planalto text of Decree 9.013/2017](https://www.planalto.gov.br/ccivil_03/_ato2015-2018/2017/decreto/d9013.htm) places interstate/international establishments under SIF, while establishments under state, district, or municipal services may execute interstate inspection when their service equivalence is recognized. The [current MAPA service page](https://www.gov.br/pt-br/servicos/solicitar-adesao-de-servico-de-inspecao-estadual-municipal-e-consorcio-publicos-municipais-ao-sisbi-poa) confirms that states, municipalities, and municipal consortia can seek equivalence. Model inspection level and SISBI equivalence as separate dimensions. + +### Trase as secondary evidence + +The current [Trase Brazil facilities page](https://trase.earth/open-data/datasets/brazil-facilities) is labeled 2026, covers 2025, reports a 1 January 2026 update, and describes a compilation from federal, state, and municipal inspection registries. Its published GeoJSON schema includes company, inspection number, IBGE municipality code, inspection level, state, status, municipality, longitude/latitude, commodity, source, export countries, facility types, capacity units/value/text, facility ID, and unique ID. + +The [Trase methodology](https://resources.trase.earth/data/facilities-data/Brazil_slaughterhouses_facilities_methods_2025_07.pdf) says its SISBI inputs included establishments, products, capacities, addresses, and geolocation; its SIF inputs included registered facilities and export-approved facilities. It states that SISBI records were joined by `ID_SISBI`, SIF records by `NR_SIF`, addresses were geocoded using the SISBI GIS API or Google Maps fallback, and the cleaned source families were transformed to a common schema and concatenated. It also states that Trase constructs an ID as `___1` and does not guarantee lifetime uniqueness after ownership/CNPJ changes. + +The captured Trase file had 18,090 features; source values were `SISBI` (10,661), `SIF database` (3,398), and `SIE database` (4,031). Inspection-level values were `SIF` (3,398), `SIE` (9,445), `SIM` (2,778), and `CONSORCIO` (2,454). There were 15,119 distinct `facility_id` values, 18,077 distinct `unique_id` values, and 13 repeated `unique_id` groups. This confirms one-to-many commodity/activity rows and the need for source-local deduplication rules. It does not establish that Trase's source snapshots, cleaning, geocodes, status, or classifications are identical to today's MAPA state. + +The Trase page says charts, graphics, maps, and other representations on its platform may be used under CC BY 4.0, while commercial data use should be discussed with Trase. That wording is not treated as blanket permission to redistribute the downloaded raw dataset or its personal/precise location fields. Any project reuse needs a terms/privacy review and explicit attribution/citation. + +## Identity, scope, and double-counting strategy + +Use source-scoped observation keys first and cross-source identity links second: + +1. SIF establishment identity: `sif:`, with CNPJ, names, dates, address, and category retained as source attributes. SIF export rows are `sif-export::::` and are never collapsed into the establishment master. +2. SISBI establishment identity: `sisbi:` when present. Keep `idServicoInspecao`, `nrRegistro`, CNPJ, and `idEndereco` as separate supporting keys. Products and capacities are child observations keyed to the SISBI establishment; they must not multiply the displayed facility count. +3. Trase identity: `trase:` or an artifact-row key. Treat its `facility_id`, CNPJ, inspection level, and inspection number as secondary evidence. A link to SIF or SISBI is an explicit reconciliation event only when the source identifier matches exactly or a human-reviewed match is recorded; name, CNPJ, address, or coordinates alone must not silently merge records. +4. Inspection level is not the same as SISBI status. Keep `SIF`, `SIE`, `SIM`, and `CONSORCIO` as authority-level values; keep SISBI equivalence/service/establishment/product states separately. A SISBI establishment can be under a state, municipal, or consortium service. +5. Count facilities by a declared unit: source-local establishment IDs for source coverage, and reviewed cross-source entities for a reconciled view. Never sum SIF rows, export-country rows, product rows, capacity rows, Trase commodity rows, or overlapping SIF/SISBI views as if each were a facility. + +No closure should be inferred from disappearance, a suspension field should not be interpreted without its date/semantics, and a source status should remain raw alongside any normalized interpretation. + +## Privacy and geocoding + +The SIF capture contains CNPJ, company names, complete addresses, CEP, telephone numbers, emails, and occurrence text. e-SISBI responses expose CNPJ/legal-person fields, names, address IDs and potentially address objects; the GIS service returns precise latitude/longitude. Some official facility addresses may be mixed residential/business sites or may identify individuals. Under `docs/ETHICS.md`, retain these fields privately for validation, screen residential/private and harmful locations, and do not publish private contacts or precise coordinates by default. A registered office or geocoder match is not proof of an operating site. + +Coordinates are a separate enrichment/evidence field. Preserve source provider (`MAPA GIS`, source-supplied, or later geocoder), query/input, retrieval timestamp, precision, result, and review state. Trase's Google-geocoded points are Trase-derived evidence and must not be presented as MAPA coordinates. Do not geocode before a terms/privacy decision; a successful geocode does not grant publication permission. Public projections should prefer municipality or reviewed coarse geometry when precise publication is not justified. + +## Artifact provenance and schema inventory + +All artifacts below are local and ignored; none is a repository fixture. Retrieval times are UTC. CSV byte sizes are the captured file sizes; API list responses did not provide a content-length header. The `HEAD 403` observation applies to SISBI list routes, while the recorded sample artifacts were captured with GET. + +| Artifact | URL / route | Retrieved | HTTP / content type | Bytes | SHA-256 | Response metadata | +| --- | --- | --- | ---: | ---: | --- | --- | +| SIF registered CSV | [MAPA resource](https://dados.agricultura.gov.br/dataset/062166e3-b515-4274-8e7d-68aadd64b820/resource/97277e92-264a-4dc0-9aea-f87b8ea93798/download/sigsifestabelecimentosregistradosnosif.csv) | 2026-09-16T06:27:27.9394921Z | 200 / `text/csv` | 11,374,764 | `6a9b944ac34f9872b00b32338d5b5f4969c724167d6e2006ec2f71b372538c5a` | ETag `"1788328931.62-11374764"`; Last-Modified 2026-09-02T06:02:11Z | +| SIF export-authorized CSV | [MAPA resource](https://dados.agricultura.gov.br/dataset/062166e3-b515-4274-8e7d-68aadd64b820/resource/fcb7f87d-0092-4a52-a44b-b3550747b4c2/download/sigsifestabelecimentosnacionais.csv) | 2026-09-16T06:27:31.7523210Z | 200 / `text/csv` | 4,864,067 | `7b0540b3f9df244d562495eba07e3be20245219caaebd5d701beba0de636d113` | ETag `"1788328873.14-4864067"`; Last-Modified 2026-09-02T06:01:13Z | +| SISBI establishment sample | [public route](https://sistemasweb.agricultura.gov.br/sisbi_api/estabelecimentos-sisbi) | 2026-09-16T06:34:53.2156653Z | 200 / `application/json; charset=UTF-8` | 25,371 | `33479012c36503b991c8d25f888e2c332bfa91927a8e2adb588e38d4a12902b9` | GET list returned 10 items; no ETag/content length; repeated response `Cache-Control: no-store, must-revalidate, no-cache, max-age=0`; HEAD 403 | +| SISBI capacity sample | [public route](https://sistemasweb.agricultura.gov.br/sisbi_api/estabs-capacidades) | 2026-09-16T06:34:54.3079161Z | 200 / `application/json; charset=UTF-8` | 13,079 | `38f5023812736a691383857e4fcd0c99956ad7ff9606ec9c3097cb2eeab90b40` | GET list returned 10 items; same no-store headers; HEAD 403 observed on list route | +| SISBI product sample | [public route](https://sistemasweb.agricultura.gov.br/sisbi_api/produtos-sisbi) | 2026-09-16T06:34:55.1203410Z | 200 / `application/json; charset=UTF-8` | 35,890 | `2b67d0d56dad29eb68da19c8399edc82c9074a99c7fd15ad7c26c04eda04ea5f` | GET list returned 10 items; same no-store headers; HEAD 403 observed on list route | +| SISBI GIS sample | [public route](https://sistemasweb.agricultura.gov.br/gis_api/gis/genderecos/4836007) | 2026-09-16T06:34:55.9739665Z | 200 / `application/json; charset=UTF-8` | 106 | `72f80b8ac3c9a199cc6ea548bd82e447aca8332f5d1cdfaee659dc3c2912fcb7` | Content-Length 106; no ETag/Last-Modified | +| Trase GeoJSON | [current dataset file](https://resources.trase.earth/data/facilities-data/2026-05-07-br_beef_logistics_map_v6.geo.json) | 2026-09-16T06:28:09.2156309Z | 200 / `application/octet-stream` | 15,888,949 | `11eb5131e0a9379c0bbbff731425726d76db74d8ac21b0356773293c75be32c5` | ETag `"f6e87a7ec515b47c44aa04c533b507cc-2"`; Last-Modified 2026-08-10T14:38:31Z | +| Trase methodology PDF | [methodology](https://resources.trase.earth/data/facilities-data/Brazil_slaughterhouses_facilities_methods_2025_07.pdf) | 2026-09-16T06:27:34.1428599Z | 200 / `application/pdf` | 520,686 | `b506873b2688f218b567d310002a91d1f98d49388982edb73975bb26ecbaa2e9` | Public PDF; methodology says source access dates 2025-03-12/14 | + +The hashes above are the captured bytes; they can be regenerated from the ignored files. The machine-readable crosswalk repeats them for automated checks. + +## Staged implementation plan + +1. **Restricted acquisition contract.** Add a Brazil source-local configuration using the shared bounded acquisition/provenance primitives. Capture URL, retrieval time, HTTP method/status, content type, ETag/Last-Modified when supplied, byte size, SHA-256, catalog/effective date, and schema fingerprint. Keep raw, parsed, normalized, quarantined, and reviewed layers distinct. +2. **Separate adapters.** Implement SIF registered, SIF export authorization, e-SISBI establishments, e-SISBI products, e-SISBI capacities, and GIS lookup as separate evidence families. Preserve all source columns and code values; classify only through reviewed codebooks. Add synthetic tests for repeated activities, missing identifiers, status changes, multilingual text, malformed rows, and mixed residential/business addresses. +3. **Operator-assisted e-SISBI validation.** Confirm pagination/query parameters, whether count endpoints are consistent with list filters, code-list meanings, effective dates, update cadence, address selection, and terms. The current browser route is enough for private reconnaissance but not a stable recurring-ingestion contract. +4. **Identity/reconciliation.** Use SIF number and `idEstabSisbi` as source-local keys. Store cross-source matches as explicit reviewed links with evidence and confidence; never merge by CNPJ/name/address alone. Keep Trase as a labeled secondary source and compare coverage by exact identifiers and aggregate counts only. +5. **Geocoding/privacy review.** Prefer source-provided MAPA GIS points after privacy screening. Any fallback geocoding must record provider/query/time/precision/review state and third-party disclosure. Suppress precise residential/private or unresolved points and propagate restrictions through caches, exports, previews, and reimports. +6. **Count and release gates.** Report counts separately by source, inspection level, authority, active/status interpretation, and facility-vs-activity unit. Run schema drift, sharp-change, duplicate, state-coordinate, and disappearance checks. Publication remains blocked until terms, privacy, project review, release approval, and authorized-maintainer availability are all documented. + +## Open questions requiring assistance + +- Obtain an operator-confirmed e-SISBI extraction contract and current codebooks; public JavaScript route names are not an API stability promise. +- Confirm MAPA dataset-specific reuse/attribution and personal-data handling beyond the catalog's displayed CC link. +- Confirm whether SIF registration number reuse, SISBI internal ID lifecycle, ownership changes, and service/establishment status codes have documented historical semantics. +- Decide, with human review, what high-level facility fields and coordinate precision are appropriate for publication under the ethics policy. +- Confirm Trase's raw-data reuse terms for this project's non-commercial/commercial use and retain Trase attribution/citation separately from MAPA attribution. diff --git a/docs/country-rehearsal-2026-09-15.json b/docs/country-rehearsal-2026-09-15.json new file mode 100644 index 0000000..d1a0d5a --- /dev/null +++ b/docs/country-rehearsal-2026-09-15.json @@ -0,0 +1,39 @@ +{ + "schema_version": "country-rehearsal-v1", + "local_sprint_date": "2026-09-15", + "observed_at_utc": "2026-09-16T06:38:26Z", + "branch": "codex/live-country-rehearsal", + "baseline": "87fde11", + "scope": "owner-authorized private/test-only candidate integration for strongest non-US country routes", + "public_exposure": false, + "release_promoted": false, + "geocoding": "disabled", + "global_gates": { + "terms": "blocked_pending_human_review", + "privacy": "blocked_pending_field_level_review", + "classification": "review_required", + "completeness": "not_claimed", + "coordinate_precision": "review_required", + "publication": "blocked", + "raw_payloads_committed": false + }, + "automation_limitations": [ + "agent-browser CLI was unavailable in this environment; existing bounded adapters and direct official fetches were used", + "Italy Python TLS failed; a bounded curl fallback fetched the official catalogued CSV", + "Germany BVL export is session/request-specific; assisted synthetic BLtU evidence was used", + "Belgium live operator acquisition was inaccessible; assisted synthetic operator/codebook evidence was used", + "CFIA direct acquisition failed certificate verification; assisted synthetic federal evidence was used" + ], + "sources": [ + {"source_id":"dk.smiley","country":"Denmark","acquisition":"live_verified","source_url":"https://pub.fvst.dk/publikationer/Smileydata.xml","retrieved_at_utc":"2026-09-16T06:29:21Z","effective_date":"2026-09-16T06:28:34Z HTTP Last-Modified","byte_size":59835715,"sha256":"4dd3e9c703cdd206b0a4e620f57f530f55325cefb0048eb9e25a973ce24c450e","input_rows":58773,"normalized_rows":58773,"quarantined_rows":0,"anomaly_counts":{"classification_requires_review":61,"coordinates_unresolved":58773},"schema_status":"pipeline_contract_verified","candidate_handoff":true,"review_packet":true,"source_health":true,"delta":"full-pipeline rerun; no delta artifact","publication":"blocked","next_action":"Review 61 validation findings, coverage/effective-date semantics, and address/coordinate privacy."}, + {"source_id":"fr.dgal.section-i","country":"France","acquisition":"live_verified","source_url":"https://fichiers-publics.agriculture.gouv.fr/dgal/ListesOfficielles/SSA1_VIAN_ONG_DOM.txt","retrieved_at_utc":"2026-09-16T06:29:12Z","effective_date":"2026-09-16T02:21:51Z HTTP Last-Modified","byte_size":204211,"sha256":"b1171561865ab664ddf18adeeed7b6993224cc2275277fdaa6e4d411dd062649","input_rows":1448,"normalized_rows":1448,"quarantined_rows":0,"schema_fingerprint":"9e390329bf11ac023f7c0d7e966491acf2a2b5b399a7c7cf968a6864c5f4696f","candidate_handoff":true,"review_packet":true,"source_health":true,"delta":"comparison failed closed: prior run-manifest missing","publication":"blocked","next_action":"Repair prior-run manifest linkage, then review category, duplicate, terms, and privacy gates."}, + {"source_id":"fr.dgal.section-ii","country":"France","acquisition":"live_verified","source_url":"https://fichiers-publics.agriculture.gouv.fr/dgal/ListesOfficielles/SSA1_VIAN_COL_LAGO.txt","retrieved_at_utc":"2026-09-16T06:29:12Z","effective_date":"2026-09-16T02:22:14Z HTTP Last-Modified","byte_size":136654,"sha256":"6c2d943024a27baa2113bb60eef6dad132d3f96ea1b001fb60ba40ac0b406fb6","input_rows":1068,"normalized_rows":1068,"quarantined_rows":0,"schema_fingerprint":"9e390329bf11ac023f7c0d7e966491acf2a2b5b399a7c7cf968a6864c5f4696f","candidate_handoff":true,"review_packet":true,"source_health":true,"delta":"comparison failed closed: prior run-manifest missing","publication":"blocked","next_action":"Repair prior-run manifest linkage, then review species/category, terms, and privacy gates."}, + {"source_id":"it.853-2004","country":"Italy","acquisition":"live_verified_curl_fallback","source_url":"https://www.dati.salute.gov.it/sites/default/files/opendata/STAB_POA_8_20260915.csv","retrieved_at_utc":"2026-09-16T06:37:05Z","effective_date":"2026-09-15","byte_size":49927230,"sha256":"2d665d355de522ff7a8f61f23c4a7eb722d6e30d41e099c557001db5aa0b266b","input_rows":47370,"normalized_rows":41844,"quarantined_rows":5526,"schema_fingerprint":"3ba24374ea6412c7240218984fac00b2e0e9f84e382ec72c19bbcef1a73c0826","anomaly_counts":{"ambiguous_repeated_recognition_activity":5526},"candidate_handoff":true,"review_packet":true,"source_health":true,"delta":{"status":"delta-ready","added":0,"changed":0,"not_observed":0,"suppressed":0},"publication":"blocked","next_action":"Resolve repeated recognition/activity identities and separately assess excluded 1069/2009 scope."}, + {"source_id":"de.locations","country":"Germany","acquisition":"assisted_synthetic","source_url":"https://www.bvl.bund.de/bltu","retrieved_at_utc":"2026-09-16T06:30:00Z","effective_date":"unknown","byte_size":627,"sha256":"cde8813830ee278031184d4771bb81e026c655853cd358f17642ef188352f96b","input_rows":3,"normalized_rows":1,"quarantined_rows":2,"schema_fingerprint":"701f66f3fe4e419727c2e0afed740660eb0e7ae429e7a969cb996d320e73011f","anomaly_counts":{"missing_activity_code":1,"unmapped_activity_code":1},"candidate_handoff":true,"review_packet":true,"source_health":true,"delta":{"status":"delta-ready","added":0,"changed":0,"not_observed":0,"suppressed":0},"publication":"blocked","next_action":"Capture a current session-bound BVL export with export-specific terms and effective-date evidence."}, + {"source_id":"be.locations","country":"Belgium","acquisition":"assisted_synthetic","source_url":"https://www.static.favv.be/bo-documents/inter_actieve_actoren_EN.csv","retrieved_at_utc":"2026-09-16T06:30:00Z","effective_date":"unknown","byte_size":679,"sha256":"a8866877ac66f555b8e79434d37ceba00f1a70adad3cbc032ff4cfdbc0a084e8","input_rows":5,"normalized_rows":3,"quarantined_rows":2,"anomaly_counts":{"address_privacy_risk":1,"unresolved_activity_code":1},"candidate_handoff":true,"review_packet":true,"source_health":true,"delta":{"status":"delta-ready","added":0,"changed":0,"not_observed":0,"suppressed":0},"publication":"blocked","next_action":"Obtain an authorized live operator/codebook pair and compare schema, row lengths, and terms."}, + {"source_id":"ca.ontario.meat-plants","country":"Canada","acquisition":"live_verified","source_url":"https://data.ontario.ca/dataset/a763088c-018d-48b7-bf47-3027a8c725b8/resource/ee6d559a-78de-40e6-b2ba-ad3c4a674b96/download/1._all_meat_plants.csv","retrieved_at_utc":"2026-09-16T06:29:12Z","effective_date":"2026-08-06T19:04:50Z HTTP Last-Modified","byte_size":130252,"sha256":"c4edfdd415812f6a914f5fb29a9f67cf2907ea6fbfe4e09ab96eae1468002adf","input_rows":460,"normalized_rows":460,"quarantined_rows":0,"schema_fingerprint":"7e103c6ebeff948204ec729d8b9ce2faebdfab74f0863017a464ce99f159b6ac","candidate_handoff":true,"review_packet":true,"source_health":true,"delta":"comparison failed closed: prior run-manifest missing","publication":"blocked","next_action":"Repair prior-run manifest linkage and complete Ontario privacy/licence review without making a national claim."}, + {"source_id":"ca.cfia.federal-meat","country":"Canada","acquisition":"assisted_synthetic","source_url":"https://active.inspection.gc.ca/scripts/meavia/reglist/download.asp?lang=e","retrieved_at_utc":"2026-09-16T06:30:00Z","effective_date":"unknown","byte_size":404,"sha256":"f0d09d2dcc2a47f025e42fe8729fb90051171f24ba777d3b8b40879a29fcb7bf","input_rows":3,"normalized_rows":2,"quarantined_rows":1,"schema_fingerprint":"c4244a8235cf0dd4febf6c572cd863fd4e45158927baea38cef2a8accf23f8e8","anomaly_counts":{"unknown_function_code":1},"candidate_handoff":true,"review_packet":true,"source_health":true,"delta":{"status":"no-previous-validated-run","added":0,"changed":0,"not_observed":0,"suppressed":0},"publication":"blocked","next_action":"Obtain an authorized current CFIA export after resolving certificate/access issues."}, + {"source_id":"fsa_approved_establishments","country":"United Kingdom","acquisition":"live_verified","source_url":"https://fsaopendata.blob.core.windows.net/opendatacatalog/Approved-Establishments-01-09-26.csv","retrieved_at_utc":"2026-09-16T06:38:26Z","effective_date":"2026-09-01","byte_size":1774417,"sha256":"d5cfec048b0f4dc4a8594b0597982f3788f10eb1b4270f9593ead8abce33b61f","input_rows":5342,"normalized_rows":4300,"quarantined_rows":1042,"schema_fingerprint":"8a78f58c004a84811e51af53fab6eaf9316a93462575a17f52b6f7a7543e1fa8","anomaly_counts":{"address_privacy_risk":11,"duplicate_id_within_nation":4,"remarks_present":999,"unknown_nation":31},"candidate_handoff":true,"review_packet":true,"source_health":true,"delta":{"status":"delta-ready","added":0,"changed":0,"not_observed":0,"suppressed":0},"publication":"blocked","next_action":"Review remarks, nation scope, duplicate IDs, privacy, and OGL/attribution gates; keep Northern Ireland separate."}, + {"source_id":"fss_approved_establishments","country":"United Kingdom","acquisition":"live_verified","source_url":"https://www.foodstandards.gov.scot/sites/default/files/2026-08/Approved%20Establishments%20in%20Scotland.csv","retrieved_at_utc":"2026-09-16T06:38:25Z","effective_date":"2026-08-11T13:55:33Z HTTP Last-Modified","byte_size":245871,"sha256":"b95b66afb112636c09f6de401054c7ea3d11e5058f34522d900c60435a125246","input_rows":725,"normalized_rows":586,"quarantined_rows":139,"anomaly_counts":{"address_privacy_risk":18,"malformed_row":2,"missing_activity":2,"missing_approval_number":2,"no_relevant_activity":125},"candidate_handoff":true,"review_packet":true,"source_health":true,"delta":{"status":"delta-ready","added":0,"changed":0,"not_observed":0,"suppressed":0},"publication":"blocked","next_action":"Review malformed/activity rows, privacy, nation scope, and OGL/attribution gates; keep Scotland separate."} + ] +} diff --git a/docs/country-rehearsal-2026-09-15.md b/docs/country-rehearsal-2026-09-15.md new file mode 100644 index 0000000..f88fbc2 --- /dev/null +++ b/docs/country-rehearsal-2026-09-15.md @@ -0,0 +1,39 @@ +# Live Country Rehearsal — 2026-09-15 + +Owner-authorized private/test-only rehearsal on branch `codex/live-country-rehearsal`, baseline `87fde11`. The artifacts were observed on 2026-09-16 UTC because the run crossed midnight Pacific. Raw, restricted, staging, and candidate-handoff payloads remain ignored local files; no release, public API, map, export, cache, or history surface was created or promoted. + +## Result + +Ten source profiles across the strongest non-US country routes completed private candidate integration. Every profile has provenance evidence, parsed/normalized/quarantined output, source-health evidence, and a human review gate. Geocoding was disabled everywhere. “Not observed” is retained as a non-closure state. + +| Profile | Acquisition and source date | Input / normalized / quarantined | Artifact bytes / SHA-256 | Automation and private state | +|---|---|---:|---|---| +| `dk.smiley` | Live Find Smiley fetch; retrieved `2026-09-16T06:29:21Z`; HTTP Last-Modified `2026-09-16T06:28:34Z` | 58,773 / 58,773 / 0 | 59,835,715 / `4dd3e9c703cdd206b0a4e620f57f530f55325cefb0048eb9e25a973ce24c450e` | Full parse → normalize → classify → validate → geocode-queue pipeline; 61 validation finding records and 58,773 unresolved coordinates; private handoff emitted | +| `fr.dgal.section-i` | Live DGAL list fetch; retrieved `2026-09-16T06:29:12Z`; file Last-Modified `2026-09-16T02:21:51Z` | 1,448 / 1,448 / 0 | 204,211 / `b1171561865ab664ddf18adeeed7b6993224cc2275277fdaa6e4d411dd062649` | Candidate-ready; bilingual composite headers supported; private handoff/review packet emitted | +| `fr.dgal.section-ii` | Live DGAL list fetch; retrieved `2026-09-16T06:29:12Z`; file Last-Modified `2026-09-16T02:22:14Z` | 1,068 / 1,068 / 0 | 136,654 / `6c2d943024a27baa2113bb60eef6dad132d3f96ea1b001fb60ba40ac0b406fb6` | Candidate-ready; bilingual composite headers supported; private handoff/review packet emitted | +| `it.853-2004` | Official catalogue/CSV fetched with bounded `curl.exe` fallback after Python TLS failure; retrieved `2026-09-16T06:37:05Z`; source date `2026-09-15` | 47,370 / 41,844 / 5,526 | 49,927,230 / `2d665d355de522ff7a8f61f23c4a7eb722d6e30d41e099c557001db5aa0b266b` | Candidate-ready; 5,526 `ambiguous_repeated_recognition_activity`; zero-change delta-ready rerun; handoff/review packet emitted | +| `de.locations` | Assisted BVL portal export using the sanitized synthetic BLtU fixture; retrieved `2026-09-16T06:30:00Z`; effective date unknown | 3 / 1 / 2 | 627 / `cde8813830ee278031184d4771bb81e026c655853cd358f17642ef188352f96b` | Portal export URL is session/request-specific; 1 unmapped and 1 missing activity-code row; zero-change delta-ready rerun; handoff/review packet emitted | +| `be.locations` | Assisted FASFC operator + activity-code fixture pair; retrieved `2026-09-16T06:30:00Z`; effective date unknown | 5 / 3 / 2 | 679 / `a8866877ac66f555b8e79434d37ceba00f1a70adad3cbc032ff4cfdbc0a084e8` | Live operator route was inaccessible in this environment; 1 address privacy risk and 1 unresolved activity code; zero-change delta-ready rerun; pair handoff/review packet emitted | +| `ca.ontario.meat-plants` | Live Ontario CSV fetch; retrieved `2026-09-16T06:29:12Z`; HTTP Last-Modified `2026-08-06T19:04:50Z` | 460 / 460 / 0 | 130,252 / `c4edfdd415812f6a914f5fb29a9f67cf2907ea6fbfe4e09ab96eae1468002adf` | Bilingual composite headers supported; private handoff/review packet emitted; one comparison attempt failed closed on a missing prior run manifest | +| `ca.cfia.federal-meat` | Assisted CFIA fixture after direct TLS certificate failure; retrieved `2026-09-16T06:30:00Z`; effective date unknown | 3 / 2 / 1 | 404 / `f0d09d2dcc2a47f025e42fe8729fb90051171f24ba777d3b8b40879a29fcb7bf` | 1 unknown function code; private handoff/review packet emitted; no previous validated run was available | +| `fsa_approved_establishments` | Live FSA England/Wales CSV; retrieved `2026-09-16T06:38:26Z`; source snapshot `2026-09-01` | 5,342 / 4,300 / 1,042 | 1,774,417 / `d5cfec048b0f4dc4a8594b0597982f3788f10eb1b4270f9593ead8abce33b61f` | Private handoff; zero-change delta-ready rerun; 999 remarks, 31 unknown-nation, 11 address-risk, 4 duplicate-ID anomalies | +| `fss_approved_establishments` | Live FSS Scotland CSV; retrieved `2026-09-16T06:38:25Z`; source Last-Modified `2026-08-11T13:55:33Z` | 725 / 586 / 139 | 245,871 / `b95b66afb112636c09f6de401054c7ea3d11e5058f34522d900c60435a125246` | Private handoff; zero-change delta-ready rerun; 125 no-relevant-activity, 18 address-risk, 2 malformed, 2 missing-activity, 2 missing-approval anomalies | + +The France, Italy, Germany, Belgium, Canada, and UK adapters now recognize the observed bilingual/composite or current source headers through explicit aliases. The Italy adapter preserves raw date text while normalizing current abbreviated Italian dates and treats source dash sentinels as unknown rather than invalid. Belgium’s row-length regression is checked per row. + +## Source routes and terms + +- France: official [DGAL lists index](https://fichiers-publics.agriculture.gouv.fr/dgal/ListesOfficielles/) with separate Section I and Section II files. +- Italy: official [Ministry 853/2004 dataset](https://www.dati.salute.gov.it/it/dataset/stabilimenti-italiani-gli-alimenti-di-origine-animale/) and its catalogued CSV; the separate 1069/2009 by-products scope remains excluded. +- UK: official [FSA catalogue](https://data.food.gov.uk/catalog/datasets/1e61736a-2a1a-4c6a-b8b1-e45912ebc8e3) plus the separate Scotland FSS route; OGL/attribution review remains distinct from source origin. +- Belgium: official [FASFC operator dataset](https://data.gov.be/fr/datasets/favv-afsca-operators); CC BY 4.0 attribution and reuse review remain open. +- Germany: official [BVL BLtU portal](https://www.bvl.bund.de/DE/Arbeitsbereiche/01_Lebensmittel/01_Aufgaben/05_GrenzueberschreitenderHandel/lm_grenzueberschrHandel_basepage.html); the export request is session-bound, so this rehearsal used assisted synthetic evidence. +- Canada: current [Ontario meat-plants dataset](https://data.ontario.ca/dataset/licensed-meat-plants) and separate CFIA federal route; Ontario and federal identities remain separate and no national completeness claim is made. + +The owner-authorized terms record applies only to this bounded private/test-only run. It does not approve redistribution, precise coordinates, addresses, contacts, public API exposure, release promotion, or publication. Human terms, privacy, classification, completeness, coordinate precision, release, and maintainer-approval gates remain blocked. + +## Rerun and release evidence + +The zero-change delta-ready reruns report `added=0`, `changed=0`, `not_observed=0`, and `suppressed=0` for Italy, Germany, Belgium, FSA, and FSS. France I/II and Ontario comparison attempts failed closed because the referenced prior `run-manifest.json` was missing; they still recorded `release_promoted=false` and all public surfaces false. No failed comparison was treated as an unchanged claim. + +No public release was created. Candidate handoffs are disposable private artifacts, not publication approval. The next operator actions are to obtain authorized live captures for Belgium/Germany/CFIA, repair the prior-run manifest linkage for France/Ontario comparisons, resolve Italy’s repeated identity collisions and excluded 1069 scope, review all address/coordinate and terms gates, and only then consider a separately approved disposable candidate import. diff --git a/docs/data-dictionary.json b/docs/data-dictionary.json new file mode 100644 index 0000000..371223d --- /dev/null +++ b/docs/data-dictionary.json @@ -0,0 +1,34 @@ +{ + "data_product_version": "uec-public-data-product-v1", + "schema_version": "uec-location-projection-v1", + "coordinate_policy": "Coordinates are emitted only for exact or city display precision; city coordinates are coarse/approximate display points and no guessed point is substituted.", + "unknown_values": "Blank or null means unknown or unavailable and is not an assertion of absence.", + "fields": [ + {"name": "facility_id", "type": "string", "required": true, "description": "Stable project facility identifier."}, + {"name": "canonical_name", "type": "string", "required": false, "description": "Project-normalized facility name."}, + {"name": "country_code", "type": "string", "required": true, "description": "ISO 3166-1 alpha-2 country code."}, + {"name": "city", "type": "string", "required": false, "description": "Public city or region label, not a street address."}, + {"name": "category", "type": "string", "required": true, "description": "Release-ruleset classification category."}, + {"name": "display_precision", "type": "string", "required": true, "description": "Public location precision: exact, city, or unmapped."}, + {"name": "latitude", "type": "number", "required": false, "description": "Public latitude; null when not allowed by display precision."}, + {"name": "longitude", "type": "number", "required": false, "description": "Public longitude; null when not allowed by display precision."}, + {"name": "lifecycle_status", "type": "string", "required": true, "description": "Observed lifecycle label; disappearance is not closure."}, + {"name": "first_observed_at", "type": "timestamp", "required": false, "description": "Earliest eligible observation in this release."}, + {"name": "last_observed_at", "type": "timestamp", "required": false, "description": "Latest eligible observation in this release."}, + {"name": "observation_count", "type": "integer", "required": false, "description": "Eligible observation count, not an animal count."}, + {"name": "source_type", "type": "string", "required": true, "description": "Evidence origin, separate from review and approval."}, + {"name": "factual_review_status", "type": "string", "required": false, "description": "Release-scoped factual review outcome or unreviewed state."}, + {"name": "privacy_screening_status", "type": "string", "required": true, "description": "Publication-safety screening outcome."}, + {"name": "project_approval", "type": "string", "required": false, "description": "Release/profile-scoped maintainer decision."}, + {"name": "reviewer_role", "type": "string", "required": false, "description": "Reviewer role label; no personal reviewer identity."}, + {"name": "publication_warning", "type": "string", "required": false, "description": "Persistent warning for community-unreviewed claims."}, + {"name": "publication_profile", "type": "string", "required": true, "description": "official, secondary, or community."}, + {"name": "release_id", "type": "string", "required": true, "description": "Immutable release identifier."}, + {"name": "release_ruleset_version", "type": "string", "required": false, "description": "Classification/projection ruleset identifier."}, + {"name": "provenance_source_id", "type": "string", "required": false, "description": "Stable source registry identifier."}, + {"name": "provenance_source_name", "type": "string", "required": false, "description": "Human-readable source name."}, + {"name": "provenance_source_url", "type": "string", "required": false, "description": "Official source URL recorded for provenance."}, + {"name": "provenance_retrieved_at", "type": "timestamp", "required": false, "description": "UTC retrieval time for the source artifact."}, + {"name": "source_rights_status", "type": "string", "required": true, "description": "cleared or attribution_required; unknown/restricted rows are excluded."} + ] +} diff --git a/docs/data-product.md b/docs/data-product.md new file mode 100644 index 0000000..85668f3 --- /dev/null +++ b/docs/data-product.md @@ -0,0 +1,46 @@ +# Public data product + +UEC public snapshots are release-scoped projections, not raw source dumps. A snapshot is available only for an explicitly promoted, non-test release/profile whose rows have passed the current publication and privacy gates. The packager fails closed when a row is suppressed, belongs to another profile/release, is not privacy-screened, is not approved for the selected profile, or has unclear/restricted reuse rights. + +## Package contents + +`pipeline/scripts/stages/export-release.py` creates a deterministic package containing: + +- `locations.csv`: stable column order, UTF-8, LF line endings, and spreadsheet-formula neutralization for text beginning with `=`, `+`, `-`, or `@`. +- `locations.geojson`: a GeoJSON `FeatureCollection` with the same public properties. Coordinates are emitted only for `exact` or `city` display precision; `city` is a coarse/approximate display point and an unmapped or restricted point is `null`. +- `data-dictionary.json`: machine-readable field semantics and coordinate/unknown-value policy. +- `manifest.json`: release/profile identity, generated and retrieved timestamps, source coverage, schema and data-product versions, row counts, review/publication state, limitations, supersession, and artifact checksums. +- `SHA256SUMS.json`: checksum sidecar covering the manifest and package artifacts. + +Run this only after an authorized release has been promoted: + +```powershell +python pipeline/scripts/stages/export-release.py RELEASE_ID ` + --profile official ` + --output-dir data/releases/RELEASE_ID-official +``` + +The database query uses a repeatable-read snapshot and repeats the release/profile, publication review, privacy screening, and current suppression gates. It does not validate, promote, deploy, or publish a release. Source `attribution` is treated as `attribution_required`; missing attribution is `unknown` and blocks packaging until reuse status is reviewed. This is a conservative source-rights gate, not a claim that attribution alone grants redistribution rights. + +## Release metadata + +`release_id` and `profile` are the citation scope. `generated_at` is the release/package generation time and `retrieved_at` is the earliest source-artifact retrieval represented by the package. `source_coverage` reports each source ID, included row count, retrieval interval, and per-source rights status. `row_counts.eligible_rows` must equal the packaged row count; the package never truncates a large release. + +Review, privacy screening, project approval, and publication are independent states. A community profile can carry a privacy-screened but factually unreviewed claim and must retain the row warning `Unreviewed community claim — not verified by Until Every Cage`. A community review does not grant project approval. No profile or export implies factual completeness, current operation, an animal count, a story-wide denominator, a project licence, or an availability SLA. + +## Verification + +Consumers should obtain the package and its trusted manifest digest through a project-controlled channel, then verify the sidecar and listed artifacts: + +```powershell +python pipeline/scripts/maintenance/verify-data-product.py data/releases/RELEASE_ID-official ` + --manifest-sha256 TRUSTED_MANIFEST_SHA256 +``` + +The Python contract also exposes `verify_package(Path(...))` for local verification. Checksums detect byte changes relative to the trusted manifest; they do not establish factual accuracy, source correctness, privacy eligibility, or reuse rights. + +## Compatibility and deprecation + +The stable public contract is the versioned `data_product_version` plus `schema_version`. Additive nullable fields may be added in a minor contract revision. Existing field meanings, identifiers, profile names, and coordinate safety rules are not changed silently. A breaking field/type/meaning change requires a new schema version and a documented migration note; old packages remain labeled rather than rewritten. Deprecated fields remain for one documented release cycle where safe, then are removed only in a new major schema version. Release IDs and artifact bytes are immutable; a correction creates a new release and records `supersedes`. + +The API remains a bounded convenience surface (`GET /api/v2/locations.csv`, maximum 1,000 rows). Reproducible bulk distribution uses the CLI package above. API and package consumers must treat `(profile, release_id, schema_version, query)` as the snapshot key and must not infer omitted rows as suppressed, closed, or absent from the underlying story. diff --git a/docs/review-packet-us-accountability.md b/docs/review-packet-us-accountability.md new file mode 100644 index 0000000..a058fad --- /dev/null +++ b/docs/review-packet-us-accountability.md @@ -0,0 +1,70 @@ +# US accountability pilot review packet + +Status: private/test-only implementation evidence, not publication approval. + +## What this pilot proves + +The bounded synthetic fixture demonstrates a deterministic candidate contract +for 12 relationships and 13 distinct typed entities. It keeps FSIS facility +and establishment-approval identifiers separate from APHIS operator and +inspection observations. It also represents legal entity, parent, brand, +violation, enforcement, laboratory, and aggregate observation nodes without +turning any of them into a facility master record. + +Each accepted relationship carries the evidence source, subject/object/evidence +source-native IDs, observation date, retrieval timestamp, relationship type, +confidence, review state, match method, and an evidence URL/excerpt. An +ownership change is represented by non-overlapping temporal relationships. + +The tests cover entity ambiguity, ownership changes, duplicate names, +subsidiaries/parents, stale evidence, conflicting identifiers, schema drift, +suppression, deterministic output, and V1 reconciliation. + +## What remains inference or blocked + +- The checked-in rows and CIK values are synthetic/sanitized. They do not + assert a live SEC, EPA, OSHA, FSIS, or APHIS match. +- A source-native ID is evidence of identity within that source, not a legal + conclusion about ownership or operation. +- An explicit reviewed link is a candidate relationship, not project approval. +- A missing source observation is not closure. An APHIS inspection or aggregate + is not an FSIS facility-master observation. +- Address, phone, DUNS, and precise coordinates remain private/pending review; + geocoding is disabled. + +## Source rights and acquisition + +FSIS uses the existing operator-assisted official-export contract because the +direct route returned HTTP 403 during reconnaissance. The pilot does not retry +around, bypass, or disguise that control. APHIS uses its existing explicit +registrations/annual-reports/inspections assisted-export profiles. Any SEC, +EPA, OSHA, or other government source needs its own official URL, retrieval +date, checksum, artifact retention decision, and terms/reuse review before a +relationship can be used outside a synthetic fixture. + +Government-sourced does not mean project-approved, current, complete, or +independently verified. The candidate run remains `private-candidate`, +`release_state=not-created`, and publication-blocked. + +## Coverage and scaling cost + +The checked-in coverage is a contract fixture only and makes no national +completeness claim. A real bounded run would require source-specific exports, +schema/count review, exact source-key reconciliation, stale/conflict scans, +privacy review, rights review, and human approval for every link family. +The dominant scaling cost is per-relationship evidence review and retention; +bulk fuzzy resolution is intentionally out of scope. No graph migration, +frontend, target scoring, activist targeting, residential-person exposure, +publication, or UI work is included. + +## Review gates + +1. Verify source authority, selected profile, URL, edition/search date, terms, + attribution, and retention for each source artifact. +2. Confirm that every relationship has source-native IDs and evidence and that + no name/address/geocoder identity inference was introduced. +3. Inspect quarantine counts and conflicts; treat stale or missing evidence as + uncertainty, not closure. +4. Complete privacy/suppression review and retain no raw rows in Git. +5. Obtain authorized project review and release approval separately. This pilot + itself never promotes a release. diff --git a/docs/source-status.json b/docs/source-status.json index 3aa3488..3adf0fe 100644 --- a/docs/source-status.json +++ b/docs/source-status.json @@ -1,6 +1,7 @@ { "schema_version": "1.0", "purpose": "Evidence-backed source readiness baseline; not a runtime health monitor or publication approval register.", + "latest_private_rehearsal": "docs/country-rehearsal-2026-09-15.json", "status_vocabulary": { "metadata": ["verified", "partial", "unknown"], "acquisition": ["not_run", "blocked", "artifact_private_only", "verified"], @@ -8,6 +9,10 @@ "publication_eligibility": ["not_assessed", "blocked", "eligible_pending_release_approval"] }, "sources": [ + {"source_id":"br.sif.registered","metadata":"verified","acquisition":"verified","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-br.md","docs/countries/br/v1-field-crosswalk.json","pipeline/source_registry.json"],"next_action":"Keep the current private SIF CSV and provenance restricted; validate repeated-row semantics, status/code meanings, source terms, privacy, reconciliation, and release approval before any adapter or publication."}, + {"source_id":"br.sif.export","metadata":"verified","acquisition":"verified","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-br.md","docs/countries/br/v1-field-crosswalk.json","pipeline/source_registry.json"],"next_action":"Model country/product authorizations as separate dated observations keyed to SIF; confirm validity/suspension semantics, terms, privacy, and no-double-counting rules before integration."}, + {"source_id":"br.sisbi.public","metadata":"verified","acquisition":"artifact_private_only","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-br.md","docs/countries/br/v1-field-crosswalk.json","pipeline/source_registry.json"],"next_action":"Use bounded GET samples only while an authorized operator confirms pagination, code lists, lifecycle/status, address linkage, cadence, terms, privacy, and project approval."}, + {"source_id":"br.trase.facilities","metadata":"verified","acquisition":"artifact_private_only","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-br.md","docs/countries/br/v1-field-crosswalk.json","pipeline/source_registry.json"],"next_action":"Keep Trase as separately labeled secondary evidence; review source lineage, geocoding, constructed IDs, raw-data terms, privacy, coverage, and any exact-ID reconciliation before use."}, {"source_id":"be.locations","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-be.md","pipeline/sources/belgium/adapter.py","pipeline/sources/belgium/refresh.py","pipeline/sources/belgium/fixtures/synthetic_operators.csv","pipeline/sources/belgium/test_adapter.py","pipeline/common/review_packet.py","docs/review-packet-belgium.md","pipeline/source_registry.json"],"next_action":"Use the assisted two-file refresh with an authorized operator capture and official activity-code CSV; compare live schema and physical row lengths to the synthetic contract. Keep category/privacy/terms gates and publication blocked."}, {"source_id":"fr.dgal.section-i","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-fr.md","docs/countries/france/dgal-853-pipeline.md","pipeline/sources/france/adapter.py","pipeline/sources/france/acquire.py","pipeline/sources/france/refresh.py","pipeline/sources/france/fixtures/section_i.csv","pipeline/common/review_packet.py","pipeline/source_registry.json"],"next_action":"Run the bounded private Section I refresh with an approved terms record or authorized capture; review category semantics, address privacy, schema drift, and release approval."}, {"source_id":"fr.dgal.section-ii","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-fr.md","docs/countries/france/dgal-853-pipeline.md","pipeline/sources/france/adapter.py","pipeline/sources/france/acquire.py","pipeline/sources/france/refresh.py","pipeline/sources/france/fixtures/section_ii.csv","pipeline/common/review_packet.py","pipeline/source_registry.json"],"next_action":"Run the bounded private Section II refresh separately from Section I; review category/species semantics, address privacy, schema drift, and release approval."}, diff --git a/docs/source-status.md b/docs/source-status.md index dbe981c..57ccc72 100644 --- a/docs/source-status.md +++ b/docs/source-status.md @@ -11,10 +11,18 @@ This is the canonical human-readable view of [`source-status.json`](source-statu No last-success timestamp is invented. Private artifacts are not proof of a public release. “Government-sourced” does not mean current, complete, project-approved, or safe to expose. +## Latest private rehearsal + +The owner-authorized 2026-09-15 live-country rehearsal completed private candidate integration for Denmark, France Sections I/II, Italy 853/2004, Germany, Belgium, Canada Ontario/CFIA, and UK FSA/FSS. It created no public surface and kept publication blocked. See the detailed [country rehearsal report](country-rehearsal-2026-09-15.md) and machine-readable [rehearsal status](country-rehearsal-2026-09-15.json). + ## Current baseline | Source ID | Metadata | Acquisition | Runtime health | Publication | Evidence / next action | |---|---|---|---|---|---| +| `br.sif.registered` | verified | verified | not_run | blocked | Current MAPA SIF registered CSV captured privately with schema/count/hash provenance; repeated activity rows, status semantics, terms, privacy, reconciliation, and project approval remain open; see `docs/country-recon-br.md` | +| `br.sif.export` | verified | verified | not_run | blocked | Current MAPA SIF export-authorized CSV captured privately; model country/product authorizations as dated observations, not facility rows, and complete terms/privacy/reconciliation review; see `docs/country-recon-br.md` | +| `br.sisbi.public` | verified | artifact_private_only | not_run | blocked | Public e-SISBI JSON/GIS routes returned bounded samples; pagination, code lists, ID lifecycle, status/effective-date semantics, terms, privacy, and cadence remain unresolved; see `docs/country-recon-br.md` | +| `br.trase.facilities` | verified | artifact_private_only | not_run | blocked | Current Trase GeoJSON/methodology captured privately as secondary evidence; keep separate from MAPA and review source lineage, geocoding, constructed IDs, terms, privacy, and coverage; see `docs/country-recon-br.md` | | `be.locations` | verified | blocked | not_run | blocked | Shared private adapter, row-length/quarantine checks, and assisted two-file refresh are covered by synthetic fixtures; obtain an authorized operator capture and compare live schema before any real run; see `docs/review-packet-belgium.md` | | `fr.dgal.section-i` | verified | not_run | not_run | blocked | DGAL Section I private adapter/refresh is implemented; run only with approved terms or authorized capture, then review category semantics, address privacy, schema drift, and release approval | | `fr.dgal.section-ii` | verified | not_run | not_run | blocked | DGAL Section II remains a separate private adapter/refresh scope; review species/category semantics, address privacy, schema drift, and release approval | diff --git a/pipeline/common/data_product.py b/pipeline/common/data_product.py new file mode 100644 index 0000000..d3dfcf1 --- /dev/null +++ b/pipeline/common/data_product.py @@ -0,0 +1,410 @@ +"""Deterministic, fail-closed packaging for public UEC release projections. + +This module deliberately accepts already projected rows rather than raw evidence. +It is also strict about the release gate so a caller cannot turn a candidate, +suppressed record, or source with unknown reuse rights into a public snapshot by +omitting a filter. +""" + +from __future__ import annotations + +import csv +import hashlib +import io +import json +import math +import re +from collections import Counter +from pathlib import Path +from typing import Any, Iterable, Mapping + + +DATA_PRODUCT_VERSION = "uec-public-data-product-v1" +SCHEMA_VERSION = "uec-location-projection-v1" +MANIFEST_VERSION = "uec-release-manifest-v2" +SUPPORTED_PROFILES = frozenset(("official", "secondary", "community")) +ALLOWED_RIGHTS = frozenset(("cleared", "attribution_required")) +FORMULA_PREFIXES = ("=", "+", "-", "@") + +CSV_FIELDS = ( + "facility_id", + "canonical_name", + "country_code", + "city", + "category", + "display_precision", + "latitude", + "longitude", + "lifecycle_status", + "first_observed_at", + "last_observed_at", + "observation_count", + "source_type", + "factual_review_status", + "privacy_screening_status", + "project_approval", + "reviewer_role", + "publication_warning", + "publication_profile", + "release_id", + "release_ruleset_version", + "provenance_source_id", + "provenance_source_name", + "provenance_source_url", + "provenance_retrieved_at", + "source_rights_status", +) + +PUBLIC_ROW_FIELDS = frozenset(CSV_FIELDS) +FORBIDDEN_ROW_FIELDS = frozenset( + ( + "raw_fields", + "raw_payload", + "source_values", + "street_address", + "postal_code", + "phone", + "private_address", + "geocoding_query", + "attachment_url", + "suppression_reason", + ) +) + + +class DataProductError(ValueError): + """A release or row is not eligible for public data-product packaging.""" + + +def canonical_json(value: Any) -> str: + """Serialize JSON in the repository-wide stable form.""" + + return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + + +def sha256_bytes(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def _required_string(metadata: Mapping[str, Any], field: str) -> str: + value = metadata.get(field) + if not isinstance(value, str) or not value.strip(): + raise DataProductError(f"release metadata field is missing: {field}") + return value + + +def validate_release_metadata( + metadata: Mapping[str, Any], + profile: str | None = None, + *, + require_projection_checksum: bool = False, +) -> dict[str, Any]: + """Validate the explicit publication contract and return a plain copy. + + ``eligible`` is intentionally required. A caller must obtain this value + from a release-scoped public projection query or an equivalent reviewed + release process; packaging never infers eligibility from row presence. + """ + + result = dict(metadata) + release_id = _required_string(result, "release_id") + selected_profile = profile or result.get("profile") + if selected_profile not in SUPPORTED_PROFILES: + raise DataProductError("release profile is unsupported") + if result.get("profile") != selected_profile: + raise DataProductError("selected profile does not match release metadata") + if result.get("eligible") is not True: + raise DataProductError("release is not explicitly eligible for public packaging") + if result.get("test_only") is True: + raise DataProductError("test-only releases cannot be packaged") + if result.get("status") not in ("promoted", "project-published"): + raise DataProductError("release must be promoted before packaging") + if result.get("publication_state") not in ("project-published", "eligible"): + raise DataProductError("release publication state is not eligible") + _required_string(result, "ruleset_version") + _required_string(result, "schema_version") + _required_string(result, "generated_at") + if not isinstance(result.get("retrieved_at"), str) or not result["retrieved_at"].strip(): + raise DataProductError("release metadata field is missing: retrieved_at") + if not isinstance(result.get("limitations"), list) or not all(isinstance(v, str) and v for v in result["limitations"]): + raise DataProductError("release limitations must be a list of non-empty strings") + if result.get("supersedes") is not None and not isinstance(result["supersedes"], str): + raise DataProductError("release supersedes must be null or a release ID") + if not isinstance(result.get("source_coverage"), list): + raise DataProductError("release source coverage must be a list") + coverage_ids = set() + for item in result["source_coverage"]: + if not isinstance(item, Mapping) or not isinstance(item.get("source_id"), str) or not item["source_id"]: + raise DataProductError("release source coverage is malformed") + if item["source_id"] in coverage_ids: + raise DataProductError("release source coverage contains duplicate source IDs") + coverage_ids.add(item["source_id"]) + if not isinstance(item.get("row_count"), int) or item["row_count"] < 0: + raise DataProductError("release source coverage row count is invalid") + if not isinstance(result.get("row_counts"), Mapping): + raise DataProductError("release row counts are missing") + if not isinstance(result["row_counts"].get("eligible_rows"), int) or result["row_counts"]["eligible_rows"] < 0: + raise DataProductError("release eligible row count is invalid") + if not isinstance(result.get("checksums"), Mapping): + raise DataProductError("release checksums are missing") + if require_projection_checksum and not re.fullmatch(r"[0-9a-f]{64}", str(result["checksums"].get("projection_sha256", ""))): + # The projection digest is calculated before the final manifest exists, + # so it is a stable input-level checksum rather than a self-reference. + raise DataProductError("release projection checksum is invalid") + result["release_id"] = release_id + result["profile"] = selected_profile + return result + + +def csv_safe_value(value: Any) -> Any: + """Prevent spreadsheet formula execution while preserving source values.""" + + if isinstance(value, str) and value.startswith(FORMULA_PREFIXES): + return "'" + value + return value + + +def _csv_cell(value: Any) -> Any: + if value is None: + return "" + return csv_safe_value(value) if isinstance(value, str) else value + + +def _coordinates(row: Mapping[str, Any]) -> list[float] | None: + precision = row.get("display_precision") + latitude, longitude = row.get("latitude"), row.get("longitude") + if precision not in ("exact", "city") or latitude is None or longitude is None: + return None + try: + lat, lon = float(latitude), float(longitude) + except (TypeError, ValueError): + raise DataProductError("public coordinates must be numeric") + if not (math.isfinite(lat) and math.isfinite(lon) and -90 <= lat <= 90 and -180 <= lon <= 180): + raise DataProductError("public coordinates are outside valid bounds") + return [lon, lat] + + +def validate_public_rows(rows: Iterable[Mapping[str, Any]], metadata: Mapping[str, Any]) -> list[dict[str, Any]]: + """Validate, whitelist, and deterministically order public projection rows.""" + + release = validate_release_metadata(metadata) + selected_profile = release["profile"] + validated: list[dict[str, Any]] = [] + seen: set[str] = set() + for index, original in enumerate(rows): + row = dict(original) + if row.get("suppressed") is True or row.get("public_access_revoked") is True: + raise DataProductError(f"row {index} is suppressed") + forbidden = sorted(FORBIDDEN_ROW_FIELDS.intersection(row)) + if forbidden: + raise DataProductError(f"row {index} contains restricted fields: {', '.join(forbidden)}") + facility_id = _required_string(row, "facility_id") + if facility_id in seen: + raise DataProductError(f"duplicate facility_id in release projection: {facility_id}") + seen.add(facility_id) + if row.get("release_id") != release["release_id"] or row.get("publication_profile") != selected_profile: + raise DataProductError(f"row {index} does not match the selected release/profile") + if row.get("publication_eligible") is not True or row.get("privacy_screening_status") != "passed": + raise DataProductError(f"row {index} is not publication-eligible and privacy-screened") + if selected_profile != "community" and row.get("project_approval") != "approved": + raise DataProductError(f"row {index} is not project-approved") + if row.get("source_rights_status") not in ALLOWED_RIGHTS: + raise DataProductError(f"row {index} has unclear or restricted source reuse rights") + if row.get("source_type") == "user_submitted" and selected_profile != "community": + raise DataProductError(f"row {index} user-submitted claim is outside the community profile") + projection = {field: row.get(field) for field in CSV_FIELDS} + projection["release_ruleset_version"] = row.get("release_ruleset_version", release["ruleset_version"]) + projection["publication_warning"] = row.get("publication_warning") or ( + "Unreviewed community claim — not verified by Until Every Cage" + if selected_profile == "community" and row.get("factual_review_status") == "unreviewed" + else None + ) + validated.append(projection) + expected = release["row_counts"]["eligible_rows"] + if len(validated) != expected: + raise DataProductError(f"row count does not match release metadata: expected {expected}, got {len(validated)}") + return sorted(validated, key=lambda row: (row["facility_id"], row["provenance_source_id"] or "")) + + +def paginate_rows(rows: Iterable[Mapping[str, Any]], limit: int = 100, cursor: str | None = None) -> tuple[list[Mapping[str, Any]], str | None]: + """Apply the same stable facility-id traversal used by API consumers. + + Callers should pass rows returned by :func:`validate_public_rows`; this + helper does not grant publication eligibility on its own. + """ + + if not isinstance(limit, int) or isinstance(limit, bool) or not 1 <= limit <= 1000: + raise DataProductError("page limit must be between 1 and 1000") + ordered = sorted(rows, key=lambda row: (str(row.get("facility_id", "")), str(row.get("provenance_source_id", "")))) + if cursor is not None: + ordered = [row for row in ordered if str(row.get("facility_id", "")) > cursor] + page = ordered[:limit] + return page, (str(page[-1]["facility_id"]) if len(ordered) > limit else None) + + +def render_csv(rows: Iterable[Mapping[str, Any]]) -> bytes: + output = io.StringIO(newline="") + writer = csv.DictWriter(output, fieldnames=CSV_FIELDS, extrasaction="raise", lineterminator="\n") + writer.writeheader() + for row in rows: + writer.writerow({field: _csv_cell(row.get(field)) for field in CSV_FIELDS}) + return output.getvalue().encode("utf-8") + + +def render_geojson(rows: Iterable[Mapping[str, Any]], metadata: Mapping[str, Any]) -> bytes: + features = [] + for row in rows: + properties = {field: row.get(field) for field in CSV_FIELDS if field not in {"latitude", "longitude"}} + geometry = None + coordinates = _coordinates(row) + if coordinates is not None: + geometry = {"coordinates": coordinates, "type": "Point"} + features.append({"geometry": geometry, "properties": properties, "type": "Feature"}) + document = { + "type": "FeatureCollection", + "features": features, + "metadata": { + "data_product_version": DATA_PRODUCT_VERSION, + "release_id": metadata["release_id"], + "profile": metadata["profile"], + "schema_version": metadata["schema_version"], + "limitations": metadata["limitations"], + }, + } + return (canonical_json(document) + "\n").encode("utf-8") + + +def data_dictionary() -> dict[str, Any]: + descriptions = { + "facility_id": "Stable project facility identifier.", + "canonical_name": "Project-normalized facility name; may be null when unknown.", + "country_code": "ISO 3166-1 alpha-2 country code.", + "city": "Public city or region label; not a street address.", + "category": "Project classification category under the release ruleset.", + "display_precision": "Public location precision: exact, city, or unmapped.", + "latitude": "Public latitude; blank when unmapped or not allowed by display precision.", + "longitude": "Public longitude; blank when unmapped or not allowed by display precision.", + "lifecycle_status": "Observed lifecycle label; disappearance is not closure.", + "first_observed_at": "Earliest retained observation included in this public release.", + "last_observed_at": "Latest retained observation included in this public release.", + "observation_count": "Count of eligible observations in this release, not an animal count.", + "source_type": "Evidence origin, separate from review and approval.", + "factual_review_status": "Scoped factual review outcome or unreviewed state.", + "privacy_screening_status": "Publication-safety screening outcome.", + "project_approval": "Scoped maintainer decision for this release/profile.", + "reviewer_role": "Role label for the recorded review, not a personal identity.", + "publication_warning": "Persistent context required for community-unreviewed claims.", + "publication_profile": "Selected publication context: official, secondary, or community.", + "release_id": "Immutable release identifier.", + "release_ruleset_version": "Classification and projection ruleset identifier.", + "provenance_source_id": "Stable source registry identifier.", + "provenance_source_name": "Human-readable source name.", + "provenance_source_url": "Official source URL recorded for provenance.", + "provenance_retrieved_at": "UTC retrieval time for the source artifact.", + "source_rights_status": "Reuse review: cleared or attribution_required; unknown rights are excluded.", + } + return { + "data_product_version": DATA_PRODUCT_VERSION, + "schema_version": SCHEMA_VERSION, + "coordinate_policy": "Coordinates are emitted only for exact or city display precision; city points are coarse/approximate and no guessed point is substituted.", + "unknown_values": "Blank/null means unknown or unavailable and is not an assertion of absence.", + "fields": [ + {"name": field, "description": descriptions[field], "nullable": field not in {"facility_id", "country_code", "category", "display_precision", "source_type", "publication_profile", "release_id"}} + for field in CSV_FIELDS + ], + } + + +def build_manifest(metadata: Mapping[str, Any], rows: list[Mapping[str, Any]], artifacts: list[dict[str, Any]]) -> dict[str, Any]: + release = validate_release_metadata(metadata, require_projection_checksum=True) + return { + "manifest_version": MANIFEST_VERSION, + "data_product_version": DATA_PRODUCT_VERSION, + "release_id": release["release_id"], + "profile": release["profile"], + "release_status": release["status"], + "test_only": release["test_only"], + "ruleset_version": release["ruleset_version"], + "schema_version": release["schema_version"], + "generated_at": release["generated_at"], + "retrieved_at": release["retrieved_at"], + "source_coverage": release["source_coverage"], + "row_counts": {**release["row_counts"], "packaged_rows": len(rows)}, + "review_state": release.get("review_state", "release-scoped review required"), + "publication_state": release["publication_state"], + "checksums": {**release["checksums"], "algorithm": "sha256", "distributed_artifacts": sorted(artifacts, key=lambda item: item["name"])}, + "limitations": release["limitations"], + "supersedes": release.get("supersedes"), + "distributed_artifacts": sorted(artifacts, key=lambda item: item["name"]), + "checksum_algorithm": "sha256", + } + + +def write_package(output_dir: Path, metadata: Mapping[str, Any], rows: Iterable[Mapping[str, Any]]) -> dict[str, Any]: + """Write a deterministic CSV/GeoJSON snapshot and verification metadata.""" + + output_dir.mkdir(parents=True, exist_ok=True) + validated = validate_public_rows(rows, metadata) + rows_by_source = Counter(row["provenance_source_id"] for row in validated) + coverage_by_source = {item["source_id"]: item["row_count"] for item in metadata["source_coverage"]} + if any(count != coverage_by_source.get(source_id) for source_id, count in rows_by_source.items()): + raise DataProductError("source coverage does not match packaged rows") + if validated and set(rows_by_source) != set(coverage_by_source): + raise DataProductError("source coverage does not match packaged sources") + csv_bytes = render_csv(validated) + geojson_bytes = render_geojson(validated, metadata) + dictionary_bytes = (canonical_json(data_dictionary()) + "\n").encode("utf-8") + projection_digest = sha256_bytes(csv_bytes + b"\n" + geojson_bytes) + enriched = dict(metadata) + enriched["checksums"] = {**dict(metadata["checksums"]), "projection_sha256": projection_digest} + artifacts = [ + {"name": "locations.csv", "sha256": sha256_bytes(csv_bytes), "byte_size": len(csv_bytes)}, + {"name": "locations.geojson", "sha256": sha256_bytes(geojson_bytes), "byte_size": len(geojson_bytes)}, + {"name": "data-dictionary.json", "sha256": sha256_bytes(dictionary_bytes), "byte_size": len(dictionary_bytes)}, + ] + manifest = build_manifest(enriched, validated, artifacts) + manifest_bytes = (canonical_json(manifest) + "\n").encode("utf-8") + files = { + "locations.csv": csv_bytes, + "locations.geojson": geojson_bytes, + "data-dictionary.json": dictionary_bytes, + "manifest.json": manifest_bytes, + } + checksums = {name: sha256_bytes(content) for name, content in files.items()} + checksums_bytes = (canonical_json({"algorithm": "sha256", "files": checksums}) + "\n").encode("utf-8") + files["SHA256SUMS.json"] = checksums_bytes + for name, content in files.items(): + (output_dir / name).write_bytes(content) + return {"manifest": manifest, "manifest_sha256": checksums["manifest.json"], "files": checksums} + + +def verify_package(output_dir: Path, expected_manifest_sha256: str | None = None) -> dict[str, Any]: + checksums_path = output_dir / "SHA256SUMS.json" + if not checksums_path.is_file(): + raise DataProductError("checksum sidecar is missing") + checksums = json.loads(checksums_path.read_text(encoding="utf-8")) + if checksums.get("algorithm") != "sha256" or not isinstance(checksums.get("files"), dict): + raise DataProductError("checksum sidecar is malformed") + for name, expected in checksums["files"].items(): + if not isinstance(name, str) or Path(name).name != name or name in {"", ".", ".."} or not re.fullmatch(r"[0-9a-f]{64}", str(expected)): + raise DataProductError("checksum sidecar contains an unsafe entry") + path = output_dir / name + if not path.is_file() or sha256_bytes(path.read_bytes()) != expected: + raise DataProductError(f"checksum verification failed: {name}") + manifest_path = output_dir / "manifest.json" + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + if manifest.get("manifest_version") != MANIFEST_VERSION or manifest.get("release_status") != "promoted" or manifest.get("test_only") is not False: + raise DataProductError("manifest is not an eligible data-product release") + if manifest.get("publication_state") != "project-published" or manifest.get("profile") not in SUPPORTED_PROFILES: + raise DataProductError("manifest publication metadata is invalid") + row_counts = manifest.get("row_counts") + if not isinstance(row_counts, dict) or row_counts.get("eligible_rows") != row_counts.get("packaged_rows"): + raise DataProductError("manifest row counts are inconsistent") + actual_manifest_sha256 = checksums["files"].get("manifest.json") + if expected_manifest_sha256 is not None and expected_manifest_sha256 != actual_manifest_sha256: + raise DataProductError("trusted manifest checksum does not match") + listed = {artifact["name"]: artifact for artifact in manifest.get("distributed_artifacts", [])} + for name, artifact in listed.items(): + if checksums["files"].get(name) != artifact.get("sha256"): + raise DataProductError(f"manifest artifact checksum mismatch: {name}") + return {"status": "verified", "manifest_sha256": actual_manifest_sha256, "artifact_count": len(listed)} diff --git a/pipeline/common/test_data_product.py b/pipeline/common/test_data_product.py new file mode 100644 index 0000000..24ec9d0 --- /dev/null +++ b/pipeline/common/test_data_product.py @@ -0,0 +1,164 @@ +import json +import tempfile +import unittest +from pathlib import Path + +from pipeline.common.data_product import ( + DataProductError, + CSV_FIELDS, + data_dictionary, + paginate_rows, + render_csv, + validate_public_rows, + verify_package, + write_package, +) + + +def metadata(profile="official", count=1): + return { + "release_id": "release-2026-09-15", + "profile": profile, + "status": "promoted", + "test_only": False, + "eligible": True, + "publication_state": "project-published", + "ruleset_version": "rules-v1", + "schema_version": "uec-location-projection-v1", + "generated_at": "2026-09-15T00:00:00Z", + "retrieved_at": "2026-09-14T00:00:00Z", + "source_coverage": [{"source_id": "source-a", "row_count": count}], + "row_counts": {"eligible_rows": count}, + "checksums": {}, + "review_state": "project-approved and privacy-screened", + "limitations": ["Synthetic fixture; not a complete denominator."], + "supersedes": None, + } + + +def row(profile="official", **overrides): + result = { + "facility_id": "00000000-0000-0000-0000-000000000001", + "canonical_name": "=Not a formula", + "country_code": "DK", + "city": "Testby", + "category": "slaughter", + "display_precision": "city", + "latitude": 55.0, + "longitude": 10.0, + "lifecycle_status": "active_observed", + "first_observed_at": "2026-01-01T00:00:00Z", + "last_observed_at": "2026-09-01T00:00:00Z", + "observation_count": 1, + "source_type": "official", + "factual_review_status": "reviewed", + "privacy_screening_status": "passed", + "project_approval": "approved", + "reviewer_role": "maintainer", + "publication_warning": None, + "publication_profile": profile, + "release_id": "release-2026-09-15", + "release_ruleset_version": "rules-v1", + "provenance_source_id": "source-a", + "provenance_source_name": "Synthetic source", + "provenance_source_url": "https://example.invalid/source", + "provenance_retrieved_at": "2026-09-14T00:00:00Z", + "source_rights_status": "attribution_required", + "publication_eligible": True, + } + result.update(overrides) + return result + + +class DataProductTests(unittest.TestCase): + def test_deterministic_csv_and_geojson_package(self): + with tempfile.TemporaryDirectory() as directory: + first = Path(directory) / "first" + second = Path(directory) / "second" + a = write_package(first, metadata(), [row()]) + b = write_package(second, metadata(), [row()]) + self.assertEqual(a["files"], b["files"]) + self.assertEqual((first / "locations.csv").read_bytes(), (second / "locations.csv").read_bytes()) + self.assertEqual((first / "locations.geojson").read_bytes(), (second / "locations.geojson").read_bytes()) + self.assertEqual(verify_package(first)["status"], "verified") + + def test_pagination_has_export_parity(self): + rows = [row(facility_id=f"00000000-0000-0000-0000-{number:012d}") for number in range(7)] + validated = validate_public_rows(rows, metadata(count=7)) + traversed = [] + cursor = None + while True: + page, cursor = paginate_rows(validated, limit=3, cursor=cursor) + traversed.extend(item["facility_id"] for item in page) + if cursor is None: + break + self.assertEqual(traversed, [item["facility_id"] for item in validated]) + + def test_formula_prefix_is_neutralized_only_in_csv(self): + rendered = render_csv([row()]).decode() + self.assertIn("'=Not a formula", rendered) + self.assertNotIn("street_address", rendered) + + def test_release_and_profile_gates_fail_closed(self): + for changes, message in [ + ({"test_only": True}, "test-only"), + ({"status": "candidate"}, "promoted"), + ({"eligible": False}, "eligible"), + ({"profile": "community"}, "profile"), + ]: + candidate = metadata() + candidate.update(changes) + with self.assertRaisesRegex(DataProductError, message): + write_package(Path(tempfile.mkdtemp()), candidate, [row()]) + + def test_rows_cannot_bypass_suppression_or_rights(self): + for changes, message in [ + ({"suppressed": True}, "suppressed"), + ({"source_rights_status": "unknown"}, "rights"), + ({"privacy_screening_status": "pending"}, "publication-eligible"), + ({"street_address": "private"}, "restricted fields"), + ({"release_id": "other"}, "selected release"), + ]: + with self.assertRaisesRegex(DataProductError, message): + write_package(Path(tempfile.mkdtemp()), metadata(), [row(**changes)]) + + def test_malformed_metadata_and_tampering_fail_verification(self): + for field in ("release_id", "generated_at", "source_coverage", "row_counts", "checksums"): + invalid = metadata() + del invalid[field] + with self.assertRaises(DataProductError): + write_package(Path(tempfile.mkdtemp()), invalid, [row()]) + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) + write_package(path, metadata(), [row()]) + (path / "locations.csv").write_bytes((path / "locations.csv").read_bytes() + b"tampered") + with self.assertRaisesRegex(DataProductError, "checksum"): + verify_package(path) + + def test_empty_release_is_valid_and_geojson_has_no_features(self): + with tempfile.TemporaryDirectory() as directory: + write_package(Path(directory), metadata(count=0), []) + geojson = json.loads((Path(directory) / "locations.geojson").read_text()) + self.assertEqual(geojson["features"], []) + + def test_mixed_rights_are_carried_per_row(self): + second = row(facility_id="00000000-0000-0000-0000-000000000002", provenance_source_id="source-b", source_rights_status="cleared") + with tempfile.TemporaryDirectory() as directory: + result = write_package(Path(directory), {**metadata(count=2), "source_coverage": [{"source_id": "source-a", "row_count": 1}, {"source_id": "source-b", "row_count": 1}]}, [row(), second]) + self.assertEqual(result["manifest"]["row_counts"]["packaged_rows"], 2) + self.assertIn("source_rights_status", (Path(directory) / "data-dictionary.json").read_text()) + + def test_large_export_is_not_silently_truncated(self): + rows = [row(facility_id=f"00000000-0000-0000-0000-{number:012d}") for number in range(1001)] + with tempfile.TemporaryDirectory() as directory: + result = write_package(Path(directory), metadata(count=len(rows)), rows) + self.assertEqual(result["manifest"]["row_counts"]["packaged_rows"], 1001) + + def test_dictionary_is_machine_readable_and_field_order_is_stable(self): + dictionary = data_dictionary() + self.assertEqual([field["name"] for field in dictionary["fields"]], list(CSV_FIELDS)) + self.assertEqual(dictionary["schema_version"], "uec-location-projection-v1") + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/common/test_source_operations.py b/pipeline/common/test_source_operations.py index eda5c79..dce7d8f 100644 --- a/pipeline/common/test_source_operations.py +++ b/pipeline/common/test_source_operations.py @@ -24,7 +24,7 @@ class SourceOperationsTests(unittest.TestCase): def test_checked_in_schedule_inventory_matches_registry(self): root = Path(__file__).parents[1] schedules = load_source_schedules(registry_path=root / "source_registry.json") - self.assertEqual(len(schedules), 16) + self.assertEqual(len(schedules), 20) self.assertEqual(schedules["dk.smiley"].stale_after_hours, 240) self.assertEqual(schedules["fr.dgal.section-i"].interval_hours, 24) self.assertIsNone(schedules["us.fsis"].interval_hours) diff --git a/pipeline/contracts/GRAPH-CANDIDATE-HANDOFF.md b/pipeline/contracts/GRAPH-CANDIDATE-HANDOFF.md new file mode 100644 index 0000000..444c4ac --- /dev/null +++ b/pipeline/contracts/GRAPH-CANDIDATE-HANDOFF.md @@ -0,0 +1,17 @@ +# Private graph candidate handoff v1 + +`graph_candidate_handoff.py` defines the smallest adapter-to-graph boundary. +Each candidate contains a source-qualified record key, preserved `source_values`, +source-native identifiers for facilities and organizations, local references for +relationships, optional claims with at least one supporting artifact/record, +and optional source-scoped crosswalks. + +The handoff is always `storage_state: private`, `privacy_status: pending`, +`review_state: review_required`, `publication_status: not_eligible`, and +`release_id: null`. It is not a database import, identity decision, review, or +publication authorization. Adapters must not add `canonical_id`, `global_id`, +or any other universal-identity assertion. Unknown relationships and claims +must carry an explicit reason. The `inspection`, `violation`, `commitment`, +`investigation`, `public_funding`, and `animal_count` claim domains are reserved +attachment points; domain-specific schemas and workflows are intentionally +deferred. diff --git a/pipeline/contracts/fixtures/synthetic_graph_candidate.json b/pipeline/contracts/fixtures/synthetic_graph_candidate.json new file mode 100644 index 0000000..e800888 --- /dev/null +++ b/pipeline/contracts/fixtures/synthetic_graph_candidate.json @@ -0,0 +1,13 @@ +{ + "contract_version": "graph-candidate-handoff-v1", + "source_id": "synthetic.us-pilot", + "source_record_key": "facility-001", + "source_row": 1, + "source_values": {"facility_name": "Synthetic Works", "operator_id": "OP-001"}, + "facilities": [{"local_ref": "facility:1", "source_identifier": {"identifier_type": "permit", "identity_scope": "source_scoped", "value": "FAC-001"}}], + "organizations": [{"local_ref": "organization:1", "source_identifier": {"identifier_type": "registration", "identity_scope": "source_scoped", "value": "OP-001"}}], + "relationships": [{"from_organization_ref": "organization:1", "target_facility_ref": "facility:1", "relationship_type": "operator", "assertion_status": "asserted", "valid_from": "2025-01-01", "valid_to": null, "observed_at": "2026-09-15T00:00:00Z", "confidence": 0.8, "review_state": "review_required"}], + "claims": [{"facility_ref": "facility:1", "claim_domain": "animal_count", "claim_kind": "annual_headcount", "value_state": "known", "claim_value": {"count": 12, "unit": "animals"}, "observed_at": "2026-01-01T00:00:00Z", "confidence": null, "support": [{"source_record_key": "facility-001"}]}], + "crosswalks": [], + "publication": {"storage_state": "private", "privacy_status": "pending", "review_state": "review_required", "publication_status": "not_eligible", "release_id": null} +} diff --git a/pipeline/contracts/graph_candidate_handoff.py b/pipeline/contracts/graph_candidate_handoff.py new file mode 100644 index 0000000..851cac5 --- /dev/null +++ b/pipeline/contracts/graph_candidate_handoff.py @@ -0,0 +1,190 @@ +"""Small, deterministic candidate shape for future country adapters. + +This is a private handoff contract only. It carries source-native identifiers +and local references; it intentionally has no universal or canonical identity +field and cannot authorize import, review, or publication. +""" +from __future__ import annotations + +import hashlib +import json +import os +from pathlib import Path +from typing import Any + + +CONTRACT_VERSION = "graph-candidate-handoff-v1" +GRAPH_DOMAINS = { + "identity", "location", "operation", "ownership", "inspection", "violation", + "commitment", "investigation", "public_funding", "animal_count", "other", +} +RELATIONSHIP_TYPES = {"operator", "owner", "parent", "brand", "supplier", "customer"} +REVIEW_STATES = {"unreviewed", "review_required", "reviewed", "accepted", "rejected"} + + +def _required_text(value: Any, field: str) -> None: + if not isinstance(value, str) or not value.strip(): + raise ValueError(f"{field} must be a non-empty string") + + +def _optional_confidence(value: Any, field: str) -> None: + if value is not None and (not isinstance(value, (int, float)) or not 0 <= value <= 1): + raise ValueError(f"{field} must be null or between 0 and 1") + + +def _assert_date_like(value: Any, field: str) -> None: + if value is not None and (not isinstance(value, str) or len(value) < 10): + raise ValueError(f"{field} must be null or an ISO date/time string") + + +def _refs(items: Any, field: str) -> set[str]: + if not isinstance(items, list): + raise ValueError(f"{field} must be a list") + refs: set[str] = set() + for item in items: + if not isinstance(item, dict): + raise ValueError(f"{field} entries must be objects") + _required_text(item.get("local_ref"), f"{field}.local_ref") + ref = item["local_ref"] + if ref in refs: + raise ValueError(f"duplicate local_ref: {ref}") + refs.add(ref) + identifier = item.get("source_identifier") + if not isinstance(identifier, dict): + raise ValueError(f"{field}.source_identifier is required") + _required_text(identifier.get("identifier_type"), f"{field}.source_identifier.identifier_type") + _required_text(identifier.get("value"), f"{field}.source_identifier.value") + if identifier.get("identity_scope", "source_scoped") != "source_scoped": + raise ValueError("source identifiers must use source_scoped identity_scope") + return refs + + +def validate_graph_candidate(candidate: dict[str, Any]) -> dict[str, Any]: + """Validate and return a candidate without inferring identities.""" + if not isinstance(candidate, dict): + raise ValueError("candidate must be an object") + if candidate.get("contract_version") != CONTRACT_VERSION: + raise ValueError(f"contract_version must be {CONTRACT_VERSION}") + for field in ("source_id", "source_record_key"): + _required_text(candidate.get(field), field) + if not isinstance(candidate.get("source_row"), int) or candidate["source_row"] < 1: + raise ValueError("source_row must be a positive integer") + if not isinstance(candidate.get("source_values"), dict): + raise ValueError("source_values must be an object") + + publication = candidate.get("publication") + expected_publication = { + "storage_state": "private", + "privacy_status": "pending", + "review_state": "review_required", + "publication_status": "not_eligible", + "release_id": None, + } + if publication != expected_publication: + raise ValueError("candidate handoffs must remain private and not eligible for publication") + + forbidden = {"canonical_id", "global_id", "universal_id", "universal_identity"} + if forbidden.intersection(candidate): + raise ValueError("candidate must not assert a universal identity") + + facilities = _refs(candidate.get("facilities"), "facilities") + organizations = _refs(candidate.get("organizations"), "organizations") + all_refs = facilities | organizations + if not facilities and not organizations: + raise ValueError("candidate must contain at least one source-native entity") + + relationships = candidate.get("relationships", []) + if not isinstance(relationships, list): + raise ValueError("relationships must be a list") + for relationship in relationships: + if not isinstance(relationship, dict): + raise ValueError("relationship entries must be objects") + status = relationship.get("assertion_status", "asserted") + if status not in {"asserted", "unknown", "disputed", "rejected"}: + raise ValueError("relationship assertion_status is invalid") + relationship_type = relationship.get("relationship_type") + if status == "unknown": + if relationship_type is not None or relationship.get("from_organization_ref") is not None: + raise ValueError("unknown relationships cannot name a type or organization") + _required_text(relationship.get("unknown_reason"), "relationship.unknown_reason") + else: + if relationship_type not in RELATIONSHIP_TYPES: + raise ValueError("relationship_type is invalid") + if relationship.get("from_organization_ref") not in organizations: + raise ValueError("relationship.from_organization_ref must reference an organization") + if relationship.get("unknown_reason") is not None: + raise ValueError("known relationships cannot include unknown_reason") + targets = [relationship.get("target_facility_ref"), relationship.get("target_organization_ref")] + if sum(target is not None for target in targets) != 1 or (targets[0] is not None and targets[0] not in facilities) or (targets[1] is not None and targets[1] not in organizations): + raise ValueError("relationship must reference exactly one valid target") + _assert_date_like(relationship.get("valid_from"), "relationship.valid_from") + _assert_date_like(relationship.get("valid_to"), "relationship.valid_to") + _required_text(relationship.get("observed_at"), "relationship.observed_at") + _optional_confidence(relationship.get("confidence"), "relationship.confidence") + if relationship.get("review_state", "review_required") not in REVIEW_STATES: + raise ValueError("relationship.review_state is invalid") + + claims = candidate.get("claims", []) + if not isinstance(claims, list): + raise ValueError("claims must be a list") + for claim in claims: + if not isinstance(claim, dict): + raise ValueError("claim entries must be objects") + if claim.get("claim_domain") not in GRAPH_DOMAINS: + raise ValueError("claim.claim_domain is invalid") + _required_text(claim.get("claim_kind"), "claim.claim_kind") + targets = [claim.get("facility_ref"), claim.get("organization_ref")] + if sum(target is not None for target in targets) != 1 or (targets[0] is not None and targets[0] not in facilities) or (targets[1] is not None and targets[1] not in organizations): + raise ValueError("claim must reference exactly one valid subject") + value_state = claim.get("value_state", "known") + if value_state not in {"known", "unknown", "not_applicable", "withheld"}: + raise ValueError("claim.value_state is invalid") + if value_state == "unknown": + _required_text(claim.get("unknown_reason"), "claim.unknown_reason") + elif claim.get("unknown_reason") is not None: + raise ValueError("known claims cannot include unknown_reason") + _required_text(claim.get("observed_at"), "claim.observed_at") + _optional_confidence(claim.get("confidence"), "claim.confidence") + if not isinstance(claim.get("support"), list) or not claim["support"]: + raise ValueError("claim.support must contain at least one artifact or source-record reference") + for support in claim["support"]: + if not isinstance(support, dict) or not (support.get("source_record_key") or support.get("artifact_sha256")): + raise ValueError("claim.support entries need source_record_key or artifact_sha256") + + for crosswalk in candidate.get("crosswalks", []): + if not isinstance(crosswalk, dict): + raise ValueError("crosswalk entries must be objects") + if crosswalk.get("left_ref") not in all_refs or crosswalk.get("right_ref") not in all_refs: + raise ValueError("crosswalk refs must reference candidate entities") + if crosswalk["left_ref"] == crosswalk["right_ref"]: + raise ValueError("crosswalk cannot link an entity to itself") + if crosswalk.get("identity_scope", "source_scoped") != "source_scoped": + raise ValueError("crosswalks must remain source-scoped") + _required_text(crosswalk.get("match_method"), "crosswalk.match_method") + _optional_confidence(crosswalk.get("confidence"), "crosswalk.confidence") + + return candidate + + +def canonical_json_bytes(candidate: dict[str, Any]) -> bytes: + validate_graph_candidate(candidate) + return (json.dumps(candidate, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + "\n").encode("utf-8") + + +def write_graph_candidate(run_dir: str | Path, candidate: dict[str, Any]) -> dict[str, Any]: + """Write one private candidate and a checksum manifest for handoff.""" + payload = canonical_json_bytes(candidate) + root = Path(run_dir) + root.mkdir(parents=True, exist_ok=True) + target = root / "graph-candidate.json" + temporary = target.with_name(target.name + ".tmp") + temporary.write_bytes(payload) + os.replace(temporary, target) + return { + "contract_version": CONTRACT_VERSION, + "source_id": candidate["source_id"], + "source_record_key": candidate["source_record_key"], + "sha256": hashlib.sha256(payload).hexdigest(), + "byte_size": len(payload), + "publication_state": "private-candidate", + } diff --git a/pipeline/contracts/test_graph_candidate_handoff.py b/pipeline/contracts/test_graph_candidate_handoff.py new file mode 100644 index 0000000..eb8bfb6 --- /dev/null +++ b/pipeline/contracts/test_graph_candidate_handoff.py @@ -0,0 +1,78 @@ +import hashlib +import tempfile +import unittest +from pathlib import Path + +from .graph_candidate_handoff import ( + CONTRACT_VERSION, + canonical_json_bytes, + validate_graph_candidate, + write_graph_candidate, +) + + +def candidate(): + return { + "contract_version": CONTRACT_VERSION, + "source_id": "synthetic.us-pilot", + "source_record_key": "facility-001", + "source_row": 1, + "source_values": {"operator_id": "OP-001", "facility_name": "Synthetic Works"}, + "facilities": [{"local_ref": "facility:1", "source_identifier": {"identifier_type": "permit", "value": "FAC-001"}}], + "organizations": [{"local_ref": "organization:1", "source_identifier": {"identifier_type": "registration", "value": "OP-001"}}], + "relationships": [{ + "from_organization_ref": "organization:1", "target_facility_ref": "facility:1", + "relationship_type": "operator", "assertion_status": "asserted", + "valid_from": "2025-01-01", "valid_to": None, "observed_at": "2026-09-15T00:00:00Z", + "confidence": 0.8, "review_state": "review_required", + }], + "claims": [{ + "facility_ref": "facility:1", "claim_domain": "animal_count", "claim_kind": "annual_headcount", + "value_state": "known", "claim_value": {"count": 12, "unit": "animals"}, + "observed_at": "2026-01-01T00:00:00Z", "confidence": None, + "support": [{"source_record_key": "facility-001"}], + }], + "crosswalks": [{ + "left_ref": "facility:1", "right_ref": "organization:1", "identity_scope": "source_scoped", + "match_method": "same source registration", "confidence": 0.5, + }], + "publication": { + "storage_state": "private", "privacy_status": "pending", "review_state": "review_required", + "publication_status": "not_eligible", "release_id": None, + }, + } + + +class GraphCandidateHandoffTests(unittest.TestCase): + def test_validates_and_writes_deterministically(self): + value = candidate() + validate_graph_candidate(value) + payload = canonical_json_bytes(value) + with tempfile.TemporaryDirectory() as directory: + manifest = write_graph_candidate(directory, value) + self.assertEqual(manifest["sha256"], hashlib.sha256(payload).hexdigest()) + self.assertEqual(Path(directory, "graph-candidate.json").read_bytes(), payload) + self.assertEqual(manifest["publication_state"], "private-candidate") + + def test_rejects_universal_identity_and_public_state(self): + value = candidate() + value["global_id"] = "do-not-accept" + with self.assertRaisesRegex(ValueError, "universal"): + validate_graph_candidate(value) + value = candidate() + value["publication"]["publication_status"] = "released" + with self.assertRaisesRegex(ValueError, "private"): + validate_graph_candidate(value) + + def test_requires_explicit_unknown_reason(self): + value = candidate() + value["relationships"] = [{ + "assertion_status": "unknown", "target_facility_ref": "facility:1", + "observed_at": "2026-09-15T00:00:00Z", + }] + with self.assertRaisesRegex(ValueError, "unknown_reason"): + validate_graph_candidate(value) + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/migrations/026_graph_entities_crosswalks.sql b/pipeline/migrations/026_graph_entities_crosswalks.sql new file mode 100644 index 0000000..7b5332d --- /dev/null +++ b/pipeline/migrations/026_graph_entities_crosswalks.sql @@ -0,0 +1,121 @@ +-- Accountability Graph foundation: distinct organizations, source-native +-- identifiers, and source-scoped crosswalks. Nothing here asserts that two +-- source identifiers are universally the same entity. + +CREATE TABLE uec.organizations ( + organization_id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + canonical_name TEXT, + country_code CHAR(2), + organization_type TEXT NOT NULL DEFAULT 'unknown' + CHECK (organization_type IN ('company', 'government', 'nonprofit', 'cooperative', 'unknown', 'other')), + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +COMMENT ON TABLE uec.organizations IS + 'Canonical organization projection; evidence about names, ownership, and relationships lives in append-only graph rows.'; + +CREATE TABLE uec.source_entity_identifiers ( + identifier_id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + source_id TEXT NOT NULL REFERENCES uec.sources(source_id), + source_record_id UUID NOT NULL REFERENCES uec.source_records(source_record_id), + entity_type TEXT NOT NULL CHECK (entity_type IN ('facility', 'organization')), + facility_id UUID REFERENCES uec.facilities(facility_id), + organization_id UUID REFERENCES uec.organizations(organization_id), + identifier_type TEXT NOT NULL, + source_identifier TEXT NOT NULL, + value_as_observed TEXT, + observed_at TIMESTAMPTZ NOT NULL, + review_state TEXT NOT NULL DEFAULT 'unreviewed' + CHECK (review_state IN ('unreviewed', 'review_required', 'reviewed', 'accepted', 'rejected')), + storage_state TEXT NOT NULL DEFAULT 'private' + CHECK (storage_state IN ('raw', 'private', 'reviewed', 'released')), + privacy_status TEXT NOT NULL DEFAULT 'pending' + CHECK (privacy_status IN ('pending', 'passed', 'failed', 'suppressed')), + publication_status TEXT NOT NULL DEFAULT 'not_eligible' + CHECK (publication_status IN ('not_eligible', 'eligible', 'released', 'suppressed')), + release_id TEXT REFERENCES uec.releases(release_id), + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + CHECK ((entity_type = 'facility' AND facility_id IS NOT NULL AND organization_id IS NULL) + OR (entity_type = 'organization' AND organization_id IS NOT NULL AND facility_id IS NULL)), + CHECK (publication_status <> 'released' OR (release_id IS NOT NULL AND storage_state = 'released' + AND review_state = 'accepted' AND privacy_status = 'passed')), + CHECK (storage_state <> 'released' OR publication_status = 'released') +); + +CREATE INDEX source_entity_identifiers_lookup_idx + ON uec.source_entity_identifiers (source_id, identifier_type, source_identifier); +CREATE INDEX source_entity_identifiers_entity_idx + ON uec.source_entity_identifiers (entity_type, facility_id, organization_id, observed_at DESC); + +CREATE TABLE uec.source_entity_crosswalks ( + crosswalk_id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + left_identifier_id UUID NOT NULL REFERENCES uec.source_entity_identifiers(identifier_id), + right_identifier_id UUID NOT NULL REFERENCES uec.source_entity_identifiers(identifier_id), + source_id TEXT NOT NULL REFERENCES uec.sources(source_id), + source_record_id UUID NOT NULL REFERENCES uec.source_records(source_record_id), + assertion_status TEXT NOT NULL DEFAULT 'review_required' + CHECK (assertion_status IN ('candidate', 'accepted', 'rejected', 'review_required', 'disputed')), + identity_scope TEXT NOT NULL DEFAULT 'source_scoped' + CHECK (identity_scope = 'source_scoped'), + match_method TEXT NOT NULL, + confidence NUMERIC(5,4) CHECK (confidence IS NULL OR confidence BETWEEN 0 AND 1), + review_state TEXT NOT NULL DEFAULT 'review_required' + CHECK (review_state IN ('unreviewed', 'review_required', 'reviewed', 'accepted', 'rejected')), + storage_state TEXT NOT NULL DEFAULT 'private' + CHECK (storage_state IN ('raw', 'private', 'reviewed', 'released')), + privacy_status TEXT NOT NULL DEFAULT 'pending' + CHECK (privacy_status IN ('pending', 'passed', 'failed', 'suppressed')), + publication_status TEXT NOT NULL DEFAULT 'not_eligible' + CHECK (publication_status IN ('not_eligible', 'eligible', 'released', 'suppressed')), + release_id TEXT REFERENCES uec.releases(release_id), + observed_at TIMESTAMPTZ NOT NULL, + note TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + CHECK (left_identifier_id <> right_identifier_id), + CHECK (publication_status <> 'released' OR (release_id IS NOT NULL AND storage_state = 'released' + AND review_state = 'accepted' AND privacy_status = 'passed')), + CHECK (storage_state <> 'released' OR publication_status = 'released') +); + +CREATE INDEX source_entity_crosswalks_pair_idx + ON uec.source_entity_crosswalks (left_identifier_id, right_identifier_id, observed_at DESC); +CREATE INDEX source_entity_crosswalks_evidence_idx + ON uec.source_entity_crosswalks (source_id, source_record_id); + +CREATE OR REPLACE FUNCTION uec.assert_graph_source_matches_record() +RETURNS trigger +LANGUAGE plpgsql +AS $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 + FROM uec.source_records + WHERE source_record_id = NEW.source_record_id + AND source_id = NEW.source_id + ) THEN + RAISE EXCEPTION 'graph source_id must match source_record_id %', NEW.source_record_id; + END IF; + RETURN NEW; +END; +$$; + +CREATE TRIGGER source_entity_identifiers_source_match + BEFORE INSERT ON uec.source_entity_identifiers + FOR EACH ROW EXECUTE FUNCTION uec.assert_graph_source_matches_record(); + +CREATE TRIGGER source_entity_crosswalks_source_match + BEFORE INSERT ON uec.source_entity_crosswalks + FOR EACH ROW EXECUTE FUNCTION uec.assert_graph_source_matches_record(); + +CREATE TRIGGER source_entity_identifiers_append_only + BEFORE UPDATE OR DELETE ON uec.source_entity_identifiers + FOR EACH ROW EXECUTE FUNCTION uec.reject_evidence_mutation(); + +CREATE TRIGGER source_entity_crosswalks_append_only + BEFORE UPDATE OR DELETE ON uec.source_entity_crosswalks + FOR EACH ROW EXECUTE FUNCTION uec.reject_evidence_mutation(); + +COMMENT ON TABLE uec.source_entity_identifiers IS + 'Source-native identifiers remain qualified by source and record; identifiers are not global IDs.'; +COMMENT ON TABLE uec.source_entity_crosswalks IS + 'Append-only, source-scoped candidate/decision links. Rejected or disputed mappings remain evidence.'; diff --git a/pipeline/migrations/027_graph_relationship_observations.sql b/pipeline/migrations/027_graph_relationship_observations.sql new file mode 100644 index 0000000..128624a --- /dev/null +++ b/pipeline/migrations/027_graph_relationship_observations.sql @@ -0,0 +1,89 @@ +-- Dated relationship observations. A later observation is a projection input; +-- it does not rewrite or erase an earlier observation. + +CREATE TABLE uec.organization_relationship_observations ( + relationship_observation_id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + source_id TEXT NOT NULL REFERENCES uec.sources(source_id), + source_record_id UUID NOT NULL REFERENCES uec.source_records(source_record_id), + from_organization_id UUID REFERENCES uec.organizations(organization_id), + target_facility_id UUID REFERENCES uec.facilities(facility_id), + target_organization_id UUID REFERENCES uec.organizations(organization_id), + relationship_type TEXT CHECK (relationship_type IN ('operator', 'owner', 'parent', 'brand', 'supplier', 'customer')), + assertion_status TEXT NOT NULL DEFAULT 'asserted' + CHECK (assertion_status IN ('asserted', 'unknown', 'disputed', 'rejected')), + unknown_reason TEXT, + valid_from DATE, + valid_to DATE, + observed_at TIMESTAMPTZ NOT NULL, + confidence NUMERIC(5,4) CHECK (confidence IS NULL OR confidence BETWEEN 0 AND 1), + review_state TEXT NOT NULL DEFAULT 'review_required' + CHECK (review_state IN ('unreviewed', 'review_required', 'reviewed', 'accepted', 'rejected')), + storage_state TEXT NOT NULL DEFAULT 'private' + CHECK (storage_state IN ('raw', 'private', 'reviewed', 'released')), + privacy_status TEXT NOT NULL DEFAULT 'pending' + CHECK (privacy_status IN ('pending', 'passed', 'failed', 'suppressed')), + publication_status TEXT NOT NULL DEFAULT 'not_eligible' + CHECK (publication_status IN ('not_eligible', 'eligible', 'released', 'suppressed')), + release_id TEXT REFERENCES uec.releases(release_id), + note TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + CHECK (valid_to IS NULL OR valid_from IS NULL OR valid_to >= valid_from), + CHECK ((assertion_status = 'unknown' + AND relationship_type IS NULL + AND from_organization_id IS NULL + AND unknown_reason IS NOT NULL AND btrim(unknown_reason) <> '') + OR (assertion_status <> 'unknown' + AND relationship_type IS NOT NULL + AND from_organization_id IS NOT NULL + AND unknown_reason IS NULL)), + CHECK ((target_facility_id IS NOT NULL) <> (target_organization_id IS NOT NULL)), + CHECK (publication_status <> 'released' OR (release_id IS NOT NULL AND storage_state = 'released' + AND review_state = 'accepted' AND privacy_status = 'passed')), + CHECK (storage_state <> 'released' OR publication_status = 'released') +); + +CREATE INDEX organization_relationship_observations_current_idx + ON uec.organization_relationship_observations + (from_organization_id, target_facility_id, target_organization_id, relationship_type, + observed_at DESC, relationship_observation_id DESC); +CREATE INDEX organization_relationship_observations_source_idx + ON uec.organization_relationship_observations (source_id, source_record_id); + +CREATE OR REPLACE VIEW uec.organization_relationship_current AS +SELECT DISTINCT ON (from_organization_id, target_facility_id, target_organization_id, relationship_type) + relationship_observation_id, + source_id, + source_record_id, + from_organization_id, + target_facility_id, + target_organization_id, + relationship_type, + assertion_status, + unknown_reason, + valid_from, + valid_to, + observed_at, + confidence, + review_state, + storage_state, + privacy_status, + publication_status, + release_id, + note, + created_at +FROM uec.organization_relationship_observations +ORDER BY from_organization_id, target_facility_id, target_organization_id, relationship_type, + observed_at DESC, relationship_observation_id DESC; + +CREATE TRIGGER organization_relationship_observations_source_match + BEFORE INSERT ON uec.organization_relationship_observations + FOR EACH ROW EXECUTE FUNCTION uec.assert_graph_source_matches_record(); + +CREATE TRIGGER organization_relationship_observations_append_only + BEFORE UPDATE OR DELETE ON uec.organization_relationship_observations + FOR EACH ROW EXECUTE FUNCTION uec.reject_evidence_mutation(); + +COMMENT ON TABLE uec.organization_relationship_observations IS + 'Append-only dated assertions, disputes, rejections, and explicit unknowns. Supplier/customer rows require evidence like every other type.'; +COMMENT ON VIEW uec.organization_relationship_current IS + 'Latest observation for each scoped endpoint/type. Different target organizations remain visible so contradictory ownership/operator observations coexist.'; diff --git a/pipeline/migrations/028_graph_claims_support.sql b/pipeline/migrations/028_graph_claims_support.sql new file mode 100644 index 0000000..72c72af --- /dev/null +++ b/pipeline/migrations/028_graph_claims_support.sql @@ -0,0 +1,107 @@ +-- Evidence-backed claims are deliberately typed but value-extensible. The +-- listed domains reserve attachment points for future work; no domain-specific +-- implementation is implied by this foundation. + +CREATE TABLE uec.claims ( + claim_id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + source_id TEXT NOT NULL REFERENCES uec.sources(source_id), + source_record_id UUID NOT NULL REFERENCES uec.source_records(source_record_id), + facility_id UUID REFERENCES uec.facilities(facility_id), + organization_id UUID REFERENCES uec.organizations(organization_id), + claim_domain TEXT NOT NULL CHECK (claim_domain IN ( + 'identity', 'location', 'operation', 'ownership', 'inspection', + 'violation', 'commitment', 'investigation', 'public_funding', 'animal_count', 'other' + )), + claim_kind TEXT NOT NULL, + value_state TEXT NOT NULL DEFAULT 'known' + CHECK (value_state IN ('known', 'unknown', 'not_applicable', 'withheld')), + claim_value JSONB NOT NULL DEFAULT '{}'::jsonb, + unknown_reason TEXT, + valid_from DATE, + valid_to DATE, + observed_at TIMESTAMPTZ NOT NULL, + confidence NUMERIC(5,4) CHECK (confidence IS NULL OR confidence BETWEEN 0 AND 1), + review_state TEXT NOT NULL DEFAULT 'review_required' + CHECK (review_state IN ('unreviewed', 'review_required', 'reviewed', 'accepted', 'disputed', 'rejected')), + storage_state TEXT NOT NULL DEFAULT 'private' + CHECK (storage_state IN ('raw', 'private', 'reviewed', 'released')), + privacy_status TEXT NOT NULL DEFAULT 'pending' + CHECK (privacy_status IN ('pending', 'passed', 'failed', 'suppressed')), + publication_status TEXT NOT NULL DEFAULT 'not_eligible' + CHECK (publication_status IN ('not_eligible', 'eligible', 'released', 'suppressed')), + release_id TEXT REFERENCES uec.releases(release_id), + note TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + CHECK ((facility_id IS NOT NULL) <> (organization_id IS NOT NULL)), + CHECK (valid_to IS NULL OR valid_from IS NULL OR valid_to >= valid_from), + CHECK ((value_state = 'unknown' AND unknown_reason IS NOT NULL AND btrim(unknown_reason) <> '') + OR (value_state <> 'unknown' AND unknown_reason IS NULL)), + CHECK (publication_status <> 'released' OR (release_id IS NOT NULL AND storage_state = 'released' + AND review_state = 'accepted' AND privacy_status = 'passed')), + CHECK (storage_state <> 'released' OR publication_status = 'released') +); + +CREATE INDEX claims_subject_kind_idx + ON uec.claims (facility_id, organization_id, claim_domain, claim_kind, observed_at DESC); +CREATE INDEX claims_source_idx ON uec.claims (source_id, source_record_id); + +CREATE TABLE uec.claim_support ( + claim_support_id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + claim_id UUID NOT NULL REFERENCES uec.claims(claim_id), + source_record_id UUID REFERENCES uec.source_records(source_record_id), + artifact_id UUID REFERENCES uec.raw_artifacts(artifact_id), + support_role TEXT NOT NULL CHECK (support_role IN ('primary', 'corroborating', 'contradicting', 'context')), + observed_at TIMESTAMPTZ NOT NULL, + storage_state TEXT NOT NULL DEFAULT 'private' + CHECK (storage_state IN ('raw', 'private', 'reviewed', 'released')), + note TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + CHECK (source_record_id IS NOT NULL OR artifact_id IS NOT NULL) +); + +CREATE INDEX claim_support_claim_idx ON uec.claim_support (claim_id, observed_at DESC); +CREATE INDEX claim_support_record_idx ON uec.claim_support (source_record_id); + +CREATE OR REPLACE VIEW uec.claim_current AS +SELECT DISTINCT ON (facility_id, organization_id, claim_domain, claim_kind, value_state, claim_value) + claim_id, + source_id, + source_record_id, + facility_id, + organization_id, + claim_domain, + claim_kind, + value_state, + claim_value, + unknown_reason, + valid_from, + valid_to, + observed_at, + confidence, + review_state, + storage_state, + privacy_status, + publication_status, + release_id, + note, + created_at +FROM uec.claims +ORDER BY facility_id, organization_id, claim_domain, claim_kind, value_state, claim_value, + observed_at DESC, claim_id DESC; + +CREATE TRIGGER claims_source_match + BEFORE INSERT ON uec.claims + FOR EACH ROW EXECUTE FUNCTION uec.assert_graph_source_matches_record(); + +CREATE TRIGGER claims_append_only + BEFORE UPDATE OR DELETE ON uec.claims + FOR EACH ROW EXECUTE FUNCTION uec.reject_evidence_mutation(); + +CREATE TRIGGER claim_support_append_only + BEFORE UPDATE OR DELETE ON uec.claim_support + FOR EACH ROW EXECUTE FUNCTION uec.reject_evidence_mutation(); + +COMMENT ON TABLE uec.claims IS + 'Append-only claims whose values may conflict. claim_domain includes reserved inspection, violation, commitment, investigation, public-funding, and animal-count attachment points.'; +COMMENT ON TABLE uec.claim_support IS + 'Append-only links to retained artifacts or source records; support_role makes corroboration and contradiction explicit.'; diff --git a/pipeline/migrations/029_graph_publication_projections.sql b/pipeline/migrations/029_graph_publication_projections.sql new file mode 100644 index 0000000..3a2fba3 --- /dev/null +++ b/pipeline/migrations/029_graph_publication_projections.sql @@ -0,0 +1,108 @@ +-- Public graph projections are opt-in, release-scoped, privacy-screened, and +-- suppression-aware. Raw/private/reviewed graph rows are never public by view +-- default, even when their source is government-originated. + +CREATE OR REPLACE VIEW uec.graph_public_relationships AS +SELECT relationship.relationship_observation_id, + relationship.release_id, + relationship.from_organization_id, + relationship.target_facility_id, + relationship.target_organization_id, + relationship.relationship_type, + relationship.assertion_status, + relationship.valid_from, + relationship.valid_to, + relationship.observed_at, + relationship.confidence, + relationship.review_state, + relationship.source_id, + relationship.source_record_id +FROM uec.organization_relationship_observations relationship +JOIN uec.releases release ON release.release_id = relationship.release_id +WHERE relationship.publication_status = 'released' + AND relationship.storage_state = 'released' + AND relationship.review_state = 'accepted' + AND relationship.privacy_status = 'passed' + AND release.status = 'promoted' + AND NOT EXISTS ( + SELECT 1 FROM uec.public_access_restricted restricted + WHERE restricted.source_record_id = relationship.source_record_id + ); + +CREATE OR REPLACE VIEW uec.graph_public_claims AS +SELECT claim.claim_id, + claim.release_id, + claim.facility_id, + claim.organization_id, + claim.claim_domain, + claim.claim_kind, + claim.value_state, + claim.claim_value, + claim.unknown_reason, + claim.valid_from, + claim.valid_to, + claim.observed_at, + claim.confidence, + claim.review_state, + claim.source_id, + claim.source_record_id +FROM uec.claims claim +JOIN uec.releases release ON release.release_id = claim.release_id +WHERE claim.publication_status = 'released' + AND claim.storage_state = 'released' + AND claim.review_state = 'accepted' + AND claim.privacy_status = 'passed' + AND release.status = 'promoted' + AND NOT EXISTS ( + SELECT 1 FROM uec.public_access_restricted restricted + WHERE restricted.source_record_id = claim.source_record_id + ); + +CREATE OR REPLACE VIEW uec.graph_publication_safety AS +SELECT 'relationship'::TEXT AS graph_kind, + relationship.relationship_observation_id AS graph_id, + relationship.release_id, + relationship.publication_status, + relationship.storage_state, + relationship.review_state, + relationship.privacy_status, + (COALESCE(release.status = 'promoted', false) + AND relationship.publication_status = 'released' + AND relationship.storage_state = 'released' + AND relationship.review_state = 'accepted' + AND relationship.privacy_status = 'passed' + AND NOT EXISTS (SELECT 1 FROM uec.public_access_restricted restricted + WHERE restricted.source_record_id = relationship.source_record_id)) AS public_eligible +FROM uec.organization_relationship_observations relationship +LEFT JOIN uec.releases release ON release.release_id = relationship.release_id +UNION ALL +SELECT 'claim'::TEXT, + claim.claim_id, + claim.release_id, + claim.publication_status, + claim.storage_state, + claim.review_state, + claim.privacy_status, + (COALESCE(release.status = 'promoted', false) + AND claim.publication_status = 'released' + AND claim.storage_state = 'released' + AND claim.review_state = 'accepted' + AND claim.privacy_status = 'passed' + AND NOT EXISTS (SELECT 1 FROM uec.public_access_restricted restricted + WHERE restricted.source_record_id = claim.source_record_id)) +FROM uec.claims claim +LEFT JOIN uec.releases release ON release.release_id = claim.release_id; + +CREATE INDEX organization_relationship_public_idx + ON uec.organization_relationship_observations (release_id, publication_status, privacy_status) + WHERE publication_status = 'released'; +CREATE INDEX claims_public_idx + ON uec.claims (release_id, claim_domain, publication_status, privacy_status) + WHERE publication_status = 'released'; + +COMMENT ON VIEW uec.graph_public_relationships IS + 'Only promoted-release relationship observations with accepted review, passed privacy, released storage, and no active source-record suppression.'; +COMMENT ON VIEW uec.graph_public_claims IS + 'Only promoted-release claims with accepted review, passed privacy, released storage, and no active source-record suppression.'; +COMMENT ON VIEW uec.graph_publication_safety IS + 'Diagnostic projection showing why each graph row is or is not eligible for public output; it never exposes raw/private payloads.'; diff --git a/pipeline/scripts/maintenance/private-environment-gate.py b/pipeline/scripts/maintenance/private-environment-gate.py index 834a70e..a6ceca5 100644 --- a/pipeline/scripts/maintenance/private-environment-gate.py +++ b/pipeline/scripts/maintenance/private-environment-gate.py @@ -152,6 +152,19 @@ def validate_release_manifest(path: Path, expected_digest: str) -> dict[str, Any names.append(item["name"]) if len(names) != len(set(names)): raise PrivateEnvironmentError("release manifest artifact inventory is invalid") + if manifest["manifest_version"] == "uec-release-manifest-v2": + for field in ("data_product_version", "schema_version", "generated_at", "retrieved_at", "publication_state", "review_state", "limitations", "row_counts", "source_coverage", "checksums"): + if field not in manifest: + raise PrivateEnvironmentError(f"release manifest v2 field is missing: {field}") + if manifest["publication_state"] != "project-published" or manifest.get("test_only") is not False: + raise PrivateEnvironmentError("release manifest v2 publication state is invalid") + counts = manifest["row_counts"] + if not isinstance(counts, dict) or counts.get("eligible_rows") != counts.get("packaged_rows"): + raise PrivateEnvironmentError("release manifest v2 row counts are invalid") + if not isinstance(manifest["limitations"], list) or not all(isinstance(value, str) and value for value in manifest["limitations"]): + raise PrivateEnvironmentError("release manifest v2 limitations are invalid") + if not isinstance(manifest["checksums"], dict) or manifest["checksums"].get("algorithm") != "sha256": + raise PrivateEnvironmentError("release manifest v2 checksums are invalid") return {"manifest_version": manifest["manifest_version"], "release_id": manifest["release_id"], "profile": manifest["profile"], "artifact_count": len(artifacts), "manifest_sha256": actual} diff --git a/pipeline/scripts/maintenance/verify-data-product.py b/pipeline/scripts/maintenance/verify-data-product.py new file mode 100644 index 0000000..04662b3 --- /dev/null +++ b/pipeline/scripts/maintenance/verify-data-product.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 +"""Verify a packaged UEC public snapshot and its optional trusted digest.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +REPOSITORY_ROOT = Path(__file__).resolve().parents[3] +if str(REPOSITORY_ROOT) not in sys.path: + sys.path.insert(0, str(REPOSITORY_ROOT)) + +from pipeline.common.data_product import DataProductError, verify_package + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("package_dir", type=Path) + parser.add_argument("--manifest-sha256", help="Trusted digest obtained from a project-controlled channel") + args = parser.parse_args() + try: + result = verify_package(args.package_dir, args.manifest_sha256) + except (DataProductError, OSError, ValueError, json.JSONDecodeError) as error: + print(json.dumps({"status": "blocked", "error": str(error)}), file=sys.stderr) + return 1 + print(json.dumps(result, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/pipeline/scripts/stages/export-release.py b/pipeline/scripts/stages/export-release.py new file mode 100644 index 0000000..ee5072b --- /dev/null +++ b/pipeline/scripts/stages/export-release.py @@ -0,0 +1,169 @@ +#!/usr/bin/env python3 +"""Package one explicitly promoted, public release as CSV and GeoJSON. + +The query is intentionally release/profile scoped and repeats the publication +and suppression gates. It never reads raw fields and never promotes a release. +""" + +from __future__ import annotations + +import argparse +import json +import os +import sys +from collections import defaultdict +from datetime import timezone +from pathlib import Path + +import psycopg + +# Keep the documented ``python pipeline/scripts/...`` invocation independent +# of the caller's PYTHONPATH. +REPOSITORY_ROOT = Path(__file__).resolve().parents[3] +if str(REPOSITORY_ROOT) not in sys.path: + sys.path.insert(0, str(REPOSITORY_ROOT)) + +from pipeline.common.data_product import SUPPORTED_PROFILES, write_package + + +def _utc(value) -> str: + if value is None: + raise ValueError("release metadata has no timestamp") + if value.tzinfo is None: + raise ValueError("release metadata timestamp has no timezone") + return value.astimezone(timezone.utc).isoformat().replace("+00:00", "Z") + + +def _rights(attribution: str | None) -> str: + return "attribution_required" if attribution and attribution.strip() else "unknown" + + +def export_release(database_url: str, release_id: str, profile: str, output_dir: Path, generated_at: str | None = None) -> dict: + if profile not in SUPPORTED_PROFILES: + raise ValueError("profile is unsupported") + with psycopg.connect(database_url) as connection: + with connection.transaction(): + connection.execute("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ") + release = connection.execute( + """ + SELECT release_id, profile, status, test_only, ruleset_version, created_at + FROM uec.releases + WHERE release_id=%s AND profile=%s + """, + (release_id, profile), + ).fetchone() + if not release: + raise ValueError("release/profile not found") + if release[2] != "promoted" or release[3]: + raise ValueError("only a non-test promoted release can be packaged") + rows = connection.execute( + """ + SELECT h.facility_id, h.canonical_name, h.country_code, h.city, + h.classification_category, h.display_precision, + ST_Y(h.display_location::geometry), ST_X(h.display_location::geometry), + h.lifecycle_status, h.first_observed_at, h.last_observed_at, + h.observation_count, h.provenance_origin_type, + review.factual_review_status, review.privacy_screening_status, + review.maintainer_approval, review.reviewer_role, + h.provenance_source_id, h.provenance_source_name, + h.provenance_source_url, h.provenance_retrieved_at, + source.attribution + FROM uec.map_facilities_display_history h + JOIN uec.releases release ON release.release_id=h.release_id + JOIN uec.publication_review_release_current review + ON review.source_record_id=h.source_record_id + AND review.release_id=h.release_id + JOIN uec.sources source ON source.source_id=h.provenance_source_id + WHERE h.release_id=%s + AND release.status='promoted' + AND release.test_only IS NOT TRUE + AND release.profile=%s + AND review.publication_eligible=true + AND review.privacy_screening_status='passed' + AND (release.profile='community' OR review.maintainer_approval='approved') + AND NOT EXISTS ( + SELECT 1 FROM uec.public_access_restricted restricted + WHERE restricted.source_record_id=h.source_record_id + ) + ORDER BY h.facility_id, h.provenance_source_id + """, + (release_id, profile), + ).fetchall() + + projection = [] + for row in rows: + rights = _rights(row[21]) + projection.append( + { + "facility_id": str(row[0]), "canonical_name": row[1], "country_code": row[2], + "city": row[3], "category": row[4], "display_precision": row[5], + "latitude": row[6], "longitude": row[7], "lifecycle_status": row[8], + "first_observed_at": _utc(row[9]) if row[9] else None, + "last_observed_at": _utc(row[10]) if row[10] else None, + "observation_count": row[11], "source_type": row[12], + "factual_review_status": row[13] or "unreviewed", + "privacy_screening_status": row[14], "project_approval": row[15], + "reviewer_role": row[16], + "publication_warning": ( + "Unreviewed community claim — not verified by Until Every Cage" + if profile == "community" and (row[13] or "unreviewed") == "unreviewed" else None + ), + "publication_profile": profile, "release_id": release_id, + "release_ruleset_version": release[4], "provenance_source_id": row[17], + "provenance_source_name": row[18], "provenance_source_url": row[19], + "provenance_retrieved_at": _utc(row[20]), "source_rights_status": rights, + "publication_eligible": True, + } + ) + by_source = defaultdict(list) + for row in projection: + by_source[row["provenance_source_id"]].append(row) + coverage = [ + { + "source_id": source_id, + "row_count": len(source_rows), + "retrieved_at": {"first": min(r["provenance_retrieved_at"] for r in source_rows), "last": max(r["provenance_retrieved_at"] for r in source_rows)}, + "rights_status": sorted({r["source_rights_status"] for r in source_rows}), + } + for source_id, source_rows in sorted(by_source.items()) + ] + retrieved = min((r["provenance_retrieved_at"] for r in projection), default=_utc(release[5])) + metadata = { + "release_id": release_id, "profile": profile, "status": "promoted", "test_only": False, + "eligible": True, "publication_state": "project-published", "ruleset_version": release[4], + "schema_version": "uec-location-projection-v1", "generated_at": generated_at or _utc(release[5]), + "retrieved_at": retrieved, "source_coverage": coverage, + "row_counts": {"eligible_rows": len(projection)}, + "checksums": {}, + "review_state": "privacy-screened; may contain community-unreviewed claims" if profile == "community" else "project-approved and privacy-screened", + "limitations": [ + "Facility projection rows are not animal counts or a complete story-wide denominator.", + "Coordinates are exact or city-level display points only; unmapped and privacy-restricted locations are omitted.", + "Source availability and government origin do not certify factual accuracy or current operation.", + "Reuse is limited to the per-source rights status in each row; no project licence is implied.", + ], + "supersedes": None, + } + # write_package computes the projection checksum after validating all rows. + return write_package(output_dir, metadata, projection) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("release_id") + parser.add_argument("--profile", required=True, choices=sorted(SUPPORTED_PROFILES)) + parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument("--database-url", default=os.environ.get("UEC_DATABASE_URL", "postgresql://uec:uec-local-development-only@localhost:5433/uec")) + parser.add_argument("--generated-at", help="Stable UTC timestamp for reproducible packaging; defaults to release creation time") + args = parser.parse_args() + try: + result = export_release(args.database_url, args.release_id, args.profile, args.output_dir, args.generated_at) + except Exception as error: + print(json.dumps({"status": "blocked", "error": str(error)}), file=sys.stderr) + return 1 + print(json.dumps({"status": "packaged", "manifest_sha256": result["manifest_sha256"], "output_dir": str(args.output_dir)}, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/pipeline/scripts/stages/promote-release.py b/pipeline/scripts/stages/promote-release.py index 6f9f90b..8cac341 100644 --- a/pipeline/scripts/stages/promote-release.py +++ b/pipeline/scripts/stages/promote-release.py @@ -50,6 +50,12 @@ def inventory_artifacts(paths: list[Path], no_distributed_artifacts: bool) -> li return sorted(artifacts, key=lambda artifact: artifact["name"]) +def utc_iso(value) -> str: + if value.tzinfo is None: + raise ValueError("database timestamp must include a timezone") + return value.astimezone(timezone.utc).isoformat().replace("+00:00", "Z") + + def promote(database_url: str, release_id: str, artifacts: list[dict]) -> dict: with psycopg.connect(database_url) as connection: with connection.transaction(): @@ -83,10 +89,54 @@ def promote(database_url: str, release_id: str, artifacts: list[dict]) -> dict: WHERE m.release_id=%s AND m.default_visible AND NOT EXISTS (SELECT 1 FROM uec.public_access_restricted x WHERE x.source_record_id=sr.source_record_id) """, (release_id,)).fetchone() + coverage_rows = connection.execute(""" + SELECT sr.source_id, count(*)::int, min(artifact.retrieved_at), max(artifact.retrieved_at), + CASE WHEN source.attribution IS NULL OR btrim(source.attribution) = '' THEN 'unknown' ELSE 'attribution_required' END + FROM uec.release_members m + JOIN uec.observations o ON o.observation_id=m.observation_id + JOIN uec.source_records sr ON sr.source_record_id=o.source_record_id + JOIN uec.raw_artifacts artifact ON artifact.artifact_id=sr.artifact_id + JOIN uec.sources source ON source.source_id=sr.source_id + WHERE m.release_id=%s AND m.default_visible + AND NOT EXISTS (SELECT 1 FROM uec.public_access_restricted x WHERE x.source_record_id=sr.source_record_id) + GROUP BY sr.source_id, source.attribution ORDER BY sr.source_id + """, (release_id,)).fetchall() created_at = connection.execute("SELECT now()").fetchone()[0] if created_at.tzinfo is None: raise ValueError("database manifest creation time must include a timezone") - manifest = {"manifest_version": "v1", "release_id": release_id, "profile": target[1], "ruleset_version": target[2], "eligible_record_count": summary[0], "source_ids": summary[1], "created_at": created_at.astimezone(timezone.utc).isoformat().replace("+00:00", "Z"), "distributed_artifacts": artifacts} + source_coverage = [ + {"source_id": source_id, "row_count": row_count, "retrieved_at": {"first": utc_iso(first), "last": utc_iso(last)}, "rights_status": rights_status} + for source_id, row_count, first, last, rights_status in coverage_rows + ] + retrieved_at = min((item["retrieved_at"]["first"] for item in source_coverage), default=utc_iso(created_at)) + manifest = { + "manifest_version": "uec-release-manifest-v2", + "data_product_version": "uec-public-data-product-v1", + "release_id": release_id, + "profile": target[1], + "release_status": "promoted", + "test_only": False, + "ruleset_version": target[2], + "schema_version": "uec-location-projection-v1", + "generated_at": utc_iso(created_at), + "retrieved_at": retrieved_at, + "source_ids": summary[1], + "source_coverage": source_coverage, + "eligible_record_count": summary[0], + "row_counts": {"eligible_rows": summary[0], "packaged_rows": summary[0]}, + "checksums": {"algorithm": "sha256", "distributed_artifacts": artifacts}, + "review_state": "privacy-screened; community claims may be unreviewed" if target[1] == "community" else "project-approved and privacy-screened", + "publication_state": "project-published", + "limitations": [ + "Facility projection rows are not animal counts or a complete story-wide denominator.", + "Coordinates and addresses remain subject to current privacy and coarse-location rules.", + "Source origin and source availability do not certify factual accuracy or current operation.", + "Artifact checksums detect byte changes but do not establish factual accuracy or reuse rights.", + ], + "supersedes": None, + "created_at": utc_iso(created_at), + "distributed_artifacts": artifacts, + } # Python's sorted-key JSON is the canonical representation shared by consumers. canonical = canonical_json(manifest) digest = hashlib.sha256(canonical.encode("utf-8")).hexdigest() diff --git a/pipeline/source_operations.json b/pipeline/source_operations.json index 612c21f..59f96a1 100644 --- a/pipeline/source_operations.json +++ b/pipeline/source_operations.json @@ -3,6 +3,9 @@ "purpose": "Private-alpha scheduling, freshness, retention, and bounded retry expectations. This file does not authorize publication.", "schedules": [ {"source_id":"be.locations","cadence":"weekly","interval_hours":168,"stale_after_hours":240,"max_attempts":3,"backoff_seconds":1,"max_backoff_seconds":30,"manual_fallback":"use the authorized two-file operator capture"}, + {"source_id":"br.sif.export","cadence":"monthly","interval_hours":720,"stale_after_hours":960,"max_attempts":1,"backoff_seconds":0,"max_backoff_seconds":0,"manual_fallback":"retain the authorized MAPA SIF export CSV capture"}, + {"source_id":"br.sif.registered","cadence":"monthly","interval_hours":720,"stale_after_hours":960,"max_attempts":1,"backoff_seconds":0,"max_backoff_seconds":0,"manual_fallback":"retain the authorized MAPA SIF registered CSV capture"}, + {"source_id":"br.sisbi.public","cadence":"unknown","interval_hours":null,"stale_after_hours":null,"max_attempts":1,"backoff_seconds":0,"max_backoff_seconds":0,"manual_fallback":"confirm the bounded e-SISBI route and terms with an authorized operator"}, {"source_id":"ca.cfia.federal-meat","cadence":"unknown","interval_hours":null,"stale_after_hours":null,"max_attempts":1,"backoff_seconds":0,"max_backoff_seconds":0,"manual_fallback":"verify the CFIA federal registry artifact before scheduling"}, {"source_id":"ca.ontario.meat-plants","cadence":"unknown","interval_hours":null,"stale_after_hours":null,"max_attempts":1,"backoff_seconds":0,"max_backoff_seconds":0,"manual_fallback":"verify the permitted Ontario artifact before scheduling"}, {"source_id":"de.locations","cadence":"unknown","interval_hours":null,"stale_after_hours":null,"max_attempts":1,"backoff_seconds":0,"max_backoff_seconds":0,"manual_fallback":"select and preserve the BVL export through the operator route"}, @@ -14,6 +17,7 @@ {"source_id":"it.853-2004","cadence":"daily","interval_hours":24,"stale_after_hours":48,"max_attempts":3,"backoff_seconds":1,"max_backoff_seconds":30,"manual_fallback":"use the catalog-discovery capture route"}, {"source_id":"mx.locations","cadence":"unknown","interval_hours":null,"stale_after_hours":null,"max_attempts":1,"backoff_seconds":0,"max_backoff_seconds":0,"manual_fallback":"resolve official directory access and terms before scheduling"}, {"source_id":"nz.locations","cadence":"unknown","interval_hours":null,"stale_after_hours":null,"max_attempts":1,"backoff_seconds":0,"max_backoff_seconds":0,"manual_fallback":"use an authorized MPI register export"}, + {"source_id":"br.trase.facilities","cadence":"unknown","interval_hours":null,"stale_after_hours":null,"max_attempts":1,"backoff_seconds":0,"max_backoff_seconds":0,"manual_fallback":"retain a bounded Trase GeoJSON capture after terms and privacy review"}, {"source_id":"uk.locations","cadence":"monthly","interval_hours":720,"stale_after_hours":960,"max_attempts":3,"backoff_seconds":1,"max_backoff_seconds":30,"manual_fallback":"use the source-specific national operator capture"}, {"source_id":"us.aphis","cadence":"unknown","interval_hours":null,"stale_after_hours":null,"max_attempts":1,"backoff_seconds":0,"max_backoff_seconds":0,"manual_fallback":"verify the APHIS export workflow"}, {"source_id":"us.fsis","cadence":"unknown","interval_hours":null,"stale_after_hours":null,"max_attempts":1,"backoff_seconds":0,"max_backoff_seconds":0,"manual_fallback":"obtain authorized FSIS export access"}, diff --git a/pipeline/source_registry.json b/pipeline/source_registry.json index 76ab644..95c3e92 100644 --- a/pipeline/source_registry.json +++ b/pipeline/source_registry.json @@ -3,6 +3,54 @@ "evidence_basis": "Repository evidence; source-specific live verification is documented in the linked evidence files.", "unknown_value": "unknown", "sources": [ + { + "source_id": "br.sif.registered", + "jurisdiction_scope": "Brazil; MAPA/DIPOA establishments registered under the federal SIF directory", + "legacy_paths": [], + "url": "https://dados.agricultura.gov.br/dataset/062166e3-b515-4274-8e7d-68aadd64b820/resource/97277e92-264a-4dc0-9aea-f87b8ea93798/download/sigsifestabelecimentosregistradosnosif.csv", + "access_method": "published semicolon-delimited CSV via MAPA CKAN; bounded GET or assisted capture", + "cadence": "monthly; catalog last-update metadata checked 2026-09-16", + "attribution_licensing_notes": "MAPA catalog displays Creative Commons Attribution; confirm dataset-specific reuse, attribution, and personal-data handling before publication", + "adapter_status": "reference_only", + "expected_artifact_schema": "UTF-8 semicolon CSV with CNPJ, legal/trade names, SIF, dates, situation, address, contacts, area/category/class, and occurrence fields; repeated rows by source activity/occurrence", + "blockers": ["Restricted raw/contact/address handling, current code semantics, dataset-specific terms, identity lifecycle, and project publication approval remain pending."] + }, + { + "source_id": "br.sif.export", + "jurisdiction_scope": "Brazil; MAPA/DIPOA SIF establishments with country/product export authorization observations", + "legacy_paths": [], + "url": "https://dados.agricultura.gov.br/dataset/062166e3-b515-4274-8e7d-68aadd64b820/resource/fcb7f87d-0092-4a52-a44b-b3550747b4c2/download/sigsifestabelecimentosnacionais.csv", + "access_method": "published semicolon-delimited CSV via MAPA CKAN; bounded GET or assisted capture", + "cadence": "monthly; catalog metadata checked 2026-09-16", + "attribution_licensing_notes": "MAPA catalog displays Creative Commons Attribution; export/product rows remain separate evidence and require terms/privacy review", + "adapter_status": "reference_only", + "expected_artifact_schema": "UTF-8 semicolon CSV with country, area, establishment, SIF, UF, municipality, product, validity, authorization-event, and suspension dates; one-to-many by SIF/country/product", + "blockers": ["Do not treat export authorization as facility status or count; terms, effective-date semantics, privacy, reconciliation, and project approval remain pending."] + }, + { + "source_id": "br.sisbi.public", + "jurisdiction_scope": "Brazil; public e-SISBI/SISBI-POA service, establishment, product, capacity, and MAPA GIS address routes", + "legacy_paths": [], + "url": "https://sistemasweb.agricultura.gov.br/sgsi/app/estabelecimentos", + "access_method": "public JavaScript client with bounded JSON GET routes under sisbi_api and GIS API; no bulk scrape performed", + "cadence": "unknown; public route and API access verified 2026-09-16", + "attribution_licensing_notes": "MAPA government source; public access does not settle API reuse, retention, privacy, or publication terms", + "adapter_status": "reference_only", + "expected_artifact_schema": "JSON establishment records keyed by idEstabSisbi with service/person/address references; separate products, capacities, inspection-service, count, and GIS coordinate routes", + "blockers": ["Confirm pagination/query contract, code lists, status/effective-date semantics, ID lifecycle, address linkage, update cadence, privacy, terms, and project approval with an authorized operator."] + }, + { + "source_id": "br.trase.facilities", + "jurisdiction_scope": "Brazil; Trase secondary compilation of SIF, SISBI, SIE, SIM, and CONSORCIO facility/activity rows", + "legacy_paths": [], + "url": "https://trase.earth/open-data/datasets/brazil-facilities", + "access_method": "published GeoJSON download; private reconnaissance capture only", + "cadence": "dataset page says 2025 data, updated 2026-01-01; future cadence unknown", + "attribution_licensing_notes": "Trase page permits platform charts/maps/representations under CC BY 4.0 and asks commercial data users to contact Trase; raw-data reuse and privacy handling remain review gates", + "adapter_status": "reference_only", + "expected_artifact_schema": "GeoJSON rows with source, inspection level/number, CNPJ, company, municipality/state, status, commodity/type, capacity, export countries, coordinates, facility_id, and constructed unique_id", + "blockers": ["Secondary source must remain separate from MAPA; validate source snapshots, geocoding provenance, terms, privacy, constructed-ID behavior, coverage, and project approval before any use."] + }, { "source_id": "be.locations", "jurisdiction_scope": "Belgium; FASFC-registered, approved, or authorized operators, including animal-origin food and other food-chain activities", diff --git a/pipeline/sources/belgium/adapter.py b/pipeline/sources/belgium/adapter.py index 224670e..cb626a6 100644 --- a/pipeline/sources/belgium/adapter.py +++ b/pipeline/sources/belgium/adapter.py @@ -179,20 +179,20 @@ def parse_bytes(self, content: bytes) -> dict[str, Any]: quarantined: list[dict[str, Any]] = [] anomalies: Counter[str] = Counter() seen: Counter[tuple[str | None, str | None]] = Counter() - prepared: list[tuple[int, dict[str, str | None], tuple[str, ...]]] = [] + prepared: list[tuple[int, dict[str, str | None], tuple[str, ...], int]] = [] row_lengths: Counter[str] = Counter() for line, values in enumerate(rows, start=2): raw = _row_values(headers, values) row_lengths[str(len(values))] += 1 codes = _split_codes(_clean(raw.get(fields["activity_code"]))) - prepared.append((line, raw, codes)) + prepared.append((line, raw, codes, len(values))) for code in codes: seen[(_clean(raw.get(fields["establishment_id"])), code)] += 1 - for line, raw, codes in prepared: + for line, raw, codes, value_count in prepared: establishment_id = _clean(raw.get(fields["establishment_id"])) name = _clean(raw.get(fields["name"])) reasons: list[str] = [] - if len(values) != len(headers) or any(value is None for value in raw.values()): + if value_count != len(headers) or any(value is None for value in raw.values()): reasons.append("malformed_row") if not establishment_id: reasons.append("missing_establishment_id") diff --git a/pipeline/sources/belgium/refresh.py b/pipeline/sources/belgium/refresh.py index ef9ad77..3d0a999 100644 --- a/pipeline/sources/belgium/refresh.py +++ b/pipeline/sources/belgium/refresh.py @@ -15,7 +15,9 @@ from pipeline.common.acquisition import AcquisitionError, fetch_source, utc_now from pipeline.common.orchestrator import run_private_lifecycle +from pipeline.common.review import write_operator_review_packet from pipeline.contracts.adapter_contract import SourceArtifact +from pipeline.contracts.candidate_handoff import write_handoff from pipeline.contracts.source_lifecycle import atomic_json from .adapter import CONFIG, BelgiumOperatorsAdapter @@ -70,6 +72,17 @@ def refresh(*, run_dir: str | Path, operators_path: str | Path | None = None, ac code_artifact = _artifact(code_meta, default_url=CONFIG["activity_code_url"], default_coverage="FASFC LAP/PAP codebook; not a facility list") adapter = BelgiumOperatorsAdapter(code_path, code_artifact) lifecycle = run_private_lifecycle(operator_path, root / "lifecycle", operator_artifact, adapter, health_as_of_utc=retrieved, previous_normalized_path=previous_normalized, review_blockers=REVIEW_BLOCKERS) + if lifecycle.get("status") == "candidate-ready": + run_root = Path(lifecycle["run_dir"]) + rows = [json.loads(line) for line in (run_root / "normalized" / "records.jsonl").read_text(encoding="utf-8").splitlines() if line] + write_handoff(run_root / "candidate-handoff", rows, operator_artifact, source_id=adapter.source_id) + write_operator_review_packet( + run_root, + lifecycle["manifest"], + source_scope=lifecycle["manifest"]["coverage"], + checks=("review candidate-handoff/normalized/records.jsonl in restricted staging", "review operator and LAP/PAP codebook provenance together", "confirm no public release or API promotion"), + blockers=("candidate is private and human-gated", "CC BY attribution and address/coordinate privacy review pending"), + ) manifest = lifecycle.get("manifest") or {} report = {"source_id": CONFIG["source_id"], "source_url": operator_artifact.source_url, "retrieved_at_utc": operator_artifact.retrieved_at_utc, "operator_sha256": operator_artifact.sha256, "activity_code_sha256": code_artifact.sha256, "input_rows": manifest.get("input_rows"), "normalized_rows": manifest.get("normalized_rows"), "quarantined_rows": manifest.get("quarantined_rows"), "operator_schema_fingerprint": manifest.get("operator_schema_fingerprint"), "activity_code_schema_fingerprint": (manifest.get("activity_codebook") or {}).get("schema_fingerprint"), "drift_alarms": [], "disappeared_not_observed_count": 0, "disappearance_semantics": "not-observed; never inferred as closure", "geocoding": "disabled", "release_state": "not-created", "publication_state": "private-candidate", "publication_eligibility": "blocked", "lifecycle_status": lifecycle.get("status"), "lifecycle_run_dir": lifecycle.get("run_dir"), "previous_normalized_supplied": previous_normalized is not None} atomic_json(root / "refresh.json", report) diff --git a/pipeline/sources/belgium/test_adapter.py b/pipeline/sources/belgium/test_adapter.py index 30ae8c7..a250123 100644 --- a/pipeline/sources/belgium/test_adapter.py +++ b/pipeline/sources/belgium/test_adapter.py @@ -33,6 +33,14 @@ def test_unknown_activity_and_privacy_risk_quarantine(self): self.assertIn("address_privacy_risk", set().union(*reasons)) self.assertIn("unresolved_activity_code", set().union(*reasons)) + def test_row_length_is_checked_per_row_not_from_previous_row(self): + raw = (ROOT / "fixtures" / "synthetic_operators.csv").read_text(encoding="utf-8") + lines = raw.splitlines() + lines[1] += ",unexpected-extra-field" + result = self._adapter().parse_bytes(("\n".join(lines) + "\n").encode()) + malformed = next(item for item in result["quarantined"] if item["record"]["source_row"] == 2) + self.assertIn("malformed_row", malformed["reasons"]) + def test_ambiguous_codebook_key_is_not_silently_selected(self): with tempfile.TemporaryDirectory() as directory: codebook = Path(directory) / "codes.csv" diff --git a/pipeline/sources/canada/adapter.py b/pipeline/sources/canada/adapter.py index d53724e..294955d 100644 --- a/pipeline/sources/canada/adapter.py +++ b/pipeline/sources/canada/adapter.py @@ -16,17 +16,17 @@ ALIASES = { - "plant_number": ("plant number", "registration number", "establishment number", "establishment id", "plant id", "registration no"), - "name": ("plant name", "operator name", "operators name", "name of operator", "establishment name", "operator", "name"), + "plant_number": ("plant number", "registration number", "establishment number", "establishment id", "plant id", "registration no", "plant number no. de l'usine"), + "name": ("plant name", "operator name", "operators name", "name of operator", "establishment name", "operator", "name", "plant name nom de l'usine"), "doing_business_as": ("doing business as", "dba name", "also doing business as name", "trade name"), - "address": ("address", "location address", "street address", "location"), - "city": ("city", "location city", "municipality", "town"), - "province": ("province", "location province", "prov", "state"), - "postal_code": ("postal code", "postcode", "zip"), - "phone": ("phone", "telephone", "telephone numbers", "contact phone"), + "address": ("address", "location address", "street address", "location", "address adresse"), + "city": ("city", "location city", "municipality", "town", "city ville"), + "province": ("province", "location province", "prov", "state", "province"), + "postal_code": ("postal code", "postcode", "zip", "postal code code postal"), + "phone": ("phone", "telephone", "telephone numbers", "contact phone", "telephone telephone"), "latitude": ("latitude", "lat", "y"), "longitude": ("longitude", "lon", "lng", "x"), - "animal_class": ("animal class", "animal classes", "species", "species processed"), + "animal_class": ("animal class", "animal classes", "species", "species processed", "animal class catégorie d'animaux"), "plant_type": ("plant type", "type", "dataset", "facility type"), "function_codes": ("function codes", "function code", "activities", "activity codes", "activity"), "status": ("status", "current status", "state"), diff --git a/pipeline/sources/canada/refresh.py b/pipeline/sources/canada/refresh.py index 3749b57..42fedc6 100644 --- a/pipeline/sources/canada/refresh.py +++ b/pipeline/sources/canada/refresh.py @@ -18,6 +18,18 @@ from .acquire import ADAPTERS, fetch_source_artifact +def _local_metadata(path: Path, adapter: Any, retrieved: str) -> dict[str, Any]: + """Reuse a verified acquisition sidecar when staging a fetched artifact.""" + sidecar = path.parent / "acquisition-metadata.json" + raw = path.read_bytes() + digest = hashlib.sha256(raw).hexdigest() + if sidecar.is_file(): + retained = json.loads(sidecar.read_text(encoding="utf-8")) + if retained.get("sha256") == digest and int(retained.get("byte_size", -1)) == len(raw): + return retained + return {"acquisition_method": "assisted_local_capture", "source_id": adapter.source_id, "artifact_path": str(path), "sha256": digest, "byte_size": len(raw), "retrieved_at_utc": retrieved, "requested_url": adapter.source_url, "final_url": adapter.source_url} + + def refresh(*, source: str, run_dir: str | Path, raw_path: str | Path | None = None, fetch: bool = False, terms_review_path: str | Path | None = None, output_root: str | Path = "data/raw", run_id: str | None = None, retrieved_at_utc: str | None = None, timeout_seconds: float = 60.0, max_bytes: int = 64 * 1024 * 1024) -> dict[str, Any]: if fetch == (raw_path is not None): raise ValueError("specify exactly one of --fetch or --raw") adapter = ADAPTERS[source](); retrieved = retrieved_at_utc or utc_now() @@ -28,8 +40,8 @@ def refresh(*, source: str, run_dir: str | Path, raw_path: str | Path | None = N else: raw = Path(raw_path).resolve() if not raw.is_file(): raise ValueError("--raw artifact must exist") - data = raw.read_bytes(); artifact = SourceArtifact(adapter.source_url, retrieved, hashlib.sha256(data).hexdigest(), len(data), code_version=adapter.adapter_version, config_version=adapter.schema_version, rights_caveat="assisted capture; current terms remain pending", privacy_caveat="private staging; privacy review pending", coverage=adapter.coverage) - metadata = {"acquisition_method": "assisted_local_capture", "source_id": adapter.source_id, "artifact_path": str(raw), "sha256": artifact.sha256, "byte_size": artifact.byte_size, "retrieved_at_utc": retrieved, "requested_url": adapter.source_url, "final_url": adapter.source_url} + metadata = _local_metadata(raw, adapter, retrieved) + data = raw.read_bytes(); artifact = SourceArtifact(str(metadata.get("final_url") or adapter.source_url), str(metadata.get("retrieved_at_utc") or retrieved), str(metadata["sha256"]), int(metadata["byte_size"]), effective_date=metadata.get("effective_date"), publication_date=metadata.get("publication_date"), code_version=adapter.adapter_version, config_version=adapter.schema_version, rights_caveat=metadata.get("rights_caveat") or "assisted capture; current terms remain pending", privacy_caveat=metadata.get("privacy_caveat") or "private staging; privacy review pending", coverage=metadata.get("coverage") or adapter.coverage, redirects=tuple(metadata.get("redirects") or ())) root = Path(run_dir); atomic_json(root / "acquisition-metadata.json", metadata); lifecycle = run_private_lifecycle(raw, root / "lifecycle", artifact, adapter, health_as_of_utc=retrieved) if lifecycle.get("status") == "candidate-ready": run_root = Path(lifecycle["run_dir"]); rows = [json.loads(line) for line in (run_root / "normalized" / "records.jsonl").read_text(encoding="utf-8").splitlines() if line]; write_handoff(run_root / "candidate-handoff", rows, artifact, source_id=adapter.source_id) diff --git a/pipeline/sources/canada/test_adapter.py b/pipeline/sources/canada/test_adapter.py index b4d6405..6973c11 100644 --- a/pipeline/sources/canada/test_adapter.py +++ b/pipeline/sources/canada/test_adapter.py @@ -11,6 +11,18 @@ class CanadaAdapterTests(unittest.TestCase): + def test_bilingual_composite_headers_from_live_ontario_file_are_supported(self): + content = ('"Plant Name_ Nom de l\'usine","Plant Number_No. de l\'usine",' + '"Address_Adresse","City_Ville","Province_Province",' + '"Postal Code_Code postal","Telephone_Telephone",Latitude,Longitude,' + '"Animal Class_Catégorie d\'animaux","Plant Type_Type",' + '"Function Codes_Codes de fonction","Status_Statut"\n' + 'Synthetic Plant,SP-001,"Industrial Road 1",Toronto,ON,M1M 1M1,' + '555-0100,43.1,-79.1,Abattoir,Abattoir,1,current\n').encode() + result = OntarioMeatPlantsAdapter().parse_bytes(content) + self.assertEqual(len(result["accepted"]), 1) + self.assertEqual(result["accepted"][0]["normalized"]["activity_categories"], ("slaughter",)) + def test_ontario_is_provincial_and_privacy_safe(self): adapter = OntarioMeatPlantsAdapter(); result = adapter.parse_file(FIXTURES / "ontario.csv") self.assertEqual(len(result["accepted"]), 2); self.assertEqual(len(result["quarantined"]), 1) diff --git a/pipeline/sources/denmark/pipeline.py b/pipeline/sources/denmark/pipeline.py index 8134e7d..ac4fb07 100644 --- a/pipeline/sources/denmark/pipeline.py +++ b/pipeline/sources/denmark/pipeline.py @@ -15,7 +15,10 @@ from pipeline.contracts.private_run import write_private_run_report from pipeline.contracts.source_health import build_health_snapshot, write_health_snapshot from pipeline.contracts.source_lifecycle import atomic_json, validate_private_manifest +from pipeline.contracts.adapter_contract import SourceArtifact +from pipeline.common.review import write_operator_review_packet from pipeline.common.review_packet import write_review_packet +from .adapter import DenmarkSmileyAdapter LOGGER = logging.getLogger("uec.denmark.pipeline") @@ -112,6 +115,7 @@ def _canonical_evidence(run_dir: Path, input_path: Path, metadata: dict, "acquisition": metadata or {"source_url": source_url, "retrieved_at_utc": retrieved}, "pipeline_started_at_utc": started_at, "pipeline_completed_at_utc": completed_at, + "coverage": "Danish Find Smiley food-business inspection listings; source rows only; no completeness claim", } validate_private_manifest(manifest) atomic_json(run_dir / "manifest.json", manifest) @@ -121,6 +125,25 @@ def _canonical_evidence(run_dir: Path, input_path: Path, metadata: dict, if retrieved: snapshot = build_health_snapshot(run_dir, as_of_utc=retrieved) write_health_snapshot(run_dir / "source-health.json", snapshot) + classified = run_dir / "03-classify" / "classified-records.jsonl" + if classified.is_file() and retrieved and source_url: + artifact = SourceArtifact( + source_url=str(source_url), retrieved_at_utc=str(retrieved), sha256=str(raw_hash), byte_size=int(raw_size), + publication_date=manifest.get("publication_date"), effective_date=manifest.get("effective_date"), + code_version=str(manifest["code_version"]), config_version=str(manifest["config_version"]), + rights_caveat="Find Smiley source attribution/currentness review remains open; private handoff only", + privacy_caveat="restricted staging; address and source-coordinate review pending", + coverage=manifest["coverage"], + ) + classified_rows = [json.loads(line) for line in classified.read_text(encoding="utf-8").splitlines() if line] + DenmarkSmileyAdapter().write_candidate_handoff(run_dir / "candidate-handoff", artifact, classified_rows) + write_operator_review_packet( + run_dir, + manifest, + source_scope=manifest["coverage"], + checks=("review candidate-handoff/normalized/records.jsonl in restricted staging", "review validation findings before any candidate import", "confirm no public release or API promotion"), + blockers=("candidate is private and human-gated", "address and coordinate publication blocked", "full pipeline validation findings require operator review"), + ) write_review_packet(run_dir, blockers={ "terms": ["Find Smiley attribution/currentness conditions are recorded; named project release approval remains open."], "privacy": ["Address and source-coordinate residential/private-location screening remains required; geocoding is separately review-gated."], diff --git a/pipeline/sources/france/adapter.py b/pipeline/sources/france/adapter.py index 18880a0..d3b8532 100644 --- a/pipeline/sources/france/adapter.py +++ b/pipeline/sources/france/adapter.py @@ -16,16 +16,16 @@ ALIASES = { - "department_number": ("department number", "department", "n departement", "numero departement", "code departement"), - "approval_number": ("approval number", "approval no", "n dagrement", "numero dagrement", "num dagrement", "agrément", "agrement"), + "department_number": ("department number", "department", "n departement", "numero departement", "code departement", "numero de département"), + "approval_number": ("approval number", "approval no", "n dagrement", "numero dagrement", "num dagrement", "agrément", "agrement", "numéro agrément/approval number"), "siret": ("siret", "siret number"), - "legal_name": ("legal name", "company name", "raison sociale", "nom de letablissement", "nom de l'etablissement", "establishment name"), + "legal_name": ("legal name", "company name", "raison sociale", "nom de letablissement", "nom de l'etablissement", "establishment name", "raison sociale - enseigne commerciale/name"), "address": ("address", "adresse", "location address"), - "postal_code": ("postal code", "code postal", "postcode"), - "commune": ("commune", "municipality", "city", "town"), - "category": ("category", "categorie", "catégorie", "establishment category"), - "associated_activities": ("associated activities", "activites associees", "activités associées", "activities", "activity"), - "species": ("species", "especes", "espèces", "animal species"), + "postal_code": ("postal code", "code postal", "postcode", "code postal/postal code"), + "commune": ("commune", "municipality", "city", "town", "commune/town"), + "category": ("category", "categorie", "catégorie", "establishment category", "catégorie/category"), + "associated_activities": ("associated activities", "activites associees", "activités associées", "activities", "activity", "activités associées/associated activities"), + "species": ("species", "especes", "espèces", "animal species", "espèce/specy"), } REQUIRED = ("approval_number", "legal_name", "commune", "category") diff --git a/pipeline/sources/france/refresh.py b/pipeline/sources/france/refresh.py index 6542c7f..9bc09d2 100644 --- a/pipeline/sources/france/refresh.py +++ b/pipeline/sources/france/refresh.py @@ -18,6 +18,18 @@ from .acquire import ADAPTERS, fetch_section +def _local_metadata(path: Path, retrieved: str) -> dict[str, Any]: + """Reuse a verified acquisition sidecar when staging a fetched artifact.""" + sidecar = path.parent / "acquisition-metadata.json" + raw = path.read_bytes() + digest = hashlib.sha256(raw).hexdigest() + if sidecar.is_file(): + retained = json.loads(sidecar.read_text(encoding="utf-8")) + if retained.get("sha256") == digest and int(retained.get("byte_size", -1)) == len(raw): + return retained + return {"acquisition_method": "assisted_local_capture", "source_id": "", "artifact_path": str(path), "sha256": digest, "byte_size": len(raw), "retrieved_at_utc": retrieved} + + def refresh(*, section: str, run_dir: str | Path, raw_path: str | Path | None = None, fetch: bool = False, terms_review_path: str | Path | None = None, output_root: str | Path = "data/raw", run_id: str | None = None, retrieved_at_utc: str | None = None, timeout_seconds: float = 60.0, max_bytes: int = 32 * 1024 * 1024) -> dict[str, Any]: if fetch == (raw_path is not None): raise ValueError("specify exactly one of --fetch or --raw") adapter = ADAPTERS[section](); retrieved = retrieved_at_utc or utc_now() @@ -29,8 +41,9 @@ def refresh(*, section: str, run_dir: str | Path, raw_path: str | Path | None = else: source = Path(raw_path).resolve() if not source.is_file(): raise ValueError("--raw artifact must exist") - raw = source.read_bytes(); artifact = SourceArtifact(adapter.source_url, retrieved, hashlib.sha256(raw).hexdigest(), len(raw), code_version=adapter.adapter_version, config_version=adapter.schema_version, rights_caveat="assisted capture; file-specific terms remain pending", privacy_caveat="private staging; privacy review pending", coverage=f"France DGAL Regulation (EC) 853/2004 Section {section}; source rows only") - metadata = {"acquisition_method": "assisted_local_capture", "source_id": adapter.source_id, "artifact_path": str(source), "sha256": artifact.sha256, "byte_size": artifact.byte_size, "retrieved_at_utc": retrieved, "requested_url": adapter.source_url, "final_url": adapter.source_url} + metadata = _local_metadata(source, retrieved) + metadata.update({"source_id": adapter.source_id, "requested_url": metadata.get("requested_url") or adapter.source_url, "final_url": metadata.get("final_url") or adapter.source_url}) + raw = source.read_bytes(); artifact = SourceArtifact(str(metadata["final_url"]), str(metadata.get("retrieved_at_utc") or retrieved), str(metadata["sha256"]), int(metadata["byte_size"]), effective_date=metadata.get("effective_date"), publication_date=metadata.get("publication_date"), code_version=adapter.adapter_version, config_version=adapter.schema_version, rights_caveat=metadata.get("rights_caveat") or "assisted capture; file-specific terms remain pending", privacy_caveat=metadata.get("privacy_caveat") or "private staging; privacy review pending", coverage=metadata.get("coverage") or f"France DGAL Regulation (EC) 853/2004 Section {section}; source rows only", redirects=tuple(metadata.get("redirects") or ())) root = Path(run_dir); atomic_json(root / "acquisition-metadata.json", metadata) lifecycle = run_private_lifecycle(source, root / "lifecycle", artifact, adapter, health_as_of_utc=retrieved) if lifecycle.get("status") == "candidate-ready": diff --git a/pipeline/sources/france/test_adapter.py b/pipeline/sources/france/test_adapter.py index 47c3616..e3a6369 100644 --- a/pipeline/sources/france/test_adapter.py +++ b/pipeline/sources/france/test_adapter.py @@ -12,6 +12,17 @@ class FranceAdapterTests(unittest.TestCase): + def test_bilingual_composite_headers_from_live_dgal_file_are_supported(self): + content = ('"Numero de département","Numéro agrément/Approval number","SIRET",' + '"Raison SOCIALE - Enseigne commerciale/Name","Adresse/Adress",' + '"Code postal/Postal code","Commune/Town","Catégorie/Category",' + '"Activités associées/Associated activities","Espèce/Specy"\n' + '01,"01.000.001",,"Synthetic Facility","Industrial Road 1",01000,' + 'Synthetic Town,SH,ABATTOIR,BOVINE\n').encode() + result = FranceDgalSectionIAdapter().parse_bytes(content) + self.assertEqual(len(result["accepted"]), 1) + self.assertEqual(result["accepted"][0]["normalized"]["activity_categories"], ("slaughter",)) + def test_section_i_preserves_source_and_quarantines_duplicate(self): adapter = FranceDgalSectionIAdapter(); result = adapter.parse_file(FIXTURES / "section_i.csv") self.assertEqual(len(result["accepted"]), 2); self.assertEqual(len(result["quarantined"]), 1) diff --git a/pipeline/sources/germany/refresh.py b/pipeline/sources/germany/refresh.py index 1db41ca..97a42c8 100644 --- a/pipeline/sources/germany/refresh.py +++ b/pipeline/sources/germany/refresh.py @@ -9,7 +9,9 @@ from pipeline.common.acquisition import AcquisitionError, fetch_source, utc_now from pipeline.common.orchestrator import run_private_lifecycle +from pipeline.common.review import write_operator_review_packet from pipeline.contracts.adapter_contract import SourceArtifact +from pipeline.contracts.candidate_handoff import write_handoff from pipeline.contracts.source_lifecycle import atomic_json from .adapter import CONFIG, BltuAdapter @@ -59,6 +61,17 @@ def refresh(*, run_dir: str | Path, raw_path: str | Path | None = None, fetch: b raw = input_path.read_bytes() artifact = SourceArtifact(source_url=str(metadata.get("final_url") or CONFIG["source_url"]), retrieved_at_utc=str(metadata["retrieved_at_utc"]), sha256=hashlib.sha256(raw).hexdigest(), byte_size=len(raw), publication_date=metadata.get("publication_date"), effective_date=metadata.get("effective_date"), code_version=adapter.adapter_version, config_version=adapter.schema_version, rights_caveat=CONFIG["terms"], privacy_caveat=metadata.get("privacy_caveat"), coverage=CONFIG["coverage"], redirects=tuple(metadata.get("redirects") or ())) lifecycle = run_private_lifecycle(input_path, Path(run_dir) / "lifecycle", artifact, adapter, health_as_of_utc=str(metadata["retrieved_at_utc"]), previous_normalized_path=previous_normalized, review_blockers=REVIEW_BLOCKERS) + if lifecycle.get("status") == "candidate-ready": + run_root = Path(lifecycle["run_dir"]) + rows = [json.loads(line) for line in (run_root / "normalized" / "records.jsonl").read_text(encoding="utf-8").splitlines() if line] + write_handoff(run_root / "candidate-handoff", rows, artifact, source_id=adapter.source_id) + write_operator_review_packet( + run_root, + lifecycle["manifest"], + source_scope=lifecycle["manifest"]["coverage"], + checks=("review candidate-handoff/normalized/records.jsonl in restricted staging", "confirm the BVL export request/session evidence", "confirm no public release or API promotion"), + blockers=("candidate is private and human-gated", "dataset-specific reuse and address/coordinate review pending"), + ) manifest = lifecycle.get("manifest") or {} report = {"source_id": CONFIG["source_id"], "source_url": artifact.source_url, "portal_url": CONFIG["portal_url"], "retrieved_at_utc": artifact.retrieved_at_utc, "sha256": artifact.sha256, "byte_size": artifact.byte_size, "input_rows": manifest.get("input_rows"), "normalized_rows": manifest.get("normalized_rows"), "quarantined_rows": manifest.get("quarantined_rows"), "schema_status": manifest.get("schema_status"), "schema_fingerprint": manifest.get("schema_fingerprint"), "drift_alarms": [], "disappearance_semantics": "not-observed; never inferred as closure", "geocoding": "disabled", "release_state": "not-created", "publication_state": "private-candidate", "publication_eligibility": "blocked", "lifecycle_status": lifecycle.get("status"), "lifecycle_run_dir": lifecycle.get("run_dir")} atomic_json(Path(run_dir) / "refresh.json", report) diff --git a/pipeline/sources/italy/it_853_adapter.py b/pipeline/sources/italy/it_853_adapter.py index 4c482be..c6d0c42 100644 --- a/pipeline/sources/italy/it_853_adapter.py +++ b/pipeline/sources/italy/it_853_adapter.py @@ -25,6 +25,11 @@ ) STATUS = {"AUTORIZZATA": "Autorizzata", "REVOCATA": "Revocata", "SOSPESA": "Sospesa"} DATE_FIELDS = ("data_inizio_attivita", "data_fine_attivita", "data_ultimo_aggiornamento") +MONTHS = { + "GEN": 1, "JAN": 1, "FEB": 2, "MAR": 3, "APR": 4, "MAG": 5, "MAY": 5, + "GIU": 6, "JUN": 6, "LUG": 7, "JUL": 7, "AGO": 8, "AUG": 8, + "SET": 9, "SEP": 9, "OTT": 10, "OCT": 10, "NOV": 11, "DIC": 12, "DEC": 12, +} def clean(value: Any) -> str | None: @@ -37,14 +42,30 @@ def row_id(row: dict[str | None, Any], occurrence: int) -> str: return hashlib.sha256(f"{payload}|{occurrence}".encode()).hexdigest() -def _date_state(value: str | None) -> str: - if not value: - return "unknown" +def _normalize_date(value: str | None) -> str | None: + """Normalize the catalog's ISO and Italian/English abbreviated dates.""" + if not value or value.strip() in {"-", "—"}: + return None + try: + return date.fromisoformat(value).isoformat() + except ValueError: + pass + parts = value.strip().upper().split("-") + if len(parts) != 3 or not parts[0].isdigit() or not parts[2].isdigit(): + return None + month = MONTHS.get(parts[1]) + if month is None: + return None try: - date.fromisoformat(value) + return date(2000 + int(parts[2]), month, int(parts[0])).isoformat() except ValueError: - return "invalid" - return "known" + return None + + +def _date_state(value: str | None) -> str: + if not value or value.strip() in {"-", "—"}: + return "unknown" + return "known" if _normalize_date(value) else "invalid" class Italy853Adapter: @@ -106,6 +127,7 @@ def parse_bytes(self, content: bytes) -> dict[str, Any]: reasons.append(f"invalid_{field}") municipality_code = clean(row.get("codice_comune")) geography_precision = "municipality-code" if municipality_code and len(municipality_code) == 6 else "unknown" + normalized_dates = {field: _normalize_date(clean(row.get(field))) for field in DATE_FIELDS} normalized = { "establishment_id": rec, "recognition_number": rec, @@ -129,7 +151,7 @@ def parse_bytes(self, content: bytes) -> dict[str, Any]: "products": clean(row.get("prodotti_abilitati")), "status": status, "status_state": "known" if status else "unknown", - "dates": {field: clean(row.get(field)) for field in DATE_FIELDS}, + "dates": normalized_dates, "date_state": {field: _date_state(clean(row.get(field))) for field in DATE_FIELDS}, "coordinates": None, "coordinate_state": "source-value-present-pending-review" if clean(row.get("longitudine")) or clean(row.get("latitudine")) else "unknown", diff --git a/pipeline/sources/italy/refresh.py b/pipeline/sources/italy/refresh.py index 3277971..533a453 100644 --- a/pipeline/sources/italy/refresh.py +++ b/pipeline/sources/italy/refresh.py @@ -10,7 +10,9 @@ from pathlib import Path from pipeline.common.orchestrator import run_private_lifecycle +from pipeline.common.review import write_operator_review_packet from pipeline.contracts.adapter_contract import SourceArtifact +from pipeline.contracts.candidate_handoff import write_handoff from pipeline.contracts.source_lifecycle import atomic_json from .acquire import CATALOG_URL, DEFAULT_MAX_BYTES, fetch @@ -88,6 +90,17 @@ def refresh( rights_caveat=facts["rights_caveat"], privacy_caveat=facts["privacy_caveat"], coverage=facts["coverage"], ) status = run_private_lifecycle(input_path, run_dir, artifact, adapter, previous_normalized_path=previous_normalized, review_blockers=REVIEW_BLOCKERS) + if status.get("status") == "candidate-ready": + run_root = Path(status["run_dir"]) + rows = [json.loads(line) for line in (run_root / "normalized" / "records.jsonl").read_text(encoding="utf-8").splitlines() if line] + write_handoff(run_root / "candidate-handoff", rows, artifact, source_id=adapter.source_id) + write_operator_review_packet( + run_root, + status["manifest"], + source_scope=status["manifest"]["coverage"], + checks=("review candidate-handoff/normalized/records.jsonl in restricted staging", "confirm no public release or API promotion", "review separate 1069/2009 coverage and source terms"), + blockers=("candidate is private and human-gated", "address, tax, and coordinate publication blocked"), + ) # Keep catalog/response/terms evidence beside the lifecycle run without # copying row payloads into QA, health, or API-shaped artifacts. if metadata: diff --git a/pipeline/sources/italy/test_it_853_adapter.py b/pipeline/sources/italy/test_it_853_adapter.py index 1b1d27a..7ec7f5b 100644 --- a/pipeline/sources/italy/test_it_853_adapter.py +++ b/pipeline/sources/italy/test_it_853_adapter.py @@ -17,8 +17,14 @@ def test_shape_drift_quarantine(self): def test_missing_date_and_geography_are_explicit(self): r=Italy853Adapter().parse_bytes((H+"\n"+row().replace("001001","001").replace("2026-09-13","")).encode())["accepted"][0] self.assertEqual(r["normalized"]["date_state"]["data_inizio_attivita"],"unknown"); self.assertEqual(r["normalized"]["geography_precision"],"unknown"); self.assertEqual(r["normalized"]["coordinate_state"],"source-value-present-pending-review") - def test_source_category_and_activity_coverage_are_explicit(self): + def test_source_category_and_activity_coverage_are_explicit(self): result=Italy853Adapter().parse_bytes((H+"\n"+row()).encode()); self.assertEqual(result["source_category_counts"], {"X": 1}); self.assertEqual(result["source_activity_counts"], {"10": 1}); self.assertTrue(result["schema_fingerprint"]) + def test_current_catalog_abbreviated_dates_normalize_without_losing_source_values(self): + content=(H+"\n"+row().replace("2026-09-13","14-LUG-22")).encode(); result=Italy853Adapter().parse_bytes(content); record=result["accepted"][0] + self.assertEqual(record["normalized"]["dates"]["data_ultimo_aggiornamento"],"2022-07-14"); self.assertEqual(record["source_values"]["data_ultimo_aggiornamento"],"14-LUG-22") + def test_catalog_dash_dates_are_unknown_not_invalid(self): + record=Italy853Adapter().parse_bytes((H+"\n"+row().replace(";;;Autorizzata;2026-09-13;",";;-;Autorizzata;2026-09-13;")).encode())["accepted"][0] + self.assertEqual(record["normalized"]["date_state"]["data_fine_attivita"],"unknown") def test_repeated_activity_quarantines_collision_without_merge(self): result=Italy853Adapter().parse_bytes((H+"\n"+row()+row()).encode()); self.assertEqual(len(result["accepted"]),1); self.assertEqual(len(result["quarantined"]),1); self.assertIn("ambiguous_repeated_recognition_activity",result["quarantined"][0]["reasons"]) def test_run_writes_contract_manifest_and_row_quarantine(self): diff --git a/pipeline/sources/us/accountability/README.md b/pipeline/sources/us/accountability/README.md new file mode 100644 index 0000000..3d00ba2 --- /dev/null +++ b/pipeline/sources/us/accountability/README.md @@ -0,0 +1,69 @@ +# US accountability pilot + +This is a private, synthetic/test-only pilot for the forthcoming graph +foundation. It writes candidate JSONL and a row-free manifest; it does not +write a graph database, create migrations, publish records, score targets, or +expose people or residential locations. + +## Bounded source scope + +The facility side starts with an FSIS establishment identifier and approval +identifier. The regulatory side starts with the already-modeled APHIS +registration and inspection identifiers. APHIS registrations, inspections, +annual reports, laboratories, and aggregate observations remain separate +evidence families. A link ledger may include a legal entity, parent, or brand +only when a source-native identifier and an explicit reviewed link event are +recorded. + +SEC, EPA, and OSHA are deferred source routes in this pilot. The checked-in +fixture uses sanitized identifiers such as `CIK-TEST-001`; it is a contract +fixture, not a claim that a live filing, environmental record, or workplace +record was matched. A production run needs the applicable terms/reuse decision, +official URL, retrieval timestamp, checksum, and retained source artifact. + +## Candidate contract + +Each relationship in `candidate/relationships.jsonl` includes: + +- typed subject/object references and source-native IDs; +- the evidence source and evidence-native ID; +- observation date and timezone-qualified retrieval date; +- relationship type, confidence, review state, match method, and evidence; +- temporal validity where ownership changes are observed; +- blocked publication and test-only markers. + +Entities are emitted separately in `candidate/entities.jsonl`. The adapter +does not collapse a facility, establishment approval, operator, legal entity, +parent, brand, inspection, violation, enforcement, laboratory, or aggregate +observation. Name-only, address-only, phone-only, fuzzy, and geocoder matches +are not defensible and quarantine. Stale, conflicting, overlapping ownership, +and suppressed/restricted relationships also quarantine. Non-overlapping +ownership versions remain as history rather than overwriting one another. + +`quarantined/relationships.jsonl` retains only the sanitized source values and +reason metadata inside the private run. No quarantine payload is checked in. + +## Run boundary + +First use the source-local assisted contracts in `pipeline/sources/us/fsis` +and `pipeline/sources/us/aphis`. FSIS direct 403 responses remain fail-closed; +the pilot never bypasses access controls. Then run the explicit reviewed link +ledger: + +```text +python -m pipeline.sources.us.accountability.refresh \ + --raw \ + --run-dir +``` + +The run emits acquisition metadata, a candidate manifest, candidate JSONL, +quarantine JSONL, an assisted-capture contract, a run status, and a human +review packet. `release_state=not-created`, `publication_state=private-candidate`, +and `publication_gate=blocked` are invariants. Geocoding is disabled. + +## Reconciliation + +`reconcile.build_v1_crosswalk` compares only the exact FSIS establishment key +to FSIS facility candidates. It is row-free and observation-only: a missing +current observation is `not-observed`, never closure; it creates no identity, +suppression, or publication decision and does not inherit V1 assumptions. diff --git a/pipeline/sources/us/accountability/__init__.py b/pipeline/sources/us/accountability/__init__.py new file mode 100644 index 0000000..0f7787b --- /dev/null +++ b/pipeline/sources/us/accountability/__init__.py @@ -0,0 +1,5 @@ +"""Private US accountability-link pilot.""" + +from .adapter import UsAccountabilityAdapter + +__all__ = ["UsAccountabilityAdapter"] diff --git a/pipeline/sources/us/accountability/adapter.py b/pipeline/sources/us/accountability/adapter.py new file mode 100644 index 0000000..3cdb1a6 --- /dev/null +++ b/pipeline/sources/us/accountability/adapter.py @@ -0,0 +1,327 @@ +"""Deterministic, private graph-candidate contract for the US pilot. + +The input is a reviewed link ledger, not a name-matching job. Each row is one +asserted relationship with two typed source-native endpoints and independent +evidence provenance. The adapter emits no database writes and no public +release. +""" +from __future__ import annotations + +import csv +import hashlib +import json +from collections import Counter +from datetime import date, datetime +from pathlib import Path +from typing import Any + +from pipeline.contracts.adapter_contract import SourceArtifact +from pipeline.contracts.source_lifecycle import atomic_json, atomic_jsonl, private_manifest + +ROOT = Path(__file__).parent +CONFIG = json.loads((ROOT / "config.json").read_text(encoding="utf-8")) + +ENTITY_TYPES = frozenset({ + "facility", "establishment_approval", "operator", "legal_entity", "parent", + "brand", "inspection", "violation", "enforcement", "laboratory", + "aggregate_observation", +}) +RELATIONSHIPS: dict[str, tuple[str, frozenset[str]]] = { + "establishment_approval_for": ("establishment_approval", frozenset({"facility"})), + "operates": ("operator", frozenset({"facility"})), + "operator_is_legal_entity": ("operator", frozenset({"legal_entity"})), + "parent_of": ("legal_entity", frozenset({"legal_entity", "parent"})), + "brand_of": ("brand", frozenset({"legal_entity"})), + "inspection_observes": ("inspection", frozenset({"facility", "operator"})), + "violation_observed_in": ("violation", frozenset({"inspection"})), + "enforcement_for": ("enforcement", frozenset({"violation"})), + "laboratory_supports": ("laboratory", frozenset({"inspection", "aggregate_observation"})), + "aggregate_describes": ("aggregate_observation", frozenset({"facility", "operator"})), +} +CONFIDENCE = frozenset({"high", "medium", "low"}) +REVIEW_STATES = frozenset({"evidence_verified", "review_required", "quarantined"}) +MATCH_METHODS = frozenset({"exact_source_id", "explicit_reviewed_link", "temporal_source_link"}) +REJECTED_MATCH_METHODS = frozenset({"name_only", "address_only", "phone_only", "fuzzy_name", "geocoder"}) +SUPPRESSION_STATES = frozenset({"eligible", "suppressed", "restricted", "unknown"}) +REQUIRED_HEADERS = ( + "subject_type", "subject_source_id", "subject_source_native_id", "subject_name", + "object_type", "object_source_id", "object_source_native_id", "object_name", + "relationship_type", "evidence_source_id", "evidence_source_native_id", + "observation_date", "retrieved_at_utc", "valid_from", "valid_to", "confidence", + "review_state", "match_method", "evidence_url", "evidence_excerpt", "suppression_state", +) +REFERENCE_DATE = date(2026, 9, 15) + + +class AccountabilityContractError(ValueError): + """The link ledger does not satisfy the candidate contract.""" + + +def _clean(value: Any) -> str | None: + if value is None: + return None + text = str(value).strip() + return text or None + + +def _iso_date(value: str | None, field: str, *, required: bool = False) -> date | None: + text = _clean(value) + if not text: + if required: + raise AccountabilityContractError(f"missing {field}") + return None + try: + return date.fromisoformat(text) + except ValueError as exc: + raise AccountabilityContractError(f"{field} must be ISO-8601 date") from exc + + +def _iso_datetime(value: str | None, field: str) -> str: + text = _clean(value) + if not text: + raise AccountabilityContractError(f"missing {field}") + try: + parsed = datetime.fromisoformat(text.replace("Z", "+00:00")) + except ValueError as exc: + raise AccountabilityContractError(f"{field} must be ISO-8601 datetime") from exc + if parsed.tzinfo is None: + raise AccountabilityContractError(f"{field} must include a timezone") + return text + + +def _entity_ref(entity_type: str, source_id: str, native_id: str) -> dict[str, str]: + return {"entity_type": entity_type, "source_id": source_id, "source_native_id": native_id} + + +def _entity_key(ref: dict[str, str]) -> tuple[str, str, str]: + return ref["entity_type"], ref["source_id"], ref["source_native_id"] + + +def _stable_entity_id(ref: dict[str, str]) -> str: + value = "|".join((ref["entity_type"], ref["source_id"], ref["source_native_id"])) + return "candidate-entity-" + hashlib.sha256(value.encode("utf-8")).hexdigest()[:24] + + +def _entity(ref: dict[str, str], name: str, *, observed_at: str, evidence: dict[str, str]) -> dict[str, Any]: + return { + "entity_id": _stable_entity_id(ref), + **ref, + "display_name": name, + "observation_date": observed_at, + "evidence": evidence, + "privacy_gate": "pending-review", + "publication_gate": "blocked", + "test_only": True, + } + + +def _relationship(row: dict[str, str], *, observed_at: str, retrieved_at: str, source_row: int) -> tuple[dict[str, Any], tuple[dict[str, Any], dict[str, Any]]]: + subject = _entity_ref(row["subject_type"], row["subject_source_id"], row["subject_source_native_id"]) + object_ = _entity_ref(row["object_type"], row["object_source_id"], row["object_source_native_id"]) + evidence = { + "source_id": row["evidence_source_id"], + "source_native_id": row["evidence_source_native_id"], + "url": row["evidence_url"], + "excerpt": row["evidence_excerpt"], + } + identity = {"subject": subject, "object": object_, "relationship_type": row["relationship_type"], "source_native_id": row["evidence_source_native_id"], "observation_date": observed_at} + relation_id = "candidate-relationship-" + hashlib.sha256(json.dumps(identity, sort_keys=True).encode()).hexdigest()[:24] + relation = { + "relationship_id": relation_id, + "relationship_type": row["relationship_type"], + "subject": subject, + "object": object_, + "source_id": row["evidence_source_id"], + "source_native_ids": { + "subject": subject["source_native_id"], + "object": object_["source_native_id"], + "evidence": row["evidence_source_native_id"], + }, + "observation_date": observed_at, + "retrieved_at_utc": retrieved_at, + "valid_from": row.get("valid_from") or None, + "valid_to": row.get("valid_to") or None, + "confidence": row["confidence"], + "review_state": row["review_state"], + "match_method": row["match_method"], + "evidence": evidence, + "source_row": source_row, + "publication_gate": "blocked", + "test_only": True, + } + return relation, ( + _entity(subject, row["subject_name"], observed_at=observed_at, evidence=evidence), + _entity(object_, row["object_name"], observed_at=observed_at, evidence=evidence), + ) + + +class UsAccountabilityAdapter: + source_id = CONFIG["source_id"] + adapter_version = CONFIG["adapter_version"] + schema_version = CONFIG["contract_version"] + + def __init__(self, *, stale_after_days: int = CONFIG["stale_after_days"], reference_date: date = REFERENCE_DATE) -> None: + if stale_after_days < 0: + raise ValueError("stale_after_days must be non-negative") + self.stale_after_days = stale_after_days + self.reference_date = reference_date + + def parse_bytes(self, content: bytes) -> dict[str, Any]: + digest = hashlib.sha256(content).hexdigest() + try: + reader = csv.DictReader(content.decode("utf-8-sig").splitlines(), strict=True) + headers = tuple(reader.fieldnames or ()) + rows = list(reader) + except (UnicodeDecodeError, csv.Error) as exc: + raise AccountabilityContractError("malformed or unsupported UTF-8 CSV") from exc + if headers != REQUIRED_HEADERS: + missing = [header for header in REQUIRED_HEADERS if header not in headers] + extra = [header for header in headers if header not in REQUIRED_HEADERS] + raise AccountabilityContractError(f"unsupported link-ledger schema; missing={missing}, extra={extra}") + if len(headers) != len(set(headers)) or any(None in row for row in rows): + raise AccountabilityContractError("schema drift: duplicate or extra link-ledger columns") + + accepted: list[dict[str, Any]] = [] + quarantined: list[dict[str, Any]] = [] + entities: dict[tuple[str, str, str], dict[str, Any]] = {} + identifiers: dict[tuple[str, str], tuple[str, str]] = {} + relationship_rows: list[tuple[dict[str, Any], dict[str, str], int]] = [] + + for line, row in enumerate(rows, 2): + reasons: list[str] = [] + try: + required = ("subject_type", "subject_source_id", "subject_source_native_id", "subject_name", "object_type", "object_source_id", "object_source_native_id", "object_name", "relationship_type", "evidence_source_id", "evidence_source_native_id", "confidence", "review_state", "match_method", "evidence_url", "evidence_excerpt", "suppression_state") + reasons.extend(f"missing_{field}" for field in required if not _clean(row.get(field))) + if row.get("subject_type") not in ENTITY_TYPES or row.get("object_type") not in ENTITY_TYPES: + reasons.append("unknown_entity_type") + relation_spec = RELATIONSHIPS.get(row.get("relationship_type", "")) + if relation_spec is None: + reasons.append("unknown_relationship_type") + elif row["subject_type"] != relation_spec[0] or row["object_type"] not in relation_spec[1]: + reasons.append("relationship_endpoint_type_mismatch") + if row.get("confidence") not in CONFIDENCE: + reasons.append("unknown_confidence") + if row.get("review_state") not in REVIEW_STATES or row.get("review_state") == "quarantined": + reasons.append("invalid_review_state") + if row.get("match_method") in REJECTED_MATCH_METHODS: + reasons.append("non_defensible_match_method") + elif row.get("match_method") not in MATCH_METHODS: + reasons.append("unknown_match_method") + if row.get("suppression_state") not in SUPPRESSION_STATES: + reasons.append("unknown_suppression_state") + elif row["suppression_state"] != "eligible": + reasons.append("suppressed_or_restricted") + observed = _iso_date(row.get("observation_date"), "observation_date", required=True) + retrieved = _iso_datetime(row.get("retrieved_at_utc"), "retrieved_at_utc") + start = _iso_date(row.get("valid_from"), "valid_from") + end = _iso_date(row.get("valid_to"), "valid_to") + if start and end and end < start: + reasons.append("valid_to_precedes_valid_from") + if observed and (self.reference_date - observed).days > self.stale_after_days: + reasons.append("stale_evidence") + if observed and datetime.fromisoformat(retrieved.replace("Z", "+00:00")).date() < observed: + reasons.append("retrieval_precedes_observation") + row_identities: list[tuple[tuple[str, str], tuple[str, str]]] = [] + for side in ("subject", "object"): + source_id, native_id = row[f"{side}_source_id"], row[f"{side}_source_native_id"] + identity_key = (source_id, native_id) + identity_value = (row[f"{side}_type"], row[f"{side}_name"]) + prior = identifiers.get(identity_key) + if prior and prior != identity_value: + reasons.append("conflicting_source_identifier") + row_identities.append((identity_key, identity_value)) + if row.get("match_method") == "explicit_reviewed_link": + same_name = [key for key, value in identifiers.items() if value[1] == row.get("object_name") and key[0] == row.get("object_source_id")] + if len(same_name) > 1 and (row["object_source_id"], row["object_source_native_id"]) not in same_name: + reasons.append("ambiguous_duplicate_name") + if not reasons: + for identity_key, identity_value in row_identities: + identifiers[identity_key] = identity_value + except AccountabilityContractError as exc: + reasons.append(str(exc).replace(" ", "_")) + observed = None + retrieved = None + + if reasons: + quarantined.append({"source_row": line, "reasons": tuple(dict.fromkeys(reasons)), "source_values": dict(row)}) + continue + relation, node_rows = _relationship(row, observed_at=observed.isoformat(), retrieved_at=retrieved, source_row=line) # type: ignore[union-attr] + entities[_entity_key(relation["subject"])] = node_rows[0] + entities[_entity_key(relation["object"])] = node_rows[1] + relationship_rows.append((relation, row, line)) + + # Ownership is history: non-overlapping periods survive. Contradictory + # operators for the same facility and overlapping validity quarantine. + ownership: dict[tuple[str, str], list[tuple[dict[str, Any], dict[str, str], int]]] = {} + for relation, row, line in relationship_rows: + if relation["relationship_type"] == "operates": + ownership.setdefault((relation["object"]["source_id"], relation["object"]["source_native_id"]), []).append((relation, row, line)) + conflicting_ids: set[str] = set() + for observations in ownership.values(): + for index, (left, _, _) in enumerate(observations): + left_start = date.fromisoformat(left["valid_from"] or left["observation_date"]) + left_end = date.fromisoformat(left["valid_to"]) if left["valid_to"] else date.max + for right, _, _ in observations[index + 1:]: + right_start = date.fromisoformat(right["valid_from"] or right["observation_date"]) + right_end = date.fromisoformat(right["valid_to"]) if right["valid_to"] else date.max + if left["subject"] != right["subject"] and max(left_start, right_start) <= min(left_end, right_end): + conflicting_ids.update((left["relationship_id"], right["relationship_id"])) + if conflicting_ids: + retained: list[tuple[dict[str, Any], dict[str, str], int]] = [] + for relation, row, line in relationship_rows: + if relation["relationship_id"] in conflicting_ids: + quarantined.append({"source_row": line, "reasons": ("overlapping_ownership_conflict",), "source_values": dict(row)}) + else: + retained.append((relation, row, line)) + relationship_rows = retained + used = {_entity_key(relation["subject"]) for relation, _, _ in retained} | {_entity_key(relation["object"]) for relation, _, _ in retained} + entities = {key: entity for key, entity in entities.items() if key in used} + + accepted = sorted((relation for relation, _, _ in relationship_rows), key=lambda relation: relation["relationship_id"]) + entity_rows = sorted(entities.values(), key=lambda entity: entity["entity_id"]) + quarantined.sort(key=lambda item: int(item["source_row"])) + return { + "accepted": accepted, + "entities": entity_rows, + "quarantined": quarantined, + "source_sha256": digest, + "schema_fingerprint": hashlib.sha256(json.dumps(headers, separators=(",", ":")).encode()).hexdigest(), + "headers": headers, + "input_rows": len(rows), + } + + def run(self, raw_path: str | Path, run_dir: str | Path, artifact: SourceArtifact) -> dict[str, Any]: + raw = Path(raw_path).read_bytes() + if artifact.sha256 != hashlib.sha256(raw).hexdigest() or artifact.byte_size != len(raw): + raise ValueError("artifact provenance mismatch") + result = self.parse_bytes(raw) + root = Path(run_dir) + _, relationships_sha, _ = atomic_jsonl(root / "candidate" / "relationships.jsonl", result["accepted"]) + _, entities_sha, _ = atomic_jsonl(root / "candidate" / "entities.jsonl", result["entities"]) + atomic_jsonl(root / "quarantined" / "relationships.jsonl", result["quarantined"]) + reasons = Counter(reason for item in result["quarantined"] for reason in item["reasons"]) + manifest = private_manifest( + source_id=self.source_id, adapter_version=self.adapter_version, + schema_version=self.schema_version, artifact=artifact, + input_rows=result["input_rows"], normalized_rows=len(result["accepted"]), + quarantined_rows=len(result["quarantined"]), normalized_sha256=relationships_sha, + parsed_sha256=entities_sha, anomaly_counts=dict(sorted(reasons.items())), + ) + manifest.update({ + "candidate_contract": "us-accountability-graph-candidate-v1", + "schema_fingerprint": result["schema_fingerprint"], + "entity_rows": len(result["entities"]), + "relationship_rows": len(result["accepted"]), + "quarantined_relationship_rows": len(result["quarantined"]), + "entity_types": dict(sorted(Counter(entity["entity_type"] for entity in result["entities"]).items())), + "relationship_types": dict(sorted(Counter(relation["relationship_type"] for relation in result["accepted"]).items())), + "candidate_relationships_sha256": relationships_sha, + "candidate_entities_sha256": entities_sha, + "test_only": True, + "graph_migration": False, + "geocoding": "disabled", + "publication_gate": "blocked", + "coverage": "Synthetic/sanitized contract fixture for FSIS, APHIS, and deferred legal-entity evidence links; not a national coverage claim", + }) + atomic_json(root / "manifest.json", manifest) + return manifest diff --git a/pipeline/sources/us/accountability/config.json b/pipeline/sources/us/accountability/config.json new file mode 100644 index 0000000..3e7f782 --- /dev/null +++ b/pipeline/sources/us/accountability/config.json @@ -0,0 +1,11 @@ +{ + "source_id": "us.accountability.pilot", + "contract_version": "us-accountability-pilot-v1", + "adapter_version": "us-accountability-candidate-v1", + "source_url": "https://www.fsis.usda.gov/inspection/establishments/meat-poultry-and-egg-product-inspection-directory", + "acquisition": "assisted-link-ledger-after-source-specific-review", + "release_allowed_by_default": false, + "entity_policy": "facilities, establishment approvals, operators, legal entities, parents, brands, inspections, violations, enforcements, laboratories, and aggregate observations remain distinct", + "match_policy": "exact source identifiers or an explicit reviewed link event; never name, address, phone, geocoder, or fuzzy matching", + "stale_after_days": 365 +} diff --git a/pipeline/sources/us/accountability/fixtures/synthetic_link_ledger.csv b/pipeline/sources/us/accountability/fixtures/synthetic_link_ledger.csv new file mode 100644 index 0000000..bf1f433 --- /dev/null +++ b/pipeline/sources/us/accountability/fixtures/synthetic_link_ledger.csv @@ -0,0 +1,13 @@ +subject_type,subject_source_id,subject_source_native_id,subject_name,object_type,object_source_id,object_source_native_id,object_name,relationship_type,evidence_source_id,evidence_source_native_id,observation_date,retrieved_at_utc,valid_from,valid_to,confidence,review_state,match_method,evidence_url,evidence_excerpt,suppression_state +establishment_approval,us.fsis,M001,Synthetic FSIS approval,facility,us.fsis,FSIS-001,Synthetic Meat Plant,establishment_approval_for,us.fsis,M001,2026-09-01,2026-09-15T00:00:00Z,2026-09-01,,high,evidence_verified,exact_source_id,https://www.fsis.usda.gov/inspection/establishments/meat-poultry-and-egg-product-inspection-directory,Sanitized fixture: approval ID explicitly identifies the establishment,eligible +operator,us.aphis,registrations:customer:1,Synthetic Operator,facility,us.fsis,FSIS-001,Synthetic Meat Plant,operates,us.aphis,registrations:00-B-0001,2026-09-01,2026-09-15T00:00:00Z,2026-09-01,2026-12-31,high,evidence_verified,explicit_reviewed_link,https://www.aphis.usda.gov/animal-care/awa-services/usda-animal-care-public-search-tool,Sanitized fixture: reviewer linked the source-native operator and facility IDs,eligible +operator,us.aphis,registrations:customer:1,Synthetic Operator,legal_entity,us.sec,CIK-TEST-001,Synthetic Foods Holdings LLC,operator_is_legal_entity,us.sec,CIK-TEST-001,2026-09-01,2026-09-15T00:00:00Z,2026-09-01,,high,evidence_verified,explicit_reviewed_link,https://www.sec.gov/edgar/search/,Sanitized fixture: exact legal-entity identifier is recorded; live SEC evidence is deferred,eligible +legal_entity,us.sec,CIK-TEST-001,Synthetic Foods Holdings LLC,parent,us.sec,CIK-TEST-PARENT,Synthetic Foods Group,parent_of,us.sec,CIK-TEST-001/parent,2026-09-01,2026-09-15T00:00:00Z,2026-09-01,,medium,review_required,explicit_reviewed_link,https://www.sec.gov/edgar/search/,Sanitized fixture: parent link is temporal and source-keyed,eligible +brand,us.brand,BR-TEST-001,Synthetic Harvest,legal_entity,us.sec,CIK-TEST-001,Synthetic Foods Holdings LLC,brand_of,us.sec,CIK-TEST-001/brand,2026-09-01,2026-09-15T00:00:00Z,2026-09-01,,medium,review_required,explicit_reviewed_link,https://www.sec.gov/edgar/search/,Sanitized fixture: brand remains separate from legal entity,eligible +inspection,us.aphis,inspections:00-C-0003:2026-02-01,Synthetic APHIS inspection,operator,us.aphis,registrations:customer:1,Synthetic Operator,inspection_observes,us.aphis,00-C-0003,2026-02-01,2026-09-15T00:00:00Z,2026-02-01,,high,evidence_verified,exact_source_id,https://www.aphis.usda.gov/animal-care/awa-services/usda-animal-care-public-search-tool,Sanitized fixture: inspection is an observation and not a facility master record,eligible +violation,us.aphis,inspections:00-C-0003:violation-1,Synthetic inspection finding,inspection,us.aphis,inspections:00-C-0003:2026-02-01,Synthetic APHIS inspection,violation_observed_in,us.aphis,00-C-0003/violation-1,2026-02-01,2026-09-15T00:00:00Z,2026-02-01,,medium,review_required,exact_source_id,https://www.aphis.usda.gov/animal-care/awa-services/usda-animal-care-public-search-tool,Sanitized fixture: finding is distinct from inspection and enforcement,eligible +enforcement,us.aphis,inspections:00-C-0003:enforcement-1,Synthetic enforcement action,violation,us.aphis,inspections:00-C-0003:violation-1,Synthetic inspection finding,enforcement_for,us.aphis,00-C-0003/enforcement-1,2026-02-15,2026-09-15T00:00:00Z,2026-02-15,,medium,review_required,exact_source_id,https://www.aphis.usda.gov/animal-care/awa-services/usda-animal-care-public-search-tool,Sanitized fixture: enforcement is not collapsed into violation,eligible +laboratory,us.aphis,lab:TEST-001,Synthetic laboratory,inspection,us.aphis,inspections:00-C-0003:2026-02-01,Synthetic APHIS inspection,laboratory_supports,us.aphis,lab:TEST-001/inspection,2026-02-01,2026-09-15T00:00:00Z,2026-02-01,,medium,review_required,exact_source_id,https://www.aphis.usda.gov/animal-care/awa-services/usda-animal-care-public-search-tool,Sanitized fixture: laboratory is a separate evidence subject,eligible +aggregate_observation,us.aphis,annual_reports:2025:aggregate-1,Synthetic annual aggregate,operator,us.aphis,registrations:customer:1,Synthetic Operator,aggregate_describes,us.aphis,annual_reports:2025,2025-12-31,2026-09-15T00:00:00Z,2025-12-31,,medium,review_required,exact_source_id,https://direct.aphis.usda.gov/awa/research-facility-report/annual-summary,Sanitized fixture: aggregate observation is not an individual facility or inspection,eligible +operator,us.aphis,registrations:customer:old,Synthetic Former Operator,facility,us.fsis,FSIS-001,Synthetic Meat Plant,operates,us.aphis,registrations:00-B-OLD,2026-06-30,2026-09-15T00:00:00Z,2026-01-01,2026-06-30,high,review_required,temporal_source_link,https://www.aphis.usda.gov/animal-care/awa-services/usda-animal-care-public-search-tool,Sanitized fixture: prior operator retained as a bounded historical relationship,eligible +operator,us.aphis,registrations:customer:1,Synthetic Operator,facility,us.fsis,FSIS-002,Synthetic Poultry Plant,operates,us.aphis,registrations:00-B-0001,2026-07-01,2026-09-15T00:00:00Z,2026-07-01,,high,evidence_verified,temporal_source_link,https://www.aphis.usda.gov/animal-care/awa-services/usda-animal-care-public-search-tool,Sanitized fixture: same operator is observed at a second source-keyed facility,eligible diff --git a/pipeline/sources/us/accountability/reconcile.py b/pipeline/sources/us/accountability/reconcile.py new file mode 100644 index 0000000..2d0970e --- /dev/null +++ b/pipeline/sources/us/accountability/reconcile.py @@ -0,0 +1,39 @@ +"""Row-free US V1-to-pilot reconciliation using FSIS keys only.""" +from __future__ import annotations + +import json +from tempfile import TemporaryDirectory +from pathlib import Path +from typing import Any + +from pipeline.reconciliation.crosswalk import compare_v1_v2 + + +def build_v1_crosswalk(v1_path: str | Path, candidate_entities_path: str | Path) -> dict[str, Any]: + """Compare legacy FSIS IDs to pilot facility IDs without merging records. + + The candidate file must already be the private adapter's entity JSONL. A + shared ``establishment_id`` is an observation comparison key only; this + function creates no identity or suppression decision. + """ + v2_rows = [] + for line in Path(candidate_entities_path).read_text(encoding="utf-8").splitlines(): + if not line.strip(): + continue + row = json.loads(line) + if row.get("entity_type") == "facility" and row.get("source_id") == "us.fsis": + v2_rows.append({ + "source_record_key": row.get("source_native_id"), + "normalized": {"coordinates": None, "classification": None, "effective_date": row.get("observation_date")}, + }) + # compare_v1_v2 accepts JSONL and emits only aggregates. A temporary file + # would make this helper awkward for callers, so use its public semantics + # through a deterministic in-memory-equivalent file beside the candidate. + with TemporaryDirectory(prefix="uec-us-crosswalk-") as directory: + temporary = Path(directory) / "candidate.jsonl" + temporary.write_text("".join(json.dumps(row, sort_keys=True) + "\n" for row in v2_rows), encoding="utf-8") + report = compare_v1_v2(v1_path, temporary, v1_key="establishment_id", v2_key="source_record_key", v1_country="US", v2_source_id="us.fsis") + report["matching"]["pilot_scope"] = "FSIS facility source-native establishment ID only" + report["matching"]["identity_inheritance"] = False + report["interpretation"]["v1_identity_assumptions_inherited"] = False + return report diff --git a/pipeline/sources/us/accountability/refresh.py b/pipeline/sources/us/accountability/refresh.py new file mode 100644 index 0000000..2485303 --- /dev/null +++ b/pipeline/sources/us/accountability/refresh.py @@ -0,0 +1,148 @@ +"""Run the US accountability link ledger through private candidate staging.""" +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path + +from pipeline.common.acquisition import utc_now +from pipeline.contracts.adapter_contract import SourceArtifact +from pipeline.contracts.source_lifecycle import atomic_json + +from .adapter import CONFIG, UsAccountabilityAdapter + + +def assisted_capture_contract() -> dict: + return { + "source_id": CONFIG["source_id"], + "method": "operator-assisted-source-link-ledger", + "steps": [ + "Acquire FSIS and APHIS exports only through their source-local assisted contracts.", + "Record source-native IDs and dates for each candidate endpoint; do not copy raw rows into Git.", + "Add a legal-entity, parent, brand, SEC, EPA, OSHA, or other government link only when a reviewer records an explicit source-native identifier and reuse/terms decision.", + "Run this ledger adapter with private staging, then inspect the row-free review packet before any disposable candidate import.", + ], + "controls": [ + "No fuzzy name, address, phone, geocoder, or access-control bypass", + "Ownership changes are temporal events; overlapping contradictory links quarantine", + "Suppressed/restricted links never enter candidate JSONL", + "No graph migration, release promotion, public API, map, export, or UI", + ], + "source_routes": { + "fsis": "https://www.fsis.usda.gov/inspection/establishments/meat-poultry-and-egg-product-inspection-directory", + "aphis": "https://www.aphis.usda.gov/animal-care/awa-services/usda-animal-care-public-search-tool", + "sec": "deferred until source-specific terms and exact CIK evidence are reviewed", + "epa_osha": "deferred; no link is emitted without a source-native identifier and reuse decision", + }, + } + + +def refresh(*, raw_path: str | Path, run_dir: str | Path, retrieved_at_utc: str | None = None, effective_date: str | None = None) -> dict: + raw = Path(raw_path).read_bytes() + retrieved = retrieved_at_utc or utc_now() + adapter = UsAccountabilityAdapter() + artifact = SourceArtifact( + source_url=CONFIG["source_url"], retrieved_at_utc=retrieved, + sha256=hashlib.sha256(raw).hexdigest(), byte_size=len(raw), + effective_date=effective_date or "unknown", code_version=adapter.adapter_version, + config_version=adapter.schema_version, + rights_caveat="Link-ledger source rights and attribution require per-source review before any reuse", + privacy_caveat="Private test-only candidate; identity, address, phone, and coordinate review pending", + coverage="Bounded explicit-link pilot; no national completeness claim", + ) + root = Path(run_dir) + metadata = { + "acquisition_method": "assisted_local_capture", + "source_id": CONFIG["source_id"], + "artifact": Path(raw_path).name, + "artifact_path": str(Path(raw_path).resolve()), + "requested_url": CONFIG["source_url"], + "final_url": CONFIG["source_url"], + "retrieved_at_utc": retrieved, + "effective_date": effective_date or "unknown", + "sha256": artifact.sha256, + "byte_size": artifact.byte_size, + "code_version": adapter.adapter_version, + "config_version": adapter.schema_version, + "retention": {"class": "restricted-research-evidence", "public_exposure": False, "review_required": True}, + } + atomic_json(root / "acquisition-metadata.json", metadata) + manifest = adapter.run(raw_path, root, artifact) + contract = assisted_capture_contract() + atomic_json(root / "assisted-capture-contract.json", contract) + packet = { + "schema_version": "us-accountability-review-packet-v1", + "source_id": CONFIG["source_id"], + "purpose": "Human review aid for a private, synthetic/test-only graph-candidate pilot; not release approval", + "what_the_pilot_proves": [ + "A facility, its FSIS establishment approval, an operator, a legal entity, a parent, and a brand can remain distinct typed candidates.", + "APHIS registration/inspection, violation, enforcement, laboratory, and aggregate observations can remain separate evidence objects.", + "Every accepted relationship carries source IDs, observation/retrieval dates, relationship type, confidence/review state, and evidence.", + "Ambiguous, stale, conflicting, or suppressed relationships are quarantined deterministically.", + ], + "what_remains_inference": [ + "No name, address, phone, geocoder, or fuzzy match establishes identity.", + "The checked-in SEC identifiers and all row values are synthetic; they do not assert a live SEC, EPA, OSHA, FSIS, or APHIS match.", + "An explicit reviewed link is a candidate relationship, not a statement that the project has approved publication or legal ownership.", + "Source disappearance is not closure, and an inspection or aggregate is not a facility-master assertion.", + ], + "source_rights_and_acquisition": { + "fsis": "Use the existing operator-assisted official export contract; direct 403 responses remain fail-closed and are never bypassed.", + "aphis": "Use the existing profile-explicit public-search assisted export; registrations, inspections, annual reports, laboratories, and aggregates remain separate.", + "sec_epa_osha": "Deferred until a source-specific terms/reuse decision and defensible source-native identifier evidence are retained with the run.", + }, + "coverage": "Synthetic/sanitized fixture only; no national completeness, currentness, or publication claim", + "scaling_cost": { + "per_relationship": "one explicit source-native link review plus provenance/evidence storage", + "per_refresh": "source acquisition, schema/count review, identity conflict scan, stale scan, quarantine review, and deterministic artifact hashing", + "unbounded_work_not_included": "bulk fuzzy entity resolution, residential exposure, target scoring, public publication, UI, or graph migration", + }, + "counts": { + "input_rows": manifest["input_rows"], + "relationships": manifest["relationship_rows"], + "entities": manifest["entity_rows"], + "quarantined_relationships": manifest["quarantined_relationship_rows"], + }, + "gates": { + "test_only": True, + "release_state": "not-created", + "publication_state": "private-candidate", + "publication_gate": "blocked", + "graph_migration": False, + "geocoding": "disabled", + }, + "row_payloads_included": False, + } + atomic_json(root / "review-packet.json", packet) + status = { + "status": "candidate-ready" if manifest["normalized_rows"] else "quarantine-only", + "run_dir": str(root), + "source_id": CONFIG["source_id"], + "release_state": "not-created", + "publication_state": "private-candidate", + "publication_gate": "blocked", + "test_only": True, + } + atomic_json(root / "run-status.json", status) + return {"manifest": manifest, "status": status} + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--raw", type=Path, required=True) + parser.add_argument("--run-dir", type=Path, required=True) + parser.add_argument("--retrieved-at-utc") + parser.add_argument("--effective-date") + args = parser.parse_args() + try: + result = refresh(raw_path=args.raw, run_dir=args.run_dir, retrieved_at_utc=args.retrieved_at_utc, effective_date=args.effective_date) + except (OSError, ValueError) as exc: + print(json.dumps({"status": "failed", "error": str(exc)}, sort_keys=True)) + return 2 + print(json.dumps({"status": result["status"]["status"], "run_dir": result["status"]["run_dir"]}, sort_keys=True)) + return 0 if result["status"]["status"] == "candidate-ready" else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/pipeline/sources/us/accountability/test_adapter.py b/pipeline/sources/us/accountability/test_adapter.py new file mode 100644 index 0000000..e56aef6 --- /dev/null +++ b/pipeline/sources/us/accountability/test_adapter.py @@ -0,0 +1,127 @@ +import csv +import hashlib +import io +import json +import tempfile +import unittest +from datetime import date +from pathlib import Path + +from pipeline.contracts.adapter_contract import SourceArtifact + +from .adapter import AccountabilityContractError, REQUIRED_HEADERS, UsAccountabilityAdapter +from .refresh import refresh + +ROOT = Path(__file__).parent +FIXTURE = ROOT / "fixtures" / "synthetic_link_ledger.csv" + + +def rows_from_fixture() -> list[dict[str, str]]: + with FIXTURE.open(newline="", encoding="utf-8") as handle: + return list(csv.DictReader(handle)) + + +def content_for(rows: list[dict[str, str]]) -> bytes: + output = io.StringIO(newline="") + writer = csv.DictWriter(output, fieldnames=REQUIRED_HEADERS, lineterminator="\n") + writer.writeheader() + writer.writerows(rows) + return output.getvalue().encode() + + +class UsAccountabilityAdapterTests(unittest.TestCase): + def test_fixture_emits_distinct_nodes_and_provenance_complete_relationships(self): + result = UsAccountabilityAdapter().parse_bytes(FIXTURE.read_bytes()) + self.assertEqual(result["input_rows"], 12) + self.assertEqual(len(result["accepted"]), 12) + self.assertEqual(len(result["quarantined"]), 0) + self.assertEqual({row["relationship_type"] for row in result["accepted"]}, { + "establishment_approval_for", "operates", "operator_is_legal_entity", "parent_of", + "brand_of", "inspection_observes", "violation_observed_in", "enforcement_for", + "laboratory_supports", "aggregate_describes", + }) + for relation in result["accepted"]: + self.assertTrue(relation["source_id"]) + self.assertEqual(set(relation["source_native_ids"]), {"subject", "object", "evidence"}) + self.assertTrue(relation["observation_date"]) + self.assertTrue(relation["retrieved_at_utc"]) + self.assertTrue(relation["evidence"]["url"]) + self.assertTrue(relation["evidence"]["excerpt"]) + self.assertEqual(relation["publication_gate"], "blocked") + types = {entity["entity_type"] for entity in result["entities"]} + self.assertTrue({"facility", "establishment_approval", "operator", "legal_entity", "parent", "brand", "inspection", "violation", "enforcement", "laboratory", "aggregate_observation"}.issubset(types)) + + def test_name_only_link_is_quarantined_and_no_identity_is_inferred(self): + rows = rows_from_fixture() + rows[1]["match_method"] = "name_only" + result = UsAccountabilityAdapter().parse_bytes(content_for(rows)) + self.assertIn("non_defensible_match_method", result["quarantined"][0]["reasons"]) + self.assertNotIn("candidate-relationship-", json.dumps(result["quarantined"][0])) + self.assertEqual(len(result["accepted"]), 11) + + def test_conflicting_identifier_and_duplicate_name_are_quarantined(self): + rows = rows_from_fixture() + conflicting = dict(rows[1]) + conflicting["object_name"] = "A different facility name" + rows.append(conflicting) + duplicate_a = dict(rows[0]) + duplicate_a.update({"object_source_native_id": "FSIS-100", "object_name": "Duplicate Facility"}) + duplicate_b = dict(rows[0]) + duplicate_b.update({"object_source_native_id": "FSIS-101", "object_name": "Duplicate Facility"}) + ambiguous = dict(rows[0]) + ambiguous.update({"object_source_native_id": "FSIS-102", "object_name": "Duplicate Facility", "evidence_source_native_id": "M004", "observation_date": "2026-09-02", "match_method": "explicit_reviewed_link"}) + rows.extend((duplicate_a, duplicate_b, ambiguous)) + result = UsAccountabilityAdapter().parse_bytes(content_for(rows)) + reasons = [reason for item in result["quarantined"] for reason in item["reasons"]] + self.assertIn("conflicting_source_identifier", reasons) + self.assertIn("ambiguous_duplicate_name", reasons) + + def test_stale_and_suppressed_evidence_quarantine(self): + rows = rows_from_fixture() + rows[0]["observation_date"] = "2024-01-01" + rows[1]["suppression_state"] = "suppressed" + result = UsAccountabilityAdapter().parse_bytes(content_for(rows)) + reasons = [reason for item in result["quarantined"] for reason in item["reasons"]] + self.assertIn("stale_evidence", reasons) + self.assertIn("suppressed_or_restricted", reasons) + + def test_non_overlapping_ownership_is_retained_but_overlap_is_quarantined(self): + rows = rows_from_fixture() + overlap = dict(rows[1]) + overlap.update({ + "subject_source_native_id": "registrations:customer:overlap", + "subject_name": "Synthetic Overlap Operator", + "evidence_source_native_id": "registrations:00-B-OVERLAP", + "valid_from": "2026-08-01", + "valid_to": "", + }) + rows.append(overlap) + result = UsAccountabilityAdapter().parse_bytes(content_for(rows)) + self.assertEqual(sum(item["reasons"] == ("overlapping_ownership_conflict",) for item in result["quarantined"]), 2) + operators = [row for row in result["accepted"] if row["relationship_type"] == "operates"] + self.assertEqual(len(operators), 2) + self.assertEqual({row["subject"]["source_native_id"] for row in operators}, {"registrations:customer:old", "registrations:customer:1"}) + + def test_run_and_assisted_refresh_are_private_and_deterministic(self): + raw = FIXTURE.read_bytes() + artifact = SourceArtifact("https://example.invalid/accountability.csv", "2026-09-15T00:00:00Z", hashlib.sha256(raw).hexdigest(), len(raw), effective_date="2026-09-15", code_version="test", config_version="test") + with tempfile.TemporaryDirectory() as directory: + first = UsAccountabilityAdapter().run(FIXTURE, Path(directory) / "one", artifact) + second = UsAccountabilityAdapter().run(FIXTURE, Path(directory) / "two", artifact) + self.assertEqual(first["normalized_sha256"], second["normalized_sha256"]) + self.assertEqual(first["parsed_sha256"], second["parsed_sha256"]) + self.assertTrue(first["test_only"]) + self.assertFalse(first["graph_migration"]) + refreshed = refresh(raw_path=FIXTURE, run_dir=Path(directory) / "refresh", retrieved_at_utc="2026-09-15T00:00:00Z") + self.assertEqual(refreshed["status"]["status"], "candidate-ready") + packet = json.loads((Path(directory) / "refresh" / "review-packet.json").read_text(encoding="utf-8")) + self.assertFalse(packet["row_payloads_included"]) + self.assertEqual(packet["counts"]["relationships"], 12) + + def test_schema_drift_fails_closed(self): + with self.assertRaises(AccountabilityContractError): + UsAccountabilityAdapter().parse_bytes(b"subject_type,object_type\nfacility,operator\n") + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/sources/us/accountability/test_reconcile.py b/pipeline/sources/us/accountability/test_reconcile.py new file mode 100644 index 0000000..33df7b0 --- /dev/null +++ b/pipeline/sources/us/accountability/test_reconcile.py @@ -0,0 +1,28 @@ +import json +import tempfile +import unittest +from pathlib import Path + +from .reconcile import build_v1_crosswalk + + +class UsCrosswalkTests(unittest.TestCase): + def test_crosswalk_is_exact_row_free_and_does_not_inherit_v1_identity(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / "v1.csv").write_text("establishment_id,name\nFSIS-001,Legacy name\nFSIS-999,Not observed\n", encoding="utf-8") + rows = [ + {"entity_type": "facility", "source_id": "us.fsis", "source_native_id": "FSIS-001", "observation_date": "2026-09-01"}, + {"entity_type": "operator", "source_id": "us.aphis", "source_native_id": "registrations:customer:1", "observation_date": "2026-09-01"}, + ] + (root / "entities.jsonl").write_text("\n".join(json.dumps(row) for row in rows) + "\n", encoding="utf-8") + report = build_v1_crosswalk(root / "v1.csv", root / "entities.jsonl") + self.assertEqual(report["counts"]["matched"], 1) + self.assertEqual(report["counts"]["not_observed_in_v2"], 1) + self.assertFalse(report["interpretation"]["not_observed_in_v2_is_closure"]) + self.assertFalse(report["interpretation"]["v1_identity_assumptions_inherited"]) + self.assertFalse(report["interpretation"]["raw_rows_in_report"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/tests/test_export_release.py b/pipeline/tests/test_export_release.py new file mode 100644 index 0000000..2782d62 --- /dev/null +++ b/pipeline/tests/test_export_release.py @@ -0,0 +1,26 @@ +import importlib.util +import unittest +from pathlib import Path + + +SCRIPT = Path(__file__).parents[1] / "scripts" / "stages" / "export-release.py" +SPEC = importlib.util.spec_from_file_location("export_release", SCRIPT) +MODULE = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(MODULE) + + +class ReleaseExportContractTests(unittest.TestCase): + def test_export_query_repeats_publication_and_suppression_gates(self): + source = SCRIPT.read_text(encoding="utf-8") + for gate in ("release.status='promoted'", "release.test_only IS NOT TRUE", "release.profile=%s", "review.publication_eligible=true", "review.privacy_screening_status='passed'", "public_access_restricted"): + self.assertIn(gate, source) + self.assertIn("REPEATABLE READ", source) + + def test_missing_attribution_is_not_called_cleared(self): + self.assertEqual(MODULE._rights(None), "unknown") + self.assertEqual(MODULE._rights(""), "unknown") + self.assertEqual(MODULE._rights("Attribution required"), "attribution_required") + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/tests/test_graph_database_contract.py b/pipeline/tests/test_graph_database_contract.py new file mode 100644 index 0000000..9a866e9 --- /dev/null +++ b/pipeline/tests/test_graph_database_contract.py @@ -0,0 +1,127 @@ +import os +import unittest +import uuid + +import psycopg + + +DATABASE_URL = os.environ.get( + "UEC_DATABASE_URL", + "postgresql://uec:uec-local-development-only@localhost:5433/uec", +) + + +class GraphDatabaseContractTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + try: + cls.connection = psycopg.connect(DATABASE_URL) + tables = {row[0] for row in cls.connection.execute( + "SELECT table_name FROM information_schema.tables WHERE table_schema = 'uec'" + ).fetchall()} + if "claims" not in tables or "organization_relationship_observations" not in tables: + raise unittest.SkipTest("graph migrations are not applied") + except psycopg.Error as error: + raise unittest.SkipTest(f"PostGIS is unavailable: {error}") + + @classmethod + def tearDownClass(cls): + if hasattr(cls, "connection"): + cls.connection.close() + + def fixture(self): + suffix = uuid.uuid4().hex + source_id = f"test.graph.{suffix}" + artifact_id = uuid.uuid4() + record_id = uuid.uuid4() + facility_id = uuid.uuid4() + organization_id = uuid.uuid4() + other_organization_id = uuid.uuid4() + self.connection.execute( + "INSERT INTO uec.sources (source_id, country_code, name, official_url, access_method) VALUES (%s, 'US', 'Synthetic graph source', 'https://example.invalid/graph', 'fixture')", + (source_id,), + ) + self.connection.execute( + "INSERT INTO uec.raw_artifacts (artifact_id, storage_key, sha256, byte_size, retrieved_at) VALUES (%s, %s, %s, 1, now())", + (artifact_id, f"test/{suffix}", suffix + "0" * (64 - len(suffix))), + ) + self.connection.execute( + "INSERT INTO uec.source_records (source_record_id, source_id, source_record_key, artifact_id, raw_fields, parsed_at) VALUES (%s, %s, %s, %s, '{}'::jsonb, now())", + (record_id, source_id, f"record-{suffix}", artifact_id), + ) + self.connection.execute( + "INSERT INTO uec.facilities (facility_id, canonical_name, country_code) VALUES (%s, 'Synthetic facility', 'US')", + (facility_id,), + ) + self.connection.execute( + "INSERT INTO uec.organizations (organization_id, canonical_name, country_code) VALUES (%s, 'Synthetic operator', 'US'), (%s, 'Synthetic alternate owner', 'US')", + (organization_id, other_organization_id), + ) + return source_id, artifact_id, record_id, facility_id, organization_id, other_organization_id + + def test_source_qualified_crosswalk_and_contradictory_claims_coexist(self): + with self.connection.transaction(): + source, artifact, record, facility, organization, alternate = self.fixture() + left = self.connection.execute( + "INSERT INTO uec.source_entity_identifiers (source_id, source_record_id, entity_type, facility_id, identifier_type, source_identifier, observed_at) VALUES (%s, %s, 'facility', %s, 'permit', 'FAC-1', now()) RETURNING identifier_id", + (source, record, facility), + ).fetchone()[0] + right = self.connection.execute( + "INSERT INTO uec.source_entity_identifiers (source_id, source_record_id, entity_type, organization_id, identifier_type, source_identifier, observed_at) VALUES (%s, %s, 'organization', %s, 'registration', 'ORG-1', now()) RETURNING identifier_id", + (source, record, organization), + ).fetchone()[0] + self.connection.execute( + "INSERT INTO uec.source_entity_crosswalks (left_identifier_id, right_identifier_id, source_id, source_record_id, match_method, observed_at) VALUES (%s, %s, %s, %s, 'synthetic review', now())", + (left, right, source, record), + ) + for count in (10, 20): + self.connection.execute( + "INSERT INTO uec.claims (source_id, source_record_id, facility_id, claim_domain, claim_kind, claim_value, observed_at) VALUES (%s, %s, %s, 'animal_count', 'annual_headcount', jsonb_build_object('count', %s), now())", + (source, record, facility, count), + ) + self.assertEqual( + self.connection.execute( + "SELECT count(*) FROM uec.claim_current WHERE facility_id = %s AND claim_kind = 'annual_headcount'", + (facility,), + ).fetchone()[0], + 2, + ) + with self.assertRaises(psycopg.Error): + with self.connection.transaction(): + self.connection.execute( + "INSERT INTO uec.source_entity_crosswalks (left_identifier_id, right_identifier_id, source_id, source_record_id, match_method, observed_at) VALUES (%s, %s, 'wrong-source', %s, 'bad', now())", + (left, right, record), + ) + + def test_unknown_relationship_requires_reason_and_public_suppression_propagates(self): + with self.connection.transaction(): + source, artifact, record, facility, organization, alternate = self.fixture() + self.connection.execute( + "INSERT INTO uec.organization_relationship_observations (source_id, source_record_id, target_facility_id, assertion_status, unknown_reason, observed_at) VALUES (%s, %s, %s, 'unknown', 'source did not identify the operator', now())", + (source, record, facility), + ) + with self.assertRaises(psycopg.Error): + with self.connection.transaction(): + self.connection.execute( + "INSERT INTO uec.organization_relationship_observations (source_id, source_record_id, target_facility_id, assertion_status, observed_at) VALUES (%s, %s, %s, 'unknown', now())", + (source, record, facility), + ) + release = f"graph-release-{uuid.uuid4().hex}" + self.connection.execute( + "INSERT INTO uec.releases (release_id, status, ruleset_version, profile, summary) VALUES (%s, 'promoted', 'graph-test', 'official', '{}'::jsonb)", + (release,), + ) + claim = self.connection.execute( + "INSERT INTO uec.claims (source_id, source_record_id, facility_id, claim_domain, claim_kind, claim_value, observed_at, review_state, storage_state, privacy_status, publication_status, release_id) VALUES (%s, %s, %s, 'operation', 'synthetic_status', '{\"status\":\"open\"}', now(), 'accepted', 'released', 'passed', 'released', %s) RETURNING claim_id", + (source, record, facility, release), + ).fetchone()[0] + self.assertEqual(self.connection.execute("SELECT count(*) FROM uec.graph_public_claims WHERE claim_id = %s", (claim,)).fetchone()[0], 1) + self.connection.execute( + "INSERT INTO uec.record_access_events (source_record_id, action, reason_category, policy_version, maintainer) VALUES (%s, 'public_access_revoked', 'privacy', 'ethics-v1', 'synthetic-test')", + (record,), + ) + self.assertEqual(self.connection.execute("SELECT count(*) FROM uec.graph_public_claims WHERE claim_id = %s", (claim,)).fetchone()[0], 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/tests/test_graph_migrations.py b/pipeline/tests/test_graph_migrations.py new file mode 100644 index 0000000..3d56b73 --- /dev/null +++ b/pipeline/tests/test_graph_migrations.py @@ -0,0 +1,51 @@ +import unittest +from pathlib import Path + + +ROOT = Path(__file__).parents[1] + + +class GraphMigrationContractTests(unittest.TestCase): + def read(self, name): + return (ROOT / "migrations" / name).read_text(encoding="utf-8") + + def test_reserved_migrations_are_present_and_ordered(self): + migrations = sorted(path.name for path in (ROOT / "migrations").glob("*.sql")) + self.assertEqual(migrations[-4:], [ + "026_graph_entities_crosswalks.sql", + "027_graph_relationship_observations.sql", + "028_graph_claims_support.sql", + "029_graph_publication_projections.sql", + ]) + + def test_entities_are_distinct_and_crosswalk_is_scoped(self): + sql = self.read("026_graph_entities_crosswalks.sql") + self.assertIn("CREATE TABLE uec.organizations", sql) + self.assertIn("entity_type TEXT NOT NULL CHECK (entity_type IN ('facility', 'organization'))", sql) + self.assertIn("identity_scope TEXT NOT NULL DEFAULT 'source_scoped'", sql) + self.assertIn("CHECK (identity_scope = 'source_scoped')", sql) + self.assertIn("source_entity_crosswalks_append_only", sql) + + def test_relationships_and_claims_preserve_uncertainty_and_state_separation(self): + relationship = self.read("027_graph_relationship_observations.sql") + claims = self.read("028_graph_claims_support.sql") + for sql in (relationship, claims): + for field in ("observed_at", "confidence", "review_state", "storage_state", "privacy_status", "publication_status"): + self.assertIn(field, sql) + self.assertIn("append_only", sql) + self.assertIn("assertion_status = 'unknown'", relationship) + self.assertIn("unknown_reason", relationship) + self.assertIn("support_role", claims) + self.assertIn("contradicting", claims) + + def test_public_projections_are_release_and_suppression_aware(self): + sql = self.read("029_graph_publication_projections.sql") + self.assertIn("release.status = 'promoted'", sql) + self.assertIn("relationship.privacy_status = 'passed'", sql) + self.assertIn("claim.privacy_status = 'passed'", sql) + self.assertIn("uec.public_access_restricted", sql) + self.assertIn("graph_publication_safety", sql) + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/tests/test_private_environment_gate.py b/pipeline/tests/test_private_environment_gate.py index 68f052a..61d3962 100644 --- a/pipeline/tests/test_private_environment_gate.py +++ b/pipeline/tests/test_private_environment_gate.py @@ -95,6 +95,20 @@ def test_manifest_rejects_path_traversal_and_migration_inventory_is_versioned(se self.assertGreater(inventory["migration_count"], 0) self.assertEqual(len(inventory["migration_inventory_sha256"]), 64) + def test_v2_manifest_requires_release_metadata_contract(self): + with tempfile.TemporaryDirectory() as directory: + paths, _ = self.fixtures(directory) + manifest = json.loads(paths["manifest.json"].read_text(encoding="utf-8")) + manifest.update({"manifest_version": "uec-release-manifest-v2", "data_product_version": "uec-public-data-product-v1", "schema_version": "uec-location-projection-v1", "generated_at": "2026-09-15T00:00:00Z", "retrieved_at": "2026-09-14T00:00:00Z", "publication_state": "project-published", "review_state": "project-approved", "limitations": ["synthetic"], "row_counts": {"eligible_rows": 0, "packaged_rows": 0}, "source_coverage": [], "checksums": {"algorithm": "sha256"}, "test_only": False}) + serialized = MODULE.canonical_json(manifest) + paths["manifest.json"].write_text(serialized, encoding="utf-8") + MODULE.validate_release_manifest(paths["manifest.json"], hashlib.sha256(serialized.encode()).hexdigest()) + del manifest["limitations"] + serialized = MODULE.canonical_json(manifest) + paths["manifest.json"].write_text(serialized, encoding="utf-8") + with self.assertRaises(MODULE.PrivateEnvironmentError): + MODULE.validate_release_manifest(paths["manifest.json"], hashlib.sha256(serialized.encode()).hexdigest()) + def test_replay_sql_is_idempotent_and_does_not_embed_sensitive_columns(self): replay_spec = importlib.util.spec_from_file_location( "replay_restriction_ledger", diff --git a/pipeline/tests/test_source_registry.py b/pipeline/tests/test_source_registry.py index 549aacd..58874be 100644 --- a/pipeline/tests/test_source_registry.py +++ b/pipeline/tests/test_source_registry.py @@ -9,8 +9,8 @@ class SourceRegistryTests(unittest.TestCase): def test_repository_registry_loads_and_references_existing_legacy_paths(self): registry = load_registry() - self.assertEqual(len(registry["sources"]), 16) - self.assertEqual(len({source["source_id"] for source in registry["sources"]}), 16) + self.assertEqual(len(registry["sources"]), 20) + self.assertEqual(len({source["source_id"] for source in registry["sources"]}), 20) def test_unknowns_are_explicit(self): registry = json.loads(REGISTRY_PATH.read_text(encoding="utf-8")) diff --git a/src/lib.rs b/src/lib.rs index 5cd6a9a..dee9a85 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -152,6 +152,7 @@ struct V2ExportRow { provenance_source_name: String, provenance_source_url: String, provenance_retrieved_at: chrono::DateTime, + source_rights_status: String, release_id: String, release_profile: String, profile_notice: String, @@ -176,6 +177,10 @@ fn export_publication_warning(profile: &str, factual_review_status: &str) -> Opt .then(|| UNREVIEWED_COMMUNITY_WARNING.to_string()) } +fn csv_safe_option(value: Option) -> Option { + value.map(csv_safe_value) +} + pub async fn get_v2_locations_export_handler( State(state): State, Query(params): Query, @@ -217,7 +222,7 @@ pub async fn get_v2_locations_export_handler( }; let release_id: String = release.get(0); let manifest_sha256: String = release.get(1); - let rows = match client.query("SELECT h.facility_id, h.canonical_name, h.country_code, h.city, h.classification_category, h.display_precision, r.factual_review_status, r.privacy_screening_status, r.maintainer_approval, r.reviewer_role, h.provenance_origin_type, h.provenance_source_id, h.provenance_source_name, h.provenance_source_url, h.provenance_retrieved_at, h.release_id FROM uec.map_facilities_display_history h JOIN uec.publication_review_release_current r ON r.source_record_id=h.source_record_id AND r.release_id=h.release_id WHERE h.release_id=$1 AND r.publication_eligible=true AND r.privacy_screening_status='passed' AND ($2='community' OR r.maintainer_approval='approved') ORDER BY h.facility_id LIMIT 1001", &[&release_id, &profile]).await { + let rows = match client.query("SELECT h.facility_id, h.canonical_name, h.country_code, h.city, h.classification_category, h.display_precision, r.factual_review_status, r.privacy_screening_status, r.maintainer_approval, r.reviewer_role, h.provenance_origin_type, h.provenance_source_id, h.provenance_source_name, h.provenance_source_url, h.provenance_retrieved_at, CASE WHEN source.attribution IS NULL OR btrim(source.attribution) = '' THEN 'unknown' ELSE 'attribution_required' END, h.release_id FROM uec.map_facilities_display_history h JOIN uec.publication_review_release_current r ON r.source_record_id=h.source_record_id AND r.release_id=h.release_id JOIN uec.sources source ON source.source_id=h.provenance_source_id WHERE h.release_id=$1 AND r.publication_eligible=true AND r.privacy_screening_status='passed' AND ($2='community' OR r.maintainer_approval='approved') ORDER BY h.facility_id LIMIT 1001", &[&release_id, &profile]).await { Ok(rows) => rows, Err(_) => return v2_error(StatusCode::SERVICE_UNAVAILABLE, "export_query_failed", "public export unavailable") }; if rows.len() > 1000 { @@ -233,25 +238,29 @@ pub async fn get_v2_locations_export_handler( if writer .serialize(V2ExportRow { facility_id: row.get(0), - canonical_name: row.get(1), - country_code: row.get(2), - city: row.get(3), - category: row.get(4), - display_precision: row.get(5), - factual_review_status: factual_review_status.clone(), - privacy_screening_status: row.get(7), - project_approval: row.get(8), - reviewer_role: row.get(9), - source_type: row.get(10), - provenance_source_id: row.get(11), - provenance_source_name: row.get(12), - provenance_source_url: row.get(13), + canonical_name: csv_safe_option(row.get(1)), + country_code: csv_safe_value(row.get(2)), + city: csv_safe_option(row.get(3)), + category: csv_safe_value(row.get(4)), + display_precision: csv_safe_value(row.get(5)), + factual_review_status: csv_safe_value(factual_review_status.clone()), + privacy_screening_status: csv_safe_value(row.get(7)), + project_approval: csv_safe_value(row.get(8)), + reviewer_role: csv_safe_option(row.get(9)), + source_type: csv_safe_value(row.get(10)), + provenance_source_id: csv_safe_value(row.get(11)), + provenance_source_name: csv_safe_value(row.get(12)), + provenance_source_url: csv_safe_value(row.get(13)), provenance_retrieved_at: row.get(14), - release_id: row.get(15), - release_profile: profile.to_string(), - profile_notice: export_profile_notice(profile).to_string(), - publication_warning: export_publication_warning(profile, &factual_review_status), - manifest_sha256: manifest_sha256.clone(), + source_rights_status: csv_safe_value(row.get(15)), + release_id: csv_safe_value(row.get(16)), + release_profile: csv_safe_value(profile.to_string()), + profile_notice: csv_safe_value(export_profile_notice(profile).to_string()), + publication_warning: csv_safe_option(export_publication_warning( + profile, + &factual_review_status, + )), + manifest_sha256: csv_safe_value(manifest_sha256.clone()), }) .is_err() { @@ -282,6 +291,8 @@ pub async fn get_v2_locations_export_handler( .header("x-uec-release-id", release_id) .header("x-uec-export-profile", profile) .header("x-uec-manifest-sha256", manifest_sha256) + .header("x-uec-data-product-version", "uec-public-data-product-v1") + .header("x-uec-schema-version", "uec-location-projection-v1") .body(axum::body::Body::from(body)) .unwrap() .into_response() @@ -1070,6 +1081,7 @@ pub struct V2Location { pub observation_count: Option, pub lifecycle_status: String, pub source_type: String, + pub source_rights_status: String, pub provenance_source: Option, pub release_id: String, pub release_ruleset_version: String, @@ -1352,28 +1364,30 @@ pub async fn get_v2_locations_handler( let promoted_profile: String = release.get(3); let query_limit = limit + 1; let rows = match transaction.query(r#" - SELECT facility_id, canonical_name, country_code, city, classification_category, display_precision, + SELECT history.facility_id, history.canonical_name, history.country_code, history.city, history.classification_category, history.display_precision, review.factual_review_status, review.privacy_screening_status, review.maintainer_approval, review.reviewer_role, - ST_Y(display_location::geometry), ST_X(display_location::geometry), - first_observed_at, last_observed_at, observation_count, lifecycle_status, - provenance_origin_type, map_facilities_display_history.release_id, release_ruleset_version, - provenance_source_id, provenance_source_name, provenance_source_url, provenance_retrieved_at - FROM uec.map_facilities_display_history + ST_Y(history.display_location::geometry), ST_X(history.display_location::geometry), + history.first_observed_at, history.last_observed_at, history.observation_count, history.lifecycle_status, + history.provenance_origin_type, history.release_id, history.release_ruleset_version, + history.provenance_source_id, history.provenance_source_name, history.provenance_source_url, history.provenance_retrieved_at, + CASE WHEN rights.attribution IS NULL OR btrim(rights.attribution) = '' THEN 'unknown' ELSE 'attribution_required' END + FROM uec.map_facilities_display_history AS history JOIN uec.publication_review_release_current AS review - ON review.source_record_id = map_facilities_display_history.source_record_id - AND review.release_id = map_facilities_display_history.release_id - WHERE map_facilities_display_history.release_id = $1 - AND ($2::uuid IS NULL OR facility_id > $2) - AND ($3::text IS NULL OR country_code = $3) - AND ($4::text IS NULL OR city = $4) - AND ($5::text IS NULL OR classification_category = $5) - AND ($6::text IS NULL OR display_precision = $6) - AND ($7::text IS NULL OR lifecycle_status = $7) - AND ($8::text IS NULL OR provenance_origin_type = $8) - AND ($9::text IS NULL OR lower(coalesce(canonical_name, '') || ' ' || coalesce(city, '') || ' ' || country_code || ' ' || classification_category || ' ' || coalesce(provenance_source_name, '')) LIKE '%' || lower($9) || '%' ESCAPE '\') - AND ($10::double precision IS NULL OR (display_location && ST_MakeEnvelope($10, $11, $12, $13, 4326)::geography AND ST_Intersects(display_location::geometry, ST_MakeEnvelope($10, $11, $12, $13, 4326)))) - AND ($14::double precision IS NULL OR ST_DWithin(display_location, ST_SetSRID(ST_Point($15, $16), 4326)::geography, $14 * 1000)) - ORDER BY facility_id LIMIT $17 OFFSET $18 + ON review.source_record_id = history.source_record_id + AND review.release_id = history.release_id + JOIN uec.sources rights ON rights.source_id = history.provenance_source_id + WHERE history.release_id = $1 + AND ($2::uuid IS NULL OR history.facility_id > $2) + AND ($3::text IS NULL OR history.country_code = $3) + AND ($4::text IS NULL OR history.city = $4) + AND ($5::text IS NULL OR history.classification_category = $5) + AND ($6::text IS NULL OR history.display_precision = $6) + AND ($7::text IS NULL OR history.lifecycle_status = $7) + AND ($8::text IS NULL OR history.provenance_origin_type = $8) + AND ($9::text IS NULL OR lower(coalesce(history.canonical_name, '') || ' ' || coalesce(history.city, '') || ' ' || history.country_code || ' ' || history.classification_category || ' ' || coalesce(history.provenance_source_name, '')) LIKE '%' || lower($9) || '%' ESCAPE '\') + AND ($10::double precision IS NULL OR (history.display_location && ST_MakeEnvelope($10, $11, $12, $13, 4326)::geography AND ST_Intersects(history.display_location::geometry, ST_MakeEnvelope($10, $11, $12, $13, 4326)))) + AND ($14::double precision IS NULL OR ST_DWithin(history.display_location, ST_SetSRID(ST_Point($15, $16), 4326)::geography, $14 * 1000)) + ORDER BY history.facility_id LIMIT $17 OFFSET $18 "#, &[&promoted_release_id, &cursor, ¶ms.country_code, ¶ms.region, ¶ms.category, ¶ms.display_precision, ¶ms.lifecycle_status, ¶ms.source_type, &search_text, &min_lon, &min_lat, &max_lon, &max_lat, &radius_km, &longitude, &latitude, &query_limit, &effective_offset]).await { Ok(rows) => rows, Err(_) => return v2_error(StatusCode::INTERNAL_SERVER_ERROR, "location_query_failed", "V2 location query failed"), @@ -1408,6 +1422,7 @@ pub async fn get_v2_locations_handler( observation_count: row.get(14), lifecycle_status: row.get(15), source_type: row.get(16), + source_rights_status: row.get(23), release_id: row.get(17), release_ruleset_version: row.get(18), provenance_source_id: row.get(19), @@ -1425,6 +1440,8 @@ pub async fn get_v2_locations_handler( let metadata = serde_json::json!({ "release_id": promoted_release_id, "ruleset_version": promoted_ruleset, + "data_product_version": "uec-public-data-product-v1", + "schema_version": "uec-location-projection-v1", "release_created_at": promoted_created_at, "profile": promoted_profile, "next_cursor": next_cursor, @@ -1508,17 +1525,19 @@ pub async fn get_v2_location_detail_handler( let created_at: chrono::DateTime = release.get(2); let profile: String = release.get(3); let row = match transaction.query_opt(r#" - SELECT facility_id, canonical_name, country_code, city, classification_category, display_precision, + SELECT history.facility_id, history.canonical_name, history.country_code, history.city, history.classification_category, history.display_precision, review.factual_review_status, review.privacy_screening_status, review.maintainer_approval, review.reviewer_role, - ST_Y(display_location::geometry), ST_X(display_location::geometry), - first_observed_at, last_observed_at, observation_count, lifecycle_status, - provenance_origin_type, map_facilities_display_history.release_id, release_ruleset_version, - provenance_source_id, provenance_source_name, provenance_source_url, provenance_retrieved_at - FROM uec.map_facilities_display_history + ST_Y(history.display_location::geometry), ST_X(history.display_location::geometry), + history.first_observed_at, history.last_observed_at, history.observation_count, history.lifecycle_status, + history.provenance_origin_type, history.release_id, history.release_ruleset_version, + history.provenance_source_id, history.provenance_source_name, history.provenance_source_url, history.provenance_retrieved_at, + CASE WHEN rights.attribution IS NULL OR btrim(rights.attribution) = '' THEN 'unknown' ELSE 'attribution_required' END + FROM uec.map_facilities_display_history AS history JOIN uec.publication_review_release_current AS review - ON review.source_record_id = map_facilities_display_history.source_record_id - AND review.release_id = map_facilities_display_history.release_id - WHERE facility_id = $1 AND map_facilities_display_history.release_id = $2 + ON review.source_record_id = history.source_record_id + AND review.release_id = history.release_id + JOIN uec.sources rights ON rights.source_id = history.provenance_source_id + WHERE history.facility_id = $1 AND history.release_id = $2 "#, &[&facility_id, &release_id]).await { Ok(row) => row, Err(_) => return v2_error(StatusCode::INTERNAL_SERVER_ERROR, "location_query_failed", "V2 location query failed"), @@ -1555,6 +1574,7 @@ pub async fn get_v2_location_detail_handler( observation_count: row.get(14), lifecycle_status: row.get(15), source_type: row.get(16), + source_rights_status: row.get(23), release_id: row.get(17), release_ruleset_version: row.get(18), provenance_source_id: row.get(19), @@ -1570,7 +1590,7 @@ pub async fn get_v2_location_detail_handler( ) .into_response(); } - Json(serde_json::json!({"data": item, "api_version": "v2", "meta": {"release_id": release_id, "ruleset_version": ruleset, "release_created_at": created_at, "profile": profile, "coverage_scope": "selected_promoted_release_public_facilities", "count_semantics": "This record is a public facility projection, not an animal count."}})).into_response() + Json(serde_json::json!({"data": item, "api_version": "v2", "meta": {"release_id": release_id, "ruleset_version": ruleset, "data_product_version": "uec-public-data-product-v1", "schema_version": "uec-location-projection-v1", "release_created_at": created_at, "profile": profile, "coverage_scope": "selected_promoted_release_public_facilities", "count_semantics": "This record is a public facility projection, not an animal count."}})).into_response() } #[derive(Deserialize)] @@ -1869,6 +1889,7 @@ mod v2_api_tests { observation_count: Some(2), lifecycle_status: "active_observed".into(), source_type: "official".into(), + source_rights_status: "attribution_required".into(), provenance_source: None, release_id: "test".into(), release_ruleset_version: "test".into(), @@ -1882,6 +1903,7 @@ mod v2_api_tests { assert_eq!(json["observation_count"], 2); assert_eq!(json["lifecycle_status"], "active_observed"); assert_eq!(json["source_type"], "official"); + assert_eq!(json["source_rights_status"], "attribution_required"); assert_eq!(json["category"], "slaughter"); assert_eq!(json["publication_profile"], "official"); } @@ -1904,6 +1926,7 @@ mod v2_api_tests { provenance_source_name: "Synthetic source".into(), provenance_source_url: "https://example.invalid/community".into(), provenance_retrieved_at: chrono::Utc::now(), + source_rights_status: "attribution_required".into(), release_id: "synthetic-release".into(), release_profile: "community".into(), profile_notice: export_profile_notice("community").into(), @@ -1927,6 +1950,7 @@ mod v2_api_tests { assert_eq!(value("project_approval"), "pending"); assert_eq!(value("publication_warning"), UNREVIEWED_COMMUNITY_WARNING); assert_eq!(value("profile_notice"), COMMUNITY_EXPORT_NOTICE); + assert_eq!(value("source_rights_status"), "attribution_required"); assert!(export_publication_warning("official", "unreviewed").is_none()); assert!(export_publication_warning("community", "reviewed").is_none()); } From 7613ee3b1ba660baa14224120811cb6a13c1aac8 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Wed, 16 Sep 2026 00:36:16 -0700 Subject: [PATCH 132/311] research: reconcile Australia source reconnaissance --- .../australia/artifact-metadata.json | 43 +++ .../countries/australia/source-crosswalk.json | 32 +++ docs/country-recon-au.md | 103 +++++++ docs/source-status.json | 22 ++ docs/source-status.md | 6 +- pipeline/common/test_source_operations.py | 2 +- pipeline/source_operations.json | 24 +- pipeline/source_registry.json | 264 ++++++++++++++++++ .../tests/test_australia_source_metadata.py | 34 +++ pipeline/tests/test_source_registry.py | 4 +- 10 files changed, 529 insertions(+), 5 deletions(-) create mode 100644 docs/countries/australia/artifact-metadata.json create mode 100644 docs/countries/australia/source-crosswalk.json create mode 100644 docs/country-recon-au.md create mode 100644 pipeline/tests/test_australia_source_metadata.py diff --git a/docs/countries/australia/artifact-metadata.json b/docs/countries/australia/artifact-metadata.json new file mode 100644 index 0000000..fef7aa2 --- /dev/null +++ b/docs/countries/australia/artifact-metadata.json @@ -0,0 +1,43 @@ +{ + "schema_version": "1.0", + "purpose": "Private bounded Australia source reconnaissance artifacts; not release inputs.", + "retrieved_at": "2026-09-16T00:17:32-07:00", + "artifacts": [ + { + "source_id": "au.npi.facilities", + "url": "https://data.gov.au/data/dataset/043f58e0-a188-4458-b61c-04e5b540aea4/resource/f83cdee9-ebcb-4f24-941b-34bb2f0996cf/download/facilities.csv", + "local_path": "data/raw/australia/npi-facilities.csv", + "retrieved_at": "2026-09-16T07:15:42.2821369Z", + "content_type": "text/csv", + "byte_size": 3248241, + "sha256": "a0be4c37d588b391ea81baf240f675b2c9f3d9ee0210f9c451c7371eac11d6c7", + "row_count": 8140, + "unique_key": "facility_id", + "unique_key_count": 8140, + "schema": ["facility_id", "jurisdiction_code", "jurisdiction_facility_id", "registered_business_name", "facility_name", "abn", "acn", "street_address", "suburb", "state", "postcode", "latitude", "longitude", "primary_anzsic_class_code", "primary_anzsic_class_name", "main_activities", "facility_website", "first_report_year", "latest_report_year", "latest_report_id", "latest_report_url", "reports"], + "retention": "ignored private raw artifact; do not commit or publish" + }, + { + "source_id": "au.sa.epa.licensed-activities", + "url": "https://data.sa.gov.au/data/dataset/8fdb86ff-d3d1-4f9e-85a5-bed4080d5ee1/resource/26e076f3-c37f-4089-8f28-3f7c9afd997e/download/topo_epa_activities_wgs84.geojson", + "local_path": "data/raw/australia/sa-epa-licensed-activities-2026-03-18.geojson", + "retrieved_at": "2026-09-16T00:15:42-07:00", + "content_type": "application/octet-stream (GeoJSON)", + "byte_size": 1613987, + "sha256": "53b908e02c67aaf9749bf517b4ed82c7e8af1dcc66fb87e0eb43ecb8bf12d7b0", + "feature_count": 4541, + "geometry_type": "Point", + "unique_key": "EPALICENCE", + "unique_key_count": 1695, + "schema": ["OBJECTID", "EPALICENCE", "ACTIVITY", "LICENCE_NAME", "PR_LINK", "geometry.coordinates"], + "retention": "ignored private raw artifact; CC BY 3.0 source terms still require project privacy/release review" + } + ], + "not_acquired": [ + { + "source_id": "au.daff.export-dairy-processing", + "url": "https://www.agriculture.gov.au/sites/default/files/documents/dairy-export-registered-processing-establishments.pdf", + "reason": "Current page/PDF verified through primary-source inspection; direct download timed out in the execution environment." + } + ] +} diff --git a/docs/countries/australia/source-crosswalk.json b/docs/countries/australia/source-crosswalk.json new file mode 100644 index 0000000..6b2f6a3 --- /dev/null +++ b/docs/countries/australia/source-crosswalk.json @@ -0,0 +1,32 @@ +{ + "schema_version": "australia-source-crosswalk-v1", + "country": "AU", + "checked_at_utc": "2026-09-16T07:30:00Z", + "publication_state": "private-reconnaissance-only", + "identity_policy": "Source-local identifiers are authoritative only within their source. Cross-source links require exact ABN/ACN or a reviewed combination of name, address and coordinates; names, addresses and coordinates alone never silently merge rows.", + "sources": [ + {"source_id":"au.daff.export-establishments","authority":"DAFF","role":"federal export establishment registration and meat welfare oversight","primary_url":"https://www.agriculture.gov.au/biosecurity-trade/export/from-australia/documentation-registration-licensing/establishment-registration","access":"authenticated TradeClear/Export Service or assisted request; guidance public","format":"HTML/PDF/DOCX guidance; report schema unknown","cadence":"ongoing/certificate-specific","primary_key_candidates":["establishment_number"],"identity_fields":["occupier_name","address","acn","abn","operations","products","registration_dates"],"coverage":"export-registered prescribed-goods premises, not domestic-only coverage","map_use":"site point only after authorised precision/privacy review","graph_edges":["export_authorisation","operator","approved_arrangement","welfare_oversight"],"privacy_fields":["address","occupier_name","management_control"],"overlap_group":"meat-authorisation","readiness":"reference_only","blockers":["no verified public meat bulk export","authorised access and retention terms required"]}, + {"source_id":"au.safefood.qld.accreditation","authority":"Safe Food Queensland","role":"Queensland accreditation","primary_url":"https://hub.safefood.qld.gov.au/registry/s/","access":"public browser search; bulk/API route not verified","format":"HTML search results; schema unknown","cadence":"unknown","primary_key_candidates":["accreditation_number"],"identity_fields":["name","industry_type","scheme","category","registration_status"],"coverage":"Queensland accredited food businesses, broader than slaughter","map_use":"only if result supplies reviewed site location","graph_edges":["accreditation","operator","activity_category"],"privacy_fields":["address","sole_trader_name"],"overlap_group":"meat-authorisation","readiness":"reference_only","blockers":["bounded authorised capture and terms/schema verification required"]}, + {"source_id":"au.primesafe.vic.meat-licences","authority":"PrimeSafe","role":"Victoria meat and seafood licensing","primary_url":"https://www.primesafe.vic.gov.au/licensing/about-your-licence/","access":"public search linked by authority; annual report public","format":"HTML/PDF; row schema/API not verified","cadence":"unknown; annual aggregate report","primary_key_candidates":["primesafe_licence_number"],"identity_fields":["business_name","category","location","status","dates"],"coverage":"Victoria meat, poultry, game, pet meat, rendering, seafood and transport categories","map_use":"reviewed licensed premises only","graph_edges":["licence","operator","category","complaint_context"],"privacy_fields":["address","complaint_details"],"overlap_group":"meat-authorisation","readiness":"reference_only","blockers":["current search/export route and privacy terms need confirmation"]}, + {"source_id":"au.wamia.abattoirs","authority":"WAMIA","role":"Western Australia abattoir approval list","primary_url":"https://wamia.wa.gov.au/abattoir-approvals/","access":"public HTML","format":"HTML table/list; stable row ID not verified","cadence":"page update observed July 2026","primary_key_candidates":["wamia_approval_number","source_name_location_tuple"],"identity_fields":["name","location","category","species","service_kill"],"coverage":"WA approved export and domestic abattoirs","map_use":"location only with precision review","graph_edges":["approval","operator","species","service_kill"],"privacy_fields":["location"],"overlap_group":"meat-authorisation","readiness":"reference_only","blockers":["approval number, revision and reuse terms need confirmation"]}, + {"source_id":"au.nsw.food-authority","authority":"NSW Food Authority","role":"NSW meat licensing","primary_url":"https://www.foodauthority.nsw.gov.au/help/licensing","access":"public guidance/forms; complete licensee bulk list not verified","format":"HTML/forms; schema unknown","cadence":"risk-based audit; register cadence unknown","primary_key_candidates":["nsw_food_licence_number"],"identity_fields":["business_name","site","permissions","status","dates"],"coverage":"NSW red-meat and poultry businesses for human or animal food, broad retail/processing scope","map_use":"only from authorised public site fields","graph_edges":["licence","operator","activity","inspection"],"privacy_fields":["notification_records","address","complaint_records"],"overlap_group":"meat-authorisation","readiness":"reference_only","blockers":["no verified public complete licensee export"]}, + {"source_id":"au.nsw.food-enforcement","authority":"NSW Food Authority","role":"food-law enforcement events","primary_url":"https://www.foodauthority.nsw.gov.au/offences/penalty-notices","access":"public HTML register","format":"HTML; weekly updates and one-year penalty window","cadence":"weekly stated for penalty register","primary_key_candidates":["notice_number","prosecution_case_id"],"identity_fields":["party_name","suburb","date","allegation","outcome"],"coverage":"NSW food-law notices/prosecutions, mixed business population","map_use":"event location only; never facility status","graph_edges":["enforcement_event","party","allegation","court_outcome"],"privacy_fields":["party_name","address"],"overlap_group":"enforcement","readiness":"reference_only","blockers":["preserve allegation/conviction semantics and publication windows"]}, + {"source_id":"au.pirsa.meat","authority":"PIRSA","role":"South Australia meat accreditation","primary_url":"https://pir.sa.gov.au/animal-management/food-safety-for-meat-dairy-and-eggs/meat","access":"public guidance/application route; current row export not verified","format":"HTML/PDF; schema unknown","cadence":"unknown","primary_key_candidates":["pirsa_accreditation_number"],"identity_fields":["operator","activity","conditions","dates"],"coverage":"SA slaughter, poultry, game, boning, smallgoods, storage, transport and pet meat","map_use":"only from authorised site location","graph_edges":["accreditation","operator","activity","audit"],"privacy_fields":["address","operator"],"overlap_group":"meat-authorisation","readiness":"reference_only","blockers":["public or authorised current accreditation export not verified"]}, + {"source_id":"au.sa.epa.licensed-activities","authority":"South Australia EPA","role":"environmental licence/activity geospatial overlay","primary_url":"https://data.sa.gov.au/data/dataset/8fdb86ff-d3d1-4f9e-85a5-bed4080d5ee1","access":"public GeoJSON download","format":"GeoJSON Point","cadence":"resource edition observed 2026-03-18","primary_key_candidates":["EPALICENCE","EPALICENCE+OBJECTID"],"identity_fields":["ACTIVITY","LICENCE_NAME","PR_LINK","geometry"],"coverage":"SA EPA licensed activities; approximate points and multi-activity licence rows","map_use":"overlay only after approximate-point and privacy review","graph_edges":["environmental_licence","activity","licensee"],"privacy_fields":["LICENCE_NAME","geometry"],"overlap_group":"environmental","readiness":"artifact_private_only","blockers":["4,541 features/1,695 licences captured privately; aggregate by licence, validate activity semantics and terms before integration"]}, + {"source_id":"au.tas.biosecurity-meat","authority":"Biosecurity Tasmania","role":"Tasmania meat and poultry accreditation","primary_url":"https://nre.tas.gov.au/biosecurity-tasmania/product-integrity/food-safety/meat-and-poultry","access":"public guidance/PDF; current row export not verified","format":"HTML/PDF; schema unknown","cadence":"annual/unknown","primary_key_candidates":["tas_accreditation_number"],"identity_fields":["processor_type","program","audit_dates","operator"],"coverage":"Tasmania meat, poultry, game and smallgoods processors","map_use":"reviewed approved premises only","graph_edges":["accreditation","operator","audit","welfare_guidance"],"privacy_fields":["address","operator"],"overlap_group":"meat-authorisation","readiness":"reference_only","blockers":["2019 public list is historical; current register/export required"]}, + {"source_id":"au.nt.meat-licensing","authority":"Northern Territory Government","role":"NT meat industry licences","primary_url":"https://nt.gov.au/industry/agriculture/meat-industry/domestic-abattoirs-meat-processing","access":"public guidance plus generic licence search; API/bulk route not verified","format":"HTML search; schema unknown","cadence":"licence year 1 July-30 June","primary_key_candidates":["nt_meat_licence_number"],"identity_fields":["holder","premise","licence_type","status","grant_expiry"],"coverage":"NT domestic abattoirs and meat processing, with generic register context","map_use":"reviewed licensed premise only","graph_edges":["licence","operator","activity","status_event"],"privacy_fields":["address","holder"],"overlap_group":"meat-authorisation","readiness":"reference_only","blockers":["scheme selector and permitted extract need verification"]}, + {"source_id":"au.act.food-registration","authority":"ACT Government","role":"ACT food-business registration coverage check","primary_url":"https://www.act.gov.au/business/health-licenses-and-inspections/food-businesses-and-events-registration","access":"public guidance; dedicated abattoir list not verified","format":"HTML; schema unknown","cadence":"unknown","primary_key_candidates":["act_registration_id"],"identity_fields":["premise","activity","status"],"coverage":"ACT food registration route; dedicated slaughterhouse coverage unresolved","map_use":"blocked pending source confirmation","graph_edges":["registration","operator","activity"],"privacy_fields":["address","operator"],"overlap_group":"coverage-gap","readiness":"reference_only","blockers":["authority/register for any dedicated facility layer unresolved"]}, + {"source_id":"au.npi.facilities","authority":"DCCEEW / National Pollutant Inventory","role":"national environmental/geospatial overlay","primary_url":"https://data.gov.au/data/dataset/043f58e0-a188-4458-b61c-04e5b540aea4/resource/f83cdee9-ebcb-4f24-941b-34bb2f0996cf/download/facilities.csv","access":"public catalogue download","format":"CSV, GeoJSON, KMZ","cadence":"catalogue dataset date 2026-04-01; cadence not explicit","primary_key_candidates":["facility_id","jurisdiction_code+jurisdiction_facility_id"],"identity_fields":["registered_business_name","facility_name","abn","acn","latitude","longitude","anzsic","report_ids"],"coverage":"8,140 NPI facilities in bounded 2026-04-01 snapshot; not complete slaughter coverage","map_use":"strong candidate; coordinates still require publication precision review","graph_edges":["environmental_report","operator","jurisdiction_facility","anzsic_activity"],"privacy_fields":["street_address","coordinates","abn","acn"],"overlap_group":"environmental","readiness":"reference_only","blockers":["ANZSIC is not proof of slaughter/current operation; publication and privacy review required"]}, + {"source_id":"au.nsw.epa-poeo","authority":"NSW EPA","role":"environmental licences and enforcement","primary_url":"https://apps.epa.nsw.gov.au/prpoeoapp/","access":"public search and downloadable register","format":"HTML/download; transition noted","cadence":"page updated 2026-04-28","primary_key_candidates":["poeo_licence_number","notice_id"],"identity_fields":["holder","premises","activity","status","dates","event_ids"],"coverage":"NSW scheduled premises and environmental events","map_use":"licence premises/event overlay with precision review","graph_edges":["environmental_licence","holder","audit","conviction","penalty"],"privacy_fields":["premises","holder"],"overlap_group":"environmental","readiness":"reference_only","blockers":["current export and transition scope need confirmation"]}, + {"source_id":"au.qld.environmental-authorities","authority":"Queensland DESI","role":"environmental authorities and enforcement","primary_url":"https://apps.des.qld.gov.au/public-register/","access":"public search/view/download; portal says not everything online","format":"HTML/download; API not verified","cadence":"refresh observed 2026-09-11; route cadence not guaranteed","primary_key_candidates":["environmental_authority_number","enforcement_id"],"identity_fields":["holder","activity","location","status","effective_dates"],"coverage":"QLD environmental authorities including meat processing/rendering ERA 25","map_use":"authority location overlay after review","graph_edges":["environmental_authority","holder","annual_return","enforcement"],"privacy_fields":["location","holder"],"overlap_group":"environmental","readiness":"reference_only","blockers":["portal completeness and terms require capture"]}, + {"source_id":"au.vic.epa-permissions","authority":"EPA Victoria","role":"environmental permissions","primary_url":"https://www.epa.vic.gov.au/public-registers?register=permissions","access":"public search plus ArcGIS REST layer","format":"JSON/GeoJSON/PBF and HTML","cadence":"updated overnight, up to 24-hour delay; public from 2021-07-01","primary_key_candidates":["permission_id","gis_guid","crm_guid"],"identity_fields":["holder","activity","status","modifiedon","geometry","document_url"],"coverage":"Victoria permissioned activities; pre-2021 coverage unresolved","map_use":"strong overlay; geometry precision review required","graph_edges":["permission","holder","scheduled_activity","statutory_document"],"privacy_fields":["geometry","holder","documents"],"overlap_group":"environmental","readiness":"reference_only","blockers":["production service/version and geometry semantics need validation"]}, + {"source_id":"au.nt.epa-licences","authority":"NT EPA","role":"environment protection licences","primary_url":"https://ntepa.nt.gov.au/your-business/public-registers/licences-and-approvals-register/environment-protection-licences","access":"public HTML category register and document links","format":"HTML/PDF; API not verified","cadence":"unknown","primary_key_candidates":["nt_epa_licence_number","document_id"],"identity_fields":["operator","activity","status","premises"],"coverage":"NT regulated premises including abattoirs","map_use":"document/site overlay after review","graph_edges":["epa_licence","operator","activity","document"],"privacy_fields":["premises","operator"],"overlap_group":"environmental","readiness":"reference_only","blockers":["current document identifiers and reuse terms need capture"]}, + {"source_id":"au.tas.epa-listmap","authority":"Tasmania EPA","role":"regulated-premises and monitoring overlay","primary_url":"https://epa.tas.gov.au/about-the-epa/release-of-environmental-monitoring-information/search-for-environmental-monitoring-information","access":"public search/LISTmap; stable API not verified","format":"HTML/map/documents","cadence":"documents from 2022 onward; ongoing additions","primary_key_candidates":["listmap_site_id","document_id"],"identity_fields":["operator","activity","report_date","licence_notice"],"coverage":"Tasmania abattoir, meat-processing, rendering, poultry, dairy and related regulated activity labels","map_use":"map layer pending service/schema review","graph_edges":["monitoring_document","site","operator","licence"],"privacy_fields":["site_geometry","redacted_personal_information"],"overlap_group":"environmental","readiness":"reference_only","blockers":["LISTmap service layer and download route unresolved"]}, + {"source_id":"au.nsw.animal-use","authority":"NSW DPIRD","role":"animal research-use aggregate and accreditation context","primary_url":"https://www.dpird.nsw.gov.au/dpi/animals/animal-ethics-infolink/nsw-animal-use-statistics/animal-use-data","access":"public deidentified aggregate download; facility list not public","format":"CSV/XLSX","cadence":"annual; 2024-2025 published 2026-07-20","primary_key_candidates":["report_year+category_tuple"],"identity_fields":["species","purpose","procedure","fate","year"],"coverage":"NSW research/teaching use statistics, not agriculture/slaughter premises","map_use":"none for row-level facility mapping","graph_edges":["aggregate_statistic","report_year","category"],"privacy_fields":["deidentified_aggregate","small_cell_risk"],"overlap_group":"animal-use-aggregate","readiness":"reference_only","blockers":["keep aggregate uses separate from named establishments and individual-animal claims"]}, + {"source_id":"au.vic.animal-use","authority":"Agriculture Victoria","role":"scientific-procedure licensing and aggregate statistics","primary_url":"https://agriculture.vic.gov.au/livestock-and-animals/animal-welfare-victoria/animals-used-in-research-and-teaching/licensing-to-use-animals-in-research-or-teaching/about-licensing-to-use-animals-in-research-or-teaching","access":"public guidance and annual report downloads; complete premises list not verified","format":"HTML/PDF/Word","cadence":"annual returns/report","primary_key_candidates":["sppl_id","spfl_id","sabl_id"],"identity_fields":["licence_type","holder","report_year","aggregate_categories"],"coverage":"Victoria scientific procedures, fieldwork and specified-animal breeding","map_use":"blocked for facility rows pending public list","graph_edges":["animal_use_licence","holder","annual_statistic"],"privacy_fields":["holder","premises","small_cell_risk"],"overlap_group":"animal-use-aggregate","readiness":"reference_only","blockers":["no complete public premises list verified"]}, + {"source_id":"au.asic.company-dataset","authority":"ASIC","role":"company identity crosswalk","primary_url":"https://data.gov.au/data/en/dataset/asic-companies/resource/5c3914e6-413e-4a2c-b890-bf8efe3eabf2","access":"public weekly snapshot; large download","format":"CSV/TSV/ZIP","cadence":"weekly Tuesday snapshot","primary_key_candidates":["acn","abn"],"identity_fields":["company_name","current_name","status","registration_date","deregistration_date","state"],"coverage":"selected national company-register data, not beneficial ownership or site operation","map_use":"none without upstream site relation","graph_edges":["legal_entity","name_change","registration_status"],"privacy_fields":["registered_office_context","director_data_not_in_snapshot"],"overlap_group":"organization","readiness":"reference_only","blockers":["bounded plan required before ~371.9 MiB capture; snapshot may lag ASIC Connect"]}, + {"source_id":"au.abr.lookup","authority":"Australian Business Register","role":"ABN/name/status crosswalk","primary_url":"https://abr.business.gov.au/","access":"public lookup/web service; hourly update claim","format":"HTML/XML/JSON route-dependent","cadence":"hourly for ABN Lookup","primary_key_candidates":["abn","acn"],"identity_fields":["entity_name","business_name","trading_name","active_status","entity_type","state","postcode"],"coverage":"public ABR identity fields; not detailed contact or industry data","map_use":"none alone; use only to support an upstream site link","graph_edges":["abn_entity","business_name","acn_association","status_event"],"privacy_fields":["sole_trader_name","postcode"],"overlap_group":"organization","readiness":"reference_only","blockers":["use only with upstream ABN/ACN; active ABN does not prove operation or ownership"]}, + {"source_id":"au.animal-welfare-and-use","authority":"Australian state and territory authorities","role":"animal-welfare enforcement and animal-use context","primary_url":"https://www.agriculture.gov.au/agriculture-land/animal/welfare/state","access":"jurisdiction-specific web pages, reports, and case records","format":"HTML/PDF/XLSX; jurisdiction-specific","cadence":"jurisdiction-specific","primary_key_candidates":["jurisdiction+case_or_report_id"],"identity_fields":["jurisdiction","event_date","status","outcome","source_url"],"coverage":"non-uniform welfare enforcement and selected aggregate animal-use evidence","map_use":"event/map only after privacy and location review","graph_edges":["welfare_event","animal_use_report","jurisdiction","reviewed_facility_link"],"privacy_fields":["case_narrative","personal_information","sensitive_location"],"overlap_group":"animal-use-aggregate","readiness":"reference_only","blockers":["No national facility-level feed; allegations and aggregates remain separate evidence types"]} + ], + "bounded_artifacts": [{"source_id":"au.npi.facilities","relative_path":"data/raw/australia/npi-facilities.csv","metadata_path":"data/raw/australia/metadata.json","sha256":"a0be4c37d588b391ea81baf240f675b2c9f3d9ee0210f9c451c7371eac11d6c7","bytes":3248241,"input_rows":8140,"release":"private ignored artifact; no row-level publication"},{"source_id":"au.sa.epa.licensed-activities","relative_path":"data/raw/australia/sa-epa-licensed-activities-2026-03-18.geojson","metadata_path":"docs/countries/australia/artifact-metadata.json","sha256":"53b908e02c67aaf9749bf517b4ed82c7e8af1dcc66fb87e0eb43ecb8bf12d7b0","bytes":1613987,"input_rows":4541,"release":"private ignored artifact; no row-level publication"}] +} diff --git a/docs/country-recon-au.md b/docs/country-recon-au.md new file mode 100644 index 0000000..3bc438a --- /dev/null +++ b/docs/country-recon-au.md @@ -0,0 +1,103 @@ +# Australia source reconnaissance + +Status: private reconnaissance and metadata handoff only; no Australia row-level release, adapter, candidate import, or publication approval. Checked 2026-09-16 UTC. The report records current primary routes, not a complete or current national facility census. + +## Decision summary + +Australia does not expose one authoritative public national slaughterhouse register. The useful model is a set of source-local evidence layers: + +1. DAFF's Establishment Register is the strongest federal export-registration authority, but the current operational route is authenticated TradeClear/Export Service rather than a public bulk meat list. Treat it as an assisted, restricted source until an authorised export/report route is confirmed. +2. State and territory meat regulators are the primary domestic layer. Queensland Safe Food has a searchable public accreditation register; Victoria PrimeSafe has a public licence-search route and publishes category totals; Western Australia's WAMIA publishes a current HTML list of approved abattoirs. NSW, South Australia, Tasmania, the Northern Territory, and the ACT publish regulatory guidance and/or general registers but no verified national-style public abattoir export was found. +3. The National Pollutant Inventory (NPI) is the best first automation candidate for a geospatial accountability overlay: its current catalogue exposes CSV, GeoJSON, and KMZ, a stable `facility_id`, jurisdiction facility IDs, ABN/ACN, coordinates, ANZSIC, activities, and report links under CC BY 4.0. It is an emissions-reporting population, not a slaughterhouse master. +4. Environmental permissions and enforcement should be joined as evidence edges, not collapsed into facility status. NSW POEO, Queensland environmental authorities, Victoria EPA permissions, NT EPA licences, and Tasmania LISTmap/monitoring routes expose different identifiers and histories. Their coverage and access semantics differ. +5. Animal-use evidence is mostly aggregate. NSW publishes deidentified annual CSV/XLSX statistics and regulates accredited research establishments; Victoria publishes annual-use reports and describes Scientific Procedures Premises/Fieldwork and Specified Animals Breeding licences. Neither should be converted into facility points without a separate public establishment register and privacy review. +6. ABN Lookup and the weekly ASIC Company Dataset support exact organization crosswalks, but neither is a beneficial-ownership graph. Public ABR data omits detailed contacts and industry code; ASIC's public snapshot is selected company-register data and may lag real-time ASIC Connect. + +Publication remains blocked. Current evidence is sufficient to build a metadata-only registry and an NPI adapter contract; it is not sufficient to publish Australian facility rows, precise coordinates, animal-use claims, ownership claims, or enforcement conclusions. + +## Source matrix + +| Source ID | Primary route and role | Access / format / cadence | Source-local identity and useful fields | Coverage and integration difficulty | Current blocker | +|---|---|---|---|---|---| +| `au.daff.export-establishments` | [DAFF export establishment registration](https://www.agriculture.gov.au/biosecurity-trade/export/from-australia/documentation-registration-licensing/establishment-registration); federal prescribed-goods registration and export meat oversight | TradeClear/Export Service is authenticated; public pages are HTML/PDF/DOCX guidance; no public meat bulk endpoint verified; registration is ongoing or certificate-specific | Establishment number; certificate/registration dates; occupier legal entity; ACN/ABN; premises name/address; registered operations/products; management/control; approved arrangement | Federal export premises, including slaughter, boning, processing, chilling, holding, packing, and storage; not domestic-only coverage; medium-high | Obtain an authorised current ER report/export, terms, schema, and permitted retention. Do not scrape authenticated systems or infer public access from guidance pages. +| `au.safefood.qld.accreditation` | [Safe Food Queensland Public Register](https://hub.safefood.qld.gov.au/registry/s/); accreditation evidence | Browser search; fields shown include accreditation number, name, industry type/scheme, category, and registration; no stable bulk/API route verified; cadence unknown | Exact accreditation number within Safe Food; name/category/industry and registration status; preserve result timestamp | Queensland accredited food businesses; includes more than slaughterhouses and can overlap other meat/food classes; medium | Capture one authorised bounded result/export, verify schema, category codebook, terms, and address/privacy semantics. +| `au.primesafe.vic.meat-licences` | [PrimeSafe licence categories](https://www.primesafe.vic.gov.au/licensing/about-your-licence/) and [2024-25 annual report](https://www.primesafe.vic.gov.au/wp-content/uploads/2025/10/Primesafe-annual-report-2024-25.pdf); Victorian meat/seafood licensing | Public search is linked by PrimeSafe; annual report is PDF; search/API cadence unknown; 2024-25 report gives category totals, not rows | PrimeSafe licence number if returned; licence category, business/facility name, location, status and dates where exposed | Victoria meat, poultry, game, pet-meat, rendering, seafood and transport categories; high overlap and category breadth; medium | Identify the current search URL and permitted export route; do not treat annual category totals as facility rows. PrimeSafe says investigations, including welfare complaints, are confidential. +| `au.wamia.abattoirs` | [WAMIA approved abattoirs](https://wamia.wa.gov.au/abattoir-approvals/); WA approval-to-operate list | Public HTML page; update timestamp was observed on page as July 2026; no API or stable row identifier verified | Approval list entry keyed provisionally by exact source name plus WAMIA category; source fields include location, export tier/domestic, species processed, service-kill flag | Western Australian approved abattoirs; the page is narrow and source-name keyed; low-medium acquisition, high identity fragility | Confirm page revision, row schema, approval number, and reuse terms. Exact source names are not safe as global IDs; changes of owner/name need new observations. +| `au.nsw.food-authority` | [NSW Food Authority licensing](https://www.foodauthority.nsw.gov.au/help/licensing) and [lists/registers](https://www.foodauthority.nsw.gov.au/about-us/lists-and-registers); domestic meat regulator | Public licensing guidance and forms; no public complete licensee CSV/list verified; licence/notification records are not equivalent; audit cadence varies by risk | NSW Food Authority licence/customer number only if lawfully exposed; business name, permissions, site, status/dates only after schema capture | NSW businesses handling/processing/packing/storing red meat or poultry for humans or animals, including retail; broad and not a public facility master; high | Licensing records are not a public bulk route in the verified pages. Do not use private notification records or infer coverage from annual totals. +| `au.nsw.food-enforcement` | [Penalty notice register](https://www.foodauthority.nsw.gov.au/offences/penalty-notices) and [prosecutions](https://www.foodauthority.nsw.gov.au/offences/prosecutions); enforcement/accountability | HTML search/register; penalty register says weekly updates and one-year publication window; prosecutions generally two years; fields include notice/case, trade name, suburb/council, date, party served | Notice number or prosecution/case identifier; retain event date, register type, allegation/outcome, source URL and publication window; never make it a facility status | Enforcement observations for NSW food law, mostly mixed food businesses; useful as event edges, not proof of ongoing operation, guilt beyond stated court outcome, or animal-welfare wrongdoing; medium | The register changes and includes names that may be business/party names. Screen personal names and preserve allegation versus conviction semantics; no row-level capture was retained. +| `au.pirsa.meat` | [PIRSA meat processing information](https://pir.sa.gov.au/animal-management/food-safety-for-meat-dairy-and-eggs/meat); SA accreditation and food-safety oversight | HTML/PDF application and guidance; no public current accreditation list or API verified; cadence unknown | Accreditation number/certificate only if lawfully exposed; permitted activities, conditions, audit observations and dates | South Australian slaughter, poultry, game, boning, smallgoods, storage, transport, pet meat and related businesses; medium-high due to absent public list | Confirm whether PIRSA can provide a public or authorised export. The page establishes scope and accreditation, not current facility rows. +| `au.sa.epa.licensed-activities` | [SA EPA Licensed Activities dataset](https://data.sa.gov.au/data/dataset/8fdb86ff-d3d1-4f9e-85a5-bed4080d5ee1); environmental licence/activity overlay | Public GeoJSON; resource observed last updated 2026-03-18; 4,541 Point features and 1,695 unique `EPALICENCE` values in bounded private capture | `EPALICENCE` is licence-level key; retain `(EPALICENCE, OBJECTID)` for activity features; properties include `ACTIVITY`, `LICENCE_NAME`, `PR_LINK`; points are explicitly approximate | South Australian EPA licensed activities, broader than animal agriculture; useful accountability/map overlay, not a meat-facility master; medium | Publisher warns points may be approximate and omit latest information. Aggregate by licence, preserve activity rows, screen names/coordinates, and confirm CC BY 3.0/terms before integration. +| `au.tas.biosecurity-meat` | [Tasmania Meat and Poultry](https://nre.tas.gov.au/biosecurity-tasmania/product-integrity/food-safety/meat-and-poultry) and [Livestock Processing Taskforce](https://nre.tas.gov.au/biosecurity-tasmania/tasmanian-livestock-processing-taskforce); Biosecurity Tasmania accreditation and welfare guidance | HTML/PDF; accreditation and audited food-safety programs; no public current facility export verified; annual/unknown | Accreditation/certificate number if exposed; processor type, approved program, audit/observation dates; keep taskforce guidance separate from current licensee records | Tasmania meat, poultry and game processors, smallgoods, and commercial livestock processing; high because older public lists are stale and current rows are not exposed | The 2019 feasibility appendix lists willing premises but is explicitly historical/legacy and must not be presented as current. Confirm current register/export and source terms. +| `au.nt.meat-licensing` | [NT domestic abattoirs and meat processing](https://nt.gov.au/industry/agriculture/meat-industry/domestic-abattoirs-meat-processing), [meat industry licence service](https://service.nt.gov.au/services/business-industry/agriculture/apply-for-a-meat-industry-licence), [public register](https://licensingnt.nt.gov.au/PublicRegister/PublicRegister/LicenceSearch.aspx) | Public guidance plus generic public-register search; meat licences run 1 July–30 June; no stable bulk/API route verified | NT licence number; licence type (domestic/export abattoir, processing, game/pet); holder, premise, grant/expiry/suspension/cancellation where lawfully returned | NT abattoir and processing licences; narrow but likely precise; medium-high due to generic register/search semantics | Verify the licence-scheme selector and permitted extract. A licence is evidence of authorisation, not proof of current throughput or welfare performance. +| `au.act.food-registration` | [ACT food-business registration](https://www.act.gov.au/business/health-licenses-and-inspections/food-businesses-and-events-registration); ACT regulatory route | Public registration guidance; no dedicated current abattoir list/API verified | ACT registration/permit identifier only if a public source exposes one; premise and activity fields remain schema unknown | ACT is a coverage gap for a dedicated slaughterhouse layer; low volume does not justify inference from absence | Identify the ACT Health/local-government authority and any public register or planning/environment dataset before acquisition. +| `au.npi.facilities` | [NPI facility catalogue](https://data.gov.au/data/dataset/043f58e0-a188-4458-b61c-04e5b540aea4) and [CSV resource](https://data.gov.au/data/dataset/043f58e0-a188-4458-b61c-04e5b540aea4/resource/f83cdee9-ebcb-4f24-941b-34bb2f0996cf/download/facilities.csv); national geospatial/environment overlay | Current CSV, GeoJSON and KMZ; catalogue updated 2026-04-01; cadence not explicitly stated; observed CSV 8,140 rows | `facility_id` primary within snapshot; `jurisdiction_code` + `jurisdiction_facility_id` source tuple; ABN/ACN; facility and business names; coordinates; ANZSIC; report IDs/URLs | Australia-wide NPI reporting facilities with known pollutant reporting; 8,140-row observed snapshot; 100 ANZSIC 1111 meat processing, 28 1112 poultry processing, 9 1113 cured meat/smallgoods, 414 0171 poultry farming (meat); low-medium | NPI is not complete facility/slaughter coverage; ANZSIC is a reported classification, not proof of activity. Confirm dataset-specific reuse/privacy review and coordinate publication precision. +| `au.nsw.epa-poeo` | [NSW POEO public register](https://apps.epa.nsw.gov.au/prpoeoapp/) and [register description](https://www.epa.nsw.gov.au/Licensing-and-Regulation/Public-registers/about-prpoeo); environmental licences and enforcement | HTML search plus downloadable full licence list; transition to a new register noted; current page updated 2026-04-28 | Licence number; licence holder; premises/activity; status; applications/notices/audits/convictions/penalties; review dates | NSW scheduled industrial/environmental premises, including some abattoirs and animal-processing activities; strong accountability overlay, medium | Confirm current export URL, transition scope, fields, and terms; link historical holders as separate events because the register warns enforcement may relate to previous holders. +| `au.qld.environmental-authorities` | [Queensland Environmental Protection Act public register](https://apps.des.qld.gov.au/public-register/) and [EA search](https://apps.des.qld.gov.au/public-register/search/ea.php); environmental authority and enforcement | Search, view and download; includes current, cancelled and surrendered permits, applications, annual returns, PRCP and enforcement; data refreshed per portal/search; no API contract verified | Permit/EA number; holder; activity/industry; location/postcode/LGA; effective dates; status; enforcement number and associated authority | Queensland environmentally relevant activities, including meat processing/rendering (ERA 25); strong evidence-edge layer, medium | Portal says not everything is online and offers information requests. Do not treat current holder as historical holder or infer closure from absence. +| `au.vic.epa-permissions` | [Victoria EPA public registers](https://www.epa.vic.gov.au/public-registers?register=permissions) and public [ArcGIS layer metadata](https://serverdev.arcgis.epa.vic.gov.au/arcgis/rest/services/Publicregister/pub_vec_feat/MapServer/layers); environmental permissions | Search plus ArcGIS REST layer with JSON/GeoJSON support; registers updated overnight (up to 24-hour delay); public list from 1 July 2021 | Permission/statutory document ID, GIS/CRM GUID, permission type/status, modified date, scheduled activity, holder, activity description, location geometry, documents | Victoria permissioned activities; suitable for map/graph overlay, but polygons/locations and holders need privacy/precision review; medium | Validate production service hostname/version, query filters, geometry semantics and pre-2021 coverage; keep permission holder, site and activity claims separate. +| `au.nt.epa-licences` | [NT EPA environment protection licences](https://ntepa.nt.gov.au/your-business/public-registers/licences-and-approvals-register/environment-protection-licences) | HTML category register with licence pages; cadence unknown | Licence/document ID, operator, activity type/status, premises documents; abattoir category explicitly listed | NT regulated premises, including abattoirs and aquaculture; useful but not complete meat licensing; medium | Capture current page/document identifiers and terms; link to NT meat licence only after explicit identity review. +| `au.tas.epa-listmap` | [Tasmania EPA monitoring search](https://epa.tas.gov.au/about-the-epa/release-of-environmental-monitoring-information/search-for-environmental-monitoring-information) and LISTmap regulated-premises layer | Web search/LISTmap; documents from 2022 onward added ongoing; some documents redacted; no stable API verified | LISTmap regulated-premises/site and regulatory-document IDs; operator/activity; environmental licence/notice; report date | Tasmania regulated food production and animal/plant processing, including abattoir, meat processing, rendering, poultry farm, dairy and aquaculture; medium-high | Identify LISTmap service layer and permitted download route; preserve redaction and date limits; do not infer a facility from a monitoring document alone. +| `au.nsw.animal-use` | [NSW animal-use data](https://www.dpird.nsw.gov.au/dpi/animals/animal-ethics-infolink/nsw-animal-use-statistics/animal-use-data) and [accreditation/licensing](https://www.dpird.nsw.gov.au/dpi/animals/animal-ethics-infolink/accreditation/accreditation-and-licencing); animal research oversight | Deidentified aggregated CSV/XLSX; annual publication; 2024–2025 title, 2024-01-01–2025-12-31 coverage, published 2026-07-20; establishment accreditation/licensing is not a public bulk register | Aggregate rows by species/purpose/procedure/fate; establishment accreditation and AEC are separate source claims; no public facility ID in the published aggregate | NSW research/teaching use, not animal agriculture facility coverage; 10,567,233 animal-use events reported for 2024 in the page notes, with fish driving most increase; low for context, unsuitable for facility points | Keep aggregate statistics separate from named organizations and from slaughter. A “use” count is not an individual-animal count and is not a location or welfare-violation finding. +| `au.vic.animal-use` | [Victoria scientific-procedure licensing](https://agriculture.vic.gov.au/livestock-and-animals/animal-welfare-victoria/animals-used-in-research-and-teaching/licensing-to-use-animals-in-research-or-teaching/about-licensing-to-use-animals-in-research-or-teaching) and [2024 statistics](https://agriculture.vic.gov.au/livestock-and-animals/animal-welfare-victoria/animals-used-in-research-and-teaching/animal-use-statistics) | Licence guidance plus annual PDF/Word report; annual returns from SPPL/SPFL/SABL holders | SPPL/SPFL/SABL licence identity if lawfully obtained; annual report aggregate categories; preserve licence type and observation dates | Victoria scientific procedures, fieldwork and specified-animal breeding; no complete public premises list verified; medium-high | Obtain an authorised public licence list if one exists; do not use annual-use reports to identify or map individual establishments. +| `au.asic.company-dataset` | [ASIC Company Dataset](https://data.gov.au/data/en/dataset/asic-companies/resource/5c3914e6-413e-4a2c-b890-bf8efe3eabf2); organization identity crosswalk | Weekly Tuesday snapshot described by catalogue; CSV/TSV/ZIP; current observed extract 2026-04-14, CSV about 371.9 MiB; CC BY 3.0 Australia | ACN; ABN; company name/current name; type/class/subclass; status; registration/deregistration; previous state; current-name dates | National company-register subset; strong exact organization identity, not beneficial ownership or operating-site evidence; medium due to large file and tab-delimited caveat | Do not bulk-fetch without an approved bounded plan. Preserve snapshot date and delimiter; ASIC says real-time ASIC Connect may be more current. +| `au.abr.lookup` | [ABN Lookup](https://abr.business.gov.au/) and [ABR public-data services](https://www.abr.gov.au/government-agencies/accessing-abr-data/abr-data-products-and-services); ABN status and name crosswalk | Public one/multiple lookup and web services; ABN Lookup updated hourly; no login for public search; web-service terms/rate details must be recorded per route | ABN; entity/business/trading names; active/cancelled; entity type; state/postcode; ACN/ARBN/ARSN/ARFN when associated | Public ABR identity only; no detailed address, phone, email or industry code; medium for exact joins, high privacy risk if used to expose individuals | Use only when an upstream source supplies ABN/ACN. Never search by or publish a private individual's identity; active ABN does not prove facility operation or ownership/control. + +## Private NPI artifact and bounded diagnostics + +The ignored artifact is `data/raw/australia/npi-facilities.csv`; its tracked metadata is [`data/raw/australia/metadata.json`](../data/raw/australia/metadata.json). It was captured read-only from the catalogue CSV route after an ordinary network attempt was refused by the execution environment and an approved retry succeeded. No raw rows are committed. + +| Observation | Value | +|---|---| +| Retrieval | 2026-09-16T07:15:42.2821369Z | +| HTTP | 200; `text/csv; charset=utf-8`; `inline; filename=facilities.csv` | +| Last-Modified / ETag | `Tue, 31 Mar 2026 07:24:24 GMT` / `W/"1774941864.963-3248241-3908178080"` | +| Cache / server | `public, max-age=60, immutable`; nginx via CloudFront | +| Bytes / SHA-256 | 3,248,241 / `a0be4c37d588b391ea81baf240f675b2c9f3d9ee0210f9c451c7371eac11d6c7` | +| Input rows / columns | 8,140 / 22 | +| Header SHA-256 | `8f5ae1618ec0a863d0647d48a4606bea82c9c68317b153503c43968aeb4b6d06` | +| Coordinate completeness | 8,140 rows had both latitude and longitude in this snapshot; this is not a publication decision | +| State counts | ACT 54; NSW 1,630; NT 208; QLD 1,884; SA 770; TAS 311; VIC 1,636; WA 1,647 | +| Relevant ANZSIC text counts | 0171 Poultry Farming (Meat) 414; 1111 Meat Processing 100; 1112 Poultry Processing 28; 1113 Cured Meat and Smallgoods Manufacturing 9; 1192 Prepared Animal and Bird Feed Manufacturing 59 | +| Latest report-year values | 1998–2024; source values include financial-year strings such as `2024/2025` | + +The bounded SA EPA artifact is `data/raw/australia/sa-epa-licensed-activities-2026-03-18.geojson`; tracked integrity metadata is in [`docs/countries/australia/artifact-metadata.json`](countries/australia/artifact-metadata.json). It contains 4,541 Point features, 1,695 unique licence numbers, five source properties plus geometry, and 231 rows in the broad `Food Production and Animal Plant Processing` activity. Text screening found 29 feature rows with meat/poultry/abattoir/rendering-like licence names; that is not a meat count. The publisher describes the points as approximate and says the dataset may not include the latest information. + +The NPI catalogue identifies CC BY 4.0 International and says the facility resource contains names, point locations, and primary ANZSIC 2006 classification. The source page also lists ABN/ACN, facility and jurisdiction IDs, activities, report IDs, and report URLs in the data dictionary. Confirm dataset-specific attribution and privacy handling before any public map or export. + +## Identity, deduplication, map and graph design + +Use source-local keys first and global links second: + +- NPI: exact `facility_id`; retain `jurisdiction_code` + `jurisdiction_facility_id` as a second source key and keep the snapshot/report-year observation. Do not deduplicate two NPI facilities by name, address, ABN or coordinates. +- DAFF: establishment number and certificate/registration observation; a changed physical location receives a new establishment number according to DAFF guidance. Keep approved operations/products and management/control as separate claims. +- WAMIA, PrimeSafe, Safe Food, PIRSA, Tasmania and NT: exact licence/accreditation/approval number when exposed. Until then, a source-scoped name is an unresolved candidate key only, never a global identity. +- Environmental sources: licence/permit/permission document ID is the event/authority key. Keep current holder and historical holder as versioned relationships; do not overwrite a prior holder. +- ASIC/ABR: ACN and ABN are organization identity keys. An ABN/ACN match is an exact crosswalk candidate, not proof that the company operates the mapped premises, owns it, or controls it. +- Text/address/coordinate similarity is a review signal only. A proposed `same_as` edge needs at least two independent source keys or a documented human review. Conflicts become unresolved edges or quarantine, not silent merges. + +Map projections should default to source-approved, privacy-eligible facilities and coarse precision where required. NPI points are exact source coordinates but must be screened for residences, mixed-use sites, and harmful disclosure. Environmental geometries may be polygons or schedules rather than operating-site points. A missing current row means `not_observed`, never closure. Graph edges should include source ID, source observation time, claim type, exact source identifier, evidence URL, confidence/review outcome, and publication scope. + +## Legacy comparison and uncertainty + +No Australian legacy file or checked-in Australia dataset exists in the repository baseline, so there is no valid V1-to-current row reconciliation to report. The Tasmania 2019 feasibility appendix found during reconnaissance is historical material that names premises willing to be listed at that time; it is not current coverage. Older DAFF meat notices and ELMER guidance describe regulatory processes but do not substitute for a current Establishment Register extract. + +The following claims must remain explicitly bounded: + +- Government source origin does not certify accuracy, completeness, operation, welfare, or wrongdoing. +- ANZSIC, environmental activity, or licence category is not proof of slaughter or animal use. +- Enforcement registers distinguish allegations, penalty notices, prosecutions and court outcomes; a record is not a general finding about the business or its current owner. +- Animal-use annual counts are counts of reported uses, potentially repeating animals across years/projects; they are deidentified aggregates, not facility totals. +- Public business identity data can include sole traders or individuals. Keep only organization-level fields needed for a reviewed crosswalk and suppress personal details. + +## Recommended staged implementation + +Stage 1: implement a private NPI adapter using exact source IDs, CSV/GeoJSON schema fingerprinting, provenance manifests, coordinate privacy checks, and a no-publication review packet. Add metadata-only candidate contracts for WAMIA and the Queensland/Victoria public searches. + +Stage 2: obtain authorised bounded captures for Safe Food Queensland, PrimeSafe, NT licensing, NSW POEO, Queensland EAs, Victoria EPA ArcGIS, and Tasmania LISTmap. Record response URL, UTC retrieval, effective/publication date, headers, byte size, SHA-256, row/document counts, schema, terms and privacy decisions before parsing. + +Stage 3: request or arrange a lawful DAFF Establishment Register report/export and current domestic accreditation routes for NSW, SA and Tasmania. Keep federal export, state domestic, environment, enforcement, animal-use and ownership layers separate; integrate only through reviewed edges. + +Stage 4: use ABN/ACN joins to reduce duplicate organization representations, then perform human review of site-versus-entity, historical holder changes, residential/mixed-use addresses, coordinate precision and publication eligibility. No automated fuzzy ownership inference. + +## Safety and publication boundary + +All proposed outputs are government-sourced observations pending project review. No Australia source is project-approved or project-published. Raw NPI and any future captures remain in ignored private storage. Public defaults must exclude unreviewed or unapproved claims, suppress private/residential details and precise coordinates when required, and label source origin, observation date, review state, uncertainty and limitations. Acquisition success is not publication authorization. diff --git a/docs/source-status.json b/docs/source-status.json index 3adf0fe..8d524ae 100644 --- a/docs/source-status.json +++ b/docs/source-status.json @@ -25,6 +25,28 @@ {"source_id":"de.locations","metadata":"partial","acquisition":"artifact_private_only","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/germany-source-assessment.md","pipeline/sources/germany/adapter.py","pipeline/sources/germany/refresh.py","pipeline/common/review_packet.py","docs/review-packet-germany.md","pipeline/source_registry.json"],"next_action":"Use the stable BVL landing to select an export, then run the assisted/private refresh with duplicate-approval and coordinate-precision checks. The export URL, reuse terms, privacy, and project approval remain unresolved; no release or public API exposure is allowed."}, {"source_id":"ca.ontario.meat-plants","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ca.md","docs/countries/canada/meat-plants-pipeline.md","pipeline/sources/canada/adapter.py","pipeline/sources/canada/acquire.py","pipeline/sources/canada/refresh.py","pipeline/sources/canada/fixtures/ontario.csv","pipeline/common/review_packet.py","pipeline/source_registry.json"],"next_action":"Run the bounded private Ontario refresh; keep plant/contact/coordinate fields restricted pending privacy and licence review, and do not generalize Ontario coverage nationally."}, {"source_id":"ca.cfia.federal-meat","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ca.md","docs/countries/canada/meat-plants-pipeline.md","pipeline/sources/canada/adapter.py","pipeline/sources/canada/acquire.py","pipeline/sources/canada/refresh.py","pipeline/sources/canada/fixtures/cfia.csv","pipeline/common/review_packet.py","pipeline/source_registry.json"],"next_action":"Run the bounded private CFIA registry refresh; validate the live export schema/function codes, keep federal scope separate from provincial scope, and retain publication gates."}, + {"source_id":"au.daff.export-establishments","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-au.md","docs/countries/australia/artifact-metadata.json","docs/countries/australia/source-crosswalk.json","pipeline/source_registry.json"],"next_action":"Obtain an authorised bounded current DAFF establishment report/export, preserve commodity/list type, record response metadata/hash/bytes, and keep export-only scope and privacy/terms/release gates explicit."}, + {"source_id":"au.npi.facilities","metadata":"verified","acquisition":"artifact_private_only","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-au.md","data/raw/australia/metadata.json","docs/countries/australia/artifact-metadata.json","docs/countries/australia/source-crosswalk.json","pipeline/source_registry.json","pipeline/tests/test_australia_source_metadata.py"],"next_action":"Add a private deterministic NPI facility/report adapter and synthetic contract tests; preserve annual release/correction history and do not present NPI as complete facility coverage."}, + {"source_id":"au.sa.epa.licensed-activities","metadata":"verified","acquisition":"artifact_private_only","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-au.md","docs/countries/australia/artifact-metadata.json","docs/countries/australia/source-crosswalk.json","pipeline/source_registry.json"],"next_action":"Validate licence-level aggregation, multi-activity child rows, approximate-point semantics, and privacy/terms before any map or graph integration."}, + {"source_id":"au.vic.epa-permissions","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-au.md","docs/countries/australia/source-crosswalk.json","pipeline/source_registry.json"],"next_action":"Capture a bounded ArcGIS REST query and matching public-register record; validate pagination, identifiers, location types, terms, privacy, and permission-to-facility relationship semantics."}, + {"source_id":"au.nsw.epa-poeo","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-au.md","docs/countries/australia/source-crosswalk.json","pipeline/source_registry.json"],"next_action":"Resolve the current transition-aware full-list/search route, capture one bounded edition, and keep licence/application/notice/enforcement record types separate."}, + {"source_id":"au.safefood.qld.accreditation","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-au.md","docs/countries/australia/source-crosswalk.json","pipeline/source_registry.json"],"next_action":"Use an assisted register search to capture accreditation-number/business/activity/status fields, confirm bulk/export behavior and terms, and keep annual counts separate from facility rows."}, + {"source_id":"au.nsw.food-authority","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-au.md","docs/countries/australia/source-crosswalk.json","pipeline/source_registry.json"],"next_action":"Treat current meat-sector category counts as context only; locate a permitted current facility lookup/export before adapter work and do not infer closure or named facilities from aggregates."}, + {"source_id":"au.animal-welfare-and-use","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-au.md","docs/countries/australia/source-crosswalk.json","pipeline/source_registry.json"],"next_action":"Keep welfare enforcement and animal-use evidence jurisdiction-specific; acquire only deidentified/terms-permitted aggregates or reviewed case records and model them as observations/events, not facility masters."}, + {"source_id":"au.primesafe.vic.meat-licences","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-au.md","docs/countries/australia/source-crosswalk.json","pipeline/source_registry.json"],"next_action":"Identify the current PrimeSafe search/export route and capture one bounded result with licence, category, status and site fields; keep annual totals separate."}, + {"source_id":"au.wamia.abattoirs","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-au.md","docs/countries/australia/source-crosswalk.json","pipeline/source_registry.json"],"next_action":"Capture the current WAMIA HTML list, confirm approval numbers/revision and reuse terms, and keep source names as unresolved candidates until reviewed."}, + {"source_id":"au.nsw.food-enforcement","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-au.md","docs/countries/australia/source-crosswalk.json","pipeline/source_registry.json"],"next_action":"Model penalty/prosecution entries as time-bounded enforcement events; preserve allegation versus court outcome and suppress personal details."}, + {"source_id":"au.pirsa.meat","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-au.md","docs/countries/australia/source-crosswalk.json","pipeline/source_registry.json"],"next_action":"Confirm whether PIRSA can provide a current permitted accreditation export; do not derive facility rows from guidance or application forms."}, + {"source_id":"au.tas.biosecurity-meat","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-au.md","docs/countries/australia/source-crosswalk.json","pipeline/source_registry.json"],"next_action":"Locate a current Tasmania accreditation route; keep the 2019 feasibility appendix legacy and separate from current observations."}, + {"source_id":"au.nt.meat-licensing","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-au.md","docs/countries/australia/source-crosswalk.json","pipeline/source_registry.json"],"next_action":"Validate the NT licence-scheme selector and obtain an authorised bounded extract; treat licence as authorisation evidence only."}, + {"source_id":"au.act.food-registration","metadata":"unknown","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-au.md","docs/countries/australia/source-crosswalk.json","pipeline/source_registry.json"],"next_action":"Identify the ACT authority and any public facility register before claiming ACT coverage; absence is not closure."}, + {"source_id":"au.qld.environmental-authorities","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-au.md","docs/countries/australia/source-crosswalk.json","pipeline/source_registry.json"],"next_action":"Capture one bounded Queensland EA search/download and validate completeness, activity codes, status history and holder/site identity."}, + {"source_id":"au.nt.epa-licences","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-au.md","docs/countries/australia/source-crosswalk.json","pipeline/source_registry.json"],"next_action":"Capture current NT EPA licence/document identifiers and terms, then link to NT meat licences only through explicit review."}, + {"source_id":"au.tas.epa-listmap","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-au.md","docs/countries/australia/source-crosswalk.json","pipeline/source_registry.json"],"next_action":"Resolve the Tasmania LISTmap service layer and permitted download route; preserve redaction/date limits and keep monitoring documents separate."}, + {"source_id":"au.nsw.animal-use","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-au.md","docs/countries/australia/source-crosswalk.json","pipeline/source_registry.json"],"next_action":"Use deidentified annual aggregates only as context; keep them separate from facility points, slaughter, and individual-animal claims."}, + {"source_id":"au.vic.animal-use","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-au.md","docs/countries/australia/source-crosswalk.json","pipeline/source_registry.json"],"next_action":"Keep Victoria scientific-procedure licence guidance and aggregate reports separate; locate a public premises list before any facility mapping."}, + {"source_id":"au.asic.company-dataset","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-au.md","docs/countries/australia/source-crosswalk.json","pipeline/source_registry.json"],"next_action":"Do not bulk-fetch the large ASIC snapshot without an approved bounded plan; use ACN/ABN only as an upstream identity crosswalk, never as proof of site ownership."}, + {"source_id":"au.abr.lookup","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-au.md","docs/countries/australia/source-crosswalk.json","pipeline/source_registry.json"],"next_action":"Query only when an upstream source supplies an ABN/ACN; record lookup time and terms, and suppress sole-trader personal details."}, {"source_id":"es.locations","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-es.md","pipeline/source_registry.json"],"next_action":"Resolve a permitted current AESAN/MAPA artifact or query route, then verify export/schema, rights, effective-date semantics, sector coverage, privacy, and legacy/source boundaries before acquisition."}, {"source_id":"us.fsis","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-us.md","docs/countries/us/README.md","docs/countries/us/v1-field-crosswalk.json","pipeline/sources/us/fsis/config.json","pipeline/sources/us/fsis/adapter.py","pipeline/sources/us/fsis/refresh.py","pipeline/source_registry.json"],"next_action":"Use the assisted official export contract after the 403 blocker; record URL, retrieval/effective dates, content type, byte size, SHA-256, terms, schema fingerprint, privacy review, and reconciliation before any test-only handoff."}, {"source_id":"us.aphis","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-us.md","docs/countries/us/README.md","pipeline/sources/us/aphis/config.json","pipeline/sources/us/aphis/adapter.py","pipeline/sources/us/aphis/refresh.py","pipeline/source_registry.json"],"next_action":"Use the explicit profile-based assisted export for registrations, annual reports, or inspections; preserve each evidence type separately and complete terms, privacy, schema, and review gates."}, diff --git a/docs/source-status.md b/docs/source-status.md index 57ccc72..e747310 100644 --- a/docs/source-status.md +++ b/docs/source-status.md @@ -40,4 +40,8 @@ The owner-authorized 2026-09-15 live-country rehearsal completed private candida | `us.aphis` | verified | not_run | not_run | blocked | Profile-explicit private adapter and assisted-capture contract cover registrations, annual reports, and inspections; capture current exports and review terms/schema/privacy | | `us.inspections` | verified | not_run | not_run | blocked | APHIS inspections profile is implemented as observation evidence; capture current export and use explicit reviewable identity links only | -The machine-readable file is the source of truth for these statuses. Legacy `.locations` paths may represent composite coverage, but source identities are split where the evidence establishes separate feeds: France Section I/II, Canada Ontario/CFIA, and Italy 853/2004/1069/2009. Candidate feeds mentioned in the Mexico and New Zealand reconnaissance documents are not silently conflated into a single healthy source. Country reconnaissance documents provide evidence and next actions; they do not override this status vocabulary or authorize publication. +## Australia additions (2026-09-16) + +Australia is represented by 22 source-local evidence layers in `source-status.json` and `pipeline/source_registry.json`. All remain `publication_eligibility=blocked`; no runtime health is claimed. The bounded private artifacts are the NPI CSV (8,140 rows) and the earlier SA EPA GeoJSON capture (4,541 features / 1,695 licences). The detailed route, schema, identity, map/graph, privacy, terms, and blocker crosswalk is [`docs/countries/australia/source-crosswalk.json`](countries/australia/source-crosswalk.json); the decision report is [`docs/country-recon-au.md`](country-recon-au.md). + +The machine-readable file is the source of truth for these statuses. Legacy `.locations` paths may represent composite coverage, but source identities are split where the evidence establishes separate feeds: France Section I/II, Canada Ontario/CFIA, Italy 853/2004/1069/2009, and Australia’s state/federal/environment/animal-use layers. Candidate feeds mentioned in reconnaissance documents are not silently conflated into a single healthy source. Country reconnaissance documents provide evidence and next actions; they do not override this status vocabulary or authorize publication. diff --git a/pipeline/common/test_source_operations.py b/pipeline/common/test_source_operations.py index dce7d8f..e080838 100644 --- a/pipeline/common/test_source_operations.py +++ b/pipeline/common/test_source_operations.py @@ -24,7 +24,7 @@ class SourceOperationsTests(unittest.TestCase): def test_checked_in_schedule_inventory_matches_registry(self): root = Path(__file__).parents[1] schedules = load_source_schedules(registry_path=root / "source_registry.json") - self.assertEqual(len(schedules), 20) + self.assertEqual(len(schedules), 38) self.assertEqual(schedules["dk.smiley"].stale_after_hours, 240) self.assertEqual(schedules["fr.dgal.section-i"].interval_hours, 24) self.assertIsNone(schedules["us.fsis"].interval_hours) diff --git a/pipeline/source_operations.json b/pipeline/source_operations.json index 59f96a1..7d52739 100644 --- a/pipeline/source_operations.json +++ b/pipeline/source_operations.json @@ -21,6 +21,28 @@ {"source_id":"uk.locations","cadence":"monthly","interval_hours":720,"stale_after_hours":960,"max_attempts":3,"backoff_seconds":1,"max_backoff_seconds":30,"manual_fallback":"use the source-specific national operator capture"}, {"source_id":"us.aphis","cadence":"unknown","interval_hours":null,"stale_after_hours":null,"max_attempts":1,"backoff_seconds":0,"max_backoff_seconds":0,"manual_fallback":"verify the APHIS export workflow"}, {"source_id":"us.fsis","cadence":"unknown","interval_hours":null,"stale_after_hours":null,"max_attempts":1,"backoff_seconds":0,"max_backoff_seconds":0,"manual_fallback":"obtain authorized FSIS export access"}, - {"source_id":"us.inspections","cadence":"unknown","interval_hours":null,"stale_after_hours":null,"max_attempts":1,"backoff_seconds":0,"max_backoff_seconds":0,"manual_fallback":"verify a current inspection export"} + {"source_id":"us.inspections","cadence":"unknown","interval_hours":null,"stale_after_hours":null,"max_attempts":1,"backoff_seconds":0,"max_backoff_seconds":0,"manual_fallback":"verify a current inspection export"}, + {"source_id":"au.daff.export-establishments","cadence":"unknown","interval_hours":null,"stale_after_hours":null,"max_attempts":1,"backoff_seconds":0,"max_backoff_seconds":0,"manual_fallback":"obtain an authorised current DAFF establishment report/export"}, + {"source_id":"au.safefood.qld.accreditation","cadence":"unknown","interval_hours":null,"stale_after_hours":null,"max_attempts":1,"backoff_seconds":0,"max_backoff_seconds":0,"manual_fallback":"use an assisted Safe Food register capture"}, + {"source_id":"au.primesafe.vic.meat-licences","cadence":"unknown","interval_hours":null,"stale_after_hours":null,"max_attempts":1,"backoff_seconds":0,"max_backoff_seconds":0,"manual_fallback":"verify the current PrimeSafe search/export route"}, + {"source_id":"au.wamia.abattoirs","cadence":"unknown","interval_hours":null,"stale_after_hours":null,"max_attempts":1,"backoff_seconds":0,"max_backoff_seconds":0,"manual_fallback":"capture a bounded current WAMIA page revision"}, + {"source_id":"au.nsw.food-authority","cadence":"unknown","interval_hours":null,"stale_after_hours":null,"max_attempts":1,"backoff_seconds":0,"max_backoff_seconds":0,"manual_fallback":"verify a permitted NSW Food Authority facility route"}, + {"source_id":"au.nsw.food-enforcement","cadence":"weekly","interval_hours":168,"stale_after_hours":336,"max_attempts":1,"backoff_seconds":0,"max_backoff_seconds":0,"manual_fallback":"capture the public NSW penalty/prosecution register"}, + {"source_id":"au.pirsa.meat","cadence":"unknown","interval_hours":null,"stale_after_hours":null,"max_attempts":1,"backoff_seconds":0,"max_backoff_seconds":0,"manual_fallback":"request or verify a current PIRSA accreditation export"}, + {"source_id":"au.sa.epa.licensed-activities","cadence":"unknown","interval_hours":null,"stale_after_hours":null,"max_attempts":1,"backoff_seconds":0,"max_backoff_seconds":0,"manual_fallback":"capture the current SA EPA GeoJSON edition"}, + {"source_id":"au.tas.biosecurity-meat","cadence":"unknown","interval_hours":null,"stale_after_hours":null,"max_attempts":1,"backoff_seconds":0,"max_backoff_seconds":0,"manual_fallback":"verify a current Biosecurity Tasmania register/export"}, + {"source_id":"au.nt.meat-licensing","cadence":"annual","interval_hours":8760,"stale_after_hours":null,"max_attempts":1,"backoff_seconds":0,"max_backoff_seconds":0,"manual_fallback":"use the NT public-register meat-licence search"}, + {"source_id":"au.act.food-registration","cadence":"unknown","interval_hours":null,"stale_after_hours":null,"max_attempts":1,"backoff_seconds":0,"max_backoff_seconds":0,"manual_fallback":"resolve the ACT facility-register route before scheduling"}, + {"source_id":"au.npi.facilities","cadence":"annual","interval_hours":8760,"stale_after_hours":null,"max_attempts":1,"backoff_seconds":0,"max_backoff_seconds":0,"manual_fallback":"download the current data.gov.au NPI edition and record metadata"}, + {"source_id":"au.nsw.epa-poeo","cadence":"unknown","interval_hours":null,"stale_after_hours":null,"max_attempts":1,"backoff_seconds":0,"max_backoff_seconds":0,"manual_fallback":"use the transition-aware NSW POEO public-register route"}, + {"source_id":"au.qld.environmental-authorities","cadence":"unknown","interval_hours":null,"stale_after_hours":null,"max_attempts":1,"backoff_seconds":0,"max_backoff_seconds":0,"manual_fallback":"capture a bounded Queensland environmental-register result"}, + {"source_id":"au.vic.epa-permissions","cadence":"daily","interval_hours":24,"stale_after_hours":48,"max_attempts":1,"backoff_seconds":0,"max_backoff_seconds":0,"manual_fallback":"query the reviewed Victoria EPA ArcGIS/public-register route"}, + {"source_id":"au.nt.epa-licences","cadence":"unknown","interval_hours":null,"stale_after_hours":null,"max_attempts":1,"backoff_seconds":0,"max_backoff_seconds":0,"manual_fallback":"capture current NT EPA licence documents"}, + {"source_id":"au.tas.epa-listmap","cadence":"unknown","interval_hours":null,"stale_after_hours":null,"max_attempts":1,"backoff_seconds":0,"max_backoff_seconds":0,"manual_fallback":"resolve the Tasmania LISTmap service/download route"}, + {"source_id":"au.nsw.animal-use","cadence":"annual","interval_hours":8760,"stale_after_hours":null,"max_attempts":1,"backoff_seconds":0,"max_backoff_seconds":0,"manual_fallback":"download the current deidentified NSW animal-use aggregate report"}, + {"source_id":"au.vic.animal-use","cadence":"annual","interval_hours":8760,"stale_after_hours":null,"max_attempts":1,"backoff_seconds":0,"max_backoff_seconds":0,"manual_fallback":"obtain the current Victoria animal-use annual report"}, + {"source_id":"au.asic.company-dataset","cadence":"weekly","interval_hours":168,"stale_after_hours":336,"max_attempts":1,"backoff_seconds":0,"max_backoff_seconds":0,"manual_fallback":"use the bounded ASIC weekly snapshot plan"}, + {"source_id":"au.abr.lookup","cadence":"on_demand","interval_hours":null,"stale_after_hours":null,"max_attempts":1,"backoff_seconds":0,"max_backoff_seconds":0,"manual_fallback":"query ABR only for an upstream exact ABN/ACN crosswalk"}, + {"source_id":"au.animal-welfare-and-use","cadence":"unknown","interval_hours":null,"stale_after_hours":null,"max_attempts":1,"backoff_seconds":0,"max_backoff_seconds":0,"manual_fallback":"use jurisdiction-specific reviewed welfare/report routes"} ] } diff --git a/pipeline/source_registry.json b/pipeline/source_registry.json index 95c3e92..39253ed 100644 --- a/pipeline/source_registry.json +++ b/pipeline/source_registry.json @@ -242,6 +242,270 @@ "adapter_status": "implemented_partial", "expected_artifact_schema": "CSV inspection observations; not a facility master record and not silently joined to registrations or FSIS", "blockers": ["Verify current inspection export schema and use explicit, reviewable identity matching only; absence is not closure."] + }, + { + "source_id": "au.primesafe.vic.meat-licences", + "jurisdiction_scope": "Australia; Victoria PrimeSafe meat and seafood licences", + "legacy_paths": [], + "url": "https://www.primesafe.vic.gov.au/licensing/about-your-licence/", + "access_method": "public search linked by PrimeSafe; annual report PDF", + "cadence": "unknown; annual aggregate report", + "attribution_licensing_notes": "PrimeSafe authority and licence categories verified; search route, terms, and confidential complaint handling are separate", + "adapter_status": "reference_only", + "expected_artifact_schema": "Licence rows if lawfully exposed; licence number, category, business/site, status, location, and dates", + "blockers": ["Identify the current search/export route; annual category totals are not facility rows."] + }, + { + "source_id": "au.wamia.abattoirs", + "jurisdiction_scope": "Australia; Western Australia WAMIA approved abattoirs", + "legacy_paths": [], + "url": "https://wamia.wa.gov.au/abattoir-approvals/", + "access_method": "public HTML list", + "cadence": "page update observed July 2026", + "attribution_licensing_notes": "WAMIA approval page and 2026 guideline context verified; confirm reuse terms and revision semantics", + "adapter_status": "reference_only", + "expected_artifact_schema": "Approval list; source name, location, export tier/domestic category, species, and service-kill flag; approval number unknown", + "blockers": ["Confirm stable approval identifier and capture rules; source names are not global IDs."] + }, + { + "source_id": "au.nsw.food-enforcement", + "jurisdiction_scope": "Australia; NSW Food Authority enforcement events", + "legacy_paths": [], + "url": "https://www.foodauthority.nsw.gov.au/offences/penalty-notices", + "access_method": "public HTML register", + "cadence": "weekly stated; penalty register has one-year publication window", + "attribution_licensing_notes": "Public enforcement register; preserve allegation, penalty, prosecution, and court-outcome semantics and screen personal names", + "adapter_status": "reference_only", + "expected_artifact_schema": "Event rows; notice/case ID, party/trade name, location, date, allegation, and outcome", + "blockers": ["No facility-status inference; preserve changing publication windows and event-versus-entity distinction."] + }, + { + "source_id": "au.pirsa.meat", + "jurisdiction_scope": "Australia; South Australia PIRSA meat accreditation", + "legacy_paths": [], + "url": "https://pir.sa.gov.au/animal-management/food-safety-for-meat-dairy-and-eggs/meat", + "access_method": "public guidance/application route; current row export not verified", + "cadence": "unknown", + "attribution_licensing_notes": "PIRSA authority and accreditation scope verified; current register availability and terms need confirmation", + "adapter_status": "reference_only", + "expected_artifact_schema": "Accreditation rows if exposed; number, operator, activity, conditions, and dates", + "blockers": ["Obtain an authorised current accreditation export before integration."] + }, + { + "source_id": "au.tas.biosecurity-meat", + "jurisdiction_scope": "Australia; Tasmania Biosecurity Tasmania meat accreditation", + "legacy_paths": [], + "url": "https://nre.tas.gov.au/biosecurity-tasmania/product-integrity/food-safety/meat-and-poultry", + "access_method": "public HTML/PDF guidance; current row export not verified", + "cadence": "annual/unknown", + "attribution_licensing_notes": "Biosecurity Tasmania authority and audited-program scope verified; historical lists are explicitly non-current", + "adapter_status": "reference_only", + "expected_artifact_schema": "Accreditation/certificate rows if exposed; processor type, program, operator, audit dates, and status", + "blockers": ["2019 feasibility appendix is legacy only; confirm current register/export and terms."] + }, + { + "source_id": "au.nt.meat-licensing", + "jurisdiction_scope": "Australia; Northern Territory meat industry licensing", + "legacy_paths": [], + "url": "https://nt.gov.au/industry/agriculture/meat-industry/domestic-abattoirs-meat-processing", + "access_method": "public guidance plus generic licence search; API/bulk route not verified", + "cadence": "licence year 1 July-30 June", + "attribution_licensing_notes": "NT meat licence scope verified; generic register selector and public disclosure terms need capture", + "adapter_status": "reference_only", + "expected_artifact_schema": "Licence rows; number, type, holder/premise, grant/expiry, suspension/cancellation/status", + "blockers": ["Verify the scheme selector and permitted extract."] + }, + { + "source_id": "au.act.food-registration", + "jurisdiction_scope": "Australia; Australian Capital Territory food registration coverage check", + "legacy_paths": [], + "url": "https://www.act.gov.au/business/health-licenses-and-inspections/food-businesses-and-events-registration", + "access_method": "public guidance; dedicated abattoir list not verified", + "cadence": "unknown", + "attribution_licensing_notes": "ACT food registration guidance only; no facility publication claim", + "adapter_status": "reference_only", + "expected_artifact_schema": "Registration rows if a dedicated public route is found; schema unknown", + "blockers": ["Identify ACT Health/local-government facility and environmental routes before acquisition."] + }, + { + "source_id": "au.npi.facilities", + "jurisdiction_scope": "Australia; National Pollutant Inventory facilities", + "legacy_paths": [], + "url": "https://data.gov.au/data/dataset/043f58e0-a188-4458-b61c-04e5b540aea4", + "access_method": "public catalogue download", + "cadence": "catalogue dataset date 2026-04-01; cadence not explicit", + "attribution_licensing_notes": "Catalogue identifies CC BY 4.0; attribute Commonwealth of Australia/DCCEEW and complete privacy/release review", + "adapter_status": "reference_only", + "expected_artifact_schema": "CSV/GeoJSON/KMZ; facility_id, jurisdiction IDs, names, ABN/ACN, address, coordinates, ANZSIC, activities, report IDs/URLs", + "blockers": ["Observed NPI population is not a complete slaughter registry; no public row release approved."] + }, + { + "source_id": "au.nsw.epa-poeo", + "jurisdiction_scope": "Australia; NSW POEO environmental register", + "legacy_paths": [], + "url": "https://apps.epa.nsw.gov.au/prpoeoapp/", + "access_method": "public search and downloadable register", + "cadence": "page updated 2026-04-28; transition noted", + "attribution_licensing_notes": "NSW EPA public-register route; confirm current export and historical-holder semantics", + "adapter_status": "reference_only", + "expected_artifact_schema": "Licence/event rows; licence number, holder, premises/activity, status, applications, audits, convictions, penalties", + "blockers": ["Validate current transition/export scope before adapter work."] + }, + { + "source_id": "au.qld.environmental-authorities", + "jurisdiction_scope": "Australia; Queensland environmental authorities and enforcement", + "legacy_paths": [], + "url": "https://apps.des.qld.gov.au/public-register/", + "access_method": "public search/view/download; portal says not everything is online", + "cadence": "refresh observed 2026-09-11; not guaranteed", + "attribution_licensing_notes": "Queensland public-register route; preserve current/cancelled/surrendered status and information-request gaps", + "adapter_status": "reference_only", + "expected_artifact_schema": "Authority/event rows; EA number, holder, activity, location, status, dates, annual returns, PRCP, enforcement IDs", + "blockers": ["Completeness and terms need an authorised bounded capture."] + }, + { + "source_id": "au.vic.epa-permissions", + "jurisdiction_scope": "Australia; EPA Victoria permissions", + "legacy_paths": [], + "url": "https://www.epa.vic.gov.au/public-registers?register=permissions", + "access_method": "public search plus ArcGIS REST layer", + "cadence": "overnight; up to 24-hour delay; public from 2021-07-01", + "attribution_licensing_notes": "EPA Victoria register and ArcGIS metadata observed; validate production service/version and geometry privacy", + "adapter_status": "reference_only", + "expected_artifact_schema": "JSON/GeoJSON permission features; permission_id, GUIDs, status/type, holder, activity, modified date, URL, geometry", + "blockers": ["Validate service filters, geometry semantics, and pre-2021 coverage."] + }, + { + "source_id": "au.nt.epa-licences", + "jurisdiction_scope": "Australia; NT EPA environment-protection licences", + "legacy_paths": [], + "url": "https://ntepa.nt.gov.au/your-business/public-registers/licences-and-approvals-register/environment-protection-licences", + "access_method": "public HTML category register and document links", + "cadence": "unknown", + "attribution_licensing_notes": "NT EPA public category register; capture document identifiers and terms", + "adapter_status": "reference_only", + "expected_artifact_schema": "Licence/document rows; ID, operator, activity, status, premises, linked documents", + "blockers": ["Current document identifiers and permitted downloads need verification."] + }, + { + "source_id": "au.tas.epa-listmap", + "jurisdiction_scope": "Australia; Tasmania EPA regulated premises and monitoring", + "legacy_paths": [], + "url": "https://epa.tas.gov.au/about-the-epa/release-of-environmental-monitoring-information/search-for-environmental-monitoring-information", + "access_method": "public search/LISTmap; stable API not verified", + "cadence": "documents from 2022 onward; ongoing additions", + "attribution_licensing_notes": "Tasmania EPA route; preserve redaction and date limits", + "adapter_status": "reference_only", + "expected_artifact_schema": "Map/document rows; site/document IDs, operator/activity, report date, licence/notice, geometry", + "blockers": ["LISTmap service layer and download route unresolved."] + }, + { + "source_id": "au.nsw.animal-use", + "jurisdiction_scope": "Australia; NSW animal-use research statistics", + "legacy_paths": [], + "url": "https://www.dpird.nsw.gov.au/dpi/animals/animal-ethics-infolink/nsw-animal-use-statistics/animal-use-data", + "access_method": "public deidentified aggregate CSV/XLSX; facility list not public", + "cadence": "annual; 2024-2025 published 2026-07-20", + "attribution_licensing_notes": "NSW DPIRD aggregate statistics; CC BY 4.0 stated; small-cell and re-identification review required", + "adapter_status": "reference_only", + "expected_artifact_schema": "Aggregate category rows by year/species/purpose/procedure/fate", + "blockers": ["Do not map aggregate uses or associate them with named establishments."] + }, + { + "source_id": "au.vic.animal-use", + "jurisdiction_scope": "Australia; Victoria scientific-procedure licensing and statistics", + "legacy_paths": [], + "url": "https://agriculture.vic.gov.au/livestock-and-animals/animal-welfare-victoria/animals-used-in-research-and-teaching/licensing-to-use-animals-in-research-or-teaching/about-licensing-to-use-animals-in-research-or-teaching", + "access_method": "public guidance and annual report downloads; complete premises list not verified", + "cadence": "annual returns/report", + "attribution_licensing_notes": "Agriculture Victoria licence categories and aggregate reports; privacy review required", + "adapter_status": "reference_only", + "expected_artifact_schema": "SPPL/SPFL/SABL licence observations and separate aggregate annual report", + "blockers": ["No complete public premises list verified; keep annual-use reports separate from facilities."] + }, + { + "source_id": "au.asic.company-dataset", + "jurisdiction_scope": "Australia; ASIC selected company-register snapshot", + "legacy_paths": [], + "url": "https://data.gov.au/data/en/dataset/asic-companies/resource/5c3914e6-413e-4a2c-b890-bf8efe3eabf2", + "access_method": "public weekly snapshot; large download", + "cadence": "weekly Tuesday snapshot", + "attribution_licensing_notes": "Catalogue states CC BY 3.0 Australia; preserve snapshot date and delimiter; not beneficial-ownership data", + "adapter_status": "reference_only", + "expected_artifact_schema": "CSV/TSV/ZIP; ACN/ABN, names, type/class, status, registration/deregistration, state and current-name fields", + "blockers": ["Do not bulk-fetch ~371.9 MiB without approved bounded plan; snapshot may lag ASIC Connect."] + }, + { + "source_id": "au.abr.lookup", + "jurisdiction_scope": "Australia; ABN Lookup and ABR public identity data", + "legacy_paths": [], + "url": "https://abr.business.gov.au/", + "access_method": "public lookup/web service; route-specific rate and terms", + "cadence": "hourly update claim for ABN Lookup", + "attribution_licensing_notes": "ABR public identity route; use only for exact upstream ABN/ACN crosswalk and never infer operation/ownership", + "adapter_status": "reference_only", + "expected_artifact_schema": "Lookup response; ABN/ACN, entity/business/trading names, active status, entity type, state/postcode", + "blockers": ["No detailed contact/industry fields; individual and sole-trader privacy review required."] + }, + { + "source_id": "au.daff.export-establishments", + "jurisdiction_scope": "Australia; DAFF export-registered prescribed-goods establishments", + "legacy_paths": [], + "url": "https://www.agriculture.gov.au/biosecurity-trade/export/from-australia/documentation-registration-licensing/establishment-registration", + "access_method": "authenticated TradeClear/Export Service or assisted request; public guidance only", + "cadence": "ongoing/certificate-specific", + "attribution_licensing_notes": "DAFF authority and export scope verified; authorised access, terms, retention, and privacy review required", + "adapter_status": "reference_only", + "expected_artifact_schema": "Establishment number, occupier, address, ACN/ABN, operations/products, dates, and management/control; exact export schema unknown", + "blockers": ["No verified public meat bulk export; do not scrape authenticated systems or infer public access from guidance pages."] + }, + { + "source_id": "au.safefood.qld.accreditation", + "jurisdiction_scope": "Australia; Queensland Safe Food accreditation", + "legacy_paths": [], + "url": "https://hub.safefood.qld.gov.au/registry/s/", + "access_method": "public browser search; bulk/API route not verified", + "cadence": "unknown", + "attribution_licensing_notes": "Safe Food Queensland public register; result capture, terms, and address/privacy handling require review", + "adapter_status": "reference_only", + "expected_artifact_schema": "Accreditation number, name, industry/scheme, category, and registration status", + "blockers": ["Verify an authorised bounded capture, schema, category codebook, and reuse terms."] + }, + { + "source_id": "au.nsw.food-authority", + "jurisdiction_scope": "Australia; NSW Food Authority meat licensing", + "legacy_paths": [], + "url": "https://www.foodauthority.nsw.gov.au/help/licensing", + "access_method": "public guidance/forms; complete licensee list not verified", + "cadence": "risk-based audit; register cadence unknown", + "attribution_licensing_notes": "NSW Food Authority authority and scope verified; licensing/notification records and privacy restrictions remain distinct", + "adapter_status": "reference_only", + "expected_artifact_schema": "Licence/customer number, business/site, permissions, status, and dates if lawfully exposed", + "blockers": ["No verified public complete licensee export; do not use private notification records."] + }, + { + "source_id": "au.sa.epa.licensed-activities", + "jurisdiction_scope": "Australia; South Australia EPA licensed activities", + "legacy_paths": [], + "url": "https://data.sa.gov.au/data/dataset/8fdb86ff-d3d1-4f9e-85a5-bed4080d5ee1", + "access_method": "public GeoJSON download", + "cadence": "resource edition observed 2026-03-18", + "attribution_licensing_notes": "CC BY 3.0 Australia; publisher warns points are approximate and may omit latest information; privacy/release review required", + "adapter_status": "reference_only", + "expected_artifact_schema": "GeoJSON Point features with EPALICENCE, OBJECTID, ACTIVITY, LICENCE_NAME, PR_LINK, and geometry; aggregate by licence before review", + "blockers": ["Private artifact exists; validate activity semantics, approximate points, terms, and licence-to-site identity before integration."] + }, + { + "source_id": "au.animal-welfare-and-use", + "jurisdiction_scope": "Australia; jurisdiction-specific animal welfare enforcement and animal-use statistics", + "legacy_paths": [], + "url": "https://www.agriculture.gov.au/agriculture-land/animal/welfare/state", + "access_method": "jurisdiction-specific web pages, reports, and case records", + "cadence": "jurisdiction-specific", + "attribution_licensing_notes": "Government-sourced evidence only; allegations, personal information, sensitive locations, terms, and human review are mandatory gates", + "adapter_status": "reference_only", + "expected_artifact_schema": "Report/case observations with jurisdiction, event/report date, status/outcome, source URL, and optional reviewed facility link; aggregate use tables separate", + "blockers": ["No uniform national facility-level feed; do not turn complaints, prosecutions, or aggregate totals into canonical facility facts."] } ] } diff --git a/pipeline/tests/test_australia_source_metadata.py b/pipeline/tests/test_australia_source_metadata.py new file mode 100644 index 0000000..92c552f --- /dev/null +++ b/pipeline/tests/test_australia_source_metadata.py @@ -0,0 +1,34 @@ +import hashlib +import json +import unittest +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[2] + + +class AustraliaSourceMetadataTests(unittest.TestCase): + def test_crosswalk_ids_are_unique_and_registered(self): + crosswalk = json.loads((ROOT / "docs/countries/australia/source-crosswalk.json").read_text(encoding="utf-8")) + registry = json.loads((ROOT / "pipeline/source_registry.json").read_text(encoding="utf-8")) + crosswalk_ids = [item["source_id"] for item in crosswalk["sources"]] + registry_ids = {item["source_id"] for item in registry["sources"]} + self.assertEqual(len(crosswalk_ids), len(set(crosswalk_ids))) + self.assertTrue(set(crosswalk_ids) <= registry_ids) + self.assertEqual(crosswalk["publication_state"], "private-reconnaissance-only") + + def test_npi_metadata_matches_private_artifact(self): + metadata = json.loads((ROOT / "data/raw/australia/metadata.json").read_text(encoding="utf-8")) + artifact = ROOT / metadata["artifact"]["relative_path"] + self.assertEqual(metadata["source_id"], "au.npi.facilities") + self.assertTrue(artifact.exists()) + self.assertEqual(artifact.stat().st_size, metadata["artifact"]["byte_size"]) + digest = hashlib.sha256(artifact.read_bytes()).hexdigest() + self.assertEqual(digest, metadata["artifact"]["sha256"]) + self.assertEqual(metadata["artifact"]["input_rows"], 8140) + self.assertEqual(len(metadata["artifact"]["columns"]), 22) + self.assertEqual(metadata["bounded_observations"]["rows_with_missing_latitude_or_longitude"], 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/tests/test_source_registry.py b/pipeline/tests/test_source_registry.py index 58874be..1c0b41d 100644 --- a/pipeline/tests/test_source_registry.py +++ b/pipeline/tests/test_source_registry.py @@ -9,8 +9,8 @@ class SourceRegistryTests(unittest.TestCase): def test_repository_registry_loads_and_references_existing_legacy_paths(self): registry = load_registry() - self.assertEqual(len(registry["sources"]), 20) - self.assertEqual(len({source["source_id"] for source in registry["sources"]}), 20) + self.assertEqual(len(registry["sources"]), 38) + self.assertEqual(len({source["source_id"] for source in registry["sources"]}), 38) def test_unknowns_are_explicit(self): registry = json.loads(REGISTRY_PATH.read_text(encoding="utf-8")) From b1c4e814817be8af3bc78510a57a9effb79c464c Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Wed, 16 Sep 2026 00:37:28 -0700 Subject: [PATCH 133/311] Track Australia artifact provenance sidecar --- data/raw/australia/metadata.json | 41 ++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 data/raw/australia/metadata.json diff --git a/data/raw/australia/metadata.json b/data/raw/australia/metadata.json new file mode 100644 index 0000000..8f97927 --- /dev/null +++ b/data/raw/australia/metadata.json @@ -0,0 +1,41 @@ +{ + "schema_version": "au-private-artifact-metadata-v1", + "source_id": "au.npi.facilities", + "landing_url": "https://data.gov.au/data/dataset/043f58e0-a188-4458-b61c-04e5b540aea4", + "source_url": "https://data.gov.au/data/dataset/043f58e0-a188-4458-b61c-04e5b540aea4/resource/f83cdee9-ebcb-4f24-941b-34bb2f0996cf/download/facilities.csv", + "retrieved_at_utc": "2026-09-16T07:15:42.2821369Z", + "publication_or_dataset_date": "2026-04-01", + "http": { + "method": "GET (artifact) and HEAD (header observation)", + "status": 200, + "content_type": "text/csv; charset=utf-8", + "content_disposition": "inline; filename=facilities.csv", + "etag": "W/\"1774941864.963-3248241-3908178080\"", + "last_modified": "Tue, 31 Mar 2026 07:24:24 GMT", + "cache_control": "public, max-age=60, immutable", + "vary": "Cookie, Accept-Encoding, Origin", + "x_cache": "Hit from cloudfront", + "age": "28", + "server": "nginx" + }, + "artifact": { + "relative_path": "data/raw/australia/npi-facilities.csv", + "retention": "ignored private research artifact; never a release", + "byte_size": 3248241, + "sha256": "a0be4c37d588b391ea81baf240f675b2c9f3d9ee0210f9c451c7371eac11d6c7", + "input_rows": 8140, + "normalized_rows": null, + "quarantined_rows": null, + "header_sha256": "8f5ae1618ec0a863d0647d48a4606bea82c9c68317b153503c43968aeb4b6d06", + "columns": ["facility_id", "jurisdiction_code", "jurisdiction_facility_id", "registered_business_name", "facility_name", "abn", "acn", "street_address", "suburb", "state", "postcode", "latitude", "longitude", "primary_anzsic_class_code", "primary_anzsic_class_name", "main_activities", "facility_website", "first_report_year", "latest_report_year", "latest_report_id", "latest_report_url", "reports"] + }, + "bounded_observations": { + "state_counts": {"ACT": 54, "NSW": 1630, "NT": 208, "QLD": 1884, "SA": 770, "TAS": 311, "VIC": 1636, "WA": 1647}, + "anzsic_class_counts_relevant_text": {"0171 Poultry Farming (Meat)": 414, "1111 Meat Processing": 100, "1112 Poultry Processing": 28, "1113 Cured Meat and Smallgoods Manufacturing": 9, "1192 Prepared Animal and Bird Feed Manufacturing": 59}, + "rows_with_missing_latitude_or_longitude": 0, + "distinct_facility_id": 8140, + "latest_report_year_range": [1998, 2024], + "warning": "Counts are private snapshot diagnostics. ANZSIC class is reported business classification, not proof of slaughter, current operation, animal use, ownership, or regulatory approval." + }, + "terms": "Catalogue identifies CC BY 4.0 International; attribution to the Commonwealth of Australia/DCCEEW is required. Confirm dataset-specific reuse and privacy review before any publication." +} From c6bb671e37564e57db8ffadbde2600acffa37d7f Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Wed, 16 Sep 2026 00:45:08 -0700 Subject: [PATCH 134/311] test: align Australia integration contracts --- pipeline/common/test_source_operations.py | 2 +- .../tests/test_australia_source_metadata.py | 24 +++++++++++++++---- pipeline/tests/test_source_registry.py | 4 ++-- 3 files changed, 23 insertions(+), 7 deletions(-) diff --git a/pipeline/common/test_source_operations.py b/pipeline/common/test_source_operations.py index e080838..71256c9 100644 --- a/pipeline/common/test_source_operations.py +++ b/pipeline/common/test_source_operations.py @@ -24,7 +24,7 @@ class SourceOperationsTests(unittest.TestCase): def test_checked_in_schedule_inventory_matches_registry(self): root = Path(__file__).parents[1] schedules = load_source_schedules(registry_path=root / "source_registry.json") - self.assertEqual(len(schedules), 38) + self.assertEqual(len(schedules), 42) self.assertEqual(schedules["dk.smiley"].stale_after_hours, 240) self.assertEqual(schedules["fr.dgal.section-i"].interval_hours, 24) self.assertIsNone(schedules["us.fsis"].interval_hours) diff --git a/pipeline/tests/test_australia_source_metadata.py b/pipeline/tests/test_australia_source_metadata.py index 92c552f..6fdf462 100644 --- a/pipeline/tests/test_australia_source_metadata.py +++ b/pipeline/tests/test_australia_source_metadata.py @@ -1,5 +1,6 @@ import hashlib import json +import subprocess import unittest from pathlib import Path @@ -21,10 +22,25 @@ def test_npi_metadata_matches_private_artifact(self): metadata = json.loads((ROOT / "data/raw/australia/metadata.json").read_text(encoding="utf-8")) artifact = ROOT / metadata["artifact"]["relative_path"] self.assertEqual(metadata["source_id"], "au.npi.facilities") - self.assertTrue(artifact.exists()) - self.assertEqual(artifact.stat().st_size, metadata["artifact"]["byte_size"]) - digest = hashlib.sha256(artifact.read_bytes()).hexdigest() - self.assertEqual(digest, metadata["artifact"]["sha256"]) + if artifact.exists(): + self.assertEqual(artifact.stat().st_size, metadata["artifact"]["byte_size"]) + digest = hashlib.sha256(artifact.read_bytes()).hexdigest() + self.assertEqual(digest, metadata["artifact"]["sha256"]) + else: + ignored = subprocess.run( + ["git", "check-ignore", "--quiet", "--", str(artifact.relative_to(ROOT))], + cwd=ROOT, + check=False, + ) + tracked = subprocess.run( + ["git", "ls-files", "--error-unmatch", "--", str(artifact.relative_to(ROOT))], + cwd=ROOT, + check=False, + capture_output=True, + text=True, + ) + self.assertEqual(ignored.returncode, 0) + self.assertNotEqual(tracked.returncode, 0) self.assertEqual(metadata["artifact"]["input_rows"], 8140) self.assertEqual(len(metadata["artifact"]["columns"]), 22) self.assertEqual(metadata["bounded_observations"]["rows_with_missing_latitude_or_longitude"], 0) diff --git a/pipeline/tests/test_source_registry.py b/pipeline/tests/test_source_registry.py index 1c0b41d..d2a273b 100644 --- a/pipeline/tests/test_source_registry.py +++ b/pipeline/tests/test_source_registry.py @@ -9,8 +9,8 @@ class SourceRegistryTests(unittest.TestCase): def test_repository_registry_loads_and_references_existing_legacy_paths(self): registry = load_registry() - self.assertEqual(len(registry["sources"]), 38) - self.assertEqual(len({source["source_id"] for source in registry["sources"]}), 38) + self.assertEqual(len(registry["sources"]), 42) + self.assertEqual(len({source["source_id"] for source in registry["sources"]}), 42) def test_unknowns_are_explicit(self): registry = json.loads(REGISTRY_PATH.read_text(encoding="utf-8")) From b84c7d9f4872f2e83a0e4c5413966c5db354f3f0 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Wed, 16 Sep 2026 01:12:56 -0700 Subject: [PATCH 135/311] Harden E2E fixture retry cleanup --- pipeline/tests/e2e/fixture.py | 7 +++++++ pipeline/tests/test_e2e_fixture.py | 24 ++++++++++++++++++++++++ 2 files changed, 31 insertions(+) create mode 100644 pipeline/tests/test_e2e_fixture.py diff --git a/pipeline/tests/e2e/fixture.py b/pipeline/tests/e2e/fixture.py index 3378fe7..62f4b6d 100644 --- a/pipeline/tests/e2e/fixture.py +++ b/pipeline/tests/e2e/fixture.py @@ -38,6 +38,12 @@ def __init__(self): self.dev_preview_token = "uec-e2e-preview-token" self.test_release_id = None + def _ensure_build_temp(self): + """Recreate per-run paths after a failed-start cleanup before retrying.""" + if self.build_temp is None: + self.build_temp = tempfile.TemporaryDirectory(prefix="uec-e2e-cargo-") + self.cargo_target_dir = Path(self.build_temp.name) + def command(self, *args): return ["docker", "compose", "-p", self.project, "-f", str(COMPOSE), *args] @@ -45,6 +51,7 @@ def compose_env(self): env = os.environ.copy(); env["UEC_E2E_DB_PORT"] = str(self.db_port); return env def start(self, migration_files=None, wait_for_ready=True): + self._ensure_build_temp() try: print(f"[e2e] starting {self.project}", flush=True) startup = subprocess.run(self.command("up", "-d", "--wait"), cwd=ROOT, capture_output=True, text=True, env=self.compose_env()) diff --git a/pipeline/tests/test_e2e_fixture.py b/pipeline/tests/test_e2e_fixture.py new file mode 100644 index 0000000..f9520d0 --- /dev/null +++ b/pipeline/tests/test_e2e_fixture.py @@ -0,0 +1,24 @@ +import unittest + +from pipeline.tests.e2e.fixture import E2EEnvironment + + +class E2EFixtureLifecycleTests(unittest.TestCase): + def test_retry_recreates_build_temp_after_failed_start_cleanup(self): + environment = E2EEnvironment() + original_target = environment.cargo_target_dir + environment.build_temp.cleanup() + environment.build_temp = None + + try: + environment._ensure_build_temp() + self.assertIsNotNone(environment.build_temp) + self.assertNotEqual(environment.cargo_target_dir, original_target) + self.assertTrue(environment.cargo_target_dir.exists()) + finally: + environment.build_temp.cleanup() + environment.build_temp = None + + +if __name__ == "__main__": + unittest.main() From 4cbd0fe4fe6ff6f35fe838702c81d1f7d0adf469 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Wed, 16 Sep 2026 01:07:48 -0700 Subject: [PATCH 136/311] Document Netherlands source reconnaissance --- data/manifests/nl-source-artifacts.json | 28 +++++ docs/countries/nl/v1-field-crosswalk.json | 61 ++++++++++ docs/country-recon-nl.md | 131 ++++++++++++++++++++++ docs/source-status.json | 13 ++- docs/source-status.md | 11 ++ pipeline/common/test_source_operations.py | 2 +- pipeline/source-inventory.csv | 11 ++ pipeline/source_operations.json | 13 ++- pipeline/source_registry.json | 13 ++- pipeline/tests/test_source_registry.py | 4 +- 10 files changed, 281 insertions(+), 6 deletions(-) create mode 100644 data/manifests/nl-source-artifacts.json create mode 100644 docs/countries/nl/v1-field-crosswalk.json create mode 100644 docs/country-recon-nl.md diff --git a/data/manifests/nl-source-artifacts.json b/data/manifests/nl-source-artifacts.json new file mode 100644 index 0000000..b907fa4 --- /dev/null +++ b/data/manifests/nl-source-artifacts.json @@ -0,0 +1,28 @@ +{ + "schema_version": "1.0", + "country": "NL", + "purpose": "Private bounded retrieval manifest. Bulk/raw files remain ignored and are not a release or publication authorization.", + "retrieval_as_of_utc": "2026-09-16T07:55:00Z", + "artifacts": [ + {"source_id":"nl.nvwa.approved-food","path":"data/raw/nl-nvwa/2026-09-16/control.xml","url":"https://www.nvwa.nl/site/binaries/content/assets/site-content/webapp-data/lijsten-erkende-bedrijven/stuurbestand-lijsten-erkende-bedrijven","retrieved_at_utc":"2026-09-16T07:43:08.5297219Z","http":"200; application/xml;charset=UTF-8; Last-Modified=Tue, 15 Sep 2026 06:40:18 GMT; ETag=W/\"1781939336758\"; Cache-Control=public,max-age=300","bytes":378439,"sha256":"dc54010c807542f2a817032936a668e2edcb5984483f33f0fd52176c4508ed87","rows":null,"schema":"XML control file; list codes, labels, SOAP endpoint, language and pagination configuration"}, + {"source_id":"nl.nvwa.approved-food","path":"data/raw/nl-nvwa/2026-09-16/overig_303.xml","url":"https://e-certnl.nvwa.nl/VINDBAARHEID_EBF-Berichtenboek-context-root/Berichtenboek","retrieved_at_utc":"2026-09-16T07:45:08.7006126Z","http":"200; XML SOAP response; Date=Wed, 16 Sep 2026 07:45:42 GMT","bytes":100137,"sha256":"3653b1fa80b39ce4de5e4af385fd9ca87507405105390e83b9df44373940c811","rows":118,"unique_source_keys":115,"schema":"SOAP response; recognition number, location, approval/activity/product/species, lifecycle, category and regulatory fields"}, + {"source_id":"nl.nvwa.approved-food","path":"data/raw/nl-nvwa/2026-09-16/overig_304.xml","url":"https://e-certnl.nvwa.nl/VINDBAARHEID_EBF-Berichtenboek-context-root/Berichtenboek","retrieved_at_utc":"2026-09-16T07:45:08.7006126Z","http":"200; XML SOAP response","bytes":376998,"sha256":"db52b332dde3dff86710e3d811e445da6e3e32bdb0409e9fb3fd72b584c2e70e","rows":470,"unique_source_keys":470,"schema":"SOAP response; domestic-ungulate cutting observations"}, + {"source_id":"nl.nvwa.approved-food","path":"data/raw/nl-nvwa/2026-09-16/overig_305.xml","url":"https://e-certnl.nvwa.nl/VINDBAARHEID_EBF-Berichtenboek-context-root/Berichtenboek","retrieved_at_utc":"2026-09-16T07:45:08.7006126Z","http":"200; XML SOAP response","bytes":20865,"sha256":"10ff4400afeb3ce04c151977c8a0ffd039d55bf32bedff412901826f687e2432","rows":25,"unique_source_keys":23,"schema":"SOAP response; poultry/rabbit slaughter observations"}, + {"source_id":"nl.nvwa.approved-food","path":"data/raw/nl-nvwa/2026-09-16/overig_306.xml","url":"https://e-certnl.nvwa.nl/VINDBAARHEID_EBF-Berichtenboek-context-root/Berichtenboek","retrieved_at_utc":"2026-09-16T07:45:08.7006126Z","http":"200; XML SOAP response","bytes":333342,"sha256":"7d351ab599d26dadc688d467654735bd4ee485ac5d14128f0001244ce86988b0","rows":425,"unique_source_keys":356,"schema":"SOAP response; poultry/rabbit cutting observations"}, + {"source_id":"nl.nvwa.approved-food","path":"data/raw/nl-nvwa/2026-09-16/overig_307.xml","url":"https://e-certnl.nvwa.nl/VINDBAARHEID_EBF-Berichtenboek-context-root/Berichtenboek","retrieved_at_utc":"2026-09-16T07:45:08.7006126Z","http":"200; XML SOAP response","bytes":14199,"sha256":"b4558c1f6a5d1c637dfb1111a5c444879fa2c1be57039bfb26cabfc0dc6758fc","rows":16,"unique_source_keys":14,"schema":"SOAP response; farmed-game slaughter observations"}, + {"source_id":"nl.nvwa.approved-food","path":"data/raw/nl-nvwa/2026-09-16/overig_308.xml","url":"https://e-certnl.nvwa.nl/VINDBAARHEID_EBF-Berichtenboek-context-root/Berichtenboek","retrieved_at_utc":"2026-09-16T07:45:08.7006126Z","http":"200; XML SOAP response","bytes":80636,"sha256":"2a807bcb273189507522cd4f7382a5cab4b8ede5d6767c77c1b11c196ed90a71","rows":102,"unique_source_keys":102,"schema":"SOAP response; farmed-game cutting observations"}, + {"source_id":"nl.nvwa.approved-food","path":"data/raw/nl-nvwa/2026-09-16/overig_309.xml","url":"https://e-certnl.nvwa.nl/VINDBAARHEID_EBF-Berichtenboek-context-root/Berichtenboek","retrieved_at_utc":"2026-09-16T07:45:08.7006126Z","http":"200; XML SOAP response","bytes":14894,"sha256":"3be0e8283bd56bc53cb704fcee0198ac481eb75755c727512bdcab66bd87285c","rows":17,"unique_source_keys":10,"schema":"SOAP response; wild-game processing observations"}, + {"source_id":"nl.nvwa.approved-food","path":"data/raw/nl-nvwa/2026-09-16/overig_310.xml","url":"https://e-certnl.nvwa.nl/VINDBAARHEID_EBF-Berichtenboek-context-root/Berichtenboek","retrieved_at_utc":"2026-09-16T07:45:08.7006126Z","http":"200; XML SOAP response","bytes":67567,"sha256":"a54952e40d2922799f2b3ce96dd49c2d4079a8de8f0ca85b886945273dc0fcad","rows":86,"unique_source_keys":86,"schema":"SOAP response; wild-game cutting observations"}, + {"source_id":"nl.nvwa.welfare-enforcement","path":"data/raw/nl/nvwa-welfare-2025.html","url":"https://www.nvwa.nl/over-de-nvwa/publicaties/jaarbeeld-2025/dierenwelzijn","retrieved_at_utc":"2026-09-16T07:49:17.1708485Z","http":"200; text/html","bytes":254984,"sha256":"ab9a000250f5f579d53f89b9e58a42f6feb56bfcb8488fe46f1883ee9e33865c","rows":null,"schema":"2025 yearbook aggregate welfare inspections and measures"}, + {"source_id":"nl.nvwa.welfare-enforcement","path":"data/raw/nl/nvwa-dierproeven-2025.html","url":"https://www.nvwa.nl/onderwerpen/dier/dierproeven-voor-onderzoek/inspectieresultaten/2025","retrieved_at_utc":"2026-09-16T07:49:18.5957477Z","http":"200; text/html","bytes":223281,"sha256":"d3bd4797f046e60c41d2d85221d38cf7aa889010d82b14b29df94555a7ede1c9","rows":null,"schema":"2025 animal-experiment inspection-result page; institution/inspection/audit aggregate evidence"}, + {"source_id":"nl.nvwa.welfare-enforcement","path":"data/raw/nl/nvwa-slaughter-enforcement-2024.pdf","url":"https://www.nvwa.nl/site/binaries/site-content/collections/documents/eten-drinken-roken/vlees-en-vleesproducten/naleefmonitor/tabellenboek-roodvlees-slachthuizen-met-permanent-toezicht-juli-december-2024/tabellenboek-roodvleesslachthuizen-permanent%2Btoezicht-juli-december-2024.pdf","retrieved_at_utc":"2026-09-16T07:49:42.5857119Z","http":"200; application/pdf","bytes":412995,"sha256":"b8b456608cb56b1c15e3d5bce3315c77584b9489e1d57eff71d9ec790e1943ed","pages":24,"schema":"Red-meat compliance tables; Jul-Dec 2024; inspections, warnings and boeterapporten"}, + {"source_id":"nl.nvwa.welfare-enforcement","path":"data/raw/nl/nvwa-zo-doende-2023.pdf","url":"https://www.nvwa.nl/onderwerpen/dier/dierproeven-voor-onderzoek/jaaroverzicht-dierproeven-en-proefdieren-zo-doende","retrieved_at_utc":"2026-09-16T07:49:18.1642401Z","http":"200; application/pdf","bytes":836656,"sha256":"d48c33892599271e2d80d914761531692ffbce07223edade7793b78e92775e9d","pages":null,"schema":"2023 annual animal-experiment report"}, + {"source_id":"nl.cokz.dairy-eggs","path":"data/raw/nl/cokz-dairy-register.html","url":"https://cokz.nl/zuivel/register/register-erkende-boerderij-zuivelbedrijven/","retrieved_at_utc":"2026-09-16T07:54:53.3441281Z","http":"200; text/html; Last-Modified=Wed, 16 Sep 2026 07:43:34 GMT; Cache-Control=must-revalidate,max-age=3","bytes":107964,"sha256":"31310cdd6b70284693bfbb4bd8f43cd1c265e1cc2126367ea0419e81b5bd3143","rows":357,"schema":"HTML table; Name, Trade, Address, Zip code, Place, Approval No."}, + {"source_id":"nl.cbs.slaughter","path":"data/raw/nl/cbs-slaughter-data.json","url":"https://opendata.cbs.nl/ODataApi/OData/7123slac/TypedDataSet","retrieved_at_utc":"2026-09-16T07:47:50.9430455Z","http":"200; application/json; Date=Wed, 16 Sep 2026 07:51:00 GMT; Cache-Control=public,max-age=1800; Content-Length=773928","bytes":773928,"sha256":"6182e8346ff61a73ec878f3ba748401ce34057eed891c344b1c9d44a3911d935","rows":6162,"schema":"OData; Slachtdieren, Perioden, AantalSlachtingen_1, GeslachtGewicht_2; monthly aggregate"}, + {"source_id":"nl.cbs.livestock","path":"data/raw/nl/cbs-livestock-data.json","url":"https://opendata.cbs.nl/ODataApi/OData/84952NED/TypedDataSet","retrieved_at_utc":"2026-09-16T07:47:52.2389706Z","http":"200; application/json; Date=Wed, 16 Sep 2026 07:51:02 GMT; Cache-Control=public,max-age=1800; Content-Length=92358","bytes":92358,"sha256":"849ca13f056e022747b5dc4e185db71284ad0353464282a04b33518878eea275","rows":1003,"schema":"OData; Landbouwdieren, Perioden, Veestapel_1; twice-yearly aggregate"}, + {"source_id":"eu.eurostat.nl-slaughter","path":"data/raw/nl/eurostat-slaughter-nl.json","url":"https://ec.europa.eu/eurostat/api/dissemination/statistics/1.0/data/apro_mt_pann?geo=NL&lang=en","retrieved_at_utc":"2026-09-16T07:47:45.2574680Z","http":"200; application/json; Date=Wed, 16 Sep 2026 07:51:03 GMT; lastUpdated=2026-09-03T14:00:00Z","bytes":56379,"sha256":"65e46170c7548e98e22757cb2f06ba3b23a6ed4943800882b61bb448239fdb21","rows":null,"schema":"Eurostat dissemination JSON; aggregate dimensions freq, meatitem, meat, unit, geo, time"}, + {"source_id":"nl.pdok.omgevingswet","path":"data/raw/nl/pdok-omgevingswet-root.json","url":"https://api.pdok.nl/omgevingswet/omgevingsdocumenten/ogc/v2?f=html&lang=nl","retrieved_at_utc":"2026-09-16T07:47:44.3621555Z","http":"200; application/json; Date=Wed, 16 Sep 2026 07:51:05 GMT; Cache-Control=public,max-age=3600,no-transform","bytes":3306,"sha256":"6f17e1afc70bcc56248c5a128a30fc8b479adc880724d5c110cf97a935a21385","rows":null,"collections":10,"schema":"OGC API metadata; RD geometry/document identifier collections"}, + {"source_id":"nl.pdok.omgevingswet","path":"data/raw/nl/pdok-omgevingswet-collections.json","url":"https://api.pdok.nl/omgevingswet/omgevingsdocumenten/ogc/v2/collections?f=json&lang=nl","retrieved_at_utc":"2026-09-16T07:47:45.7282745Z","http":"200; application/json","bytes":24014,"sha256":"de60d5d5d92480d8b0a8e407c413ce506a7a15634f56ffcd8afecba79b33d252","rows":10,"schema":"OGC API collections; current/full and non-production environments"}, + {"source_id":"nl.koop.local-permits","path":"data/raw/nl/overheid-sru-slachthuis.xml","url":"https://zoek.officielebekendmakingen.nl/sru/Search","retrieved_at_utc":"2026-09-16T07:47:46.6066245Z","http":"diagnostic response; invalid search field keyword","bytes":563,"sha256":"0382360ef4103d3b1b9a91e69ea8209674a0096d8ee11a5b87c5643ac13513da","rows":0,"schema":"SRU diagnostics only; no publication rows retained"} + ] +} diff --git a/docs/countries/nl/v1-field-crosswalk.json b/docs/countries/nl/v1-field-crosswalk.json new file mode 100644 index 0000000..1f4f744 --- /dev/null +++ b/docs/countries/nl/v1-field-crosswalk.json @@ -0,0 +1,61 @@ +{ + "schema_version": "1.0", + "country": "NL", + "purpose": "Private reconnaissance crosswalk for future Netherlands integration; not a release mapping.", + "legacy_snapshot": { + "status": "no Netherlands V1 file found", + "rows": 0, + "columns": 0, + "paths_checked": ["static_data", "Old CSVs", "dirty-datasets"], + "notes": "The repository V1 country set has no static_data/nl or other verified Netherlands facility extract. No legacy reconciliation was performed." + }, + "source_local_identities": { + "nvwa_approved_food": { + "facility_candidate_key": "erkenningsnummer", + "observation_key": ["lijstcode", "erkenningsnummer", "specificatiecode", "activiteit", "producttype", "diersoortEn"], + "location_fields": ["adres", "postcode", "plaats"], + "activity_fields": ["categorie", "activiteit", "producttype", "diersoortEn", "regelgeving", "erkenningssoort"], + "lifecycle_fields": ["aanvangsdatum", "opheffingsdatum"], + "notes": "The viewer groups repeated observations by recognition number. Preserve the underlying one-to-many approval/activity observations; recognition number is a source-local candidate key, not a globally proven facility ID." + }, + "cokz_registers": { + "facility_candidate_key": "Approval No.", + "location_fields": ["Address", "Zip code", "Place"], + "organization_fields": ["Name", "Trade"], + "notes": "COKZ is a separate competent-authority register family for dairy and eggs. Do not union with NVWA rows without an explicit reviewed identity link." + }, + "rvo_ir": { + "facility_candidate_key": "UBN", + "location_fields": ["location data"], + "notes": "UBN/I&R data is access-controlled and is not a public release source. Use only through an authorized, purpose-approved route." + }, + "kvk_identity": { + "organization_key": "kvkNummer", + "establishment_key": "vestigingsnummer", + "link_fields": ["postcodeRegio", "activiteiten.sbiCode", "actief", "datumAanvang"], + "notes": "KVK is identity/link evidence, not a facility master. Sole-proprietor and mixed-address records can be personal data; protected fields and UBO data are excluded." + }, + "planning_and_permits": { + "document_key": "DSO/KOOP publication or omgevingsdocument identifier", + "geometry_key": "PDOK/DSO geometry identifier", + "notes": "Planning/permit evidence is document- and geometry-centered. It does not supply a national animal-facility master key." + } + }, + "evidence_crosswalk": { + "facility_and_activity": ["NVWA recognition observations", "COKZ approval rows", "authorized RVO UBN evidence"], + "organization": ["KVK identity evidence", "source-published operator/trade-name fields"], + "inspection_observation": ["NVWA welfare yearbook", "NVWA animal-experiment inspection page", "NVWA red-meat tables"], + "enforcement_observation": ["NVWA measure/warning/boeterapport fields and published report period"], + "aggregate_claim": ["CBS 7123slac", "CBS 84952NED", "Eurostat apro_mt_pann"], + "permit_or_planning_claim": ["DSO/PDOK omgevingsdocument evidence", "KOOP official publication evidence"] + }, + "overlap_rules": [ + "Deduplicate only within an NVWA list by recognition number for facility-candidate counts; retain every source observation and list code.", + "Never sum XML observation rows as facilities: product, activity, species, and specification rows repeat a recognition number.", + "Do not union recognition numbers, COKZ Approval No., RVO UBN, KVK numbers, and EU/TRACES identifiers without a reviewed link record.", + "Keep CBS and Eurostat aggregates outside facility tables; retain provisional, unknown, and secret status symbols.", + "Keep inspections, enforcement, permits, and claims as dated evidence observations; absence from a publication is not closure or non-compliance proof.", + "Do not geocode NVWA addresses automatically for release; preserve source precision and privacy review state." + ], + "release_state": "private-reconnaissance-only" +} diff --git a/docs/country-recon-nl.md b/docs/country-recon-nl.md new file mode 100644 index 0000000..174bed4 --- /dev/null +++ b/docs/country-recon-nl.md @@ -0,0 +1,131 @@ +# Netherlands source reconnaissance + +Status: private research and integration planning only. As of 2026-09-16. No release, deployment, public map, or publication approval is implied. + +## Decision + +The Netherlands is a strong next-country implementation candidate, with a current official NVWA approval source that is machine-readable once its SOAP contract is implemented, useful current welfare/enforcement publications, and stable CBS/Eurostat aggregate context. Integration difficulty is medium-high rather than low: the approval source is a SOAP viewer with repeated observation rows, the country has separate NVWA and COKZ competent-authority register families, RVO data is access-controlled, KVK is identity evidence rather than a facility master, and there is no verified national open permit/facility export. + +Recommendation: implement a private NVWA adapter first, then add dated evidence layers for NVWA enforcement and COKZ. Keep CBS/Eurostat, DSO/PDOK, KOOP, KVK and RVO as explicitly separate sources. Keep publication blocked until source terms, privacy, identity links, schema drift, and review gates are complete. + +The strongest subsequent reconnaissance candidate is Ireland. The official FSAI approved-premises page explicitly routes users to DAFM, HSE and SFPA establishment lists and states that approved establishments receive unique approval numbers; this is a promising multi-authority source shape for a dedicated lane, but it has not been acquired or integration-tested here. See the [FSAI approved food premises page](https://www.fsai.ie/enforcement-and-legislation/official-controls/mancp/approved-food-premises) and [FSAI approval guidance](https://www.fsai.ie/business-advice/starting-a-food-business/approval-of-a-new-business). + +## V1 and legacy boundary + +No verified Netherlands V1 facility file was found under `static_data`, `Old CSVs`, or `dirty-datasets`. The Netherlands crosswalk therefore records zero legacy rows and zero legacy columns, with no attempted reconciliation. This is an explicit “no legacy snapshot” result, not evidence that no Dutch facilities exist. + +The repository’s older V1 facility shape cannot safely be used as the target model. The new source package must preserve approval observations, activities, products, species and lifecycle values before any candidate facility view is produced. + +## Primary source inventory + +| Source | Authority / class | Access and current evidence | Identity and scope | Integration decision | +|---|---|---|---|---| +| [NVWA approved establishments](https://www.nvwa.nl/onderwerpen/dier/slachthuis-uitsnijderij/overzicht-erkende-slachthuizen-en-uitsnijderijen) | Dutch official | Public viewer; control XML and SOAP endpoint captured privately | `erkenningsnummer` plus one-to-many approval/activity observations; slaughter, cutting, game, meat and other food categories | First adapter; deduplicate only within source and retain every observation | +| [NVWA animal welfare](https://www.nvwa.nl/over-de-nvwa/publicaties/jaarbeeld-2025/dierenwelzijn) and [animal-experiment results](https://www.nvwa.nl/onderwerpen/dier/dierproeven-voor-onderzoek/inspectieresultaten/2025) | Dutch official | Current 2025 HTML publications captured privately | Dated aggregate and institution/inspection evidence, not a facility master | Separate inspection/enforcement evidence tables | +| [NVWA red-meat compliance tables](https://www.nvwa.nl/documenten/eten-drinken-roken/vlees-en-vleesproducten/naleefmonitor/tabellenboek-roodvlees-slachthuizen-met-permanent-toezicht-juli-december-2024) | Dutch official | 24-page PDF published 2025-04-22; effective period Jul-Dec 2024 | Individual slaughterhouse compliance tables; publication says facility NAW came from KVK | Keep period and publication date; reviewed identity link only | +| [COKZ](https://cokz.nl/) registers | Dutch official delegated regulator | Current HTML register captured; register pages expose update dates and tables | `Approval No.` and address/name fields; dairy, farm dairy, eggs and egg products | Separate source family; do not union with NVWA by name/address alone | +| [RVO I&R](https://www.rvo.nl/form/bestanden-webservices) | Dutch official, restricted | WSDL/XSD and service documentation; no data acquired | UBN/location/animal records; authorized users only for restricted data | Blocked pending purpose-specific authorization | +| [KVK open dataset/API](https://developers.kvk.nl/nl/documentation/open-dataset-basis-bedrijfsgegevens-api) | Dutch official identity source | API route and CC BY 4.0 open dataset documented; no calls made | `kvkNummer`, branch identifiers where authorized, activity/SBI and status fields | Link evidence only; no facility inference or UBO use | +| [CBS slaughter 7123slac](https://opendata.cbs.nl/ODataApi/OData/7123slac) | Dutch official statistics | Public OData; current data captured | 6,162 aggregate rows; monthly species/count/weight | Claims/context only; never facility rows | +| [CBS livestock 84952NED](https://opendata.cbs.nl/ODataApi/OData/84952NED) | Dutch official statistics | Public OData; current data captured | 1,003 aggregate rows; twice-yearly agricultural-business scope | Claims/context only; preserve provisional status | +| [Eurostat slaughter API](https://ec.europa.eu/eurostat/api/dissemination/statistics/1.0/data/apro_mt_pann?geo=NL&lang=en) | EU official mirror | Current NL JSON captured; last-updated metadata retained | Multidimensional aggregate, no facility IDs | Harmonized context only; never add to CBS/NVWA counts | +| [PDOK/DSO Omgevingswet](https://api.pdok.nl/omgevingswet/omgevingsdocumenten/ogc/v2?f=html&lang=nl) | Dutch official planning/geometry | Public OGC API; 10 collections observed; production metadata updated 2026-09-15 | RD geometry/document identifiers; legal content comes from DSO services | Planning evidence layer, not a facility master | +| [KOOP official publications](https://data.overheid.nl/dataset/officiele-bekendmakingen) / [SRU](https://zoek.officielebekendmakingen.nl/sru/Search) | Dutch official publications | Daily publication surface; bounded test query returned diagnostics because `keyword` is unsupported | Publication/document IDs, authority, dates, text and links; local coverage varies | Implement collection-specific CQL; no national completeness claim | +| [TRACES approved establishments](https://food.ec.europa.eu/food-safety/biological-safety/food-hygiene/approved-eu-food-establishments_en) | EU official mirror/standard | National links and TRACES context verified; no separate NL artifact acquired | EU establishment approval/activity layer | Keep as mirror provenance, never a second facility population | + +Secondary compilations were not used as facility authority. No third-party geocoded directory, commercial company list, or constructed facility compilation is part of this package. + +## NVWA approval route and bounded results + +The NVWA list page describes approved establishments under Regulation (EC) 853/2004. The control file publishes list codes and the Berichtenboek SOAP endpoint. The current viewer’s request shape is a POST with `Content-Type: text/xml`, a list code, language, and pagination (`cvgOffset`, `cvgLimit`, observed limit 10,000). The response parser exposes `erkenningsnummer`, `handelsnaam`, `adres`, `postcode`, `plaats`, `categorie`, `activiteit`, `producttype`, `diersoortEn`, `regelgeving`, `erkenningssoort`, `specificatiecode`, `aanvangsdatum`, `opheffingsdatum`, remarks and related fields. + +The current client groups observations by `erkenningsnummer`. That grouping must not be mistaken for a flat facility export: activity/product/species/specification observations repeat source identities. The bounded responses were parsed privately as follows. + +| List code | NVWA label / scope | XML observations | Advertised/unique recognition numbers | Repeated observations | +|---|---|---:|---:|---:| +| `overig_303` | Domestic ungulate slaughter | 118 | 115 | 3 groups | +| `overig_304` | Domestic ungulate cutting | 470 | 470 | 0 | +| `overig_305` | Poultry/rabbit slaughter | 25 | 23 | 2 groups | +| `overig_306` | Poultry/rabbit cutting | 425 | 356 | 69 | +| `overig_307` | Farmed-game slaughter | 16 | 14 | 1 group | +| `overig_308` | Farmed-game cutting | 102 | 102 | 0 | +| `overig_309` | Wild-game processing | 17 | 10 | 3 groups | +| `overig_310` | Wild-game cutting | 86 | 86 | 0 | + +These numbers are source-response diagnostics, not a national facility total. The same recognition number can be present across list families, and an observation can represent a product/activity/species authorization rather than a distinct site. Counts must therefore be reported as “observations” or “unique recognition numbers within list,” never summed as facilities. + +No geocoding was performed. No closure was inferred from missing or null `opheffingsdatum`; lifecycle semantics require source review. Preserve all nulls and source status values. + +## Welfare, inspection and enforcement evidence + +The NVWA 2025 welfare yearbook reports 4,418 welfare inspections, 3,136 unique companies and 943 companies receiving a measure; it also reports 137 animal-experiment inspections in 2025, plus the published severity and response breakdown. The 2025 animal-experiment result page identifies 52 institutions/companies, 137 inspections plus 4 audits, and 69 license holders. These are dated publication claims and should not be converted into facility rows without a source-local identity. + +The [Zo Doende annual overview](https://www.nvwa.nl/onderwerpen/dier/dierproeven-voor-onderzoek/jaaroverzicht-dierproeven-en-proefdieren-zo-doende) provides 2023 aggregate animal-experiment context (413,746 experiments). It is not a current facility list. The red-meat compliance PDF covers July-December 2024 and was published in April 2025; it includes individual slaughterhouse tables, inspections, official warnings and boeterapporten, but is only one part of NVWA enforcement. Keep its effective period distinct from the 2025 yearbook. + +## Statistics and publication state + +CBS table 7123slac is monthly, modified 2026-08-21, and currently covers January 1990 through June 2026. It has `Slachtdieren`, `Perioden`, `AantalSlachtingen_1` and `GeslachtGewicht_2`; status markers include provisional and unknown/unreliable/secret values. CBS table 84952NED is twice-yearly, modified 2026-08-26, covers April 2018 through April 2026, and is limited to agricultural businesses above the table’s size threshold. Neither table has named facilities or coordinates. + +Eurostat’s `apro_mt_pann` API is a harmonized multidimensional aggregate mirror. The captured response reports dimensions including frequency, meat item, meat, unit, geo and time and a last-updated property observed on 2026-09-03. Eurostat and CBS must remain separate statistical provenance layers, not additive facility or throughput sources. + +## Environment, permits and planning + +PDOK’s Omgevingswet OGC API exposes production current/full collections, RD geometry and document/geometry identifiers; the service metadata states daily update behavior, no authentication/cost, and CC0 1.0 for that geometry surface. The DSO developer register documents omgevingsdocument, geometry, publication and permit-related APIs, with some APIs requiring an API key. The geometry is only meaningful with DSO document context and legal interpretation. + +KOOP/Overheid.nl provides official publications and local permits/announcements through collection-specific SRU routes. The bounded test request used an unsupported generic `keyword` field and produced a diagnostic response with zero rows. The next run must select the correct collection (for example local permits) and use documented CQL. Local/provincial publication feeds are scoped official evidence, not proof of a complete national animal-facility permit register. + +## Identity, privacy and overlap + +KVK’s documented open dataset includes business identity, activity and status fields and is described under CC BY 4.0; subscribed search/base/branch APIs require an API key. KVK warns that business names, numbers and addresses can become personal data for sole proprietorships or mixed addresses, and protected/private fields and UBO data are restricted. Use KVK as an identity-link evidence source keyed to a source-local recognition/approval/establishment identifier. Do not infer that a KVK record is an animal facility, and do not publish protected fields. + +RVO I&R is a service/documentation surface, not an open facility dump. UBN, holder, stall/manager, location and animal information must be treated as restricted until authorization and purpose are documented. COKZ is a distinct official delegated regulator and its HTML registers expose a current farm-dairy table with 357 data rows in the private capture; this is not evidence for all dairy/egg coverage or a reason to merge COKZ and NVWA. + +The overlap policy is: + +- NVWA: deduplicate recognition numbers only for within-list facility-candidate diagnostics; preserve one-to-many observations. +- COKZ, RVO, KVK and TRACES: preserve source-local identifiers and add only explicit reviewed link records. +- CBS and Eurostat: aggregate claims only; preserve flags, scope and effective period. +- NVWA inspection/enforcement, DSO/KOOP permits and planning: dated evidence observations; absence is not closure, compliance or non-compliance proof. +- Addresses and geometry: retain source precision and review privacy before any map or release; do not automatic-geocode for public output. + +## Private artifact and provenance record + +The complete manifest is [data/manifests/nl-source-artifacts.json](../data/manifests/nl-source-artifacts.json). Bulk/raw captures remain ignored under `data/raw/` and are not committed. + +| Artifact group | Private paths | Size/rows | Provenance and schema | +|---|---|---|---| +| NVWA control and eight approval responses | `data/raw/nl-nvwa/2026-09-16/` | 378,439-byte control; responses 14,199–376,998 bytes; observations as table above | SHA-256, URLs, timestamps and response headers in manifest; XML control/SOAP schemas | +| NVWA current welfare/animal experiments | `data/raw/nl/nvwa-welfare-2025.html`, `nvwa-dierproeven-2025.html` | 254,984 and 223,281 bytes | 200 HTML captures; dated 2025 publication schema | +| NVWA enforcement reports | `data/raw/nl/nvwa-slaughter-enforcement-2024.pdf`, `nvwa-zo-doende-2023.pdf` | 412,995 and 836,656 bytes | 200 PDF captures; 24 pages for red-meat table; effective periods preserved | +| COKZ farm-dairy register | `data/raw/nl/cokz-dairy-register.html` | 107,964 bytes; 357 data rows | 200 HTML; last-modified 2026-09-16; six bilingual columns | +| CBS OData | `data/raw/nl/cbs-slaughter-*`, `cbs-livestock-*` | 6,162 and 1,003 data rows | JSON metadata, properties, categories, periods and data captured; response headers in manifest | +| Eurostat | `data/raw/nl/eurostat-slaughter-nl.json` | 56,379 bytes | 200 JSON; multidimensional aggregate | +| PDOK/DSO | `data/raw/nl/pdok-omgevingswet-root.json`, `*-collections.json` | 3,306 and 24,014 bytes; 10 collections | 200 JSON; public cache headers and production update metadata | +| KOOP/SRU | `data/raw/nl/overheid-sru-slachthuis.xml` | 563 bytes; 0 rows | Diagnostic only: unsupported `keyword`; retained to document access behavior | + +The manifest records retrieval timestamps, URLs, HTTP observations, byte sizes, SHA-256 checksums, response schemas and bounded row/collection counts. No facility names, addresses, coordinates, or row-level business data are reproduced in this report. + +## Integration difficulty and staged plan + +Difficulty: medium-high. + +1. Build a private NVWA acquisition/parser contract from the control XML; fingerprint namespaces and fields, paginate explicitly, retain raw observations, and test counts against the eight captured responses. +2. Add a normalized source-local approval-observation table keyed by recognition number plus list code and observation dimensions. Generate facility candidates only as a derived, reviewable view. +3. Add NVWA inspection/enforcement evidence as dated observations, preserving publication/effective periods and avoiding name-only joins. +4. Add COKZ as a separate register adapter; add KVK only through explicit identity-link review. Keep RVO blocked unless authorization is supplied. +5. Add CBS/Eurostat claim adapters and validation for provisional/unknown/secret flags; keep them outside facility counts. +6. Add PDOK/DSO and KOOP document evidence with collection-specific queries and legal/document review; do not promise national permit completeness. +7. Run source-specific privacy, terms, coverage, duplicate, lifecycle and release review. Only then prepare a release candidate; the current package remains private. + +## Open blockers + +- NVWA SOAP schema, pagination, code-list completeness, lifecycle/status semantics and terms need a repeatable adapter contract. +- Recognition numbers are strong source-local keys but cross-authority links are not established. +- COKZ has multiple register families with register-specific dates and no verified uniform API/export contract. +- RVO I&R requires authorization; restricted location/holder/animal data cannot be scraped or exposed. +- KVK API choice, query limits, attribution and personal-data handling need a per-run review record. +- The detailed red-meat enforcement source is effective Jul-Dec 2024, not current 2025 enforcement. +- KOOP’s initial SRU query was rejected because of unsupported field syntax; local permit coverage is heterogeneous. +- No national open permit/facility master or source-approved geocoding policy was verified. +- No Netherlands V1 file exists for reconciliation, and no publication approval exists. + diff --git a/docs/source-status.json b/docs/source-status.json index 8d524ae..b5d1326 100644 --- a/docs/source-status.json +++ b/docs/source-status.json @@ -50,6 +50,17 @@ {"source_id":"es.locations","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-es.md","pipeline/source_registry.json"],"next_action":"Resolve a permitted current AESAN/MAPA artifact or query route, then verify export/schema, rights, effective-date semantics, sector coverage, privacy, and legacy/source boundaries before acquisition."}, {"source_id":"us.fsis","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-us.md","docs/countries/us/README.md","docs/countries/us/v1-field-crosswalk.json","pipeline/sources/us/fsis/config.json","pipeline/sources/us/fsis/adapter.py","pipeline/sources/us/fsis/refresh.py","pipeline/source_registry.json"],"next_action":"Use the assisted official export contract after the 403 blocker; record URL, retrieval/effective dates, content type, byte size, SHA-256, terms, schema fingerprint, privacy review, and reconciliation before any test-only handoff."}, {"source_id":"us.aphis","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-us.md","docs/countries/us/README.md","pipeline/sources/us/aphis/config.json","pipeline/sources/us/aphis/adapter.py","pipeline/sources/us/aphis/refresh.py","pipeline/source_registry.json"],"next_action":"Use the explicit profile-based assisted export for registrations, annual reports, or inspections; preserve each evidence type separately and complete terms, privacy, schema, and review gates."}, - {"source_id":"us.inspections","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-us.md","docs/countries/us/README.md","pipeline/sources/us/aphis/config.json","pipeline/sources/us/aphis/adapter.py","pipeline/sources/us/aphis/refresh.py","pipeline/source_registry.json"],"next_action":"Capture the APHIS inspections profile through the documented public-search route; treat rows as observations, not a facility master, and use explicit reviewable identity matching only."} + {"source_id":"us.inspections","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-us.md","docs/countries/us/README.md","pipeline/sources/us/aphis/config.json","pipeline/sources/us/aphis/adapter.py","pipeline/sources/us/aphis/refresh.py","pipeline/source_registry.json"],"next_action":"Capture the APHIS inspections profile through the documented public-search route; treat rows as observations, not a facility master, and use explicit reviewable identity matching only."}, + {"source_id":"nl.nvwa.approved-food","metadata":"verified","acquisition":"artifact_private_only","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-nl.md","docs/countries/nl/v1-field-crosswalk.json","data/manifests/nl-source-artifacts.json","pipeline/source_registry.json"],"next_action":"Implement SOAP only after confirming coverage, repeated observations, lifecycle, terms, privacy and identity policy."}, + {"source_id":"nl.nvwa.welfare-enforcement","metadata":"verified","acquisition":"artifact_private_only","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-nl.md","data/manifests/nl-source-artifacts.json","pipeline/source_registry.json"],"next_action":"Keep 2025 aggregate welfare and 2024 detailed compliance evidence dated and separate; review identity links and release terms."}, + {"source_id":"nl.cokz.dairy-eggs","metadata":"verified","acquisition":"artifact_private_only","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-nl.md","docs/countries/nl/v1-field-crosswalk.json","data/manifests/nl-source-artifacts.json","pipeline/source_registry.json"],"next_action":"Confirm COKZ register families, update semantics, terms and overlap with NVWA before modeling."}, + {"source_id":"nl.rvo.ir","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-nl.md","pipeline/source_registry.json"],"next_action":"Do not acquire until purpose-specific authorization and privacy/terms review exists; keep UBN separate."}, + {"source_id":"nl.kvk.hvds","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-nl.md","docs/countries/nl/v1-field-crosswalk.json","pipeline/source_registry.json"],"next_action":"Confirm API route and privacy/terms review; use KVK only for reviewed identity links, never UBO or facility proof."}, + {"source_id":"nl.cbs.slaughter","metadata":"verified","acquisition":"artifact_private_only","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-nl.md","data/manifests/nl-source-artifacts.json","pipeline/source_registry.json"],"next_action":"Keep monthly aggregate claims outside facility tables and retain CBS flags and scope."}, + {"source_id":"nl.cbs.livestock","metadata":"verified","acquisition":"artifact_private_only","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-nl.md","data/manifests/nl-source-artifacts.json","pipeline/source_registry.json"],"next_action":"Keep twice-yearly aggregate data separate from named facilities and preserve provisional status."}, + {"source_id":"eu.eurostat.nl-slaughter","metadata":"verified","acquisition":"artifact_private_only","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-nl.md","data/manifests/nl-source-artifacts.json","pipeline/source_registry.json"],"next_action":"Use only as harmonized aggregate context; retain dimensions/flags and do not double-count CBS."}, + {"source_id":"nl.pdok.omgevingswet","metadata":"verified","acquisition":"artifact_private_only","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-nl.md","data/manifests/nl-source-artifacts.json","pipeline/source_registry.json"],"next_action":"Resolve DSO document identity and legal scope before treating geometry as permit evidence."}, + {"source_id":"nl.koop.local-permits","metadata":"verified","acquisition":"artifact_private_only","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-nl.md","data/manifests/nl-source-artifacts.json","pipeline/source_registry.json"],"next_action":"Replace keyword query with collection-specific CQL and document incomplete local coverage."}, + {"source_id":"eu.traces.approved-establishments","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-nl.md","pipeline/source_registry.json"],"next_action":"Verify NL-specific TRACES export and stable IDs only if needed; never merge as additional facilities."} ] } diff --git a/docs/source-status.md b/docs/source-status.md index e747310..670a206 100644 --- a/docs/source-status.md +++ b/docs/source-status.md @@ -39,6 +39,17 @@ The owner-authorized 2026-09-15 live-country rehearsal completed private candida | `us.fsis` | verified | blocked | not_run | blocked | Private adapter and assisted-capture contract are implemented; current CSV access returned 403, so obtain an authorized export and record provenance/schema/privacy/reconciliation before test-only handoff | | `us.aphis` | verified | not_run | not_run | blocked | Profile-explicit private adapter and assisted-capture contract cover registrations, annual reports, and inspections; capture current exports and review terms/schema/privacy | | `us.inspections` | verified | not_run | not_run | blocked | APHIS inspections profile is implemented as observation evidence; capture current export and use explicit reviewable identity links only | +| `nl.nvwa.approved-food` | verified | artifact_private_only | not_run | blocked | Current control XML and eight SOAP list captures are private; repeated observation semantics, terms, privacy and adapter contract remain open; see `docs/country-recon-nl.md` | +| `nl.nvwa.welfare-enforcement` | verified | artifact_private_only | not_run | blocked | Current 2025 welfare/animal-experiment pages and dated 2024/2023 PDFs captured privately; keep effective periods and evidence types separate | +| `nl.cokz.dairy-eggs` | verified | artifact_private_only | not_run | blocked | Current COKZ HTML register captured privately; register-family coverage, terms and overlap with NVWA remain open | +| `nl.rvo.ir` | verified | blocked | not_run | blocked | RVO I&R is authorization-gated; no public scrape or restricted UBN/location data exposure | +| `nl.kvk.hvds` | verified | not_run | not_run | blocked | KVK route and privacy/terms review remain open; use only for reviewed identity links, never UBO or facility proof | +| `nl.cbs.slaughter` | verified | artifact_private_only | not_run | blocked | Current monthly aggregate OData captured privately; keep outside facility tables and preserve status flags | +| `nl.cbs.livestock` | verified | artifact_private_only | not_run | blocked | Current twice-yearly aggregate OData captured privately; agricultural-business scope is not a facility master | +| `eu.eurostat.nl-slaughter` | verified | artifact_private_only | not_run | blocked | Current NL aggregate mirror captured privately; use only as harmonized context and do not double-count CBS | +| `nl.pdok.omgevingswet` | verified | artifact_private_only | not_run | blocked | Current OGC metadata/collections captured privately; resolve DSO legal/document context before permit evidence | +| `nl.koop.local-permits` | verified | artifact_private_only | not_run | blocked | Initial SRU request returned unsupported-field diagnostics; implement collection-specific CQL and retain local coverage limits | +| `eu.traces.approved-establishments` | verified | not_run | not_run | blocked | EU mirror context verified; no separate NL artifact; never merge as additional facilities | ## Australia additions (2026-09-16) diff --git a/pipeline/common/test_source_operations.py b/pipeline/common/test_source_operations.py index 71256c9..960ea46 100644 --- a/pipeline/common/test_source_operations.py +++ b/pipeline/common/test_source_operations.py @@ -24,7 +24,7 @@ class SourceOperationsTests(unittest.TestCase): def test_checked_in_schedule_inventory_matches_registry(self): root = Path(__file__).parents[1] schedules = load_source_schedules(registry_path=root / "source_registry.json") - self.assertEqual(len(schedules), 42) + self.assertEqual(len(schedules), 53) self.assertEqual(schedules["dk.smiley"].stale_after_hours, 240) self.assertEqual(schedules["fr.dgal.section-i"].interval_hours, 24) self.assertIsNone(schedules["us.fsis"].interval_hours) diff --git a/pipeline/source-inventory.csv b/pipeline/source-inventory.csv index 6ebf25f..8b0580d 100644 --- a/pipeline/source-inventory.csv +++ b/pipeline/source-inventory.csv @@ -12,3 +12,14 @@ uk.locations,GB,static_data/uk/locations.csv;static_data/uk/locations.csv.backup us.fsis,US,static_data/us/locations.csv,USDA FSIS MPI directory and supplemental establishment-demographic data,operator-assisted official CSV export; bounded direct fetch with terms review,implemented_partial,Current route verified; direct links returned 403, so acquire an authorized edition and preserve provenance/schema/privacy review before test-only handoff. us.aphis,US,static_data/us/aphis_data_final.csv,USDA APHIS Animal Care Public Search Tool,operator-assisted profile-explicit export,implemented_partial,Registrations, annual reports, inspections, laboratories, and aggregate summaries remain separate evidence types. us.inspections,US,static_data/us/inspection_reports.csv,USDA APHIS inspection-reports public search,operator-assisted inspection export,implemented_partial,Inspection reports are observations, not facility master records; use explicit reviewable identity links only. +nl.nvwa.approved-food,NL,,NVWA approved food establishment lists,public control XML plus SOAP POST,source_candidate,Recognition/activity observations retained separately; implement SOAP and review terms/privacy before release. +nl.nvwa.welfare-enforcement,NL,,NVWA welfare and compliance publications,official HTML/PDF retrieval,source_candidate,2025 summary and 2024 detailed period are evidence observations; not a facility master. +nl.cokz.dairy-eggs,NL,,COKZ dairy and egg registers,official HTML registers,source_candidate,Separate delegated regulator register family; terms and overlap with NVWA require review. +nl.rvo.ir,NL,,RVO I&R services,authorized web service,source_candidate,Restricted UBN/location/animal data; no public scrape or exposure. +nl.kvk.hvds,NL,,KVK business identity APIs,public open dataset or subscribed API,source_candidate,Identity/link evidence only; privacy and API terms remain gates. +nl.cbs.slaughter,NL,,CBS table 7123slac,public OData API,source_candidate,Monthly aggregate slaughter counts/weight; never create facility rows. +nl.cbs.livestock,NL,,CBS table 84952NED,public OData API,source_candidate,Twice-yearly aggregate livestock counts; agricultural-business scope only. +eu.eurostat.nl-slaughter,NL,,Eurostat apro_mt_pann,public dissemination JSON API,source_candidate,EU aggregate mirror; keep separate from CBS and NVWA. +nl.pdok.omgevingswet,NL,,PDOK/DSO omgevingsdocumenten,public OGC API and DSO APIs,source_candidate,Document/geometry evidence; no national animal-facility master. +nl.koop.local-permits,NL,,KOOP/Overheid.nl official publications,collection-specific SRU CQL,source_candidate,Daily official publication surface with local coverage variation; initial query returned diagnostic. +eu.traces.approved-establishments,NL,,European Commission TRACES/IMSOC,publication surface/national links,source_candidate,EU mirror context only; never double-count national NVWA observations. diff --git a/pipeline/source_operations.json b/pipeline/source_operations.json index 7d52739..bce17ff 100644 --- a/pipeline/source_operations.json +++ b/pipeline/source_operations.json @@ -43,6 +43,17 @@ {"source_id":"au.vic.animal-use","cadence":"annual","interval_hours":8760,"stale_after_hours":null,"max_attempts":1,"backoff_seconds":0,"max_backoff_seconds":0,"manual_fallback":"obtain the current Victoria animal-use annual report"}, {"source_id":"au.asic.company-dataset","cadence":"weekly","interval_hours":168,"stale_after_hours":336,"max_attempts":1,"backoff_seconds":0,"max_backoff_seconds":0,"manual_fallback":"use the bounded ASIC weekly snapshot plan"}, {"source_id":"au.abr.lookup","cadence":"on_demand","interval_hours":null,"stale_after_hours":null,"max_attempts":1,"backoff_seconds":0,"max_backoff_seconds":0,"manual_fallback":"query ABR only for an upstream exact ABN/ACN crosswalk"}, - {"source_id":"au.animal-welfare-and-use","cadence":"unknown","interval_hours":null,"stale_after_hours":null,"max_attempts":1,"backoff_seconds":0,"max_backoff_seconds":0,"manual_fallback":"use jurisdiction-specific reviewed welfare/report routes"} + {"source_id":"au.animal-welfare-and-use","cadence":"unknown","interval_hours":null,"stale_after_hours":null,"max_attempts":1,"backoff_seconds":0,"max_backoff_seconds":0,"manual_fallback":"use jurisdiction-specific reviewed welfare/report routes"}, + {"source_id":"nl.nvwa.approved-food","cadence":"unknown","interval_hours":null,"stale_after_hours":null,"max_attempts":1,"backoff_seconds":0,"max_backoff_seconds":0,"manual_fallback":"use the private NVWA SOAP capture after schema and terms review"}, + {"source_id":"nl.nvwa.welfare-enforcement","cadence":"annual/periodic","interval_hours":null,"stale_after_hours":null,"max_attempts":1,"backoff_seconds":0,"max_backoff_seconds":0,"manual_fallback":"retain the dated official HTML/PDF publication capture"}, + {"source_id":"nl.cokz.dairy-eggs","cadence":"unknown","interval_hours":null,"stale_after_hours":null,"max_attempts":1,"backoff_seconds":0,"max_backoff_seconds":0,"manual_fallback":"obtain a reviewed COKZ register capture"}, + {"source_id":"nl.rvo.ir","cadence":"unknown","interval_hours":null,"stale_after_hours":null,"max_attempts":1,"backoff_seconds":0,"max_backoff_seconds":0,"manual_fallback":"authorized RVO operator route only"}, + {"source_id":"nl.kvk.hvds","cadence":"on-demand","interval_hours":null,"stale_after_hours":null,"max_attempts":1,"backoff_seconds":0,"max_backoff_seconds":0,"manual_fallback":"use the reviewed KVK API route"}, + {"source_id":"nl.cbs.slaughter","cadence":"monthly","interval_hours":720,"stale_after_hours":960,"max_attempts":3,"backoff_seconds":1,"max_backoff_seconds":30,"manual_fallback":"use the public CBS OData endpoint"}, + {"source_id":"nl.cbs.livestock","cadence":"twice-yearly","interval_hours":4380,"stale_after_hours":5760,"max_attempts":3,"backoff_seconds":1,"max_backoff_seconds":30,"manual_fallback":"use the public CBS OData endpoint"}, + {"source_id":"eu.eurostat.nl-slaughter","cadence":"dataset-defined","interval_hours":null,"stale_after_hours":null,"max_attempts":1,"backoff_seconds":0,"max_backoff_seconds":0,"manual_fallback":"use the public Eurostat dissemination API"}, + {"source_id":"nl.pdok.omgevingswet","cadence":"daily","interval_hours":24,"stale_after_hours":48,"max_attempts":3,"backoff_seconds":1,"max_backoff_seconds":30,"manual_fallback":"use the public PDOK OGC API"}, + {"source_id":"nl.koop.local-permits","cadence":"daily","interval_hours":24,"stale_after_hours":48,"max_attempts":3,"backoff_seconds":1,"max_backoff_seconds":30,"manual_fallback":"use collection-specific KOOP SRU CQL"}, + {"source_id":"eu.traces.approved-establishments","cadence":"unknown","interval_hours":null,"stale_after_hours":null,"max_attempts":1,"backoff_seconds":0,"max_backoff_seconds":0,"manual_fallback":"verify the NL TRACES publication route"} ] } diff --git a/pipeline/source_registry.json b/pipeline/source_registry.json index 39253ed..2e2d5cb 100644 --- a/pipeline/source_registry.json +++ b/pipeline/source_registry.json @@ -506,6 +506,17 @@ "adapter_status": "reference_only", "expected_artifact_schema": "Report/case observations with jurisdiction, event/report date, status/outcome, source URL, and optional reviewed facility link; aggregate use tables separate", "blockers": ["No uniform national facility-level feed; do not turn complaints, prosecutions, or aggregate totals into canonical facility facts."] - } + }, + {"source_id":"nl.nvwa.approved-food","jurisdiction_scope":"Netherlands; NVWA approved food establishments, slaughter/cutting lists","legacy_paths":[],"url":"https://www.nvwa.nl/site/binaries/content/assets/site-content/webapp-data/lijsten-erkende-bedrijven/stuurbestand-lijsten-erkende-bedrijven","access_method":"public control XML plus SOAP POST; bounded private capture","cadence":"unknown; control file observed current 2026-09-15","attribution_licensing_notes":"Official NVWA; confirm reuse, privacy and project approval before release.","adapter_status":"reference_only","expected_artifact_schema":"XML control file and SOAP observations with recognition number, location, activities, species, lifecycle and regulatory fields","blockers":["Implement SOAP adapter; confirm pagination, coverage, lifecycle semantics, terms, privacy and identity policy."]}, + {"source_id":"nl.nvwa.welfare-enforcement","jurisdiction_scope":"Netherlands; NVWA welfare, animal-experiment and red-meat compliance publications","legacy_paths":[],"url":"https://www.nvwa.nl/over-de-nvwa/publicaties/jaarbeeld-2025/dierenwelzijn","access_method":"official HTML pages and PDF tables","cadence":"annual summary; detailed tables are period-specific","attribution_licensing_notes":"Official NVWA; preserve publication/effective period and confirm reuse/privacy.","adapter_status":"reference_only","expected_artifact_schema":"Narrative/aggregate pages and dated compliance-table PDFs","blockers":["Model inspections and enforcement separately; absence is not closure; review identity links."]}, + {"source_id":"nl.cokz.dairy-eggs","jurisdiction_scope":"Netherlands; COKZ approved dairy, farm-dairy, egg and egg-product establishments","legacy_paths":[],"url":"https://cokz.nl/","access_method":"official HTML registers","cadence":"register-specific update dates; uniform API cadence unknown","attribution_licensing_notes":"Official delegated regulator; terms and machine reuse not verified.","adapter_status":"reference_only","expected_artifact_schema":"HTML tables with name, trade name, address, postcode, place and Approval No.","blockers":["Confirm all register families, lifecycle, terms and overlap with NVWA."]}, + {"source_id":"nl.rvo.ir","jurisdiction_scope":"Netherlands; RVO I&R animal and UBN/location services","legacy_paths":[],"url":"https://www.rvo.nl/form/bestanden-webservices","access_method":"authorized web services and WSDL/XSD","cadence":"24/7 service; refresh cadence unknown","attribution_licensing_notes":"Official RVO; holder, location and animal data may be restricted/personal.","adapter_status":"reference_only","expected_artifact_schema":"Authorized I&R messages with UBN, location and animal registrations","blockers":["Obtain purpose-specific authorization; do not scrape or expose restricted data."]}, + {"source_id":"nl.kvk.hvds","jurisdiction_scope":"Netherlands; KVK business and establishment identity/link evidence","legacy_paths":[],"url":"https://developers.kvk.nl/nl/documentation/open-dataset-basis-bedrijfsgegevens-api","access_method":"open dataset API or subscribed APIs; no calls made","cadence":"on-demand; provider update cadence per run","attribution_licensing_notes":"Open dataset documented CC BY 4.0; API terms and privacy restrictions apply.","adapter_status":"reference_only","expected_artifact_schema":"Business identity/activity fields including kvkNummer, branch ID, active/date and SBI activity","blockers":["Confirm terms/privacy; use only for reviewed identity links, never facility proof or UBO."]}, + {"source_id":"nl.cbs.slaughter","jurisdiction_scope":"Netherlands; CBS monthly aggregate slaughter statistics","legacy_paths":[],"url":"https://opendata.cbs.nl/ODataApi/OData/7123slac","access_method":"public OData API","cadence":"monthly; new figures about two months after reference month","attribution_licensing_notes":"Official CBS; cite CBS and verify dataset-specific reuse terms.","adapter_status":"reference_only","expected_artifact_schema":"OData aggregate Slachtdieren/Perioden/count/weight","blockers":["Keep outside facility tables; preserve provisional/unknown/secret flags."]}, + {"source_id":"nl.cbs.livestock","jurisdiction_scope":"Netherlands; CBS aggregate livestock statistics","legacy_paths":[],"url":"https://opendata.cbs.nl/ODataApi/OData/84952NED","access_method":"public OData API","cadence":"twice-yearly; latest periods may be provisional","attribution_licensing_notes":"Official CBS; cite CBS and verify dataset-specific reuse terms.","adapter_status":"reference_only","expected_artifact_schema":"OData aggregate Landbouwdieren/Perioden/Veestapel_1","blockers":["Do not interpret as named facilities or coordinates; preserve scope/status."]}, + {"source_id":"eu.eurostat.nl-slaughter","jurisdiction_scope":"EU statistical mirror; Netherlands aggregate slaughter context","legacy_paths":[],"url":"https://ec.europa.eu/eurostat/api/dissemination/statistics/1.0/data/apro_mt_pann?geo=NL&lang=en","access_method":"public dissemination JSON API","cadence":"dataset-defined; retain last-updated metadata","attribution_licensing_notes":"Official EU mirror; never count in addition to CBS or NVWA.","adapter_status":"reference_only","expected_artifact_schema":"Multidimensional JSON with frequency, meat, unit, geo and time","blockers":["Use only for harmonized aggregate context and preserve flags."]}, + {"source_id":"nl.pdok.omgevingswet","jurisdiction_scope":"Netherlands; DSO/PDOK planning geometry/document evidence","legacy_paths":[],"url":"https://api.pdok.nl/omgevingswet/omgevingsdocumenten/ogc/v2?f=html&lang=nl","access_method":"public PDOK OGC API plus DSO context APIs","cadence":"daily; production collections observed updated 2026-09-15","attribution_licensing_notes":"PDOK metadata states CC0 1.0; DSO terms and legal interpretation remain gates.","adapter_status":"reference_only","expected_artifact_schema":"OGC collections/features with RD geometry and document identifiers","blockers":["Resolve document/legal scope; no national animal-facility master implied."]}, + {"source_id":"nl.koop.local-permits","jurisdiction_scope":"Netherlands; official publications and local permit/announcement evidence","legacy_paths":[],"url":"https://zoek.officielebekendmakingen.nl/sru/Search","access_method":"KOOP/Overheid.nl SRU and local feeds","cadence":"daily publication surface; local coverage varies","attribution_licensing_notes":"Official publication evidence; preserve authority, identity, dates and state.","adapter_status":"reference_only","expected_artifact_schema":"SRU publication records with document IDs, authority, dates, text and links","blockers":["Use collection-specific CQL; initial keyword query returned diagnostics; no complete national permit coverage claim."]}, + {"source_id":"eu.traces.approved-establishments","jurisdiction_scope":"EU; TRACES/IMSOC approved-establishment mirror","legacy_paths":[],"url":"https://food.ec.europa.eu/food-safety/biological-safety/food-hygiene/approved-eu-food-establishments_en","access_method":"TRACES publication surface and national links","cadence":"competent-authority updates; timestamp each access","attribution_licensing_notes":"Official EU mirror; do not double-count national NVWA observations.","adapter_status":"reference_only","expected_artifact_schema":"Jurisdiction-specific establishment approval/activity export","blockers":["Verify NL export/API and stable IDs before use; keep separate provenance layer."]} ] } diff --git a/pipeline/tests/test_source_registry.py b/pipeline/tests/test_source_registry.py index d2a273b..07d39fc 100644 --- a/pipeline/tests/test_source_registry.py +++ b/pipeline/tests/test_source_registry.py @@ -9,8 +9,8 @@ class SourceRegistryTests(unittest.TestCase): def test_repository_registry_loads_and_references_existing_legacy_paths(self): registry = load_registry() - self.assertEqual(len(registry["sources"]), 42) - self.assertEqual(len({source["source_id"] for source in registry["sources"]}), 42) + self.assertEqual(len(registry["sources"]), 53) + self.assertEqual(len({source["source_id"] for source in registry["sources"]}), 53) def test_unknowns_are_explicit(self): registry = json.loads(REGISTRY_PATH.read_text(encoding="utf-8")) From c8a76cb8ec6b0d2000818e2b9fabc73e1b42ec08 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Wed, 16 Sep 2026 02:31:45 -0700 Subject: [PATCH 137/311] Fix Netherlands report trailing whitespace --- docs/country-recon-nl.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/country-recon-nl.md b/docs/country-recon-nl.md index 174bed4..285da7f 100644 --- a/docs/country-recon-nl.md +++ b/docs/country-recon-nl.md @@ -128,4 +128,3 @@ Difficulty: medium-high. - KOOP’s initial SRU query was rejected because of unsupported field syntax; local permit coverage is heterogeneous. - No national open permit/facility master or source-approved geocoding policy was verified. - No Netherlands V1 file exists for reconciliation, and no publication approval exists. - From f5162994c8b1ff7f2a39a8b195d90be3ddc43a75 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Wed, 16 Sep 2026 02:49:25 -0700 Subject: [PATCH 138/311] Add Ireland source reconnaissance --- data/manifests/ireland-source-artifacts.json | 24 +++ .../countries/ireland/v1-field-crosswalk.json | 31 +++ docs/country-recon-ie.md | 95 +++++++++ docs/source-status.json | 18 +- docs/source-status.md | 4 + pipeline/common/test_source_operations.py | 2 +- pipeline/source_operations.json | 18 +- pipeline/source_registry.json | 194 +++++++++++++++++- pipeline/tests/test_ireland_recon_metadata.py | 93 +++++++++ pipeline/tests/test_source_registry.py | 4 +- 10 files changed, 477 insertions(+), 6 deletions(-) create mode 100644 data/manifests/ireland-source-artifacts.json create mode 100644 docs/countries/ireland/v1-field-crosswalk.json create mode 100644 docs/country-recon-ie.md create mode 100644 pipeline/tests/test_ireland_recon_metadata.py diff --git a/data/manifests/ireland-source-artifacts.json b/data/manifests/ireland-source-artifacts.json new file mode 100644 index 0000000..c9f9876 --- /dev/null +++ b/data/manifests/ireland-source-artifacts.json @@ -0,0 +1,24 @@ +{ + "schema_version": "uec-source-artifact-manifest-v1", + "country_code": "IE", + "checked_at_utc": "2026-09-16T09:35:42.6672214Z", + "retention_policy": "metadata-only in repository; no bulk raw rows, addresses, names, or coordinates committed", + "capture_method": "Official pages were verified through bounded browser research. Direct PowerShell HTTP was unavailable in this environment. Browser snapshot hashes cover the captured accessibility views, not the source bytes.", + "publication_state": "blocked; no release artifact created", + "artifacts": [ + {"artifact_id":"ie-fsai-approved-directory-page","source_id":"ie.fsai.approved-directory","official_url":"https://www.fsai.ie/enforcement-and-legislation/official-controls/mancp/approved-food-premises","retrieved_at_utc":"2026-09-16T09:35:42.6672214Z","capture_state":"metadata_only","source_artifact_sha256":null,"snapshot_sha256":null,"byte_size":null,"content_type":"unknown","provenance_note":"Official coordinating page verified via current search/browser research; no source bytes retained."}, + {"artifact_id":"ie-dafm-approved-establishments-page","source_id":"ie.dafm.approved-establishments","official_url":"https://www.gov.ie/en/department-of-agriculture-food-and-the-marine/publications/dafm-approved-establishments/","retrieved_at_utc":"2026-09-16T09:35:42.6672214Z","capture_state":"browser_snapshot_only","source_artifact_sha256":null,"snapshot_sha256":"e0f1f22c7235bea5dc533b313876f07c79a14b0dd1b59d5c11aa3e41fad09273","snapshot_characters":7532,"byte_size":null,"content_type":"text/html; rendered accessibility snapshot","provenance_note":"Rendered publication page showed last updated 2026-09-15 and three current workbook links; workbook bytes not retained."}, + {"artifact_id":"ie-hse-approvals-page","source_id":"ie.hse.low-throughput-meat","official_url":"https://oapi.fsai.ie/HSEApprovedEstablishments.aspx","retrieved_at_utc":"2026-09-16T09:35:42.6672214Z","capture_state":"browser_snapshot_only","source_artifact_sha256":null,"snapshot_sha256":"ea42bcfa2958302d4fc1e023b010d4774149243f0315703065a901f7fc0752a6","snapshot_characters":64226,"byte_size":null,"content_type":"text/html; rendered accessibility snapshot","observed":{"last_refreshed":"2026-09-16","approval_number_nodes":72},"provenance_note":"Count is distinct Approval_Number nodes in the rendered page; child activity/species rows are not counted as facilities."}, + {"artifact_id":"ie-sfpa-approved-establishments-page","source_id":"ie.sfpa.approved-establishments","official_url":"https://www.sfpa.ie/What-We-Do/Seafood-Safety/Registration-Approval-of-Businesses/List-of-Approved-Establishments/Approved-Establishments","retrieved_at_utc":"2026-09-16T09:35:42.6672214Z","capture_state":"browser_snapshot_only","source_artifact_sha256":null,"snapshot_sha256":"2872114cb3f70d345fe5edd54a8c16d1112d8d1f6c0e4e6d189435a0b94ce313","snapshot_characters":15204,"byte_size":null,"content_type":"text/html; rendered accessibility snapshot","observed":{"updated":"2026-09-15","page_entries":184},"provenance_note":"Paginated table entry count; approval-number variants and facility identity require validation."}, + {"artifact_id":"ie-sfpa-freezer-vessels-page","source_id":"ie.sfpa.freezer-vessels","official_url":"https://www.sfpa.ie/What-We-Do/Seafood-Safety/Registration-Approval-of-Businesses/List-of-Approved-Establishments/Approved-Freezer-Vessels","retrieved_at_utc":"2026-09-16T09:35:42.6672214Z","capture_state":"browser_snapshot_only","source_artifact_sha256":null,"snapshot_sha256":"fbd6fd24f209833171a074424cd9e5cff032ab1277b3b9a03a1c07f540cdfda9","snapshot_characters":14379,"byte_size":null,"content_type":"text/html; rendered accessibility snapshot","observed":{"updated":"2026-09-15","page_entries":50},"provenance_note":"Vessel entries are a separate entity class and are not fixed facility points."}, + {"artifact_id":"ie-sfpa-factory-vessels-page","source_id":"ie.sfpa.factory-vessels","official_url":"https://www.sfpa.ie/What-We-Do/Seafood-Safety/Registration-Approval-of-Businesses/List-of-Approved-Establishments/Approved-Factory-Vessels","retrieved_at_utc":"2026-09-16T09:35:42.6672214Z","capture_state":"browser_snapshot_only","source_artifact_sha256":null,"snapshot_sha256":"15cd1c490828d5e47e1752ef33a04835fe390ac23c5e441245202e02191c54b5","snapshot_characters":9602,"byte_size":null,"content_type":"text/html; rendered accessibility snapshot","observed":{"page_entries":1},"provenance_note":"Factory-vessel table rendered one entry; no source bytes retained."}, + {"artifact_id":"ie-epa-leap-metadata","source_id":"ie.epa.leap","official_url":"https://www.epa.ie/our-services/compliance--enforcement/whats-happening/leap-online/","retrieved_at_utc":"2026-09-16T09:35:42.6672214Z","capture_state":"metadata_only","source_artifact_sha256":null,"snapshot_sha256":null,"byte_size":null,"content_type":"unknown","provenance_note":"Official LEAP guidance/API families verified; no record bulk capture performed."}, + {"artifact_id":"ie-planning-npad-metadata","source_id":"ie.planning.npad","official_url":"https://www.myplan.ie/national-planning-application-map-viewer/","retrieved_at_utc":"2026-09-16T09:35:42.6672214Z","capture_state":"metadata_only","source_artifact_sha256":null,"snapshot_sha256":null,"byte_size":null,"content_type":"unknown","provenance_note":"Official map and weekly-upload semantics verified; no application rows or geometry retained."}, + {"artifact_id":"ie-cro-companies-metadata","source_id":"ie.cro.companies","official_url":"https://opendata.cro.ie/dataset/companies","retrieved_at_utc":"2026-09-16T09:35:42.6672214Z","capture_state":"metadata_only","source_artifact_sha256":null,"snapshot_sha256":null,"byte_size":null,"content_type":"unknown","provenance_note":"Official open-data catalog metadata verified; no company rows retained."}, + {"artifact_id":"ie-cso-slaughterings-metadata","source_id":"ie.cso.livestock-slaughterings","official_url":"https://www.cso.ie/en/statistics/agriculture/livestockslaughterings/","retrieved_at_utc":"2026-09-16T09:35:42.6672214Z","capture_state":"metadata_only","source_artifact_sha256":null,"snapshot_sha256":null,"byte_size":null,"content_type":"unknown","provenance_note":"Current CSO release family and aggregate scope verified; no table export retained."}, + {"artifact_id":"ie-dafm-beef-kill-metadata","source_id":"ie.dafm.national-beef-kill","official_url":"https://opendata.agriculture.gov.ie/dataset/national-beef-kill-figures","retrieved_at_utc":"2026-09-16T09:35:42.6672214Z","capture_state":"metadata_only","source_artifact_sha256":null,"snapshot_sha256":null,"byte_size":null,"content_type":"unknown","provenance_note":"Catalog describes a weekly resource with 2020-2024 coverage and 2024-08-01 update metadata; no rows retained."}, + {"artifact_id":"ie-dafm-funding-context","source_id":"ie.dafm.seafood-processing-funding","official_url":"https://www.gov.ie/en/department-of-agriculture-food-and-the-marine/press-releases/minister-dooley-announces-opening-of-the-seafood-processing-capital-investment-scheme/","retrieved_at_utc":"2026-09-16T09:35:42.6672214Z","capture_state":"metadata_only","source_artifact_sha256":null,"snapshot_sha256":null,"byte_size":null,"content_type":"unknown","provenance_note":"Programme context only; no beneficiary or award rows retained."}, + {"artifact_id":"ie-fsai-enforcement-context","source_id":"ie.fsai.enforcement-orders","official_url":"https://www.fsai.ie/news-and-alerts/latest-news/fourteen-enforcement-orders-served-on-food-bus-%281%29","retrieved_at_utc":"2026-09-16T09:35:42.6672214Z","capture_state":"metadata_only","source_artifact_sha256":null,"snapshot_sha256":null,"byte_size":null,"content_type":"unknown","provenance_note":"Current enforcement-notice route verified; no named-subject rows retained."}, + {"artifact_id":"ie-dafm-welfare-context","source_id":"ie.dafm.animal-welfare-controls","official_url":"https://www.gov.ie/en/department-of-agriculture-food-and-the-marine/press-releases/statement-on-garda-investigation-into-alleged-offences-of-deception/","retrieved_at_utc":"2026-09-16T09:35:42.6672214Z","capture_state":"metadata_only","source_artifact_sha256":null,"snapshot_sha256":null,"byte_size":null,"content_type":"unknown","provenance_note":"DAFM control responsibilities verified as context; no facility-linked welfare events retained."} + ] +} diff --git a/docs/countries/ireland/v1-field-crosswalk.json b/docs/countries/ireland/v1-field-crosswalk.json new file mode 100644 index 0000000..b5c5101 --- /dev/null +++ b/docs/countries/ireland/v1-field-crosswalk.json @@ -0,0 +1,31 @@ +{ + "schema_version": "ie-recon-crosswalk-1", + "country_code": "IE", + "checked_at_utc": "2026-09-16T09:35:42.6672214Z", + "status": "reconnaissance_only; metadata_and_bounded_browser_observations; no_raw_source_bytes; no_adapter; no_release", + "canonical_entity_rules": { + "facility": "One reviewed operating premises or vessel identity; never one row per activity/species/category.", + "source_key": "Preserve the authority's approval/registration number as a string, including punctuation and EC suffix; add source_family and entity_class.", + "activity_observation": "Child observation keyed by source_key plus source activity/category/species values; repeated rows are expected and must not inflate facility counts.", + "cross_source_link": "Only a reviewed edge using stable authority identifiers, explicit evidence, match method, reviewer, and confidence; name similarity alone is insufficient." + }, + "sources": [ + {"source_id":"ie.fsai.approved-directory","official_url":"https://www.fsai.ie/enforcement-and-legislation/official-controls/mancp/approved-food-premises","coverage":"Coordinator page linking DAFM, HSE, and SFPA approved-premises lists","entity_class":"directory","observed_counts":{},"field_crosswalk":[{"source_field":"competent-authority link","canonical_field":"source_route","status":"observed"}],"adapter_status":"not_started","privacy":"restricted_pending_review","rights":"unresolved"}, + {"source_id":"ie.dafm.approved-establishments","official_url":"https://assets.gov.ie/static/documents/09fe3ad4/AllApprovedPlants_2026_5.xlsx","coverage":"DAFM meat establishments incorporating fish, egg, and dairy under S.I. 22 of 2020","entity_class":"facility","observed_counts":{"publication_last_updated":"2026-09-15","row_count":null,"count_status":"not_captured"},"field_crosswalk":[{"source_field":"approval/registration number","canonical_field":"source_establishment_key","status":"to_verify_from_workbook"},{"source_field":"establishment name","canonical_field":"source_name","status":"to_verify_from_workbook"},{"source_field":"activity/species","canonical_field":"activity_observation","status":"expected_one_to_many"},{"source_field":"status/effective date","canonical_field":"source_status_observation","status":"to_verify_from_workbook"}],"adapter_status":"not_started","privacy":"restricted_pending_review","rights":"unresolved"}, + {"source_id":"ie.dafm.former-la-establishments","official_url":"https://assets.gov.ie/static/documents/09fe3ad4/AllApprovedPlants_2026_Formerly_LA_Plants_1.xlsx","coverage":"Former local-authority meat establishments incorporating fish, egg, and dairy","entity_class":"facility","observed_counts":{"publication_last_updated":"2026-09-15","row_count":null,"count_status":"not_captured"},"field_crosswalk":[{"source_field":"approval/registration number","canonical_field":"source_establishment_key","status":"to_verify_from_workbook"},{"source_field":"activity/species","canonical_field":"activity_observation","status":"expected_one_to_many"}],"adapter_status":"not_started","privacy":"restricted_pending_review","rights":"unresolved"}, + {"source_id":"ie.dafm.milk-dairy-establishments","official_url":"https://assets.gov.ie/static/documents/09fe3ad4/1._Milk_Dairy_Establishments_Registered_and_or_Approved_11th_August_2026.xlsx","coverage":"DAFM milk and dairy establishments approved and/or registered under Hygiene Regulations","entity_class":"facility","observed_counts":{"filename_date":"2026-08-11","row_count":null,"count_status":"not_captured"},"field_crosswalk":[{"source_field":"approval versus registration","canonical_field":"source_status_observation","status":"to_verify_from_workbook"},{"source_field":"activity","canonical_field":"activity_observation","status":"expected_one_to_many"}],"adapter_status":"not_started","privacy":"restricted_pending_review","rights":"unresolved"}, + {"source_id":"ie.hse.low-throughput-meat","official_url":"https://oapi.fsai.ie/HSEApprovedEstablishments.aspx","coverage":"HSE-supervised low-throughput meat processors","entity_class":"facility","observed_counts":{"last_refreshed":"2026-09-16","approval_number_nodes":72,"count_semantics":"distinct Approval_Number values observed in rendered page; activity/species child rows excluded"},"field_crosswalk":[{"source_field":"Approval_Number","canonical_field":"source_establishment_key","status":"observed_string"},{"source_field":"Premises Trading Name","canonical_field":"source_name","status":"observed"},{"source_field":"Address","canonical_field":"restricted_source_address","status":"observed_restricted"},{"source_field":"County","canonical_field":"source_region","status":"observed"},{"source_field":"Primary Business Type","canonical_field":"source_activity_group","status":"observed"},{"source_field":"Activity","canonical_field":"activity_observation.activity","status":"observed_repeated"},{"source_field":"Species","canonical_field":"activity_observation.species","status":"observed_repeated"}],"activity_model":{"parent_key":"source_establishment_key","child_key":"activity_observation_key","deduplication":"Do not collapse distinct activity/species values; do not count children as facilities."},"adapter_status":"not_started","privacy":"restricted_pending_review","rights":"unresolved"}, + {"source_id":"ie.sfpa.approved-establishments","official_url":"https://www.sfpa.ie/What-We-Do/Seafood-Safety/Registration-Approval-of-Businesses/List-of-Approved-Establishments/Approved-Establishments","coverage":"SFPA approved seafood establishments under Regulation (EC) No 853/2004","entity_class":"facility","observed_counts":{"updated":"2026-09-15","page_entries":184,"count_semantics":"paginated table entries; distinct facility identity remains to validate against approval-number variants"},"field_crosswalk":[{"source_field":"Approval Number","canonical_field":"source_establishment_key","status":"observed_string_preserve_raw"},{"source_field":"Establishment Name","canonical_field":"source_name","status":"observed"},{"source_field":"Address","canonical_field":"restricted_source_address","status":"observed_restricted"},{"source_field":"County","canonical_field":"source_region","status":"observed"},{"source_field":"Categories","canonical_field":"activity_observation.category","status":"observed_repeated_or_multivalue"},{"source_field":"Activity Codes","canonical_field":"activity_observation.activity_code","status":"observed"},{"source_field":"Certificate Number","canonical_field":"source_certificate_key","status":"observed"}],"activity_model":{"parent_key":"source_establishment_key","child_key":"activity_observation_key","deduplication":"Preserve EC and non-EC approval text; derive a stable key only after explicit normalization rule."},"adapter_status":"not_started","privacy":"restricted_pending_review","rights":"copyright_all_rights_reserved_footer_review_required"}, + {"source_id":"ie.sfpa.freezer-vessels","official_url":"https://www.sfpa.ie/What-We-Do/Seafood-Safety/Registration-Approval-of-Businesses/List-of-Approved-Establishments/Approved-Freezer-Vessels","coverage":"SFPA approved freezer vessels","entity_class":"vessel","observed_counts":{"updated":"2026-09-15","page_entries":50,"count_semantics":"paginated vessel entries"},"field_crosswalk":[{"source_field":"Approval Number","canonical_field":"source_vessel_key","status":"observed_string_preserve_raw"},{"source_field":"Name of Vessel","canonical_field":"source_name","status":"observed"},{"source_field":"Categories","canonical_field":"activity_observation.category","status":"observed"},{"source_field":"Activity Codes","canonical_field":"activity_observation.activity_code","status":"observed"},{"source_field":"Certificate Number","canonical_field":"source_certificate_key","status":"observed"}],"adapter_status":"not_started","privacy":"restricted_pending_review","rights":"copyright_all_rights_reserved_footer_review_required"}, + {"source_id":"ie.sfpa.factory-vessels","official_url":"https://www.sfpa.ie/What-We-Do/Seafood-Safety/Registration-Approval-of-Businesses/List-of-Approved-Establishments/Approved-Factory-Vessels","coverage":"SFPA approved factory vessels","entity_class":"vessel","observed_counts":{"page_entries":1,"count_semantics":"rendered table entries"},"field_crosswalk":[{"source_field":"Approval Number","canonical_field":"source_vessel_key","status":"observed_string_preserve_raw"},{"source_field":"Vessel Name","canonical_field":"source_name","status":"observed"},{"source_field":"Categories","canonical_field":"activity_observation.category","status":"observed"},{"source_field":"Activity Codes","canonical_field":"activity_observation.activity_code","status":"observed"}],"adapter_status":"not_started","privacy":"restricted_pending_review","rights":"copyright_all_rights_reserved_footer_review_required"}, + {"source_id":"ie.epa.leap","official_url":"https://www.epa.ie/our-services/compliance--enforcement/whats-happening/leap-online/","coverage":"EPA licences, site profiles, returns, visits, incidents, complaints, non-compliances, and investigations","entity_class":"accountability","observed_counts":{"publication_lag":"most records published 30 calendar days after creation","row_count":null},"field_crosswalk":[{"source_field":"licence/site/profile ID","canonical_field":"source_accountability_key","status":"API_family_documented"},{"source_field":"site/facility","canonical_field":"reviewed_subject_ref","status":"requires_explicit_match"},{"source_field":"inspection/incident/non-compliance","canonical_field":"accountability_observation","status":"separate_child_event"}],"adapter_status":"not_started","privacy":"restricted_pending_review","rights":"EPA_LEAP_terms_required"}, + {"source_id":"ie.planning.npad","official_url":"https://www.myplan.ie/national-planning-application-map-viewer/","coverage":"National planning application map and local-authority application data","entity_class":"planning_event","observed_counts":{"cadence":"weekly upload stated by MyPlan help","row_count":null},"field_crosswalk":[{"source_field":"application number","canonical_field":"source_planning_key","status":"expected_verify"},{"source_field":"authority/status/decision","canonical_field":"planning_event_state","status":"expected_verify"},{"source_field":"point/polygon","canonical_field":"approximate_geometry","status":"approximate_not_site_specific"}],"adapter_status":"not_started","privacy":"applicant_data_restricted_pending_review","rights":"byline_and_splash_conditions_review_required"}, + {"source_id":"ie.cro.companies","official_url":"https://opendata.cro.ie/dataset/companies","coverage":"CRO company and business-name identity records","entity_class":"corporate_identity","observed_counts":{"cadence":"daily","row_count":null},"field_crosswalk":[{"source_field":"company number","canonical_field":"source_company_key","status":"expected_verify"},{"source_field":"company name/status","canonical_field":"corporate_identity_observation","status":"expected_verify"},{"source_field":"registered office/officer","canonical_field":"restricted_identity_detail","status":"do_not_publish_by_default"}],"adapter_status":"not_started","privacy":"restricted_pending_review","rights":"catalog_describes_CC_BY_4_0_verify_live_terms"}, + {"source_id":"ie.cso.livestock-slaughterings","official_url":"https://www.cso.ie/en/statistics/agriculture/livestockslaughterings/","coverage":"CSO aggregate slaughterings and meat-supply statistics","entity_class":"aggregate_statistic","observed_counts":{"periodicity":"monthly","facility_count":null},"field_crosswalk":[{"source_field":"period","canonical_field":"statistic_period","status":"expected_verify"},{"source_field":"species","canonical_field":"statistic_species","status":"expected_verify"},{"source_field":"heads/tonnes","canonical_field":"statistic_measure","status":"expected_verify"}],"adapter_status":"not_started","privacy":"aggregate","rights":"table_terms_and_attribution_review_required"}, + {"source_id":"ie.dafm.national-beef-kill","official_url":"https://opendata.agriculture.gov.ie/dataset/national-beef-kill-figures","coverage":"DAFM aggregate beef-kill figures by approved processing plants","entity_class":"aggregate_statistic","observed_counts":{"periodicity":"weekly","catalog_cutoff":"2024","facility_count":null},"field_crosswalk":[{"source_field":"week/plant/category","canonical_field":"statistic_dimension","status":"to_verify"},{"source_field":"kill figure","canonical_field":"statistic_measure","status":"to_verify"}],"adapter_status":"not_started","privacy":"aggregate_or_pseudonymous_plant_dimension_review_required","rights":"catalog_terms_and_historical_cutoff_review_required"}, + {"source_id":"ie.dafm.seafood-processing-funding","official_url":"https://www.gov.ie/en/department-of-agriculture-food-and-the-marine/press-releases/minister-dooley-announces-opening-of-the-seafood-processing-capital-investment-scheme/","coverage":"Seafood-processing capital-investment programme context","entity_class":"funding_context","observed_counts":{"award_rows":null},"field_crosswalk":[{"source_field":"scheme/year","canonical_field":"funding_programme","status":"observed_context"},{"source_field":"award/beneficiary","canonical_field":"reviewed_funding_observation","status":"not_captured"}],"adapter_status":"not_started","privacy":"beneficiary_data_restricted_pending_review","rights":"scheme_specific_review_required"}, + {"source_id":"ie.fsai.enforcement-orders","official_url":"https://www.fsai.ie/news-and-alerts/latest-news/fourteen-enforcement-orders-served-on-food-bus-%281%29","coverage":"FSAI enforcement-order notices and context","entity_class":"enforcement_event","observed_counts":{"facility_count":null},"field_crosswalk":[{"source_field":"order type/date","canonical_field":"enforcement_observation","status":"event_only"},{"source_field":"named subject","canonical_field":"reviewed_subject_ref","status":"privacy_and_identity_review_required"}],"adapter_status":"not_started","privacy":"restricted_pending_review","rights":"notice_scope_and_legal_review_required"}, + {"source_id":"ie.dafm.animal-welfare-controls","official_url":"https://www.gov.ie/en/department-of-agriculture-food-and-the-marine/press-releases/statement-on-garda-investigation-into-alleged-offences-of-deception/","coverage":"DAFM official-veterinary, animal-welfare, transport, hygiene, and ABP control scope","entity_class":"welfare_context","observed_counts":{"row_level_facility_events":null},"field_crosswalk":[{"source_field":"control responsibility","canonical_field":"authority_scope","status":"observed_context"},{"source_field":"inspection/event","canonical_field":"welfare_observation","status":"not_captured"}],"adapter_status":"not_started","privacy":"restricted_pending_review","rights":"event_artifact_and_publication_review_required"} + ], + "shared_canonical_fields": ["country_code","source_id","source_key","entity_class","source_name","source_region","restricted_source_address","source_status_observation","activity_observation","source_accountability_key","source_planning_key","source_company_key","evidence_ref","observed_at_utc","privacy_state","rights_state","review_state"] +} diff --git a/docs/country-recon-ie.md b/docs/country-recon-ie.md new file mode 100644 index 0000000..546745b --- /dev/null +++ b/docs/country-recon-ie.md @@ -0,0 +1,95 @@ +# Ireland source reconnaissance + +Status: reconnaissance complete, private metadata only, no adapter, no release. Checked 2026-09-16. The machine-readable [field crosswalk](countries/ireland/v1-field-crosswalk.json) and [artifact manifest](../data/manifests/ireland-source-artifacts.json) are the implementation handoff. No raw bulk data, addresses, coordinates, company rows, or named enforcement rows are committed. + +## Decision summary + +Ireland is a strong but split source landscape. FSAI is the directory/coordinating layer; DAFM, HSE, and SFPA own different approval populations. EPA LEAP, planning, CRO, statistics, funding, and enforcement/welfare material are useful accountability or context layers, not additional facility masters. The recommended decision is to proceed only with a bounded, operator-authorized capture phase, preserving each authority and entity class separately. + +| Source family | Current evidence | Integration decision | +|---|---|---| +| FSAI | Approved-premises page says products of animal origin requiring Regulation (EC) 853/2004 approval must appear in up-to-date competent-authority lists, and links DAFM, HSE, and SFPA | Coordinator only; do not add a fourth facility population | +| DAFM | Current publication page last updated 2026-09-15 and exposes three XLSX families: main establishments, former-LA establishments, and milk/dairy establishments | Three source identities; workbook headers/counts still unverified | +| HSE | Rendered low-throughput directory last refreshed 2026-09-16; 72 distinct `Approval_Number` nodes observed | Facility parent plus repeated activity/species children; no address release | +| SFPA | Approved establishments updated 2026-09-15 with 184 table entries; freezer vessels 50; factory vessels 1 | Establishment, freezer-vessel, and factory-vessel entity classes stay separate | +| EPA / planning / CRO | LEAP exposes licence/compliance events, MyPlan exposes ten years of applications with weekly uploads, CRO catalog describes daily company data | Accountability and identity edges only; explicit reviewed joins | +| Statistics / funding / enforcement / welfare | CSO/DAFM aggregate measures; DAFM seafood funding context; FSAI orders; DAFM control responsibilities | Context/event layers; never facility counts or proof of current operation | + +## Official routes and schemas + +The FSAI [approved food premises directory](https://www.fsai.ie/enforcement-and-legislation/official-controls/mancp/approved-food-premises) explains the competent-authority split. FSAI’s [approval guidance](https://www.fsai.ie/business-advice/starting-a-food-business/approval-of-a-new-business) identifies slaughterhouses, meat processors, egg/dairy businesses, and fish processors as typical approval cases. Its [establishments subject to approval](https://www.fsai.ie/enforcement-and-legislation/legislation/food-legislation/meat-fresh-meat/establishments-subject-to-approval) page distinguishes approval under 853/2004 from registration under 852/2004. + +DAFM’s [approved establishments publication](https://www.gov.ie/en/department-of-agriculture-food-and-the-marine/publications/dafm-approved-establishments/) says the Hygiene Package took effect 1 January 2006, was transposed by S.I. 432/2009 and revised by S.I. 22/2020, and that DAFM inspects existing and new meat establishments. The page links: + +- [main workbook](https://assets.gov.ie/static/documents/09fe3ad4/AllApprovedPlants_2026_5.xlsx), incorporating fish, egg, and dairy; +- [former local-authority workbook](https://assets.gov.ie/static/documents/09fe3ad4/AllApprovedPlants_2026_Formerly_LA_Plants_1.xlsx); and +- [milk/dairy workbook](https://assets.gov.ie/static/documents/09fe3ad4/1._Milk_Dairy_Establishments_Registered_and_or_Approved_11th_August_2026.xlsx). + +The links are verified, but the workbooks were not downloaded in this run. No count, header, status code, effective date, or overlap is inferred from a filename or page title. + +The [HSE/FSAI directory](https://oapi.fsai.ie/HSEApprovedEstablishments.aspx) explicitly renders `Approval_Number`, `Premises Trading Name`, `Address`, `County`, `Primary Business Type`, `Activity`, and `Species`. Its 72 approval-number count excludes the many repeated activity/species rows. The source therefore requires a parent approval record and a child activity observation, with `Not Stated` and `Unknown` preserved as source values. + +SFPA’s [approved-establishments table](https://www.sfpa.ie/What-We-Do/Seafood-Safety/Registration-Approval-of-Businesses/List-of-Approved-Establishments/Approved-Establishments) renders `Approval Number`, `Establishment Name`, `Address`, `County`, `Categories`, `Activity Codes`, and `Certificate Number`; it defines `AH`, `DC`, `FFPP`, `PC`, `PP`, and `CS`. It displayed 184 entries and an update date of 15 September 2026. The [freezer-vessel](https://www.sfpa.ie/What-We-Do/Seafood-Safety/Registration-Approval-of-Businesses/List-of-Approved-Establishments/Approved-Freezer-Vessels) table displayed 50 entries, while the [factory-vessel](https://www.sfpa.ie/What-We-Do/Seafood-Safety/Registration-Approval-of-Businesses/List-of-Approved-Establishments/Approved-Factory-Vessels) table displayed 1. SFPA approval strings sometimes show both an EC form and a non-EC form; preserve that raw value and normalize only in a reviewed transform. + +## Identity, counts, and overlap + +The safe key is `(source_id, source-local approval/registration number, entity_class)`. DAFM’s three workbook families must not be unioned until overlap is measured. HSE approvals, SFPA establishments, and SFPA vessels must not be deduplicated by name or address. Approval, activity, category, species, certificate, and status are observations attached to a parent identity, not separate facilities. + +The report’s observed values are deliberately not a national total: + +- HSE: 72 distinct approval-number nodes in the rendered page; +- SFPA: 184 approved-establishment table entries, 50 freezer-vessel entries, and 1 factory-vessel entry; +- DAFM: count unavailable until a bounded workbook capture and schema fingerprint; +- FSAI: no independent rows; it is a coordinator; +- EPA, NPAD, CRO, enforcement, and funding: no facility count claimed; +- CSO and DAFM beef-kill: aggregate statistics only. + +The first implementation must produce both `source_row_count` and `distinct_parent_count`, plus `repeated_child_count`, `quarantined_count`, and an overlap report. A cross-source link needs an explicit evidence packet: source IDs, raw identifiers, reviewed name/address evidence kept privately, match method, reviewer, confidence, and unresolved alternatives. A shared trading name, registered office, certificate, county, or approximate location is not enough. + +## Accountability graph and map usefulness + +The graph potential is high: competent authority → approval; approval → activity/species/category; SFPA approval → certificate; facility/vessel → EPA licence or compliance event; facility/company → CRO identity; site → planning application; sector/time → CSO or DAFM aggregate; facility/event → reviewed enforcement or welfare observation. Keep graph edges typed and dated. Absence of a public event is not evidence of no event. + +Map usefulness is medium for fixed premises because HSE, DAFM, and SFPA expose address fields, but no source coordinates were verified. It is low for vessels as fixed points. EPA and MyPlan geometry is an accountability/development clue, not a substitute for source address review; MyPlan itself warns that portal points/polygons are approximate and unsuitable for site-specific decisions. Geocoding is disabled until source terms, privacy eligibility, residential/mixed-use risk, and publication approval are separately passed. A successful geocode would not authorize publication. + +## Environmental, planning, corporate, statistics, funding, and controls + +The EPA [licence search](https://www.epa.ie/our-services/licensing/licencesearch/) and [LEAP guidance](https://www.epa.ie/our-services/compliance--enforcement/whats-happening/leap-online/) cover licence/site profiles, inspections, returns, incidents, complaints, non-compliances, and compliance investigations. EPA’s [API documentation](https://data-stg.epa.ie/api-list/leap-open-data/) describes separate endpoint families and identifier-based retrieval. The [LEAP terms](https://www.epa.ie/publications/compliance--enforcement/licensees/performance/LEAP-Online-Terms-%26-Conditions-23-May-2023.pdf) and personal-data controls are a hard gate against bulk copying. + +The [MyPlan national planning map](https://www.myplan.ie/national-planning-application-map-viewer/) and [help page](https://www.myplan.ie/help/) say applications are sourced from 31 local authorities and uploaded weekly. Planning records evidence applications, development history, and decisions; they do not prove that an approved food establishment operates. Applicant/personal data and approximate geometry stay restricted. + +The CRO [company open-data catalog](https://opendata.cro.ie/dataset/companies) describes daily machine-readable company records and CC BY 4.0. Use company number as an identity edge only. A [CORE search](https://cro.ie/services-and-help/using-services/) or registered office is not operating-premises evidence, and officer data should not enter the released model. + +The [CSO livestock slaughterings releases](https://www.cso.ie/en/statistics/agriculture/livestockslaughterings/) provide monthly aggregate context; CSO background notes say monthly data come from DAFM and include DAFM and local-authority approved plants. The [DAFM national beef-kill catalog](https://opendata.agriculture.gov.ie/dataset/national-beef-kill-figures) is described as weekly but the observed resource metadata is historical through 2024 with a 1 August 2024 update. Neither is a current facility census. + +Funding is context only. DAFM’s [Seafood Processing Capital Investment Scheme](https://www.gov.ie/en/department-of-agriculture-food-and-the-marine/press-releases/minister-dooley-announces-opening-of-the-seafood-processing-capital-investment-scheme/) describes 2025 support under the Ireland Seafood Development Programme and EMFAF. It must not be joined to establishments by name alone. DAFM’s [control statement](https://www.gov.ie/en/department-of-agriculture-food-and-the-marine/press-releases/statement-on-garda-investigation-into-alleged-offences-of-deception/) describes permanent official-veterinary/technical presence and checks including identification, hygiene, animal welfare/transport, remedies, and animal-by-product disposal. FSAI’s [enforcement notices](https://www.fsai.ie/news-and-alerts/latest-news/fourteen-enforcement-orders-served-on-food-bus-%281%29) and the [FSAI 2024 annual report](https://www.fsai.ie/getmedia/14f8726b-1f98-4d44-90d9-dcdf96d49cef/FSAI-Annual-Report-2024-ENG-Final-Accessible_1.pdf?ext=.pdf) are event/aggregate context. No current row-level facility-linked welfare feed was verified. + +## Access, provenance, privacy, and blockers + +The checked-in manifest records exact official URLs, retrieval time, capture state, byte-size/hash nullability, and browser accessibility-snapshot hashes. A snapshot hash is not presented as a source-byte hash. Direct PowerShell HTTP was blocked by the execution proxy, and browser-rendered workbook links did not yield a retained file. This is an access blocker, not evidence that the sources lack data. + +Primary blockers are: + +1. authorized bounded capture of the three DAFM workbooks and headers; +2. export/pagination and cadence contracts for HSE and SFPA; +3. workbook/table status and effective-date semantics; +4. rights and attribution confirmation, especially SFPA copyright and EPA LEAP terms; +5. privacy review for addresses, mixed residential premises, registered offices, vessels, applicants, officers, and enforcement subjects; +6. source overlap and company/licence/planning identity review; and +7. human release approval. No public map, API, promotion, or deployment is authorized by this reconnaissance. + +## Staged pipeline plan and difficulty + +1. Operator-authorized capture: retain each source artifact privately with URL, retrieval/effective dates, content type, bytes, SHA-256, and terms record. +2. Parse: preserve raw values and source-local IDs; record schema fingerprints and pagination completeness. +3. Normalize: create parent facility/vessel records and one-to-many activity/category/species observations; never overwrite source values. +4. Validate: check identifiers, row lengths, status/effective dates, duplicate approvals, missingness, and source-specific counts. +5. Reconcile: emit candidate overlaps only; require reviewed evidence for cross-source links to EPA, planning, or CRO. +6. Enrich privately: apply coarse geography only after privacy/terms review; keep precise coordinates quarantined. +7. Review and release gate: inspect QA, provenance, privacy, rights, accountability semantics, and publication eligibility before any disposable candidate import or release. + +Implementation difficulty is medium-high. The approval topology is understandable, but three competent authorities, three DAFM workbook families, repeated child rows, mobile vessels, approximate planning geometry, and identity/privacy constraints make a production adapter materially harder than a single national CSV. Adapter implementation is intentionally not started until bounded artifacts are lawfully captured. + +## Recommended next country + +Germany is the recommended next country after Ireland. The repository already has a typed BVL route and partial adapter/refresh scaffolding, so its remaining work is bounded export capture, terms confirmation, duplicate-approval validation, and privacy review. That is a lower incremental implementation cost than starting Ireland’s multi-authority adapter family. This recommendation is operational, not a claim that Germany is more complete or more publishable. diff --git a/docs/source-status.json b/docs/source-status.json index b5d1326..6e7e2c7 100644 --- a/docs/source-status.json +++ b/docs/source-status.json @@ -61,6 +61,22 @@ {"source_id":"eu.eurostat.nl-slaughter","metadata":"verified","acquisition":"artifact_private_only","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-nl.md","data/manifests/nl-source-artifacts.json","pipeline/source_registry.json"],"next_action":"Use only as harmonized aggregate context; retain dimensions/flags and do not double-count CBS."}, {"source_id":"nl.pdok.omgevingswet","metadata":"verified","acquisition":"artifact_private_only","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-nl.md","data/manifests/nl-source-artifacts.json","pipeline/source_registry.json"],"next_action":"Resolve DSO document identity and legal scope before treating geometry as permit evidence."}, {"source_id":"nl.koop.local-permits","metadata":"verified","acquisition":"artifact_private_only","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-nl.md","data/manifests/nl-source-artifacts.json","pipeline/source_registry.json"],"next_action":"Replace keyword query with collection-specific CQL and document incomplete local coverage."}, - {"source_id":"eu.traces.approved-establishments","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-nl.md","pipeline/source_registry.json"],"next_action":"Verify NL-specific TRACES export and stable IDs only if needed; never merge as additional facilities."} + {"source_id":"eu.traces.approved-establishments","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-nl.md","pipeline/source_registry.json"],"next_action":"Verify NL-specific TRACES export and stable IDs only if needed; never merge as additional facilities."}, + {"source_id":"ie.fsai.approved-directory","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ie.md","docs/countries/ireland/v1-field-crosswalk.json","data/manifests/ireland-source-artifacts.json","pipeline/source_registry.json"],"next_action":"Use FSAI only as the coordinator; capture and review the linked DAFM, HSE, and SFPA artifacts separately."}, + {"source_id":"ie.dafm.approved-establishments","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ie.md","docs/countries/ireland/v1-field-crosswalk.json","data/manifests/ireland-source-artifacts.json","pipeline/source_registry.json"],"next_action":"Obtain an authorized bounded workbook capture, headers, schema fingerprint, row counts, terms, privacy review, and lifecycle semantics."}, + {"source_id":"ie.dafm.former-la-establishments","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ie.md","docs/countries/ireland/v1-field-crosswalk.json","data/manifests/ireland-source-artifacts.json","pipeline/source_registry.json"],"next_action":"Capture separately from the main DAFM workbook and measure overlap before any deduplication."}, + {"source_id":"ie.dafm.milk-dairy-establishments","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ie.md","docs/countries/ireland/v1-field-crosswalk.json","data/manifests/ireland-source-artifacts.json","pipeline/source_registry.json"],"next_action":"Capture the dated workbook and preserve approval-versus-registration and dairy activity semantics."}, + {"source_id":"ie.hse.low-throughput-meat","metadata":"verified","acquisition":"artifact_private_only","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ie.md","docs/countries/ireland/v1-field-crosswalk.json","data/manifests/ireland-source-artifacts.json","pipeline/source_registry.json"],"next_action":"Use the bounded page observation as metadata only; obtain a lawful repeatable export/capture contract and keep repeated activity/species rows as children."}, + {"source_id":"ie.sfpa.approved-establishments","metadata":"verified","acquisition":"artifact_private_only","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ie.md","docs/countries/ireland/v1-field-crosswalk.json","data/manifests/ireland-source-artifacts.json","pipeline/source_registry.json"],"next_action":"Capture all paginated rows privately, confirm rights, and validate approval-number variant and facility semantics."}, + {"source_id":"ie.sfpa.freezer-vessels","metadata":"verified","acquisition":"artifact_private_only","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ie.md","docs/countries/ireland/v1-field-crosswalk.json","data/manifests/ireland-source-artifacts.json","pipeline/source_registry.json"],"next_action":"Keep vessel rows separate from fixed establishments and review address/rights handling."}, + {"source_id":"ie.sfpa.factory-vessels","metadata":"verified","acquisition":"artifact_private_only","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ie.md","docs/countries/ireland/v1-field-crosswalk.json","data/manifests/ireland-source-artifacts.json","pipeline/source_registry.json"],"next_action":"Keep factory-vessel identity separate; capture and validate the one-entry table through a permitted route."}, + {"source_id":"ie.epa.leap","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ie.md","docs/countries/ireland/v1-field-crosswalk.json","pipeline/source_registry.json"],"next_action":"Resolve LEAP API identifiers and terms; use it only for reviewed accountability edges and events."}, + {"source_id":"ie.planning.npad","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ie.md","docs/countries/ireland/v1-field-crosswalk.json","pipeline/source_registry.json"],"next_action":"Capture application-level records separately; preserve approximate geometry and applicant privacy."}, + {"source_id":"ie.cro.companies","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ie.md","docs/countries/ireland/v1-field-crosswalk.json","pipeline/source_registry.json"],"next_action":"Verify the live daily schema and use company number for reviewed identity edges only."}, + {"source_id":"ie.cso.livestock-slaughterings","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ie.md","docs/countries/ireland/v1-field-crosswalk.json","pipeline/source_registry.json"],"next_action":"Capture aggregate monthly measures with coverage notes; never treat them as facility rows."}, + {"source_id":"ie.dafm.national-beef-kill","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ie.md","docs/countries/ireland/v1-field-crosswalk.json","pipeline/source_registry.json"],"next_action":"Retain the observed 2024 historical cutoff and verify the resource before any aggregate use."}, + {"source_id":"ie.dafm.seafood-processing-funding","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ie.md","docs/countries/ireland/v1-field-crosswalk.json","pipeline/source_registry.json"],"next_action":"Keep programme context separate; capture awards only after scheme-specific privacy and terms review."}, + {"source_id":"ie.fsai.enforcement-orders","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ie.md","docs/countries/ireland/v1-field-crosswalk.json","pipeline/source_registry.json"],"next_action":"Treat notices as dated event evidence and require reviewed subject identity before linkage."}, + {"source_id":"ie.dafm.animal-welfare-controls","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ie.md","docs/countries/ireland/v1-field-crosswalk.json","pipeline/source_registry.json"],"next_action":"Keep the DAFM statement as control-scope context until a current row-level welfare artifact is lawfully captured and reviewed."} ] } diff --git a/docs/source-status.md b/docs/source-status.md index 670a206..6b7788b 100644 --- a/docs/source-status.md +++ b/docs/source-status.md @@ -15,6 +15,10 @@ No last-success timestamp is invented. Private artifacts are not proof of a publ The owner-authorized 2026-09-15 live-country rehearsal completed private candidate integration for Denmark, France Sections I/II, Italy 853/2004, Germany, Belgium, Canada Ontario/CFIA, and UK FSA/FSS. It created no public surface and kept publication blocked. See the detailed [country rehearsal report](country-rehearsal-2026-09-15.md) and machine-readable [rehearsal status](country-rehearsal-2026-09-15.json). +## Ireland reconnaissance update + +The 2026-09-16 Ireland reconnaissance verified the current FSAI/DAFM/HSE/SFPA source topology and adjacent EPA, planning, CRO, CSO, funding, enforcement, and welfare context. It captured only bounded browser observations: HSE 72 distinct approval-number nodes, SFPA 184 approved-establishment entries, 50 freezer-vessel entries, and 1 factory-vessel entry. DAFM’s three workbook links were verified from the publication page, but workbook bytes, headers, and counts were not captured. See [Ireland reconnaissance](country-recon-ie.md), the [Ireland crosswalk](countries/ireland/v1-field-crosswalk.json), and the [Ireland artifact manifest](../data/manifests/ireland-source-artifacts.json). All 16 Ireland sources remain publication-blocked; no adapter or release artifact exists. + ## Current baseline | Source ID | Metadata | Acquisition | Runtime health | Publication | Evidence / next action | diff --git a/pipeline/common/test_source_operations.py b/pipeline/common/test_source_operations.py index 960ea46..acfff27 100644 --- a/pipeline/common/test_source_operations.py +++ b/pipeline/common/test_source_operations.py @@ -24,7 +24,7 @@ class SourceOperationsTests(unittest.TestCase): def test_checked_in_schedule_inventory_matches_registry(self): root = Path(__file__).parents[1] schedules = load_source_schedules(registry_path=root / "source_registry.json") - self.assertEqual(len(schedules), 53) + self.assertEqual(len(schedules), 69) self.assertEqual(schedules["dk.smiley"].stale_after_hours, 240) self.assertEqual(schedules["fr.dgal.section-i"].interval_hours, 24) self.assertIsNone(schedules["us.fsis"].interval_hours) diff --git a/pipeline/source_operations.json b/pipeline/source_operations.json index bce17ff..d8f3beb 100644 --- a/pipeline/source_operations.json +++ b/pipeline/source_operations.json @@ -54,6 +54,22 @@ {"source_id":"eu.eurostat.nl-slaughter","cadence":"dataset-defined","interval_hours":null,"stale_after_hours":null,"max_attempts":1,"backoff_seconds":0,"max_backoff_seconds":0,"manual_fallback":"use the public Eurostat dissemination API"}, {"source_id":"nl.pdok.omgevingswet","cadence":"daily","interval_hours":24,"stale_after_hours":48,"max_attempts":3,"backoff_seconds":1,"max_backoff_seconds":30,"manual_fallback":"use the public PDOK OGC API"}, {"source_id":"nl.koop.local-permits","cadence":"daily","interval_hours":24,"stale_after_hours":48,"max_attempts":3,"backoff_seconds":1,"max_backoff_seconds":30,"manual_fallback":"use collection-specific KOOP SRU CQL"}, - {"source_id":"eu.traces.approved-establishments","cadence":"unknown","interval_hours":null,"stale_after_hours":null,"max_attempts":1,"backoff_seconds":0,"max_backoff_seconds":0,"manual_fallback":"verify the NL TRACES publication route"} + {"source_id":"eu.traces.approved-establishments","cadence":"unknown","interval_hours":null,"stale_after_hours":null,"max_attempts":1,"backoff_seconds":0,"max_backoff_seconds":0,"manual_fallback":"verify the NL TRACES publication route"}, + {"source_id":"ie.fsai.approved-directory","cadence":"unknown","interval_hours":null,"stale_after_hours":null,"max_attempts":1,"backoff_seconds":0,"max_backoff_seconds":0,"manual_fallback":"use the operator-assisted FSAI authority-link capture"}, + {"source_id":"ie.dafm.approved-establishments","cadence":"unknown","interval_hours":null,"stale_after_hours":null,"max_attempts":1,"backoff_seconds":0,"max_backoff_seconds":0,"manual_fallback":"use the authorized DAFM main workbook capture"}, + {"source_id":"ie.dafm.former-la-establishments","cadence":"unknown","interval_hours":null,"stale_after_hours":null,"max_attempts":1,"backoff_seconds":0,"max_backoff_seconds":0,"manual_fallback":"use the authorized DAFM former-LA workbook capture"}, + {"source_id":"ie.dafm.milk-dairy-establishments","cadence":"unknown","interval_hours":null,"stale_after_hours":null,"max_attempts":1,"backoff_seconds":0,"max_backoff_seconds":0,"manual_fallback":"use the authorized DAFM milk/dairy workbook capture"}, + {"source_id":"ie.hse.low-throughput-meat","cadence":"unknown","interval_hours":null,"stale_after_hours":null,"max_attempts":1,"backoff_seconds":0,"max_backoff_seconds":0,"manual_fallback":"use a bounded HSE browser capture or authorized export"}, + {"source_id":"ie.sfpa.approved-establishments","cadence":"unknown","interval_hours":null,"stale_after_hours":null,"max_attempts":1,"backoff_seconds":0,"max_backoff_seconds":0,"manual_fallback":"use a bounded SFPA paginated-table capture"}, + {"source_id":"ie.sfpa.freezer-vessels","cadence":"unknown","interval_hours":null,"stale_after_hours":null,"max_attempts":1,"backoff_seconds":0,"max_backoff_seconds":0,"manual_fallback":"use a bounded SFPA freezer-vessel capture"}, + {"source_id":"ie.sfpa.factory-vessels","cadence":"unknown","interval_hours":null,"stale_after_hours":null,"max_attempts":1,"backoff_seconds":0,"max_backoff_seconds":0,"manual_fallback":"use a bounded SFPA factory-vessel capture"}, + {"source_id":"ie.epa.leap","cadence":"unknown","interval_hours":null,"stale_after_hours":null,"max_attempts":1,"backoff_seconds":0,"max_backoff_seconds":0,"manual_fallback":"resolve LEAP API identifiers and use reviewed record capture"}, + {"source_id":"ie.planning.npad","cadence":"weekly","interval_hours":168,"stale_after_hours":240,"max_attempts":1,"backoff_seconds":0,"max_backoff_seconds":0,"manual_fallback":"use the authorized NPAD/MyPlan weekly service capture"}, + {"source_id":"ie.cro.companies","cadence":"daily","interval_hours":24,"stale_after_hours":48,"max_attempts":1,"backoff_seconds":0,"max_backoff_seconds":0,"manual_fallback":"use the reviewed CRO open-data/API identity capture"}, + {"source_id":"ie.cso.livestock-slaughterings","cadence":"monthly","interval_hours":720,"stale_after_hours":960,"max_attempts":1,"backoff_seconds":0,"max_backoff_seconds":0,"manual_fallback":"use the bounded CSO aggregate release capture"}, + {"source_id":"ie.dafm.national-beef-kill","cadence":"weekly","interval_hours":168,"stale_after_hours":240,"max_attempts":1,"backoff_seconds":0,"max_backoff_seconds":0,"manual_fallback":"use the bounded historical DAFM beef-kill resource capture"}, + {"source_id":"ie.dafm.seafood-processing-funding","cadence":"unknown","interval_hours":null,"stale_after_hours":null,"max_attempts":1,"backoff_seconds":0,"max_backoff_seconds":0,"manual_fallback":"capture only reviewed programme metadata or award records"}, + {"source_id":"ie.fsai.enforcement-orders","cadence":"unknown","interval_hours":null,"stale_after_hours":null,"max_attempts":1,"backoff_seconds":0,"max_backoff_seconds":0,"manual_fallback":"capture a reviewed FSAI notice event"}, + {"source_id":"ie.dafm.animal-welfare-controls","cadence":"unknown","interval_hours":null,"stale_after_hours":null,"max_attempts":1,"backoff_seconds":0,"max_backoff_seconds":0,"manual_fallback":"use an authorized DAFM welfare/control artifact if one is approved"} ] } diff --git a/pipeline/source_registry.json b/pipeline/source_registry.json index 2e2d5cb..29dd35d 100644 --- a/pipeline/source_registry.json +++ b/pipeline/source_registry.json @@ -517,6 +517,198 @@ {"source_id":"eu.eurostat.nl-slaughter","jurisdiction_scope":"EU statistical mirror; Netherlands aggregate slaughter context","legacy_paths":[],"url":"https://ec.europa.eu/eurostat/api/dissemination/statistics/1.0/data/apro_mt_pann?geo=NL&lang=en","access_method":"public dissemination JSON API","cadence":"dataset-defined; retain last-updated metadata","attribution_licensing_notes":"Official EU mirror; never count in addition to CBS or NVWA.","adapter_status":"reference_only","expected_artifact_schema":"Multidimensional JSON with frequency, meat, unit, geo and time","blockers":["Use only for harmonized aggregate context and preserve flags."]}, {"source_id":"nl.pdok.omgevingswet","jurisdiction_scope":"Netherlands; DSO/PDOK planning geometry/document evidence","legacy_paths":[],"url":"https://api.pdok.nl/omgevingswet/omgevingsdocumenten/ogc/v2?f=html&lang=nl","access_method":"public PDOK OGC API plus DSO context APIs","cadence":"daily; production collections observed updated 2026-09-15","attribution_licensing_notes":"PDOK metadata states CC0 1.0; DSO terms and legal interpretation remain gates.","adapter_status":"reference_only","expected_artifact_schema":"OGC collections/features with RD geometry and document identifiers","blockers":["Resolve document/legal scope; no national animal-facility master implied."]}, {"source_id":"nl.koop.local-permits","jurisdiction_scope":"Netherlands; official publications and local permit/announcement evidence","legacy_paths":[],"url":"https://zoek.officielebekendmakingen.nl/sru/Search","access_method":"KOOP/Overheid.nl SRU and local feeds","cadence":"daily publication surface; local coverage varies","attribution_licensing_notes":"Official publication evidence; preserve authority, identity, dates and state.","adapter_status":"reference_only","expected_artifact_schema":"SRU publication records with document IDs, authority, dates, text and links","blockers":["Use collection-specific CQL; initial keyword query returned diagnostics; no complete national permit coverage claim."]}, - {"source_id":"eu.traces.approved-establishments","jurisdiction_scope":"EU; TRACES/IMSOC approved-establishment mirror","legacy_paths":[],"url":"https://food.ec.europa.eu/food-safety/biological-safety/food-hygiene/approved-eu-food-establishments_en","access_method":"TRACES publication surface and national links","cadence":"competent-authority updates; timestamp each access","attribution_licensing_notes":"Official EU mirror; do not double-count national NVWA observations.","adapter_status":"reference_only","expected_artifact_schema":"Jurisdiction-specific establishment approval/activity export","blockers":["Verify NL export/API and stable IDs before use; keep separate provenance layer."]} + {"source_id":"eu.traces.approved-establishments","jurisdiction_scope":"EU; TRACES/IMSOC approved-establishment mirror","legacy_paths":[],"url":"https://food.ec.europa.eu/food-safety/biological-safety/food-hygiene/approved-eu-food-establishments_en","access_method":"TRACES publication surface and national links","cadence":"competent-authority updates; timestamp each access","attribution_licensing_notes":"Official EU mirror; do not double-count national NVWA observations.","adapter_status":"reference_only","expected_artifact_schema":"Jurisdiction-specific establishment approval/activity export","blockers":["Verify NL export/API and stable IDs before use; keep separate provenance layer."]}, + { + "source_id": "ie.fsai.approved-directory", + "jurisdiction_scope": "Ireland; FSAI coordinating directory for competent-authority approved animal-origin food establishments", + "legacy_paths": [], + "url": "https://www.fsai.ie/enforcement-and-legislation/official-controls/mancp/approved-food-premises", + "access_method": "official directory page; bounded browser capture or operator-assisted authority exports", + "cadence": "unknown", + "attribution_licensing_notes": "FSAI explains the approval obligation and links to DAFM, HSE, and SFPA lists; the directory page is not itself a facility master and reuse/privacy terms require review", + "adapter_status": "not_started", + "expected_artifact_schema": "HTML directory metadata with authority links; no facility rows expected", + "blockers": ["Do not count this coordinator page as an additional establishment source; verify linked authority exports, terms, privacy, and publication approval."] + }, + { + "source_id": "ie.dafm.approved-establishments", + "jurisdiction_scope": "Ireland; DAFM-approved or registered meat establishments including fish, egg, and dairy under S.I. 22 of 2020", + "legacy_paths": [], + "url": "https://assets.gov.ie/static/documents/09fe3ad4/AllApprovedPlants_2026_5.xlsx", + "access_method": "official gov.ie publication link; bounded workbook capture or operator-assisted download", + "cadence": "unknown; publication page last updated 2026-09-15", + "attribution_licensing_notes": "DAFM government publication; workbook-specific reuse, attribution, personal-data handling, and redistribution terms must be confirmed before release", + "adapter_status": "not_started", + "expected_artifact_schema": "XLSX establishment workbook; exact headers, repeated activity/species rows, status, approval, and effective-date semantics must be captured and fingerprinted", + "blockers": ["Workbook bytes and headers were not captured in this run; do not infer row counts or schema from the publication title."] + }, + { + "source_id": "ie.dafm.former-la-establishments", + "jurisdiction_scope": "Ireland; former local-authority meat establishments including fish, egg, and dairy under S.I. 22 of 2020", + "legacy_paths": [], + "url": "https://assets.gov.ie/static/documents/09fe3ad4/AllApprovedPlants_2026_Formerly_LA_Plants_1.xlsx", + "access_method": "official gov.ie publication link; bounded workbook capture or operator-assisted download", + "cadence": "unknown; publication page last updated 2026-09-15", + "attribution_licensing_notes": "DAFM government publication; workbook-specific reuse, attribution, personal-data handling, and redistribution terms must be confirmed before release", + "adapter_status": "not_started", + "expected_artifact_schema": "XLSX establishment workbook; preserve former-LA scope and source-local identifiers; exact headers and lifecycle semantics to verify", + "blockers": ["Keep former-LA coverage separate from the main DAFM workbook until overlap and authority ownership are reviewed."] + }, + { + "source_id": "ie.dafm.milk-dairy-establishments", + "jurisdiction_scope": "Ireland; DAFM milk and dairy establishments approved and/or registered under the Hygiene Regulations", + "legacy_paths": [], + "url": "https://assets.gov.ie/static/documents/09fe3ad4/1._Milk_Dairy_Establishments_Registered_and_or_Approved_11th_August_2026.xlsx", + "access_method": "official gov.ie publication link; bounded workbook capture or operator-assisted download", + "cadence": "unknown; workbook filename dated 2026-08-11", + "attribution_licensing_notes": "DAFM government publication; workbook-specific reuse, attribution, personal-data handling, and redistribution terms must be confirmed before release", + "adapter_status": "not_started", + "expected_artifact_schema": "XLSX dairy establishment workbook; exact headers, approval/registration distinction, and repeated activity rows to verify", + "blockers": ["Do not union dairy rows with meat rows without preserving source family and approval-versus-registration semantics."] + }, + { + "source_id": "ie.hse.low-throughput-meat", + "jurisdiction_scope": "Ireland; low-throughput meat processors under HSE supervision", + "legacy_paths": [], + "url": "https://oapi.fsai.ie/HSEApprovedEstablishments.aspx", + "access_method": "rendered official HSE/FSAI directory; bounded browser capture", + "cadence": "unknown; page displayed last refreshed 2026-09-16", + "attribution_licensing_notes": "Public HSE/FSAI directory; address and trading-name fields are restricted pending terms/privacy review and publication approval", + "adapter_status": "not_started", + "expected_artifact_schema": "HTML tables with Approval_Number, Premises Trading Name, Address, County, Primary Business Type, Activity, and Species; repeated activity/species rows per approval", + "blockers": ["Page-level refresh date is observed but no documented cadence or export contract was found; preserve all repeated child observations and do not publish addresses."] + }, + { + "source_id": "ie.sfpa.approved-establishments", + "jurisdiction_scope": "Ireland; SFPA establishments approved under Regulation (EC) No 853/2004 for fishery products and live bivalve molluscs", + "legacy_paths": [], + "url": "https://www.sfpa.ie/What-We-Do/Seafood-Safety/Registration-Approval-of-Businesses/List-of-Approved-Establishments/Approved-Establishments", + "access_method": "rendered official SFPA paginated table; bounded browser capture", + "cadence": "unknown; table displayed updated 2026-09-15", + "attribution_licensing_notes": "SFPA page is public but footer states copyright/all rights reserved; obtain reuse permission/interpretation and complete privacy review before redistribution", + "adapter_status": "not_started", + "expected_artifact_schema": "Paginated HTML table with Approval Number, Establishment Name, Address, County, Categories, Activity Codes, and Certificate Number", + "blockers": ["Pagination/export behavior, rights, and exact facility-versus-approval row semantics require a bounded artifact capture and review."] + }, + { + "source_id": "ie.sfpa.freezer-vessels", + "jurisdiction_scope": "Ireland; SFPA freezer vessels approved under Regulation (EC) No 853/2004", + "legacy_paths": [], + "url": "https://www.sfpa.ie/What-We-Do/Seafood-Safety/Registration-Approval-of-Businesses/List-of-Approved-Establishments/Approved-Freezer-Vessels", + "access_method": "rendered official SFPA paginated table; bounded browser capture", + "cadence": "unknown; table displayed updated 2026-09-15", + "attribution_licensing_notes": "SFPA page is public but footer states copyright/all rights reserved; vessel and address fields are restricted pending rights/privacy review", + "adapter_status": "not_started", + "expected_artifact_schema": "Paginated HTML table with Approval Number, Name of Vessel, Address, County, Categories, Activity Codes, and Certificate Number", + "blockers": ["Vessel identity is mobile/non-fixed; do not treat the address as a stable facility location or sum vessels with establishments."] + }, + { + "source_id": "ie.sfpa.factory-vessels", + "jurisdiction_scope": "Ireland; SFPA factory vessels approved under Regulation (EC) No 853/2004", + "legacy_paths": [], + "url": "https://www.sfpa.ie/What-We-Do/Seafood-Safety/Registration-Approval-of-Businesses/List-of-Approved-Establishments/Approved-Factory-Vessels", + "access_method": "rendered official SFPA table; bounded browser capture", + "cadence": "unknown; page is live and no cadence is stated", + "attribution_licensing_notes": "SFPA page is public but footer states copyright/all rights reserved; vessel data requires rights/privacy review", + "adapter_status": "not_started", + "expected_artifact_schema": "HTML table with Approval Number, Vessel Name, Address, County, Categories, and Activity Codes", + "blockers": ["Keep factory-vessel rows as a separate entity class; do not infer a fixed premises or blend with freezer-vessel rows."] + }, + { + "source_id": "ie.epa.leap", + "jurisdiction_scope": "Ireland; EPA IE/IPC licensed sites and public LEAP licensing, compliance, and enforcement records", + "legacy_paths": [], + "url": "https://www.epa.ie/our-services/compliance--enforcement/whats-happening/leap-online/", + "access_method": "official LEAP search and documented public API families; bounded record retrieval after identifier review", + "cadence": "unknown; most records are published 30 calendar days after creation", + "attribution_licensing_notes": "EPA LEAP terms and conditions govern use of environmental information; do not bulk-copy or expose personal/precise information without legal/privacy review", + "adapter_status": "not_started", + "expected_artifact_schema": "Licence/site profiles plus separate licences, returns, site visits, incidents, complaints, non-compliances, and compliance investigations keyed by EPA IDs", + "blockers": ["LEAP is an accountability layer, not a food-establishment census; resolve API identifiers, terms, privacy, and explicit cross-source matching."] + }, + { + "source_id": "ie.planning.npad", + "jurisdiction_scope": "Ireland; National Planning Application Map and local-authority planning application data", + "legacy_paths": [], + "url": "https://www.myplan.ie/national-planning-application-map-viewer/", + "access_method": "official map/open-data/service routes; bounded application-level capture", + "cadence": "weekly; MyPlan help states data are uploaded weekly", + "attribution_licensing_notes": "MyPlan allows public information distribution/copying with byline credit, subject to data-use conditions; applicant/personal data and approximate GIS locations require review", + "adapter_status": "not_started", + "expected_artifact_schema": "Planning application identifier, authority, applicant, description, dates, status/decision, and approximate point/polygon; exact service schema to verify", + "blockers": ["Planning applications show development history or intent, not proof of operating approval; keep applications separate from facility entities and do not use approximate geometry for site-specific decisions."] + }, + { + "source_id": "ie.cro.companies", + "jurisdiction_scope": "Ireland; Companies Registration Office CORE company/business-name identity register", + "legacy_paths": [], + "url": "https://opendata.cro.ie/dataset/companies", + "access_method": "official open-data bulk/API route or CORE search; bounded identity capture", + "cadence": "daily; catalog describes daily updates", + "attribution_licensing_notes": "CRO open-data company dataset is described as CC BY 4.0; officer/personal details remain restricted and company identity is not proof of a facility or operation", + "adapter_status": "not_started", + "expected_artifact_schema": "Machine-readable company records; exact live fields/schema fingerprint to verify, with company number/name/status separated from registered-office and officer data", + "blockers": ["Name similarity is not a merge; require reviewed company-number links and keep registered offices distinct from operating premises."] + }, + { + "source_id": "ie.cso.livestock-slaughterings", + "jurisdiction_scope": "Ireland; CSO aggregate livestock slaughterings and meat supply statistics", + "legacy_paths": [], + "url": "https://www.cso.ie/en/statistics/agriculture/livestockslaughterings/", + "access_method": "official CSO release/table route; bounded aggregate capture", + "cadence": "monthly", + "attribution_licensing_notes": "CSO statistics are aggregate context; table-specific terms and attribution must be retained with any downstream use", + "adapter_status": "not_started", + "expected_artifact_schema": "Aggregate period/species/slaughtered-heads and tonnage measures; no facility identity", + "blockers": ["Do not join aggregate statistics to facility rows or interpret totals as a facility census; preserve CSO coverage notes."] + }, + { + "source_id": "ie.dafm.national-beef-kill", + "jurisdiction_scope": "Ireland; DAFM national beef kill figures by approved processing plants, aggregate release", + "legacy_paths": [], + "url": "https://opendata.agriculture.gov.ie/dataset/national-beef-kill-figures", + "access_method": "official open-data catalog/resource route; bounded aggregate capture", + "cadence": "weekly; current catalog resource described as 2020-2024 and last updated 2024-08-01", + "attribution_licensing_notes": "DAFM open-data catalog indicates open-data reuse for the resource; verify current resource terms and preserve its historical cutoff", + "adapter_status": "not_started", + "expected_artifact_schema": "Aggregate week/plant/species or category kill figures; exact current resource fields and cutoff to verify", + "blockers": ["Catalog resource is historical through 2024 in the observed metadata; do not present it as a current facility status feed or inflate facility counts."] + }, + { + "source_id": "ie.dafm.seafood-processing-funding", + "jurisdiction_scope": "Ireland; DAFM seafood-processing capital-investment funding context under Ireland Seafood Development Programme/EMFAF", + "legacy_paths": [], + "url": "https://www.gov.ie/en/department-of-agriculture-food-and-the-marine/press-releases/minister-dooley-announces-opening-of-the-seafood-processing-capital-investment-scheme/", + "access_method": "official programme/press-release metadata; no facility award rows captured", + "cadence": "unknown; programme/competition dependent", + "attribution_licensing_notes": "Government/EU programme context; award-level terms, personal data, and beneficiary publication rules require scheme-specific review", + "adapter_status": "not_started", + "expected_artifact_schema": "Programme metadata and reviewed award records only; never a facility master", + "blockers": ["Keep funding context separate from approvals; do not link by name without a reviewed company identifier and award relationship."] + }, + { + "source_id": "ie.fsai.enforcement-orders", + "jurisdiction_scope": "Ireland; FSAI enforcement-order notices and aggregate official-control enforcement context", + "legacy_paths": [], + "url": "https://www.fsai.ie/news-and-alerts/latest-news/fourteen-enforcement-orders-served-on-food-bus-%281%29", + "access_method": "official notice/news route; bounded notice capture after scope review", + "cadence": "unknown; notice publication cadence is not a source-health guarantee", + "attribution_licensing_notes": "FSAI notice content is enforcement context; person/business names and allegations require legal, privacy, and factual review before any linkage", + "adapter_status": "not_started", + "expected_artifact_schema": "Notice/order event, date, authority, order type, and reviewed subject reference; no inferred facility master", + "blockers": ["Do not infer absence of an order from absence of a notice; preserve event scope and link only after reviewable identity resolution."] + }, + { + "source_id": "ie.dafm.animal-welfare-controls", + "jurisdiction_scope": "Ireland; DAFM official-veterinary and animal-welfare control context at approved slaughter plants", + "legacy_paths": [], + "url": "https://www.gov.ie/en/department-of-agriculture-food-and-the-marine/press-releases/statement-on-garda-investigation-into-alleged-offences-of-deception/", + "access_method": "official statement and annual-report routes; bounded contextual capture", + "cadence": "unknown; no row-level public welfare feed verified", + "attribution_licensing_notes": "DAFM statement describes control responsibilities; use only as authority/scope evidence unless a specific current inspection artifact is lawfully obtained and reviewed", + "adapter_status": "not_started", + "expected_artifact_schema": "Contextual control statement or separately identified inspection/enforcement event; not a facility census", + "blockers": ["Current row-level facility-linked welfare/enforcement observations were not captured; do not convert qualitative controls or aggregate reports into facility claims."] + } ] } diff --git a/pipeline/tests/test_ireland_recon_metadata.py b/pipeline/tests/test_ireland_recon_metadata.py new file mode 100644 index 0000000..9a21aee --- /dev/null +++ b/pipeline/tests/test_ireland_recon_metadata.py @@ -0,0 +1,93 @@ +import hashlib +import json +import re +import unittest +from pathlib import Path + + +ROOT = Path(__file__).parents[2] +REGISTRY = ROOT / "pipeline/source_registry.json" +OPERATIONS = ROOT / "pipeline/source_operations.json" +STATUS = ROOT / "docs/source-status.json" +CROSSWALK = ROOT / "docs/countries/ireland/v1-field-crosswalk.json" +MANIFEST = ROOT / "data/manifests/ireland-source-artifacts.json" + + +class IrelandReconMetadataTests(unittest.TestCase): + def setUp(self): + self.registry = json.loads(REGISTRY.read_text(encoding="utf-8")) + self.operations = json.loads(OPERATIONS.read_text(encoding="utf-8")) + self.status = json.loads(STATUS.read_text(encoding="utf-8")) + self.crosswalk = json.loads(CROSSWALK.read_text(encoding="utf-8")) + self.manifest = json.loads(MANIFEST.read_text(encoding="utf-8")) + + def test_ireland_source_inventory_is_consistent(self): + expected = { + "ie.fsai.approved-directory", + "ie.dafm.approved-establishments", + "ie.dafm.former-la-establishments", + "ie.dafm.milk-dairy-establishments", + "ie.hse.low-throughput-meat", + "ie.sfpa.approved-establishments", + "ie.sfpa.freezer-vessels", + "ie.sfpa.factory-vessels", + "ie.epa.leap", + "ie.planning.npad", + "ie.cro.companies", + "ie.cso.livestock-slaughterings", + "ie.dafm.national-beef-kill", + "ie.dafm.seafood-processing-funding", + "ie.fsai.enforcement-orders", + "ie.dafm.animal-welfare-controls", + } + registry_ids = {item["source_id"] for item in self.registry["sources"]} + schedule_ids = {item["source_id"] for item in self.operations["schedules"]} + status_ids = {item["source_id"] for item in self.status["sources"]} + crosswalk_ids = {item["source_id"] for item in self.crosswalk["sources"]} + self.assertEqual(expected, registry_ids & expected) + self.assertEqual(expected, schedule_ids & expected) + self.assertEqual(expected, status_ids & expected) + self.assertEqual(expected, crosswalk_ids) + + def test_observed_counts_are_semantically_bounded(self): + observed = { + item["source_id"]: item.get("observed_counts", {}) + for item in self.crosswalk["sources"] + } + self.assertEqual(observed["ie.hse.low-throughput-meat"]["approval_number_nodes"], 72) + self.assertEqual(observed["ie.sfpa.approved-establishments"]["page_entries"], 184) + self.assertEqual(observed["ie.sfpa.freezer-vessels"]["page_entries"], 50) + self.assertEqual(observed["ie.sfpa.factory-vessels"]["page_entries"], 1) + self.assertIsNone(observed["ie.dafm.approved-establishments"]["row_count"]) + self.assertIsNone(observed["ie.cso.livestock-slaughterings"]["facility_count"]) + + def test_manifest_hashes_and_retention_are_explicit(self): + digest = re.compile(r"^[0-9a-f]{64}$") + self.assertEqual(self.manifest["publication_state"], "blocked; no release artifact created") + for artifact in self.manifest["artifacts"]: + self.assertIsNone(artifact["source_artifact_sha256"]) + snapshot_hash = artifact["snapshot_sha256"] + if snapshot_hash is not None: + self.assertRegex(snapshot_hash, digest) + self.assertGreater(artifact["snapshot_characters"], 0) + self.assertNotIn("raw/", artifact["official_url"]) + + def test_crosswalk_requires_activity_children_for_repeated_sources(self): + by_id = {item["source_id"]: item for item in self.crosswalk["sources"]} + for source_id in ( + "ie.hse.low-throughput-meat", + "ie.sfpa.approved-establishments", + ): + source = by_id[source_id] + self.assertIn("activity_model", source) + self.assertEqual(source["activity_model"]["child_key"], "activity_observation_key") + self.assertIn("source_key", self.crosswalk["canonical_entity_rules"]) + + def test_metadata_files_have_stable_content_digests_for_change_detection(self): + for path in (CROSSWALK, MANIFEST): + digest = hashlib.sha256(path.read_bytes()).hexdigest() + self.assertRegex(digest, r"^[0-9a-f]{64}$") + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/tests/test_source_registry.py b/pipeline/tests/test_source_registry.py index 07d39fc..f9ae10b 100644 --- a/pipeline/tests/test_source_registry.py +++ b/pipeline/tests/test_source_registry.py @@ -9,8 +9,8 @@ class SourceRegistryTests(unittest.TestCase): def test_repository_registry_loads_and_references_existing_legacy_paths(self): registry = load_registry() - self.assertEqual(len(registry["sources"]), 53) - self.assertEqual(len({source["source_id"] for source in registry["sources"]}), 53) + self.assertEqual(len(registry["sources"]), 69) + self.assertEqual(len({source["source_id"] for source in registry["sources"]}), 69) def test_unknowns_are_explicit(self): registry = json.loads(REGISTRY_PATH.read_text(encoding="utf-8")) From 74c5665812b54be00b35b9996784e73527e91a26 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Wed, 16 Sep 2026 08:52:59 -0700 Subject: [PATCH 139/311] Add Poland source reconnaissance --- data/manifests/pl-source-artifacts.json | 23 ++++ .../raw/poland/20260916T000000Z/metadata.json | 23 ++++ docs/countries/pl/source-crosswalk.json | 30 ++++ docs/country-recon-pl.md | 130 ++++++++++++++++++ docs/source-status.json | 14 +- docs/source-status.md | 4 + pipeline/poland/__init__.py | 1 + pipeline/poland/test_metadata.py | 45 ++++++ pipeline/source_registry.json | 14 +- 9 files changed, 282 insertions(+), 2 deletions(-) create mode 100644 data/manifests/pl-source-artifacts.json create mode 100644 data/raw/poland/20260916T000000Z/metadata.json create mode 100644 docs/countries/pl/source-crosswalk.json create mode 100644 docs/country-recon-pl.md create mode 100644 pipeline/poland/__init__.py create mode 100644 pipeline/poland/test_metadata.py diff --git a/data/manifests/pl-source-artifacts.json b/data/manifests/pl-source-artifacts.json new file mode 100644 index 0000000..3e09fbd --- /dev/null +++ b/data/manifests/pl-source-artifacts.json @@ -0,0 +1,23 @@ +{ + "schema_version": "1.0", + "country": "PL", + "purpose": "Private bounded reconnaissance manifest. No raw source rows, addresses, coordinates or personal contact data are committed or retained by this run.", + "retrieval_as_of_utc": "2026-09-16T15:49:07Z", + "publication_state": "blocked; private reconnaissance only", + "network_observation": "Shell HTTP requests were refused. First-party pages were verified through the browser/web route. The GIW XLS route was observed but its browser navigation timed out; no source XLS body was retained.", + "artifacts": [ + {"source_id":"pl.private.recon-metadata","path":"data/raw/poland/20260916T000000Z/metadata.json","url":null,"retrieved_at_utc":"2026-09-16T15:49:07Z","capture_status":"tracked_metadata_only","http":"not_applicable","bytes":4951,"sha256":"f8ac4988fa84a2c3f5f57d1cac1dd461ebef21c5d01f283661e8ead2d7fc3405","rows":null,"schema":"JSON route/count/privacy/provenance metadata; contains no source row payload"}, + {"source_id":"pl.giw.approved-food","path":null,"url":"https://www.wetgiw.gov.pl/handel-eksport-import/listy-zakladow","retrieved_at_utc":"2026-09-16T15:49:07Z","capture_status":"rendered_metadata_only","http":"page observed; export route observed; direct source bytes not retained","bytes":null,"sha256":null,"rows":{"section_0":279,"section_I":1050,"section_II":559,"section_III":20,"section_IV":45,"section_V":687,"section_VI":899,"section_VII":0,"section_VIII":265,"section_IX":678,"section_X":407,"section_XI":8,"section_XII":263,"section_XIII":219,"section_XIV":10,"section_XV":15,"section_XVI":3,"combined_meat":1686,"adapted_853":49},"schema":"Rendered table fields: LP, WNI, Nazwa, Adres, kat., gatunki, powiązana działalność, uwagi, Regulation 2023/594 art.44"}, + {"source_id":"pl.giw.registered-food","path":null,"url":"https://www.wetgiw.gov.pl/nadzor-weterynaryjny/wykaz-zakladow-rejestrowanych","retrieved_at_utc":"2026-09-16T15:49:07Z","capture_status":"route_only","http":"page observed; no list body retained","bytes":null,"sha256":null,"rows":14,"schema":"Fourteen list families; list-specific schemas unresolved"}, + {"source_id":"pl.giw.abp","path":null,"url":"https://www.wetgiw.gov.pl/handel-eksport-import/niespozywcze-produkty-pochodzenia-zwierzecego","retrieved_at_utc":"2026-09-16T15:49:07Z","capture_status":"route_only","http":"GIW page and linked pasze route observed","bytes":null,"sha256":null,"rows":null,"schema":"ABP approval/registration list; exact fields unresolved"}, + {"source_id":"pl.giw.rrw","path":null,"url":"https://www.wetgiw.gov.pl/publikacje/rrw-sprawozdawczosc-statystyczna/printpage","retrieved_at_utc":"2026-09-16T15:49:07Z","capture_status":"route_only","http":"publication/index observed","bytes":null,"sha256":null,"rows":null,"schema":"RRW-3, RRW-5, RRW-6 report/table families"}, + {"source_id":"pl.gus.slaughter","path":null,"url":"https://stat.gov.pl/obszary-tematyczne/rolnictwo-lesnictwo/produkcja-zwierzeca-zwierzeta-gospodarskie/uboje-zwierzat-gospodarskich-w-2025-r-,16,1.html","retrieved_at_utc":"2026-09-16T15:49:07Z","capture_status":"route_only","http":"publication page observed","bytes":null,"sha256":null,"rows":null,"schema":"XLSX aggregate tables from R-09U; species, slaughter measure, live/slaughter weight, period"}, + {"source_id":"pl.gios.ippc","path":null,"url":"https://www.gov.pl/web/gios/instalacje-wymagajace-uzyskania-pozwolenia-zintegrowanego","retrieved_at_utc":"2026-09-16T15:49:07Z","capture_status":"route_only","http":"central index observed; 16 regional links","bytes":null,"sha256":null,"rows":16,"schema":"Regional IPPC installation/permit lists; formats vary"}, + {"source_id":"pl.gdos.eia","path":null,"url":"https://www.gov.pl/web/gdos/bazy-danych-o-ocenach-oddzialywania-na-srodowisko","retrieved_at_utc":"2026-09-16T15:49:07Z","capture_status":"access_blocked_for_reproducible_capture","http":"source states access from outside Poland is blocked","bytes":null,"sha256":null,"rows":null,"schema":"EIA proceedings/documents; no facility master"}, + {"source_id":"pl.geoportal.urban-planning","path":null,"url":"https://www.geoportal.gov.pl/aktualnosci/nowe-uslugi-w-geoportalu-rejestr-urbanistyczny/","retrieved_at_utc":"2026-09-16T15:49:07Z","capture_status":"route_only","http":"announcement and published register URL observed","bytes":null,"sha256":null,"rows":null,"schema":"WMS/GML planning acts and geometries"}, + {"source_id":"pl.arimr.processing-support","path":null,"url":"https://www.gov.pl/web/arimr/startuje-wsparcie-dla-przetworcow","retrieved_at_utc":"2026-09-16T15:49:07Z","capture_status":"route_only","http":"current call page observed","bytes":null,"sha256":null,"rows":null,"schema":"Call/application/beneficiary/project evidence; not facility authorization"}, + {"source_id":"pl.krs.open-api","path":null,"url":"https://prs.ms.gov.pl/krs/openApi","retrieved_at_utc":"2026-09-16T15:49:07Z","capture_status":"route_only","http":"open API announcement and endpoint observed; no API request made","bytes":null,"sha256":null,"rows":null,"schema":"KRS legal-entity response; exact API schema unresolved"}, + {"source_id":"pl.gus.regon-bir","path":null,"url":"https://api.stat.gov.pl/Home/RegonApi?lang=en","retrieved_at_utc":"2026-09-16T15:49:07Z","capture_status":"route_only","http":"BIR1 documentation observed; no authorized query made","bytes":null,"sha256":null,"rows":null,"schema":"SOAP/API lookup by REGON, NIP or KRS"}, + {"source_id":"eu.traces.pl-approved-food","path":null,"url":"https://food.ec.europa.eu/food-safety/biological-safety/food-hygiene/approved-eu-food-establishments_en","retrieved_at_utc":"2026-09-16T15:49:07Z","capture_status":"route_only","http":"EU publication page observed","bytes":null,"sha256":null,"rows":null,"schema":"TRACES approved-establishment mirror; Poland national authority remains GIW"} + ] +} diff --git a/data/raw/poland/20260916T000000Z/metadata.json b/data/raw/poland/20260916T000000Z/metadata.json new file mode 100644 index 0000000..517d296 --- /dev/null +++ b/data/raw/poland/20260916T000000Z/metadata.json @@ -0,0 +1,23 @@ +{ + "schema_version": "poland-private-recon-metadata-v1", + "country": "PL", + "retrieved_at_utc": "2026-09-16T15:49:07Z", + "capture_scope": "route and rendered-page metadata only; no source row payload retained", + "acquisition_note": "Shell HTTP was refused. First-party GIW pages and counts were verified through the browser/web route. The GIW XLS export route was observed but did not complete as a bounded local download.", + "source_observations": [ + {"source_id":"pl.giw.approved-food","route":"https://www.wetgiw.gov.pl/handel-eksport-import/listy-zakladow","observed_sections":{"0":279,"I":1050,"II":559,"III":20,"IV":45,"V":687,"VI":899,"VII":0,"VIII":265,"IX":678,"X":407,"XI":8,"XII":263,"XIII":219,"XIV":10,"XV":15,"XVI":3,"meat_combined":1686,"adapted_853":49},"row_identity":"WNI (Weterynaryjny Numer Identyfikacyjny) in rendered register; activity/species/category fields are one-to-many observations","schema_fields":["LP","WNI","Nazwa","Adres","kat.","gatunki","powiązana działalność","uwagi","Rozp. KE 2023/594 – art. 44"],"capture_status":"metadata_only"}, + {"source_id":"pl.giw.registered-food","route":"https://www.wetgiw.gov.pl/nadzor-weterynaryjny/wykaz-zakladow-rejestrowanych","observed_lists":14,"row_identity":"list-specific source identity not yet verified","schema_fields":["list-specific identifier","name","address","activity","status/date fields unknown"],"capture_status":"route_only"}, + {"source_id":"pl.giw.abp","route":"https://www.wetgiw.gov.pl/handel-eksport-import/niespozywcze-produkty-pochodzenia-zwierzecego","observed_polish_route":"https://pasze.wetgiw.gov.pl/uppz1/demo/index.php?l=en","row_identity":"ABP approval/registration number if exposed; exact export schema unresolved","capture_status":"route_only"}, + {"source_id":"pl.giw.rrw","route":"https://www.wetgiw.gov.pl/publikacje/rrw-sprawozdawczosc-statystyczna/printpage","observed_reports":["RRW-3","RRW-5","RRW-6"],"row_identity":"report/table/effective-year; not a facility master","capture_status":"route_only"}, + {"source_id":"pl.gus.slaughter","route":"https://stat.gov.pl/obszary-tematyczne/rolnictwo-lesnictwo/produkcja-zwierzeca-zwierzeta-gospodarskie/uboje-zwierzat-gospodarskich-w-ubojniach-i-rzezniach-w-2025-r-,16,1.html","observed_reference_period":"2025","observed_frequency":"monthly","row_identity":"aggregate species/measure/period; all slaughterhouses in reporting obligation, no facility rows retained","capture_status":"route_only"}, + {"source_id":"pl.gios.ippc","route":"https://www.gov.pl/web/gios/instalacje-wymagajace-uzyskania-pozwolenia-zintegrowanego","observed_regional_links":16,"row_identity":"regional installation/permit identifiers; no national facility key verified","capture_status":"route_only"}, + {"source_id":"pl.gdos.eia","route":"https://www.gov.pl/web/gdos/bazy-danych-o-ocenach-oddzialywania-na-srodowisko","observed_access":"public BIP search; page states access from outside Poland is blocked","row_identity":"proceeding/document identifiers; no facility master","capture_status":"route_only"}, + {"source_id":"pl.geoportal.urban-planning","route":"https://www.geoportal.gov.pl/aktualnosci/nowe-uslugi-w-geoportalu-rejestr-urbanistyczny/","observed_layers":["general municipal plans","local spatial development plans","landscape resolutions","voivodeship spatial plans","landscape audits"],"row_identity":"planning act/spatial dataset identifiers","capture_status":"route_only"}, + {"source_id":"pl.arimr.processing-support","route":"https://www.gov.pl/web/arimr/startuje-wsparcie-dla-przetworcow","observed_current_window":"2026-09-01 through 2026-09-30","row_identity":"application/beneficiary/project identifiers; not facility authorization","capture_status":"route_only"}, + {"source_id":"pl.krs.open-api","route":"https://prs.ms.gov.pl/krs/openApi","observed_access":"open API announced by Ministry of Justice; endpoint documentation/access not exercised","row_identity":"KRS number","capture_status":"route_only"}, + {"source_id":"pl.gus.regon-bir","route":"https://api.stat.gov.pl/Home/RegonApi?lang=en","observed_access":"BIR1 API; registration/user key required for service use","row_identity":"REGON/NIP/KRS","capture_status":"route_only"}, + {"source_id":"eu.traces.pl-approved-food","route":"https://food.ec.europa.eu/food-safety/biological-safety/food-hygiene/approved-eu-food-establishments_en","observed_role":"EU mirror/publication surface with national competent-authority responsibility","row_identity":"TRACES establishment identifier if exported; do not union with GIW","capture_status":"route_only"} + ], + "privacy": {"raw_source_rows_retained":false,"addresses_retained":false,"coordinates_retained":false,"personal_contacts_retained":false}, + "provenance": {"primary_source_citations_in":"docs/country-recon-pl.md","machine_crosswalk":"docs/countries/pl/source-crosswalk.json","tracked_manifest":"data/manifests/pl-source-artifacts.json"} +} diff --git a/docs/countries/pl/source-crosswalk.json b/docs/countries/pl/source-crosswalk.json new file mode 100644 index 0000000..39fa6a3 --- /dev/null +++ b/docs/countries/pl/source-crosswalk.json @@ -0,0 +1,30 @@ +{ + "schema_version": "poland-source-crosswalk-v1", + "country": "PL", + "checked_at_utc": "2026-09-16T15:49:07Z", + "publication_state": "private-reconnaissance-only", + "legacy": {"rows_found": 0,"paths_checked":["static_data","Old CSVs","dirty-datasets"],"interpretation":"No Poland V1 snapshot was found. This is not evidence that Poland has no facilities."}, + "identity_policy": "WNI, approval number, permit number, proceeding ID, KRS, REGON and statistical keys remain source-local. Exact cross-source joins require an explicit reviewed identity link; names, addresses, coordinates, activity rows, products, species and statistics never silently merge or inflate facility counts.", + "sources": [ + {"source_id":"pl.giw.approved-food","authority":"Główny Inspektorat Weterynarii","role":"national approved food-establishment register under Regulation (EC) 853/2004","primary_url":"https://www.wetgiw.gov.pl/handel-eksport-import/listy-zakladow","access":"rendered HTML register with filters and XLS export route; XLS download timed out in browser and shell HTTP was refused","format":"HTML table plus XLS route","cadence":"unknown; current page observed 2026-09-16","primary_key_candidates":["WNI"],"identity_fields":["WNI","Nazwa","Adres","kat.","gatunki","powiązana działalność","uwagi"],"coverage":"Sections 0–XVI plus combined meat view; current rendered counts recorded in manifest","map_use":"candidate site only after privacy/precision review; no geocoding in reconnaissance","graph_edges":["approved_under_853_2004","operator_name_as_source_value","activity","species","product","regulatory_note"],"privacy_fields":["address","contact details if present","sole-trader names"],"overlap_group":"giw-food-approval","readiness":"reference_only","blockers":["Acquire bounded XLS/HTML artifact with headers and hashes","resolve WNI lifecycle and repeat-row semantics","review terms, privacy, category codes and release approval"]}, + {"source_id":"pl.giw.registered-food","authority":"Główny Inspektorat Weterynarii / Inspekcja Weterynaryjna","role":"registered animal-origin food-sector lists","primary_url":"https://www.wetgiw.gov.pl/nadzor-weterynaryjny/wykaz-zakladow-rejestrowanych","access":"official index links to 14 list families; list-specific exports/routes unresolved","format":"list-specific HTML/downloads; schema varies","cadence":"unknown; current index observed 2026-09-16","primary_key_candidates":["list-specific veterinary identifier"],"identity_fields":["list-specific identifier","name","address","activity","status/date fields"],"coverage":"storage, honey, game collection, transport, trade/intermediary, vessels, eggs, hunting districts, composite food, snails, egg farms, own-use slaughter, marginal/local/limited, retail and direct sale","map_use":"separate labeled evidence layers, not a single approved-facility total","graph_edges":["registered_activity","transport","collection_point","egg_farm","own_use_slaughter"],"privacy_fields":["address","sole-trader/operator names"],"overlap_group":"giw-registered","readiness":"reference_only","blockers":["Capture list-specific schemas and terms","do not union registration with 853/2004 approval"]}, + {"source_id":"pl.giw.abp","authority":"Główny Inspektorat Weterynarii / Inspekcja Weterynaryjna","role":"animal by-product approved/registered establishment evidence","primary_url":"https://www.wetgiw.gov.pl/handel-eksport-import/niespozywcze-produkty-pochodzenia-zwierzecego","access":"official GIW page links to a separate pasze.wetgiw.gov.pl register; public route observed, export contract unresolved","format":"web register; possible TRACES-aligned list","cadence":"unknown","primary_key_candidates":["ABP approval/registration number"],"identity_fields":["approval/registration number","operator","address","ABP activity/category","status/date"],"coverage":"plants/operators handling animal by-products and derived products; legally distinct from food approvals","map_use":"permit/evidence overlay after precision/privacy review","graph_edges":["abp_approval","category_1_2_3_activity","derived_product","operator"],"privacy_fields":["address","operator identity where personal"],"overlap_group":"abp","readiness":"reference_only","blockers":["Verify current downloadable/API route and list format","preserve Regulation (EC) 1069/2009 scope and do not add to food counts"]}, + {"source_id":"pl.giw.rrw","authority":"Główny Inspektorat Weterynarii","role":"veterinary inspection statistics, welfare, official controls and meat examination","primary_url":"https://www.wetgiw.gov.pl/publikacje/rrw-sprawozdawczosc-statystyczna/printpage","access":"official publication/index route; current files not acquired","format":"RRW reports/tables","cadence":"annual reporting cycle; exact release cadence varies","primary_key_candidates":["report","table","year"],"identity_fields":["report year","RRW-3/RRW-5/RRW-6 section","activity","control/nonconformity/outcome"],"coverage":"RRW-3 includes animal welfare and controls/enforcement; RRW-5 covers establishments producing animal-origin products; RRW-6 covers ante/post-mortem examination and unfit meat","map_use":"aggregate/accountability context; only facility-linked when a source-local ID is present","graph_edges":["inspection","nonconformity","administrative_or_criminal_proceeding","welfare_control","meat_exam_result"],"privacy_fields":["case details","small-cell or named operator information"],"overlap_group":"giw-inspection","readiness":"reference_only","blockers":["Acquire current report artifacts and schema","keep aggregate reports separate from facility rows and distinguish allegation/outcome"]}, + {"source_id":"pl.gus.slaughter","authority":"Statistics Poland (GUS)","role":"national slaughter statistics","primary_url":"https://stat.gov.pl/obszary-tematyczne/rolnictwo-lesnictwo/produkcja-zwierzeca-zwierzeta-gospodarskie/uboje-zwierzat-gospodarskich-w-ubojniach-i-rzezniach-w-2025-r-,16,1.html","access":"official publication with XLSX table; current page observed","format":"XLSX and publication HTML","cadence":"monthly collection; 2025 publication dated 2026-03-02","primary_key_candidates":["species","measure","period"],"identity_fields":["species","total/ritual slaughter","live/slaughter weight","month/year"],"coverage":"reporting obligation covers all slaughterhouses conducting livestock slaughter in Poland; published product is aggregate","map_use":"no facility points; country/species/time context only","graph_edges":["aggregate_slaughter","species","period"],"privacy_fields":["small-cell/statistical disclosure flags if supplied"],"overlap_group":"statistics","readiness":"reference_only","blockers":["Do not infer named facilities or join to GIW by totals","preserve revisions and statistical flags"]}, + {"source_id":"pl.gios.ippc","authority":"Główny Inspektorat Ochrony Środowiska and 16 WIOŚ","role":"integrated-permit installation index","primary_url":"https://www.gov.pl/web/gios/instalacje-wymagajace-uzyskania-pozwolenia-zintegrowanego","access":"central official page links to 16 regional WIOŚ registers; no national export verified","format":"regional HTML/PDF/XLSX varies","cadence":"regional/unknown","primary_key_candidates":["permit number","installation identifier"],"identity_fields":["installation","operator","address","industry","permit authority","permit date/status"],"coverage":"installations requiring integrated permits under Polish and EU IED rules; broader than animal agriculture","map_use":"environmental permit overlay; source geometry/precision review required","graph_edges":["integrated_permit","installation","operator","emission_or_monitoring_condition"],"privacy_fields":["operator/address where mixed or personal"],"overlap_group":"environment","readiness":"reference_only","blockers":["Capture regional schemas and version dates","link permits to GIW WNI only by reviewed evidence"]}, + {"source_id":"pl.gdos.eia","authority":"General Directorate for Environmental Protection (GDOŚ)","role":"environmental-impact-assessment proceedings and decisions","primary_url":"https://www.gov.pl/web/gdos/bazy-danych-o-ocenach-oddzialywania-na-srodowisko","access":"public BIP database/search; source states access from outside Poland is blocked","format":"web database and linked proceedings/documents","cadence":"authorities should enter information within 30 days; current public release cadence not independently verified","primary_key_candidates":["proceeding/document ID"],"identity_fields":["proceeding","decision/document","authority","project","dates","status"],"coverage":"strategic EIA, project EIA/re-assessment and Natura 2000 proceedings","map_use":"planning/accountability evidence only; not a facility master","graph_edges":["eia_proceeding","decision","public_participation","authority","project"],"privacy_fields":["applicant/address/document text"],"overlap_group":"environment-planning","readiness":"reference_only","blockers":["Access restriction blocked reproducible acquisition here","review document-level privacy and completeness before use"]}, + {"source_id":"pl.geoportal.urban-planning","authority":"GUGiK/Geoportal and Ministry of Development and Technology Urban Register","role":"spatial planning and zoning context","primary_url":"https://www.geoportal.gov.pl/aktualnosci/nowe-uslugi-w-geoportalu-rejestr-urbanistyczny/","access":"public WMS/portal and published Urban Register; service/API contract requires separate capture","format":"WMS/GML/spatial datasets","cadence":"regular updates by local-government units; legacy services transition by end of September 2026","primary_key_candidates":["planning act/document ID","geometry ID"],"identity_fields":["plan/act","municipality","geometry","legal status","publication/effective date"],"coverage":"general municipal plans, local spatial development plans, landscape resolutions, voivodeship plans and landscape audits","map_use":"zoning context around reviewed facility points; never imply zoning proves operation","graph_edges":["zoning","planning_act","municipality","effective_date"],"privacy_fields":["parcel-level geometry where sensitive"],"overlap_group":"environment-planning","readiness":"reference_only","blockers":["Capture production service metadata and terms","preserve legal/effective dates and avoid geometry-only interpretation"]}, + {"source_id":"pl.arimr.processing-support","authority":"Agency for Restructuring and Modernisation of Agriculture (ARiMR)","role":"agricultural processing investment support and funding context","primary_url":"https://www.gov.pl/web/arimr/startuje-wsparcie-dla-przetworcow","access":"official call/public guidance; beneficiary/project-level dataset not acquired","format":"HTML/PDF/call lists; schema varies","cadence":"call-specific; current 2026 call window observed 2026-09-01 to 2026-09-30","primary_key_candidates":["application","agreement","beneficiary/project ID"],"identity_fields":["beneficiary","project","measure","amount","dates","voivodeship"],"coverage":"processing and marketing investment support under PS WPR; not an operating or approval register","map_use":"funding/accountability edge after privacy and beneficiary handling review","graph_edges":["funded_project","beneficiary","processing_investment","public_support"],"privacy_fields":["sole-trader beneficiary","project address"],"overlap_group":"funding","readiness":"reference_only","blockers":["Acquire permitted award/project data only if necessary","do not treat funding as proof of facility activity or compliance"]}, + {"source_id":"pl.krs.open-api","authority":"Ministry of Justice","role":"corporate identity and court-register crosswalk","primary_url":"https://prs.ms.gov.pl/krs/openApi","access":"open API announced by Ministry; exact schema/auth/rate route not exercised","format":"API; current/full KRS information subject to RODO limits","cadence":"on-demand; registry changes as filed","primary_key_candidates":["KRS number"],"identity_fields":["KRS","legal name","status","registered office","representation","NIP/REGON where present"],"coverage":"entities entered in KRS; not all operators and not facility proof","map_use":"no direct map use; exact organization-link evidence only","graph_edges":["legal_entity","registered_office","representation","status_event"],"privacy_fields":["registered-office overlap with residence","representative personal data"],"overlap_group":"corporate-identity","readiness":"reference_only","blockers":["Confirm API documentation, terms and RODO filtering","never infer operating site, ownership/control or beneficial ownership from KRS alone"]}, + {"source_id":"pl.gus.regon-bir","authority":"Statistics Poland (GUS)","role":"REGON/NIP/KRS identity lookup","primary_url":"https://api.stat.gov.pl/Home/RegonApi?lang=en","access":"BIR1 web service; registration/user key and usage limits documented","format":"SOAP/API responses","cadence":"continuously updated register; query/on-demand","primary_key_candidates":["REGON","NIP","KRS"],"identity_fields":["REGON","NIP","KRS","name","address/activity fields by authorization"],"coverage":"national-economy entities; not a facility or approval register","map_use":"no direct map use; exact identity crosswalk only","graph_edges":["legal_entity","tax_identifier","regon_identifier","activity_code"],"privacy_fields":["sole trader names/address","restricted fields"],"overlap_group":"corporate-identity","readiness":"reference_only","blockers":["Obtain authorized key or operator capture","respect documented 3-per-second and hourly limits; keep identity data restricted"]}, + {"source_id":"eu.traces.pl-approved-food","authority":"European Commission DG SANTE / TRACES-IMSOC","role":"EU mirror/publication surface for approved establishments","primary_url":"https://food.ec.europa.eu/food-safety/biological-safety/food-hygiene/approved-eu-food-establishments_en","access":"public page links to TRACES/national lists; Poland national source remains GIW authority","format":"TRACES publication/search/export; exact Poland export not verified","cadence":"competent-authority updates; timestamp each access","primary_key_candidates":["TRACES establishment ID","national approval/WNI"],"identity_fields":["country","establishment","approval number","activity/sector","status/date"],"coverage":"EU approved-food mirror; not a second population to add to GIW","map_use":"cross-check/lineage only","graph_edges":["eu_approval_mirror","national_authority","activity"],"privacy_fields":["address/operator fields if exposed"],"overlap_group":"giw-food-approval","readiness":"reference_only","blockers":["Verify Poland-specific export/API and stable identifiers","keep mirror provenance separate and never double-count"]} + ], + "facility_count_rules": [ + "Count a candidate facility only from a source-local facility/approval identifier after source-specific deduplication.", + "Preserve each GIW activity/species/category/product row as an observation keyed to WNI; do not sum section counts.", + "GUS, RRW aggregates, ARiMR funding, EIA/planning records and environmental permits are claims/events/overlays, not facility rows.", + "A missing row is not closure; lifecycle requires source status/date evidence.", + "Addresses and coordinates remain restricted until privacy and precision review; no geocoding was performed." + ], + "next_country_recommendation": {"country":"IE","reason":"FSAI currently exposes a coordinated approved-food landing page with DAFM, HSE and SFPA authority lists, and the HSE page provides a refreshed, structured approval-number/trading-name/address/business-type/activity/species view. This is a strong multi-authority adapter rehearsal, subject to the same privacy, terms and no-double-counting gates.","sources":["https://www.fsai.ie/enforcement-and-legislation/official-controls/mancp/approved-food-premises","https://oapi.fsai.ie/HSEApprovedEstablishments.aspx"]} +} diff --git a/docs/country-recon-pl.md b/docs/country-recon-pl.md new file mode 100644 index 0000000..4b6a73a --- /dev/null +++ b/docs/country-recon-pl.md @@ -0,0 +1,130 @@ +# Poland source reconnaissance + +Status: private research and integration planning only. Checked 2026-09-16. No release, deployment, public map, publication approval, or claim of national completeness is implied. + +## Decision + +Poland is a strong implementation candidate for the animal-products lane. The Chief Veterinary Inspectorate (GIW) exposes a national approved-establishment register with a stable-looking veterinary identifier (`WNI`), section filters, species/category/activity fields, and an XLS export route. The rendered current view is unusually useful for a source-local facility candidate layer, but it is not a clean one-row-per-facility data product: the same WNI can occur across sections and the displayed activity/species/product values are observations attached to that source identity. + +Integration difficulty is medium-high. The food register is accessible in HTML but its XLS export could not be captured in this environment; registered-food and animal-by-product lists are separate families; inspection statistics are report-shaped; environmental permits are regional; the EIA database states that access from outside Poland is blocked; and KRS/REGON are organization-identity services rather than facility masters. Keep all publication blocked until terms, privacy, schema drift, identity links, and review gates are complete. + +The strongest next-country recommendation is Ireland. The [Food Safety Authority of Ireland approved-premises page](https://www.fsai.ie/enforcement-and-legislation/official-controls/mancp/approved-food-premises) coordinates DAFM, HSE, and SFPA lists, while the [HSE approved-establishments view](https://oapi.fsai.ie/HSEApprovedEstablishments.aspx) currently exposes a refreshed structured view with approval number, trading name, address, business type, activity, and species. That is a good multi-authority adapter rehearsal after Poland, not a claim that Ireland is complete or publication-ready. + +## Primary source inventory + +| Source | Authority / class | Access and current evidence | Identity and scope | Integration decision | +|---|---|---|---|---| +| [GIW approved establishments](https://www.wetgiw.gov.pl/handel-eksport-import/listy-zakladow) and [GIW register UI](https://zywnosc.wetgiw.gov.pl/spi/zatw/index.php?sekcja=2&lng=0) | Polish official, Chief Veterinary Inspectorate | Rendered HTML with filters and an XLS route; current views observed 2026-09-16; direct source bytes were not retained because shell HTTP was refused and the browser export timed out | `WNI`, name, address, category, species, linked activity, notes; Sections 0–XVI and combined meat view | First adapter target. Preserve source observations; count facilities only after source-local WNI handling | +| [GIW registered establishments](https://www.wetgiw.gov.pl/nadzor-weterynaryjny/wykaz-zakladow-rejestrowanych) | Polish official | Index observed with 14 list families and separate links; list-specific schemas not acquired | Registered food-sector activities including game collection, transport, egg farms, own-use slaughter, marginal/local/limited and retail | Separate adapters/list families; never union registration rows with 853/2004 approvals | +| [GIW animal by-products](https://www.wetgiw.gov.pl/handel-eksport-import/niespozywcze-produkty-pochodzenia-zwierzecego) | Polish official / Inspekcja Weterynaryjna | GIW page and linked [ABP register](https://pasze.wetgiw.gov.pl/uppz1/demo/index.php?l=en) observed; export contract unresolved | Regulation (EC) 1069/2009 / 142/2011 ABP establishments and operators | Separate ABP evidence layer; no addition to food-facility totals | +| [GIW RRW statistical reporting](https://www.wetgiw.gov.pl/publikacje/rrw-sprawozdawczosc-statystyczna/printpage) | Polish official | Current publication/index route observed | RRW-3: welfare, controls and enforcement; RRW-5: animal-origin product establishments; RRW-6: official animal/meat examination | Dated reports/events and aggregates, not a facility master | +| [GUS slaughter 2025](https://stat.gov.pl/obszary-tematyczne/rolnictwo-lesnictwo/produkcja-zwierzeca-zwierzeta-gospodarskie/uboje-zwierzat-gospodarskich-w-ubojniach-i-rzezniach-w-2025-r-,16,1.html) | Polish official, Statistics Poland | Current publication dated 2026-03-02; XLSX attachment; monthly collection | Aggregate slaughter counts and live/slaughter weight from R-09U; reporting obligation covers all slaughterhouses conducting livestock slaughter | Statistics/context only; no named facilities or map points | +| [GIOŚ integrated permits index](https://www.gov.pl/web/gios/instalacje-wymagajace-uzyskania-pozwolenia-zintegrowanego) | Polish official, GIOŚ plus 16 WIOŚ | Central page lists 16 regional registers; regional formats and cadence vary | IPPC installations and permits under Polish/EU IED rules | Environmental permit overlay; regional acquisition required | +| [GDOŚ EIA database](https://www.gov.pl/web/gdos/bazy-danych-o-ocenach-oddzialywania-na-srodowisko) | Polish official | Public BIP database/search described; page explicitly states access from outside Poland is blocked | EIA proceedings/documents, decisions, authorities and project dates | Strong accountability evidence if access is available from Poland; no facility master | +| [Geoportal Urban Register announcement](https://www.geoportal.gov.pl/aktualnosci/nowe-uslugi-w-geoportalu-rejestr-urbanistyczny/) | Polish official, GUGiK/MRIT and local governments | Public WMS/register route; announcement says regular updates and transition by end of September 2026 | General plans, local plans, landscape resolutions, voivodeship plans, landscape audits | Planning/zoning context around reviewed sites, never proof of operation | +| [ARiMR processing support](https://www.gov.pl/web/arimr/startuje-wsparcie-dla-przetworcow) and [PS WPR](https://www.gov.pl/web/rolnictwo/plan-strategiczny-dla-wspolnej-polityki-rolnej-na-lata-2023-27) | Polish official | Current 2026 call page observed; 1–30 September 2026 application window | Funding calls, beneficiaries and projects; not operating authorizations | Separate funding edges; do not infer activity, compliance or throughput | +| [KRS open API](https://prs.ms.gov.pl/krs/openApi) | Polish official, Ministry of Justice | API announced; schema/auth/rate behavior not exercised | Legal entities, KRS number, status and registered information subject to RODO | Exact legal-entity crosswalk only; no operating-site inference | +| [REGON BIR1 API](https://api.stat.gov.pl/Home/RegonApi?lang=en) | Polish official, GUS | Documentation observed; registration/user key required; limits documented | Lookup by REGON, NIP or KRS | Identity evidence only; retain restricted fields and query provenance | +| [EU approved-food establishments](https://food.ec.europa.eu/food-safety/biological-safety/food-hygiene/approved-eu-food-establishments_en) and [TRACES](https://food.ec.europa.eu/horizontal-topics/traces/modules_en) | EU official mirror | Public publication/search surface; Poland national authority remains GIW | EU approval mirror and establishment/activity identifiers if exported | Lineage/cross-check only; never double-count GIW | + +No secondary facility compilation, commercial directory, or third-party geocoded list was used. + +## GIW route, schema and counts + +The source index lists separate GIW views for Section 0 (general scope), Sections I–XVI of Regulation (EC) 853/2004, a combined meat view (I, II, III, IV, V, VI, XII, XIII), and a separate adapted-measures view. The rendered table exposes `LP`, `WNI`, `Nazwa`, `Adres`, `kat.`, `gatunki`, `powiązana działalność`, `uwagi`, and an Article 44 field. Section I exposes category codes `SH` (slaughterhouse) and `CP` (cutting plant), species codes including bovine, goat, sheep, porcine and equine, and filter dimensions for province, WNI, name, Article 44 and TSE exceptions. + +| GIW rendered view | Current displayed rows | Interpretation | +|---|---:|---| +| Section 0 | 279 | General activity scope; not a slaughterhouse-only count | +| Section I: domestic ungulates | 1,050 | Approval-register rows keyed by WNI; slaughter/cutting categories and species observations | +| Section II: poultry and lagomorphs | 559 | Approval-register rows keyed by WNI; poultry/rabbit activity/species observations | +| Section III: farmed game | 20 | Approval-register rows | +| Section IV: wild game | 45 | Approval-register rows | +| Section V: minced meat/raw preparations/MSM | 687 | Product/activity approval rows | +| Section VI: meat products | 899 | Product/activity approval rows | +| Section VII: live bivalve molluscs | 0 | No rows in the current rendered view; absence is not a closure conclusion | +| Section VIII: fishery products | 265 | Approval-register rows | +| Section IX: raw milk/dairy | 678 | Approval-register rows | +| Section X: eggs/egg products | 407 | Approval-register rows | +| Section XI: frog legs/snails | 8 | Approval-register rows | +| Section XII: animal fats/greaves | 263 | Product/activity approval rows | +| Section XIII: processed stomachs/bladders/intestines | 219 | Product/activity approval rows | +| Section XIV: gelatin | 10 | Approval-register rows | +| Section XV: collagen | 15 | Approval-register rows | +| Section XVI: highly refined products | 3 | Approval-register rows | +| Combined meat view | 1,686 | Best bounded meat candidate view observed; still source-view rows, not a release count | +| Adapted 853/2004 measures | 49 | Separate legal/measure scope; do not merge into combined meat total | + +The section counts sum to 5,407 for Sections 0–XVI only, but that arithmetic is intentionally not a facility total. Sections overlap by WNI and by activity/product/species. The combined meat view is the preferred diagnostic for meat candidates because it is a single source view, but it still requires export-level uniqueness checks and lifecycle review. No address, coordinate, or contact payload was retained in this run. + +## Inspection, welfare and enforcement + +GIW’s RRW index states that RRW-3 includes animal welfare, controls, nonconformities and administrative/criminal proceedings; RRW-5 covers activities and sanitary condition of animal-origin product establishments; RRW-6 covers official pre- and post-mortem examination and causes of meat being deemed unfit. These reports are valuable accountability evidence, but their row grain is report/table/year or control observation, not automatically `WNI`. Keep the original report and effective year, and distinguish inspection, nonconformity, allegation, administrative action, criminal proceeding and outcome. + +The GIW [animal-origin food page](https://www.wetgiw.gov.pl/nadzor-weterynaryjny/zywnosc-pochodzenia-zwierzecego/printpage) and [registered-establishment page](https://www.wetgiw.gov.pl/nadzor-weterynaryjny/wykaz-zakladow-rejestrowanych) establish the broader official-control surface. A source disappearance or missing current row remains `not_observed`; it is never closure, compliance or non-compliance proof. + +## Slaughter statistics and funding + +GUS’s 2025 slaughter publication says the R-09U results cover all slaughterhouses and abattoirs conducting livestock slaughter in Poland, present total and ritual slaughter in head-counts and live/slaughter weight, and are collected monthly. GUS is therefore a valuable national aggregate context layer but cannot be joined to a named GIW site by total, species, or period. Preserve revision/status information in any later capture. + +ARiMR’s current processing-support notice states that applications for investment in processing and placing agricultural products on the market were accepted from 1–30 September 2026 through PUE. The PS WPR page provides the program context. Funding can support an accountability graph edge (`funded_project` → beneficiary/project) when a lawful project dataset is acquired, but it does not prove an operating facility, current approval, throughput, welfare performance, or wrongdoing. + +## Environment, permits and planning + +GIOŚ states that WIOŚ maintain the registers of installations subject to integrated permits and publishes 16 regional links. This is a real official environmental source family, but not a single national facility export; formats, identifiers and update cadence must be captured per voivodeship. Treat permit holder, installation, permit document, emission condition, monitoring and enforcement as separate observations and preserve historical holders. + +GDOŚ describes the EIA database as a statutory system containing strategic EIA, project EIA/re-assessment and Natura 2000 proceedings, sourced from the authorities conducting those proceedings. It also states that access from outside Poland is blocked. This is a concrete acquisition blocker for the current environment, not evidence that the database is empty or incomplete. + +The Geoportal announcement says the Urban Register provides plans and spatial data for general municipal plans, local plans, landscape resolutions, voivodeship plans and landscape audits, updated regularly by local-government units. Use this for zoning and planning context around a reviewed facility point. Geometry-only joins are insufficient: preserve the planning act, legal status, publication/effective date and municipality. + +## Identity, privacy and overlap policy + +- GIW `WNI` is the primary source-local key. Preserve one-to-many observations by section, category, activity, product, species and note. Deduplicate only exact WNI within a declared source view, and never sum section rows into facilities. +- GIW registered-food and ABP identities stay in separate namespaces. A shared name/address is only a review signal. +- RRW, GUS, ARiMR, EIA, IPPC and planning rows are evidence events/claims/overlays, not facility rows. +- KRS `KRS` and GUS `REGON`/`NIP` are exact organization identifiers. A successful identity lookup does not show that the organization operates a GIW site, owns it, controls it, or is a beneficial owner. +- Addresses and coordinates require privacy and precision review. A registered office, sole-trader address, geocoder match, or mixed residential/business site is not automatically an operating facility. No geocoding was performed. +- Keep source origin, project review, project approval and publication as separate fields. All current entries are government-sourced or EU-mirror-sourced evidence, not project-approved or project-published records. + +### Map value + +The GIW WNI layer can support a reviewed, source-qualified map of approved establishment candidates, with category/species/activity facets and coarse or suppressed location states. The environmental and planning layers can show permit, EIA and zoning context without pretending those documents identify a facility. GUS and RRW can provide aggregate regional/time context, not points. Missing or changed observations must render as dated source states rather than inferred closures. + +### Accountability-graph value + +Poland has a useful graph shape: `WNI` approval observations connect to activity/species/product claims; RRW adds inspection and outcome events; GIOŚ/WIOŚ adds permit and monitoring edges; GDOŚ adds EIA proceedings; Geoportal adds planning acts; ARiMR adds funding-project relationships; KRS/REGON add exact legal-entity evidence; and TRACES adds an EU mirror lineage edge. Each edge must carry source ID, source-local identifier, observation/effective date, evidence URL, review outcome and publication scope. A graph makes those different claims visible without collapsing them into a single “official facility” fact. + +## Legacy boundary and crosswalk + +No Poland V1 file was found under `static_data`, `Old CSVs`, or `dirty-datasets`. The machine-readable crosswalk at [`docs/countries/pl/source-crosswalk.json`](countries/pl/source-crosswalk.json) records `rows_found=0` and the checked paths. This is an explicit no-legacy-snapshot result, not evidence that Poland has no facilities. No fuzzy name/address/coordinate matching was attempted. + +## Private artifact and provenance record + +The tracked manifest is [`data/manifests/pl-source-artifacts.json`](../data/manifests/pl-source-artifacts.json). The only private artifact created is the metadata-only run note at `data/raw/poland/20260916T000000Z/metadata.json`; it contains route observations, displayed counts, schema notes and privacy assertions, not source row payloads. It is ignored as a raw-path artifact except for its explicitly tracked `metadata.json` file. The manifest records its byte size and SHA-256 and records null bytes/hashes for source pages whose bodies were not captured. + +The shell network refusal and browser XLS timeout are blockers that must remain visible in any rerun report. Do not replace missing source hashes with hashes of rendered notes. The GIW displayed counts above are web observations, not byte-verified exports. + +## Difficulty, staged pipeline and blockers + +Difficulty: medium-high. + +1. Acquire one bounded GIW XLS export or a complete paginated HTML capture from an authorized Polish execution context; record headers, redirects, retrieval time, bytes, SHA-256, schema fingerprint and any supplied update/effective date. +2. Build a GIW adapter keyed by `(source_id, WNI, section/view, observation dimensions)`; retain raw source values and generate a separate reviewable facility-candidate projection. +3. Add registered-food and ABP adapters as separate source families, beginning with one list and one ABP category; test missing IDs, repeated observations and status/date semantics. +4. Add RRW inspection/welfare/enforcement observations and GUS aggregates with explicit effective periods, report/table keys and no facility-count joins. +5. Add one regional WIOŚ IPPC register, then GDOŚ EIA from an in-Poland/authorized context; preserve regional coverage limits. +6. Add Urban Register/Geoportal planning evidence and exact KRS/REGON links only when an upstream WNI row supplies a reviewed organization key; never query identity sources as a facility discovery mechanism. +7. Run privacy/terms/schema/coverage/duplicate/lifecycle review, then a private candidate handoff. Publication remains blocked pending authorized human review. + +Open blockers: + +- GIW XLS body, headers, byte hash, export terms and update metadata were not captured in this environment. +- GIW WNI lifecycle/status semantics and cross-section repeat rules need an export-level contract. +- Registered-food and ABP list families have unresolved per-list schemas, cadence and terms. +- RRW reports need current artifact acquisition and careful report-grain modeling. +- GIOŚ/WIOŚ IPPC coverage is regional and heterogeneous; GDOŚ EIA access is geographically blocked from outside Poland. +- KRS/REGON API registration, rate/terms and RODO filtering need a purpose-specific review. +- No national source-approved geocoding or public precision policy was verified. +- No Poland V1 file exists for reconciliation and no publication approval exists. + +All artifacts, status changes and code are scoped to this isolated branch. No source rows, addresses, coordinates, publication, deployment, promotion or public exposure were performed. diff --git a/docs/source-status.json b/docs/source-status.json index 6e7e2c7..c582844 100644 --- a/docs/source-status.json +++ b/docs/source-status.json @@ -77,6 +77,18 @@ {"source_id":"ie.dafm.national-beef-kill","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ie.md","docs/countries/ireland/v1-field-crosswalk.json","pipeline/source_registry.json"],"next_action":"Retain the observed 2024 historical cutoff and verify the resource before any aggregate use."}, {"source_id":"ie.dafm.seafood-processing-funding","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ie.md","docs/countries/ireland/v1-field-crosswalk.json","pipeline/source_registry.json"],"next_action":"Keep programme context separate; capture awards only after scheme-specific privacy and terms review."}, {"source_id":"ie.fsai.enforcement-orders","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ie.md","docs/countries/ireland/v1-field-crosswalk.json","pipeline/source_registry.json"],"next_action":"Treat notices as dated event evidence and require reviewed subject identity before linkage."}, - {"source_id":"ie.dafm.animal-welfare-controls","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ie.md","docs/countries/ireland/v1-field-crosswalk.json","pipeline/source_registry.json"],"next_action":"Keep the DAFM statement as control-scope context until a current row-level welfare artifact is lawfully captured and reviewed."} + {"source_id":"ie.dafm.animal-welfare-controls","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ie.md","docs/countries/ireland/v1-field-crosswalk.json","pipeline/source_registry.json"],"next_action":"Keep the DAFM statement as control-scope context until a current row-level welfare artifact is lawfully captured and reviewed."}, + {"source_id":"pl.giw.approved-food","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-pl.md","docs/countries/pl/source-crosswalk.json","data/manifests/pl-source-artifacts.json","data/raw/poland/20260916T000000Z/metadata.json","pipeline/source_registry.json","pipeline/poland/test_metadata.py"],"next_action":"Acquire a bounded GIW XLS/HTML artifact from an allowed context, record source hash/schema, and model WNI activity/species/product observations without summing sections or publishing addresses."}, + {"source_id":"pl.giw.registered-food","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-pl.md","docs/countries/pl/source-crosswalk.json","pipeline/source_registry.json"],"next_action":"Capture one registered-list family at a time; preserve list-specific identifiers and keep registration separate from 853/2004 approval."}, + {"source_id":"pl.giw.abp","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-pl.md","docs/countries/pl/source-crosswalk.json","pipeline/source_registry.json"],"next_action":"Verify the current ABP list route/schema and preserve Regulation 1069/2009 scope outside food-facility totals."}, + {"source_id":"pl.giw.rrw","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-pl.md","docs/countries/pl/source-crosswalk.json","pipeline/source_registry.json"],"next_action":"Acquire current RRW reports and model welfare, controls, enforcement and meat-examination evidence by report/table/year or source-local ID."}, + {"source_id":"pl.gus.slaughter","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-pl.md","docs/countries/pl/source-crosswalk.json","pipeline/source_registry.json"],"next_action":"Keep GUS R-09U aggregates separate from GIW WNI facilities and preserve statistical revisions/flags."}, + {"source_id":"pl.gios.ippc","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-pl.md","docs/countries/pl/source-crosswalk.json","pipeline/source_registry.json"],"next_action":"Capture regional WIOŚ registers one voivodeship at a time and keep permits/installations as dated environmental evidence."}, + {"source_id":"pl.gdos.eia","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-pl.md","docs/countries/pl/source-crosswalk.json","pipeline/source_registry.json"],"next_action":"Use an in-Poland or authorized context for bounded EIA acquisition; source states outside-Poland access is blocked."}, + {"source_id":"pl.geoportal.urban-planning","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-pl.md","docs/countries/pl/source-crosswalk.json","pipeline/source_registry.json"],"next_action":"Capture production Urban Register/WMS metadata and preserve planning act identity and legal/effective dates."}, + {"source_id":"pl.arimr.processing-support","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-pl.md","docs/countries/pl/source-crosswalk.json","pipeline/source_registry.json"],"next_action":"Acquire only permitted funding/project evidence; do not interpret funding as facility operation or compliance."}, + {"source_id":"pl.krs.open-api","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-pl.md","docs/countries/pl/source-crosswalk.json","pipeline/source_registry.json"],"next_action":"Confirm current KRS API contract and RODO filtering; use only for exact organization identity links supplied by an upstream source."}, + {"source_id":"pl.gus.regon-bir","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-pl.md","docs/countries/pl/source-crosswalk.json","pipeline/source_registry.json"],"next_action":"Obtain authorized BIR1 access before queries, respect documented rate limits, and keep sole-trader/restricted fields private."}, + {"source_id":"eu.traces.pl-approved-food","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-pl.md","docs/countries/pl/source-crosswalk.json","pipeline/source_registry.json"],"next_action":"Verify Poland-specific TRACES export only as a lineage/cross-check layer; never double-count GIW."} ] } diff --git a/docs/source-status.md b/docs/source-status.md index 6b7788b..72722c0 100644 --- a/docs/source-status.md +++ b/docs/source-status.md @@ -60,3 +60,7 @@ The 2026-09-16 Ireland reconnaissance verified the current FSAI/DAFM/HSE/SFPA so Australia is represented by 22 source-local evidence layers in `source-status.json` and `pipeline/source_registry.json`. All remain `publication_eligibility=blocked`; no runtime health is claimed. The bounded private artifacts are the NPI CSV (8,140 rows) and the earlier SA EPA GeoJSON capture (4,541 features / 1,695 licences). The detailed route, schema, identity, map/graph, privacy, terms, and blocker crosswalk is [`docs/countries/australia/source-crosswalk.json`](countries/australia/source-crosswalk.json); the decision report is [`docs/country-recon-au.md`](country-recon-au.md). The machine-readable file is the source of truth for these statuses. Legacy `.locations` paths may represent composite coverage, but source identities are split where the evidence establishes separate feeds: France Section I/II, Canada Ontario/CFIA, Italy 853/2004/1069/2009, and Australia’s state/federal/environment/animal-use layers. Candidate feeds mentioned in reconnaissance documents are not silently conflated into a single healthy source. Country reconnaissance documents provide evidence and next actions; they do not override this status vocabulary or authorize publication. + +## Poland additions (2026-09-16) + +Poland reconnaissance added GIW approved-food, registered-food, ABP and RRW sources; GUS slaughter statistics; GIOŚ/WIOŚ integrated permits; GDOŚ EIA; Geoportal Urban Register; ARiMR processing support; KRS and REGON identity routes; and the EU TRACES mirror. The GIW HTML views were verified with current displayed counts, but the XLS body was not acquired because shell HTTP was refused and the browser export timed out. The run therefore records a private metadata-only artifact, not a source-data capture. All Poland entries remain `publication_eligibility=blocked`, with runtime health `not_run`; GDOŚ and REGON acquisition are additionally blocked pending the source’s access/authorization conditions. See [`docs/country-recon-pl.md`](country-recon-pl.md), [`docs/countries/pl/source-crosswalk.json`](countries/pl/source-crosswalk.json), and [`data/manifests/pl-source-artifacts.json`](../data/manifests/pl-source-artifacts.json). diff --git a/pipeline/poland/__init__.py b/pipeline/poland/__init__.py new file mode 100644 index 0000000..471457e --- /dev/null +++ b/pipeline/poland/__init__.py @@ -0,0 +1 @@ +"""Poland reconnaissance metadata tests and future source adapters.""" diff --git a/pipeline/poland/test_metadata.py b/pipeline/poland/test_metadata.py new file mode 100644 index 0000000..917829f --- /dev/null +++ b/pipeline/poland/test_metadata.py @@ -0,0 +1,45 @@ +import hashlib +import json +import unittest +from pathlib import Path + + +ROOT = Path(__file__).parents[2] +MANIFEST = ROOT / "data" / "manifests" / "pl-source-artifacts.json" +PRIVATE_METADATA = ROOT / "data" / "raw" / "poland" / "20260916T000000Z" / "metadata.json" + + +class PolandMetadataIntegrityTests(unittest.TestCase): + def test_manifest_and_crosswalk_have_unique_source_ids(self): + manifest = json.loads(MANIFEST.read_text(encoding="utf-8")) + crosswalk = json.loads((ROOT / "docs" / "countries" / "pl" / "source-crosswalk.json").read_text(encoding="utf-8")) + manifest_ids = [item["source_id"] for item in manifest["artifacts"]] + crosswalk_ids = [item["source_id"] for item in crosswalk["sources"]] + self.assertEqual(len(manifest_ids), len(set(manifest_ids))) + self.assertEqual(len(crosswalk_ids), len(set(crosswalk_ids))) + self.assertEqual(set(manifest_ids) - {"pl.private.recon-metadata"}, set(crosswalk_ids)) + + def test_private_metadata_hash_and_privacy_boundary(self): + manifest = json.loads(MANIFEST.read_text(encoding="utf-8")) + metadata = json.loads(PRIVATE_METADATA.read_text(encoding="utf-8")) + entry = next(item for item in manifest["artifacts"] if item["source_id"] == "pl.private.recon-metadata") + digest = hashlib.sha256(PRIVATE_METADATA.read_bytes()).hexdigest() + self.assertEqual(entry["sha256"], digest) + self.assertEqual(entry["bytes"], PRIVATE_METADATA.stat().st_size) + privacy = metadata["privacy"] + self.assertFalse(privacy["raw_source_rows_retained"]) + self.assertFalse(privacy["addresses_retained"]) + self.assertFalse(privacy["coordinates_retained"]) + self.assertFalse(privacy["personal_contacts_retained"]) + + def test_giw_counts_are_nonnegative_and_not_facility_total(self): + metadata = json.loads(PRIVATE_METADATA.read_text(encoding="utf-8")) + giw = next(item for item in metadata["source_observations"] if item["source_id"] == "pl.giw.approved-food") + counts = giw["observed_sections"] + self.assertTrue(all(isinstance(value, int) and value >= 0 for value in counts.values())) + self.assertEqual(giw["capture_status"], "metadata_only") + self.assertIn("one-to-many", giw["row_identity"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/source_registry.json b/pipeline/source_registry.json index 29dd35d..34208e0 100644 --- a/pipeline/source_registry.json +++ b/pipeline/source_registry.json @@ -709,6 +709,18 @@ "adapter_status": "not_started", "expected_artifact_schema": "Contextual control statement or separately identified inspection/enforcement event; not a facility census", "blockers": ["Current row-level facility-linked welfare/enforcement observations were not captured; do not convert qualitative controls or aggregate reports into facility claims."] - } + }, + {"source_id":"pl.giw.approved-food","jurisdiction_scope":"Poland; GIW approved animal-origin food establishments under Regulation (EC) 853/2004","legacy_paths":[],"url":"https://www.wetgiw.gov.pl/handel-eksport-import/listy-zakladow","access_method":"official rendered HTML register with filters plus XLS export route; bounded metadata-only observation","cadence":"unknown; current views observed 2026-09-16","attribution_licensing_notes":"Polish official source; terms, privacy and project release review required","adapter_status":"reference_only","expected_artifact_schema":"HTML/XLS rows with LP, WNI, name, address, category, species, linked activity, notes and Article 44 fields; repeated observations by section/activity/species/product","blockers":["Acquire a bounded source artifact and hash; resolve WNI lifecycle/repeat semantics; do not sum section rows; review privacy/terms/release approval."]}, + {"source_id":"pl.giw.registered-food","jurisdiction_scope":"Poland; GIW registered animal-origin food-sector activities","legacy_paths":[],"url":"https://www.wetgiw.gov.pl/nadzor-weterynaryjny/wykaz-zakladow-rejestrowanych","access_method":"official index with 14 list families; list-specific routes not acquired","cadence":"unknown","attribution_licensing_notes":"Polish official source; list-specific terms/privacy review required","adapter_status":"reference_only","expected_artifact_schema":"List-specific registered-operator rows; identifier/name/address/activity/status-date fields vary","blockers":["Capture one list at a time and keep registration separate from 853/2004 approval."]}, + {"source_id":"pl.giw.abp","jurisdiction_scope":"Poland; GIW animal by-product establishments and operators","legacy_paths":[],"url":"https://www.wetgiw.gov.pl/handel-eksport-import/niespozywcze-produkty-pochodzenia-zwierzecego","access_method":"GIW page plus linked pasze.wetgiw.gov.pl register; exact export unresolved","cadence":"unknown","attribution_licensing_notes":"Polish official source under ABP rules; terms/privacy/release review required","adapter_status":"reference_only","expected_artifact_schema":"ABP approval/registration number, operator, address, category/activity, status/date","blockers":["Verify current list route and preserve 1069/2009 scope separately from food facilities."]}, + {"source_id":"pl.giw.rrw","jurisdiction_scope":"Poland; GIW veterinary statistical reporting","legacy_paths":[],"url":"https://www.wetgiw.gov.pl/publikacje/rrw-sprawozdawczosc-statystyczna/printpage","access_method":"official RRW publication/index; no current file acquired","cadence":"annual/report-specific","attribution_licensing_notes":"Polish official aggregate/evidence reports; small-cell and case privacy review required","adapter_status":"reference_only","expected_artifact_schema":"RRW-3 welfare/controls, RRW-5 animal-origin establishments, RRW-6 official meat examination by report/table/year","blockers":["Acquire current reports and model them as dated evidence, not facility master rows."]}, + {"source_id":"pl.gus.slaughter","jurisdiction_scope":"Poland; Statistics Poland slaughter statistics","legacy_paths":[],"url":"https://stat.gov.pl/obszary-tematyczne/rolnictwo-lesnictwo/produkcja-zwierzeca-zwierzeta-gospodarskie/uboje-zwierzat-gospodarskich-w-2025-r-,16,1.html","access_method":"official HTML publication with XLSX attachment; page observed","cadence":"monthly collection; 2025 publication dated 2026-03-02","attribution_licensing_notes":"GUS official statistics; preserve revisions/flags and source terms","adapter_status":"reference_only","expected_artifact_schema":"Aggregate species/period/slaughter counts and live/slaughter weight from R-09U","blockers":["Never infer named facilities or join totals to GIW WNI; keep aggregate context separate."]}, + {"source_id":"pl.gios.ippc","jurisdiction_scope":"Poland; GIOŚ and 16 WIOŚ integrated-permit installation registers","legacy_paths":[],"url":"https://www.gov.pl/web/gios/instalacje-wymagajace-uzyskania-pozwolenia-zintegrowanego","access_method":"central official page with 16 regional links; no national export verified","cadence":"regional/unknown","attribution_licensing_notes":"Polish official environmental source; regional terms/privacy review required","adapter_status":"reference_only","expected_artifact_schema":"Regional installation/permit rows with operator, address, industry, authority, status and dates","blockers":["Capture regional schemas and retain permit/installations as evidence overlays."]}, + {"source_id":"pl.gdos.eia","jurisdiction_scope":"Poland; GDOŚ environmental-impact-assessment database","legacy_paths":[],"url":"https://www.gov.pl/web/gdos/bazy-danych-o-ocenach-oddzialywania-na-srodowisko","access_method":"official BIP database/search; page states access from outside Poland is blocked","cadence":"authority entry within 30 days; public release cadence unknown","attribution_licensing_notes":"Polish official source; document privacy and access restrictions apply","adapter_status":"reference_only","expected_artifact_schema":"EIA proceeding/document/authority/project/date/status records","blockers":["Obtain an in-Poland or authorized bounded capture; no facility-master inference."]}, + {"source_id":"pl.geoportal.urban-planning","jurisdiction_scope":"Poland; Geoportal/Urban Register planning data","legacy_paths":[],"url":"https://www.geoportal.gov.pl/aktualnosci/nowe-uslugi-w-geoportalu-rejestr-urbanistyczny/","access_method":"public WMS/register announcement; service contract not acquired","cadence":"regular local-government updates; transition noted for end of September 2026","attribution_licensing_notes":"Polish official spatial data; preserve service terms and legal dates","adapter_status":"reference_only","expected_artifact_schema":"WMS/GML planning-act geometries, municipality, legal status and effective/publication dates","blockers":["Capture production service metadata and keep zoning context separate from facility identity."]}, + {"source_id":"pl.arimr.processing-support","jurisdiction_scope":"Poland; ARiMR agricultural processing investment support","legacy_paths":[],"url":"https://www.gov.pl/web/arimr/startuje-wsparcie-dla-przetworcow","access_method":"official call/guidance; beneficiary/project dataset not acquired","cadence":"call-specific; 2026 call window observed 1-30 September","attribution_licensing_notes":"Polish official funding evidence; beneficiary/privacy terms review required","adapter_status":"reference_only","expected_artifact_schema":"Call/application/agreement/beneficiary/project/amount/date observations","blockers":["Do not treat funding as facility authorization, operation or compliance."]}, + {"source_id":"pl.krs.open-api","jurisdiction_scope":"Poland; Ministry of Justice KRS legal-entity identity","legacy_paths":[],"url":"https://prs.ms.gov.pl/krs/openApi","access_method":"open API announced by Ministry; no request made","cadence":"on-demand/registry filings","attribution_licensing_notes":"Official KRS; RODO filtering and API terms review required","adapter_status":"reference_only","expected_artifact_schema":"KRS number, legal name, status, registered information and identifiers","blockers":["Confirm current API contract; exact identity link only, never operating-site inference."]}, + {"source_id":"pl.gus.regon-bir","jurisdiction_scope":"Poland; GUS REGON BIR1 identity service","legacy_paths":[],"url":"https://api.stat.gov.pl/Home/RegonApi?lang=en","access_method":"documented SOAP/API service; registration/user key required","cadence":"continuously updated register; on-demand queries","attribution_licensing_notes":"Official GUS service; respect registration, rate limits and personal-data handling","adapter_status":"reference_only","expected_artifact_schema":"Lookup response by REGON, NIP or KRS","blockers":["Obtain authorized access before queries and keep fields restricted."]}, + {"source_id":"eu.traces.pl-approved-food","jurisdiction_scope":"EU mirror; Poland approved-food establishment lineage","legacy_paths":[],"url":"https://food.ec.europa.eu/food-safety/biological-safety/food-hygiene/approved-eu-food-establishments_en","access_method":"public TRACES/publication page; Poland export not verified","cadence":"competent-authority updates","attribution_licensing_notes":"EU official mirror; do not double-count GIW","adapter_status":"reference_only","expected_artifact_schema":"TRACES establishment/activity listing keyed by EU/national approval identifiers","blockers":["Verify Poland-specific export and keep mirror provenance separate."]} ] } From 57fa8a1d2db33dae83f3f60d391eda04adb506ea Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Wed, 16 Sep 2026 09:24:17 -0700 Subject: [PATCH 140/311] Fix consolidated registry and metadata gates --- pipeline/poland/test_metadata.py | 5 +++-- pipeline/tests/test_source_registry.py | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/pipeline/poland/test_metadata.py b/pipeline/poland/test_metadata.py index 917829f..7d23959 100644 --- a/pipeline/poland/test_metadata.py +++ b/pipeline/poland/test_metadata.py @@ -23,9 +23,10 @@ def test_private_metadata_hash_and_privacy_boundary(self): manifest = json.loads(MANIFEST.read_text(encoding="utf-8")) metadata = json.loads(PRIVATE_METADATA.read_text(encoding="utf-8")) entry = next(item for item in manifest["artifacts"] if item["source_id"] == "pl.private.recon-metadata") - digest = hashlib.sha256(PRIVATE_METADATA.read_bytes()).hexdigest() + canonical_bytes = PRIVATE_METADATA.read_bytes().replace(b"\r\n", b"\n") + digest = hashlib.sha256(canonical_bytes).hexdigest() self.assertEqual(entry["sha256"], digest) - self.assertEqual(entry["bytes"], PRIVATE_METADATA.stat().st_size) + self.assertEqual(entry["bytes"], len(canonical_bytes)) privacy = metadata["privacy"] self.assertFalse(privacy["raw_source_rows_retained"]) self.assertFalse(privacy["addresses_retained"]) diff --git a/pipeline/tests/test_source_registry.py b/pipeline/tests/test_source_registry.py index f9ae10b..0532cf8 100644 --- a/pipeline/tests/test_source_registry.py +++ b/pipeline/tests/test_source_registry.py @@ -9,8 +9,8 @@ class SourceRegistryTests(unittest.TestCase): def test_repository_registry_loads_and_references_existing_legacy_paths(self): registry = load_registry() - self.assertEqual(len(registry["sources"]), 69) - self.assertEqual(len({source["source_id"] for source in registry["sources"]}), 69) + self.assertEqual(len(registry["sources"]), 81) + self.assertEqual(len({source["source_id"] for source in registry["sources"]}), 81) def test_unknowns_are_explicit(self): registry = json.loads(REGISTRY_PATH.read_text(encoding="utf-8")) From f697fa80ff2c36da7fa62eb5a7f6e56ab860d699 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Wed, 16 Sep 2026 09:07:22 -0700 Subject: [PATCH 141/311] Add row-free coordinate coverage diagnostic --- .../diagnostics/coordinate-coverage.py | 89 +++++++++++++++++++ pipeline/tests/test_coordinate_coverage.py | 43 +++++++++ 2 files changed, 132 insertions(+) create mode 100644 pipeline/scripts/diagnostics/coordinate-coverage.py create mode 100644 pipeline/tests/test_coordinate_coverage.py diff --git a/pipeline/scripts/diagnostics/coordinate-coverage.py b/pipeline/scripts/diagnostics/coordinate-coverage.py new file mode 100644 index 0000000..6ab6486 --- /dev/null +++ b/pipeline/scripts/diagnostics/coordinate-coverage.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 +"""Write a row-free aggregate report of coordinate evidence states. + +The input is a JSONL stage export. Only aggregate counts are written; source +keys, names, addresses, coordinates, queries, and geocoder responses are never +copied to the report. +""" + +from __future__ import annotations + +import argparse +import json +from collections import Counter +from pathlib import Path +from typing import Any + + +STATES = ("source_supplied", "geocoded_exact", "approximate_coarse", "unresolved", "restricted") + + +def _has_source_coordinate(record: dict[str, Any]) -> bool: + coordinates = record.get("coordinates") + return isinstance(coordinates, dict) and all( + coordinates.get(key) is not None for key in ("latitude", "longitude") + ) + + +def coordinate_state(record: dict[str, Any]) -> str: + """Classify one observation without exposing any record value.""" + if record.get("restricted") is True or record.get("privacy_status") in { + "restricted", + "failed", + "suppressed", + }: + return "restricted" + if _has_source_coordinate(record): + return "source_supplied" + geocode = record.get("geocode") + if not isinstance(geocode, dict): + geocode = {} + status = geocode.get("status") or record.get("geocoding_status") + precision = geocode.get("precision") or record.get("coordinate_precision") + if status == "accepted" and geocode.get("result") is not None: + return "geocoded_exact" if precision in (None, "exact", "rooftop", "parcel") else "approximate_coarse" + if status in {"review_required", "approximate", "coarse"} or precision in {"city", "coarse", "approximate"}: + return "approximate_coarse" + return "unresolved" + + +def build_report(input_path: Path) -> dict[str, Any]: + counts = Counter() + facility_ids: set[str] = set() + records_seen = 0 + with input_path.open(encoding="utf-8") as source: + for line in source: + if not line.strip(): + continue + record = json.loads(line) + if not isinstance(record, dict): + raise ValueError("each JSONL item must be an object") + records_seen += 1 + counts[coordinate_state(record)] += 1 + facility_id = record.get("facility_id") + if facility_id is not None: + facility_ids.add(str(facility_id)) + return { + "report_version": "coordinate-coverage-1", + "observations": records_seen, + "unique_facilities": len(facility_ids), + "coordinate_states": {state: counts[state] for state in STATES}, + "raw_rows_in_report": False, + "sensitive_fields_in_report": False, + "semantics": "Counts are observations; unique_facilities uses facility_id when present.", + } + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("input", type=Path) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + report = build_report(args.input) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(json.dumps(report, indent=2, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/pipeline/tests/test_coordinate_coverage.py b/pipeline/tests/test_coordinate_coverage.py new file mode 100644 index 0000000..b109314 --- /dev/null +++ b/pipeline/tests/test_coordinate_coverage.py @@ -0,0 +1,43 @@ +import importlib.util +import json +import tempfile +import unittest +from pathlib import Path + + +SCRIPT = Path(__file__).parents[1] / "scripts" / "diagnostics" / "coordinate-coverage.py" +SPEC = importlib.util.spec_from_file_location("coordinate_coverage", SCRIPT) +MODULE = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(MODULE) + + +class CoordinateCoverageTests(unittest.TestCase): + def test_states_prioritize_restriction_and_distinguish_exact_from_coarse(self): + self.assertEqual(MODULE.coordinate_state({"coordinates": {"latitude": 1, "longitude": 2}}), "source_supplied") + self.assertEqual(MODULE.coordinate_state({"geocode": {"status": "accepted", "result": {"x": 2}, "precision": "exact"}}), "geocoded_exact") + self.assertEqual(MODULE.coordinate_state({"geocode": {"status": "review_required", "result": {"x": 2}}}), "approximate_coarse") + self.assertEqual(MODULE.coordinate_state({"geocode": {"status": "accepted", "result": {"x": 2}, "precision": "city"}}), "approximate_coarse") + self.assertEqual(MODULE.coordinate_state({"geocode": {"status": "accepted", "result": {"x": 2}}, "restricted": True}), "restricted") + self.assertEqual(MODULE.coordinate_state({}), "unresolved") + + def test_report_counts_observations_and_unique_facilities_without_rows(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + source = root / "input.jsonl" + source.write_text("\n".join(json.dumps(item) for item in [ + {"facility_id": "a", "coordinates": {"latitude": 1, "longitude": 2}}, + {"facility_id": "a", "geocode": {"status": "unresolved"}, "name": "PRIVATE"}, + {"facility_id": "b", "geocode": {"status": "review_required", "result": {"x": 1}}}, + {"facility_id": "c", "privacy_status": "restricted", "address": "PRIVATE"}, + ]) + "\n", encoding="utf-8") + report = MODULE.build_report(source) + self.assertEqual(report["observations"], 4) + self.assertEqual(report["unique_facilities"], 3) + self.assertEqual(report["coordinate_states"], {"source_supplied": 1, "geocoded_exact": 0, "approximate_coarse": 1, "unresolved": 1, "restricted": 1}) + encoded = json.dumps(report) + self.assertNotIn("PRIVATE", encoded) + self.assertNotIn('"facility_id"', encoded) + + +if __name__ == "__main__": + unittest.main() From ae0a4c0807dae1ad3f9a0405d8107f4ba6113f24 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Wed, 16 Sep 2026 09:32:48 -0700 Subject: [PATCH 142/311] Harden migration recovery and suppression coverage --- .../diagnostics/coordinate-coverage.py | 17 +++- .../scripts/maintenance/apply-migrations.py | 43 ++++---- .../tests/e2e/test_italy_candidate_import.py | 7 +- pipeline/tests/e2e/test_public_api.py | 17 +++- pipeline/tests/test_apply_migrations.py | 99 +++++++++++++++++++ pipeline/tests/test_coordinate_coverage.py | 4 + 6 files changed, 160 insertions(+), 27 deletions(-) create mode 100644 pipeline/tests/test_apply_migrations.py diff --git a/pipeline/scripts/diagnostics/coordinate-coverage.py b/pipeline/scripts/diagnostics/coordinate-coverage.py index 6ab6486..5789594 100644 --- a/pipeline/scripts/diagnostics/coordinate-coverage.py +++ b/pipeline/scripts/diagnostics/coordinate-coverage.py @@ -10,6 +10,7 @@ import argparse import json +import math from collections import Counter from pathlib import Path from typing import Any @@ -20,8 +21,16 @@ def _has_source_coordinate(record: dict[str, Any]) -> bool: coordinates = record.get("coordinates") - return isinstance(coordinates, dict) and all( - coordinates.get(key) is not None for key in ("latitude", "longitude") + if not isinstance(coordinates, dict): + return False + latitude, longitude = coordinates.get("latitude"), coordinates.get("longitude") + if any(isinstance(value, bool) or not isinstance(value, (int, float)) for value in (latitude, longitude)): + return False + return ( + math.isfinite(latitude) + and math.isfinite(longitude) + and -90 <= latitude <= 90 + and -180 <= longitude <= 180 ) @@ -41,7 +50,9 @@ def coordinate_state(record: dict[str, Any]) -> str: status = geocode.get("status") or record.get("geocoding_status") precision = geocode.get("precision") or record.get("coordinate_precision") if status == "accepted" and geocode.get("result") is not None: - return "geocoded_exact" if precision in (None, "exact", "rooftop", "parcel") else "approximate_coarse" + if precision in {"exact", "rooftop", "parcel", "building", "address"}: + return "geocoded_exact" + return "approximate_coarse" if precision in {"city", "coarse", "approximate"} else "unresolved" if status in {"review_required", "approximate", "coarse"} or precision in {"city", "coarse", "approximate"}: return "approximate_coarse" return "unresolved" diff --git a/pipeline/scripts/maintenance/apply-migrations.py b/pipeline/scripts/maintenance/apply-migrations.py index b7c52e9..39b4235 100644 --- a/pipeline/scripts/maintenance/apply-migrations.py +++ b/pipeline/scripts/maintenance/apply-migrations.py @@ -30,36 +30,41 @@ def apply(database_url: str, directory: Path) -> list[str]: raise time.sleep(1) assert connection is not None + # Keep setup and each migration in separate transactions. A later + # migration failure must leave already-applied versions recorded so a + # retry can resume, while the failing migration and its ledger row roll + # back together. with connection: - connection.execute("CREATE SCHEMA IF NOT EXISTS uec") - connection.execute( - """ - CREATE TABLE IF NOT EXISTS uec.schema_migrations ( - version TEXT PRIMARY KEY, - sha256 CHAR(64) NOT NULL, - applied_at TIMESTAMPTZ NOT NULL DEFAULT now() + with connection.transaction(): + connection.execute("CREATE SCHEMA IF NOT EXISTS uec") + connection.execute( + """ + CREATE TABLE IF NOT EXISTS uec.schema_migrations ( + version TEXT PRIMARY KEY, + sha256 CHAR(64) NOT NULL, + applied_at TIMESTAMPTZ NOT NULL DEFAULT now() + ) + """ ) - """ - ) for path in files: version = path.stem digest = hashlib.sha256(path.read_bytes()).hexdigest() - row = connection.execute( - "SELECT sha256 FROM uec.schema_migrations WHERE version = %s", - (version,), - ).fetchone() - if row: - if row[0] != digest: - raise ValueError(f"migration checksum changed after application: {version}") - continue - sql = path.read_text(encoding="utf-8") with connection.transaction(): + row = connection.execute( + "SELECT sha256 FROM uec.schema_migrations WHERE version = %s", + (version,), + ).fetchone() + if row: + if row[0] != digest: + raise ValueError(f"migration checksum changed after application: {version}") + continue + sql = path.read_text(encoding="utf-8") connection.execute(sql) connection.execute( "INSERT INTO uec.schema_migrations (version, sha256) VALUES (%s, %s)", (version, digest), ) - applied.append(version) + applied.append(version) return applied diff --git a/pipeline/tests/e2e/test_italy_candidate_import.py b/pipeline/tests/e2e/test_italy_candidate_import.py index d4ec625..cd7233c 100644 --- a/pipeline/tests/e2e/test_italy_candidate_import.py +++ b/pipeline/tests/e2e/test_italy_candidate_import.py @@ -57,11 +57,14 @@ def test_privacy_restriction_relocks_all_surfaces(self): record=db.execute("SELECT source_record_id FROM uec.source_records WHERE source_id='it.853-2004' LIMIT 1").fetchone()[0] db.execute("INSERT INTO uec.record_access_events(source_record_id,action,reason_category,policy_version,maintainer) VALUES (%s,'public_access_revoked','privacy','ethics-v1','authorized-synthetic-operator')",(record,)); db.commit() base=f"http://127.0.0.1:{self.env.api_port}"; h={"X-UEC-Dev-Preview-Token":self.env.dev_preview_token} + for path in ("/api/dev/preview/test-release/locations?profile=official", "/api/dev/preview/test-release/discovery/facets?profile=official"): + with urllib.request.urlopen(urllib.request.Request(base+path,headers=h)) as r: + suppressed_surface=r.read().decode(); self.assertNotIn("Synthetic Italy Facility",suppressed_surface); self.assertNotIn("it.853-2004",suppressed_surface); self.assertNotIn(self.facility_id,suppressed_surface) with urllib.request.urlopen(urllib.request.Request(base+"/api/dev/preview/test-release/locations?profile=official",headers=h)) as r: self.assertEqual(json.loads(r.read())["data"],[]) with self.assertRaises(urllib.error.HTTPError) as error: urllib.request.urlopen(urllib.request.Request(base+"/api/dev/preview/test-release/locations/"+self.facility_id+"?profile=official",headers=h)) self.assertIn(error.exception.code,(404,410)) - with urllib.request.urlopen(urllib.request.Request(base+"/api/dev/preview/test-release/discovery/facets?profile=official",headers=h)) as r: self.assertNotIn('"IT"',r.read().decode()) - with urllib.request.urlopen(urllib.request.Request(base+"/api/dev/preview/test-release/locations.csv?profile=official",headers=h)) as r: self.assertEqual(len(r.read().decode().splitlines()),1) + with urllib.request.urlopen(urllib.request.Request(base+"/api/dev/preview/test-release/locations.csv?profile=official",headers=h)) as r: + csv_body=r.read().decode(); self.assertEqual(len(csv_body.splitlines()),1); self.assertNotIn("Synthetic Italy Facility",csv_body); self.assertNotIn("it.853-2004",csv_body); self.assertNotIn(self.facility_id,csv_body) with urllib.request.urlopen(base+"/api/v2/locations?profile=official") as r: self.assertEqual(json.loads(r.read())["data"],[]) if __name__=="__main__": unittest.main() diff --git a/pipeline/tests/e2e/test_public_api.py b/pipeline/tests/e2e/test_public_api.py index aeb992d..64e4adf 100644 --- a/pipeline/tests/e2e/test_public_api.py +++ b/pipeline/tests/e2e/test_public_api.py @@ -75,9 +75,20 @@ def test_filters_do_not_bypass_publication_gate(self): self.assertEqual(body["data"], []) def test_invalid_pagination_is_rejected(self): - with self.assertRaises(urllib.error.HTTPError) as error: - self.get("/api/v2/locations?limit=invalid") - self.assertEqual(error.exception.code, 400) + for query in ("limit=invalid", "offset=invalid"): + with self.subTest(query=query), self.assertRaises(urllib.error.HTTPError) as error: + self.get(f"/api/v2/locations?{query}") + self.assertEqual(error.exception.code, 400) + + def test_incomplete_or_conflicting_spatial_queries_are_rejected(self): + for query in ( + "latitude=55", + "min_lat=54&min_lon=10&max_lat=53&max_lon=11", + "latitude=55&longitude=10&radius_km=0", + ): + with self.subTest(query=query), self.assertRaises(urllib.error.HTTPError) as error: + self.get(f"/api/v2/locations?{query}") + self.assertEqual(error.exception.code, 400) def test_profile_is_explicit_and_mismatch_does_not_leak_records(self): with self.assertRaises(urllib.error.HTTPError) as error: diff --git a/pipeline/tests/test_apply_migrations.py b/pipeline/tests/test_apply_migrations.py new file mode 100644 index 0000000..3a8ab6f --- /dev/null +++ b/pipeline/tests/test_apply_migrations.py @@ -0,0 +1,99 @@ +import importlib.util +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + + +SCRIPT = Path(__file__).parents[1] / "scripts" / "maintenance" / "apply-migrations.py" +SPEC = importlib.util.spec_from_file_location("apply_migrations", SCRIPT) +MODULE = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(MODULE) + + +class Cursor: + def __init__(self, row=None): + self.row = row + + def fetchone(self): + return self.row + + +class Transaction: + def __init__(self, connection): + self.connection = connection + + def __enter__(self): + self.connection.transaction_depth += 1 + return self + + def __exit__(self, exc_type, exc_value, traceback): + self.connection.transaction_events.append((self.connection.transaction_depth, exc_type is not None)) + self.connection.transaction_depth -= 1 + return False + + +class Connection: + def __init__(self): + self.ledger = {} + self.transaction_depth = 0 + self.transaction_events = [] + self.executed = [] + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + return False + + def transaction(self): + return Transaction(self) + + def execute(self, query, params=None): + self.executed.append((query, params, self.transaction_depth)) + compact = " ".join(query.split()) + if compact.startswith("SELECT sha256 FROM uec.schema_migrations"): + return Cursor(self.ledger.get(params[0])) + if compact.startswith("INSERT INTO uec.schema_migrations"): + self.ledger[params[0]] = (params[1],) + return Cursor() + if "FAIL_MIGRATION" in query: + raise RuntimeError("synthetic migration failure") + return Cursor() + + +class MigrationRunnerTests(unittest.TestCase): + def test_failed_migration_is_not_recorded_and_retry_resumes(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / "001_first.sql").write_text("SELECT 1;\n", encoding="utf-8") + failing = root / "002_second.sql" + failing.write_text("-- FAIL_MIGRATION\nSELECT 2;\n", encoding="utf-8") + connection = Connection() + with patch.object(MODULE.psycopg, "connect", return_value=connection): + with self.assertRaisesRegex(RuntimeError, "synthetic migration failure"): + MODULE.apply("postgresql://test", root) + self.assertIn("001_first", connection.ledger) + self.assertNotIn("002_second", connection.ledger) + self.assertIn((1, True), connection.transaction_events) + + failing.write_text("SELECT 2;\n", encoding="utf-8") + with patch.object(MODULE.psycopg, "connect", return_value=connection): + self.assertEqual(MODULE.apply("postgresql://test", root), ["002_second"]) + self.assertIn("002_second", connection.ledger) + + def test_changed_applied_migration_checksum_fails_before_sql_execution(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + migration = root / "001_first.sql" + migration.write_text("SELECT 1;\n", encoding="utf-8") + connection = Connection() + connection.ledger["001_first"] = ("0" * 64,) + with patch.object(MODULE.psycopg, "connect", return_value=connection): + with self.assertRaisesRegex(ValueError, "checksum changed"): + MODULE.apply("postgresql://test", root) + self.assertFalse(any("SELECT 1;" in query for query, _, _ in connection.executed)) + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/tests/test_coordinate_coverage.py b/pipeline/tests/test_coordinate_coverage.py index b109314..09ecb53 100644 --- a/pipeline/tests/test_coordinate_coverage.py +++ b/pipeline/tests/test_coordinate_coverage.py @@ -14,7 +14,11 @@ class CoordinateCoverageTests(unittest.TestCase): def test_states_prioritize_restriction_and_distinguish_exact_from_coarse(self): self.assertEqual(MODULE.coordinate_state({"coordinates": {"latitude": 1, "longitude": 2}}), "source_supplied") + self.assertEqual(MODULE.coordinate_state({"coordinates": {"latitude": 0, "longitude": 0}}), "source_supplied") + self.assertEqual(MODULE.coordinate_state({"coordinates": {"latitude": 91, "longitude": 2}}), "unresolved") + self.assertEqual(MODULE.coordinate_state({"coordinates": {"latitude": True, "longitude": 2}}), "unresolved") self.assertEqual(MODULE.coordinate_state({"geocode": {"status": "accepted", "result": {"x": 2}, "precision": "exact"}}), "geocoded_exact") + self.assertEqual(MODULE.coordinate_state({"geocode": {"status": "accepted", "result": {"x": 2}}}), "unresolved") self.assertEqual(MODULE.coordinate_state({"geocode": {"status": "review_required", "result": {"x": 2}}}), "approximate_coarse") self.assertEqual(MODULE.coordinate_state({"geocode": {"status": "accepted", "result": {"x": 2}, "precision": "city"}}), "approximate_coarse") self.assertEqual(MODULE.coordinate_state({"geocode": {"status": "accepted", "result": {"x": 2}}, "restricted": True}), "restricted") From 2a20ed2bf64f4ee4998ff7b9504f21a872739e4d Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Wed, 16 Sep 2026 09:30:11 -0700 Subject: [PATCH 143/311] docs: add Sweden source reconnaissance --- docs/country-recon-se.md | 38 ++++++++++++++++++++ docs/source-status.json | 8 ++++- pipeline/source_registry.json | 8 ++++- pipeline/tests/test_source_registry.py | 4 +-- pipeline/tests/test_sweden_recon_metadata.py | 35 ++++++++++++++++++ 5 files changed, 89 insertions(+), 4 deletions(-) create mode 100644 docs/country-recon-se.md create mode 100644 pipeline/tests/test_sweden_recon_metadata.py diff --git a/docs/country-recon-se.md b/docs/country-recon-se.md new file mode 100644 index 0000000..8ae2799 --- /dev/null +++ b/docs/country-recon-se.md @@ -0,0 +1,38 @@ +# Sweden source reconnaissance + +Status: row-free reconnaissance only. No facility rows, raw exports, coordinates, or production ingestion were retained. Verified against public official pages on 2026-09-16. + +## Decision + +Sweden is a viable but medium/high-effort country lane. The strongest first-party candidates are the Swedish Board of Agriculture (Jordbruksverket) slaughterhouse installation table and its linked feed/animal-by-product XLSX lists. Jordbruksverket's quarterly slaughter statistics and the Swedish Environmental Protection Agency's PRTR are useful separate evidence products. A national public master of all food establishments, intensive farms, or research facilities was not verified. + +The implementation must remain fully automated from acquisition through validation, provenance, normalization, review gating, and ingestion. Browser-only portals and human-export steps are not production ingestion routes; they are reconnaissance or assisted-capture fallbacks until a stable download/API contract is verified. + +## Primary source inventory + +| Source | Authority and route | Cadence / format | Identifiers and location fields | Reuse, privacy, automation decision | +|---|---|---|---|---| +| Slaughterhouse installation numbers | [Jordbruksverket installation-number table](https://jordbruksverket.se/3608.html) | Live HTML; cadence not stated | Official installation number / former production-site or “SE-number”; slaughtered species and ombuds-reporting flag; no address or coordinates observed in the table | Source-specific reuse terms not verified; names can identify operators. Medium effort HTML adapter with schema/row-count drift alarms; reference-only until capture and terms review. | +| Feed and animal by-product facilities | [Jordbruksverket facility lists](https://jordbruksverket.se/djur/foder-och-produkter-fran-djur/listor-over-anlaggningar-for-foder-och-animaliska-biprodukter-och-darav-framstallda-produkter) | Page says lists are updated continuously; linked XLSX sections I–IX and XI–XIII plus PDF material | Official facility numbers; feed business, storage/transport, processing, incineration, biogas, composting and other categories; address/coordinate columns require header capture | Terms not verified on landing page. XLSX acquisition is the best machine-readable candidate, but category sections must remain separate and names/addresses undergo privacy review. | +| EU exchange / approved-facility context | [Jordbruksverket TRACES guidance](https://jordbruksverket.se/e-tjanster-databaser-och-appar/e-tjanster-och-databaser-djur/anmal-resor-och-handel-med-djur-och-djurprodukter-med-mera-i-traces/traces-for-sandningar-av-djur-foder-och-djurprodukter) | Portal and authority-maintained registrations | Swedish authority validates entries; national facility/register identifiers depend on the linked register | Not a verified public bulk route. High effort and access/session uncertainty; use for provenance/context and reconciliation, never as an extra facility count. | +| Slaughter statistics | [Jordbruksverket slaughter statistics](https://jordbruksverket.se/djur/djurtransportorer-och-slakterier/statistik-om-slaktade-djur-och-klassning) | Weekly reports; published/updated quarterly; XLSX current-quarter and annual/classification files | Aggregate species/quantity/classification; only largest named slaughterhouses with consent, remainder as “Other”; no coordinates | Do not infer national facility coverage from named rows. Medium effort scheduled XLSX ingestion with period/version checks; keep aggregate facts separate from facility entities. | +| Experimental-animal use | [Jordbruksverket permits and approvals](https://jordbruksverket.se/djur/ovriga-djur/forsoksdjur-och-djurforsok/tillstand-och-godkannanden) and [use/statistics guidance](https://jordbruksverket.se/djur/ovriga-djur/forsoksdjur-och-djurforsok/verksamhet-med-forsoksdjur) | Annual Swedish statistics; ALURES summaries from 2021; no stable facility export verified | Facility approvals are described, but a public national facility register was not located; ALURES summaries are anonymized and not a facility master | Sensitive research and animal-use information. Keep separate, anonymized, and aggregate; medium/high effort portal/statistics adapter only after stable route and purpose review. | +| Environmental releases and permits | [Swedish PRTR](https://www.naturvardsverket.se/en/services-and-permits/data-databases-and-applications/the-swedish-pollutant-release-and-transfer-register/) and [environmental document catalogue](https://miljodokument.naturvardsverket.se/) | PRTR annual emissions/transfers; public search; document catalogue is browser-filtered | PRTR covers about 1,300 environmentally hazardous facilities; exact export/API and facility key were not verified. Permit documents may contain local addresses/geometry | Source terms and bulk contract require confirmation. PRTR is medium/high effort; document catalogue is high effort and may have local-coverage gaps. Treat permit/PRTR evidence as separate from food approval. | +| Corporate identity linkage | [Bolagsverket company-information API](https://bolagsverket.se/apierochoppnadata/hamtaforetagsinformation/apiforatthamtaforetagsinformation.3988.html) | API/download products; auth, quotas and exact public access not verified | Organisation number; company name/status/business/address fields; do not ingest board/beneficial-owner data | Identity crosswalk only, never facility proof. Medium effort after access approval; sole-trader and mixed residential address privacy risk. | +| Food controls and approved export subset | [Livsmedelsverket control-system guidance](https://kontrollwiki.livsmedelsverket.se/artikel/593/sveriges-kontrollsystem) and [approved beef-to-Hong-Kong PDF](https://www.livsmedelsverket.se/globalassets/foretag-regler-kontroll/export-import-handel/export/hong-kong/annex-ia_list-of-approved-swedish-establishments-to-export-beef-to-hk_20240614_ol.pdf) | Annual control reporting; cited PDF dated 2024-06-14 | Control reporting is national/aggregate guidance, not a row-level register; export PDF contains approval number/name/address/activity for a destination-specific subset | Useful for schema and authority context only. Do not represent the export subset as Sweden-wide coverage; PDF extraction is brittle and stale for current use. | + +## Verification findings + +- Jordbruksverket explicitly distinguishes official installation numbers and publishes the slaughterhouse table with species/reporting fields. The table does not supply a national address/coordinate layer. +- The feed/animal-by-product landing page states continuous updates and links multiple XLSX category lists. Primary producers of feed for food-producing animals are registered through county administrative boards, so the page is not evidence of one complete national feed-farm register. +- Livsmedelsverket describes registration and approval responsibilities, but food registration is held in local authority control registers; no national row-level public endpoint was verified. +- Research-facility authorization exists, while published experimental-animal summaries are intentionally anonymized. They cannot support facility-level claims. +- PRTR and environmental documents answer a different question from food approval. They require separate provenance, coverage, and privacy decisions. + +## Automation and release gates + +Recommended order: (1) scheduled, bounded retrieval of the Jordbruksverket slaughterhouse HTML and feed/ABP XLSX links; (2) header fingerprint, section, count, and official-number validation; (3) private review of names/addresses and source terms; (4) only then candidate normalization and guarded ingestion. Slaughter statistics and PRTR should remain separate aggregate/evidence pipelines. TRACES, ALURES, Bolagsverket, and the document catalogue are not production bulk routes until access and stable contracts are verified. + +Open blockers are the absence of a verified national animal-origin establishment master, unresolved reuse terms for the main Jordbruksverket files, unverified XLSX headers/direct download stability, unknown address/coordinate availability, and lack of a public research-facility register. Publication remains blocked pending ethics, privacy, source-rights, coverage, and release approval. + +Recommended next country after Sweden: Ireland, because its official food-safety authority routes approved-establishment evidence across bounded competent-authority lists; it should be treated as a multi-authority lane rather than assumed to have one national feed/farm master. diff --git a/docs/source-status.json b/docs/source-status.json index c582844..64285fb 100644 --- a/docs/source-status.json +++ b/docs/source-status.json @@ -89,6 +89,12 @@ {"source_id":"pl.arimr.processing-support","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-pl.md","docs/countries/pl/source-crosswalk.json","pipeline/source_registry.json"],"next_action":"Acquire only permitted funding/project evidence; do not interpret funding as facility operation or compliance."}, {"source_id":"pl.krs.open-api","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-pl.md","docs/countries/pl/source-crosswalk.json","pipeline/source_registry.json"],"next_action":"Confirm current KRS API contract and RODO filtering; use only for exact organization identity links supplied by an upstream source."}, {"source_id":"pl.gus.regon-bir","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-pl.md","docs/countries/pl/source-crosswalk.json","pipeline/source_registry.json"],"next_action":"Obtain authorized BIR1 access before queries, respect documented rate limits, and keep sole-trader/restricted fields private."}, - {"source_id":"eu.traces.pl-approved-food","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-pl.md","docs/countries/pl/source-crosswalk.json","pipeline/source_registry.json"],"next_action":"Verify Poland-specific TRACES export only as a lineage/cross-check layer; never double-count GIW."} + {"source_id":"eu.traces.pl-approved-food","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-pl.md","docs/countries/pl/source-crosswalk.json","pipeline/source_registry.json"],"next_action":"Verify Poland-specific TRACES export only as a lineage/cross-check layer; never double-count GIW."}, + {"source_id":"se.jordbruksverket.slaughterhouses","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-se.md","pipeline/source_registry.json"],"next_action":"Verify live HTML stability, terms, completeness, effective-date semantics, and privacy before automated acquisition."}, + {"source_id":"se.jordbruksverket.feed-abp","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-se.md","pipeline/source_registry.json"],"next_action":"Capture linked XLSX files privately, fingerprint headers/sections, and resolve terms, coverage, and privacy."}, + {"source_id":"se.jordbruksverket.slaughter-stats","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-se.md","pipeline/source_registry.json"],"next_action":"Verify current quarterly/annual XLSX links and preserve consent-limited named versus Other scope."}, + {"source_id":"se.jordbruksverket.animal-experiments","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-se.md","pipeline/source_registry.json"],"next_action":"Keep anonymized annual/ALURES summaries separate; no facility master is verified."}, + {"source_id":"se.naturvardsverket.prtr","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-se.md","pipeline/source_registry.json"],"next_action":"Verify public export/API, facility key, completeness, terms, and separation from permit evidence."}, + {"source_id":"se.bolagsverket.company-api","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-se.md","pipeline/source_registry.json"],"next_action":"Confirm API access, quotas, terms, and privacy before identity-only crosswalk use."} ] } diff --git a/pipeline/source_registry.json b/pipeline/source_registry.json index 34208e0..ea6492d 100644 --- a/pipeline/source_registry.json +++ b/pipeline/source_registry.json @@ -721,6 +721,12 @@ {"source_id":"pl.arimr.processing-support","jurisdiction_scope":"Poland; ARiMR agricultural processing investment support","legacy_paths":[],"url":"https://www.gov.pl/web/arimr/startuje-wsparcie-dla-przetworcow","access_method":"official call/guidance; beneficiary/project dataset not acquired","cadence":"call-specific; 2026 call window observed 1-30 September","attribution_licensing_notes":"Polish official funding evidence; beneficiary/privacy terms review required","adapter_status":"reference_only","expected_artifact_schema":"Call/application/agreement/beneficiary/project/amount/date observations","blockers":["Do not treat funding as facility authorization, operation or compliance."]}, {"source_id":"pl.krs.open-api","jurisdiction_scope":"Poland; Ministry of Justice KRS legal-entity identity","legacy_paths":[],"url":"https://prs.ms.gov.pl/krs/openApi","access_method":"open API announced by Ministry; no request made","cadence":"on-demand/registry filings","attribution_licensing_notes":"Official KRS; RODO filtering and API terms review required","adapter_status":"reference_only","expected_artifact_schema":"KRS number, legal name, status, registered information and identifiers","blockers":["Confirm current API contract; exact identity link only, never operating-site inference."]}, {"source_id":"pl.gus.regon-bir","jurisdiction_scope":"Poland; GUS REGON BIR1 identity service","legacy_paths":[],"url":"https://api.stat.gov.pl/Home/RegonApi?lang=en","access_method":"documented SOAP/API service; registration/user key required","cadence":"continuously updated register; on-demand queries","attribution_licensing_notes":"Official GUS service; respect registration, rate limits and personal-data handling","adapter_status":"reference_only","expected_artifact_schema":"Lookup response by REGON, NIP or KRS","blockers":["Obtain authorized access before queries and keep fields restricted."]}, - {"source_id":"eu.traces.pl-approved-food","jurisdiction_scope":"EU mirror; Poland approved-food establishment lineage","legacy_paths":[],"url":"https://food.ec.europa.eu/food-safety/biological-safety/food-hygiene/approved-eu-food-establishments_en","access_method":"public TRACES/publication page; Poland export not verified","cadence":"competent-authority updates","attribution_licensing_notes":"EU official mirror; do not double-count GIW","adapter_status":"reference_only","expected_artifact_schema":"TRACES establishment/activity listing keyed by EU/national approval identifiers","blockers":["Verify Poland-specific export and keep mirror provenance separate."]} + {"source_id":"eu.traces.pl-approved-food","jurisdiction_scope":"EU mirror; Poland approved-food establishment lineage","legacy_paths":[],"url":"https://food.ec.europa.eu/food-safety/biological-safety/food-hygiene/approved-eu-food-establishments_en","access_method":"public TRACES/publication page; Poland export not verified","cadence":"competent-authority updates","attribution_licensing_notes":"EU official mirror; do not double-count GIW","adapter_status":"reference_only","expected_artifact_schema":"TRACES establishment/activity listing keyed by EU/national approval identifiers","blockers":["Verify Poland-specific export and keep mirror provenance separate."]}, + {"source_id":"se.jordbruksverket.slaughterhouses","jurisdiction_scope":"Sweden; Jordbruksverket slaughterhouse installation-number table","legacy_paths":[],"url":"https://jordbruksverket.se/3608.html","access_method":"public live HTML table","cadence":"not stated; timestamp each retrieval","attribution_licensing_notes":"Official Swedish authority; source-specific reuse terms not verified; privacy review required for names and addresses if later captured.","adapter_status":"reference_only","expected_artifact_schema":"HTML rows with installation/SE number, establishment name, slaughtered species, and ombuds-reporting flag; address/coordinates not observed","blockers":["Verify terms, live HTML stability, completeness, effective-date semantics, and privacy before automated acquisition."]}, + {"source_id":"se.jordbruksverket.feed-abp","jurisdiction_scope":"Sweden; Jordbruksverket feed and animal-by-product facility lists","legacy_paths":[],"url":"https://jordbruksverket.se/djur/foder-och-produkter-fran-djur/listor-over-anlaggningar-for-foder-och-animaliska-biprodukter-och-darav-framstallda-produkter","access_method":"official landing page with linked XLSX/PDF downloads","cadence":"continuous updates stated by publisher","attribution_licensing_notes":"Official Swedish authority; file-specific reuse terms not verified; operator names/addresses may require privacy review.","adapter_status":"reference_only","expected_artifact_schema":"Sectioned XLSX/PDF lists with official facility number, operator/site identity, approval/registration category, activity and address fields; coordinates unknown","blockers":["Capture and fingerprint each linked file, verify headers/section semantics, direct-download stability, terms, and national coverage."]}, + {"source_id":"se.jordbruksverket.slaughter-stats","jurisdiction_scope":"Sweden; Jordbruksverket aggregate slaughter and classification statistics","legacy_paths":[],"url":"https://jordbruksverket.se/djur/djurtransportorer-och-slakterier/statistik-om-slaktade-djur-och-klassning","access_method":"official XLSX downloads and explanatory HTML","cadence":"weekly reporting; publication/update quarterly","attribution_licensing_notes":"Official Swedish authority; source-specific reuse terms not verified; preserve consent-limited naming and aggregate scope.","adapter_status":"reference_only","expected_artifact_schema":"Periodized XLSX aggregate species, quantity, weight/classification and named-largest/Other dimensions","blockers":["Do not infer facility completeness from consent-based named subset; verify file URLs, schema, revisions and terms."]}, + {"source_id":"se.jordbruksverket.animal-experiments","jurisdiction_scope":"Sweden; experimental-animal permits, approvals and use statistics","legacy_paths":[],"url":"https://jordbruksverket.se/djur/ovriga-djur/forsoksdjur-och-djurforsok/verksamhet-med-forsoksdjur","access_method":"official guidance, annual statistics and ALURES summaries","cadence":"annual statistics; ALURES summaries from 2021","attribution_licensing_notes":"Sensitive animal-use/research evidence; anonymization, purpose limitation and privacy review required; terms not fully verified.","adapter_status":"reference_only","expected_artifact_schema":"Anonymized project/use summaries with year, species, count, purpose and severity; no facility master assumed","blockers":["No public national facility export verified; resolve stable statistics route and keep facility-level research claims out of canonical entities."]}, + {"source_id":"se.naturvardsverket.prtr","jurisdiction_scope":"Sweden; Swedish Pollutant Release and Transfer Register and environmental document catalogue","legacy_paths":[],"url":"https://www.naturvardsverket.se/en/services-and-permits/data-databases-and-applications/the-swedish-pollutant-release-and-transfer-register/","access_method":"public search service and browser-filtered document catalogue","cadence":"annual PRTR emissions/transfers; document publication varies","attribution_licensing_notes":"Official Swedish environmental authority; bulk reuse terms/API not verified; location and permit-document privacy review required.","adapter_status":"reference_only","expected_artifact_schema":"Facility/environmental observations with reporting year, emissions/transfers, authority, document IDs and optional reviewed site link","blockers":["Verify export/API, stable facility key, completeness, terms, and separation of PRTR from permit/food approval evidence."]}, + {"source_id":"se.bolagsverket.company-api","jurisdiction_scope":"Sweden; Bolagsverket corporate identity and organization-number API","legacy_paths":[],"url":"https://bolagsverket.se/apierochoppnadata/hamtaforetagsinformation/apiforatthamtaforetagsinformation.3988.html","access_method":"official API/download documentation; access not exercised","cadence":"provider-defined; auth and quota cadence unknown","attribution_licensing_notes":"Identity linkage only; API terms, personal-data handling, and sole-trader/mixed-address suppression required.","adapter_status":"reference_only","expected_artifact_schema":"Organization-number keyed identity/status/business/address response with retrieval timestamp","blockers":["Confirm account/auth, rate limits, terms, field availability and privacy before automated crosswalk use; never treat corporate identity as facility proof."]} ] } diff --git a/pipeline/tests/test_source_registry.py b/pipeline/tests/test_source_registry.py index 0532cf8..337c7cf 100644 --- a/pipeline/tests/test_source_registry.py +++ b/pipeline/tests/test_source_registry.py @@ -9,8 +9,8 @@ class SourceRegistryTests(unittest.TestCase): def test_repository_registry_loads_and_references_existing_legacy_paths(self): registry = load_registry() - self.assertEqual(len(registry["sources"]), 81) - self.assertEqual(len({source["source_id"] for source in registry["sources"]}), 81) + self.assertEqual(len(registry["sources"]), 87) + self.assertEqual(len({source["source_id"] for source in registry["sources"]}), 87) def test_unknowns_are_explicit(self): registry = json.loads(REGISTRY_PATH.read_text(encoding="utf-8")) diff --git a/pipeline/tests/test_sweden_recon_metadata.py b/pipeline/tests/test_sweden_recon_metadata.py new file mode 100644 index 0000000..64de8e4 --- /dev/null +++ b/pipeline/tests/test_sweden_recon_metadata.py @@ -0,0 +1,35 @@ +import json +import unittest +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[2] +SWEDEN_IDS = { + "se.jordbruksverket.slaughterhouses", + "se.jordbruksverket.feed-abp", + "se.jordbruksverket.slaughter-stats", + "se.jordbruksverket.animal-experiments", + "se.naturvardsverket.prtr", + "se.bolagsverket.company-api", +} + + +class SwedenReconMetadataTests(unittest.TestCase): + def test_sweden_sources_are_registered_and_row_free(self): + registry = json.loads((ROOT / "pipeline/source_registry.json").read_text(encoding="utf-8-sig")) + self.assertTrue(SWEDEN_IDS.issubset({source["source_id"] for source in registry["sources"]})) + recon = (ROOT / "docs/country-recon-se.md").read_text(encoding="utf-8") + self.assertIn("No facility rows", recon) + self.assertNotIn("Anläggnings-nummer", recon) + self.assertIn("fully automated", recon) + + def test_sweden_sources_are_publication_blocked(self): + status = json.loads((ROOT / "docs/source-status.json").read_text(encoding="utf-8-sig")) + by_id = {source["source_id"]: source for source in status["sources"]} + for source_id in SWEDEN_IDS: + self.assertEqual(by_id[source_id]["publication_eligibility"], "blocked") + self.assertEqual(by_id[source_id]["acquisition"], "not_run") + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file From 215f963923307dbb1e0c3fa7b8bf805ae85c7b36 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Wed, 16 Sep 2026 09:35:41 -0700 Subject: [PATCH 144/311] docs: add Norway source reconnaissance --- docs/country-recon-no.md | 31 ++++++++++++++++++++ docs/source-status.json | 10 ++++++- pipeline/source_registry.json | 10 ++++++- pipeline/tests/test_norway_recon_metadata.py | 29 ++++++++++++++++++ pipeline/tests/test_source_registry.py | 4 +-- 5 files changed, 80 insertions(+), 4 deletions(-) create mode 100644 docs/country-recon-no.md create mode 100644 pipeline/tests/test_norway_recon_metadata.py diff --git a/docs/country-recon-no.md b/docs/country-recon-no.md new file mode 100644 index 0000000..4986cdd --- /dev/null +++ b/docs/country-recon-no.md @@ -0,0 +1,31 @@ +# Norway source reconnaissance + +Status: row-free reconnaissance only. No facility rows, raw exports, coordinates, or production ingestion were retained. Official routes checked 2026-09-16. + +## Decision + +Norway is a strong medium-effort candidate for automated integration. Mattilsynet publishes approved food/feed/animal-by-product lists as CSV-backed pages, Fiskeridirektoratet exposes aquaculture APIs and a point feature service, Brønnøysundregistrene publishes organization-number APIs, and Statistics Norway (SSB) provides open Statbank APIs. Approval, aquaculture authorization, corporate identity, environmental permits, inspections, and statistics remain separate evidence products. + +The required pipeline is fully automated end-to-end: scheduled retrieval, artifact hashing, schema validation, provenance, normalization, privacy/review gates, and guarded ingestion. Browser searches/manual exports are reconnaissance or assisted fallbacks, not production routes. + +## Source inventory + +| Source | Official route and findings | Cadence / format / identifiers | Location, terms, privacy, automation | +|---|---|---|---| +| Approved animal-origin establishments | [Mattilsynet approved products/businesses](https://www.mattilsynet.no/godkjente-produkter-og-virksomheter), including [Section 0 CSV](https://www.mattilsynet.no/godkjente-produkter-og-virksomheter/food-of-animal-origin-section-0-general-activity-establishment) and [Section VI CSV](https://www.mattilsynet.no/godkjente-produkter-og-virksomheter/food-of-animal-origin-section-VI-meat-products) | Most lists state daily updates; CSV fields include approval-from date, organization/town, activities, associated activities, municipality, approval ID, category and list IDs | Address/town and organization names present; no coordinates observed. Confirm file-specific terms and privacy. Strong candidate, medium effort, drift alarms required. | +| Feed and ABP | [Mattilsynet feed-sector/TSE list](https://www.mattilsynet.no/godkjente-produkter-og-virksomheter/forvarefeed-sector-approved-and-registered-feed-companies-tse) | Sectioned approved/registered lists covering slaughter, processing, PAP and aquaculture feed; CSV/HTML links | Official approval/registration number expected; coordinates unknown. Separate categories; verify terms/completeness. Medium effort. | +| Aquaculture | [Fiskeridirektoratet Aquaculture Register](https://www.fiskeridir.no/registre/akvakulturregisteret), [API catalogue](https://api.fiskeridir.no/catalog/), [locality feature layer](https://gis.fiskeridir.no/server/rest/services/Yggdrasil/Akvakulturregisteret/MapServer/0) | Public API plus XLSX/CSV; ArcGIS service supports JSON/GeoJSON/PBF, pagination and spatial queries | Point geometry in EPSG:25833; locality/permit/species identifiers require header capture. Verify terms and geometry semantics. Strongest route, medium effort. | +| Farm/intensive agriculture | [Landbruksregisteret](https://www.landbruksdirektoratet.no/nb/jordbruk/kart-og-register/landbruksregisteret) and [farm/livestock statistics](https://www.landbruksdirektoratet.no/nb/statistikk-og-utviklingstrekk/utvikling-i-jordbruket/jordbruksforetak-jordbruksareal-og-husdyr) | Register covers farms, persons, enterprises and national identifiers; public aggregate/downloadable data, but no public facility-level export verified | Property/person relationships are sensitive. Do not infer intensive-farm rows from aggregate reports; blocked pending authorized route. | +| Animal experimentation | [Mattilsynet annual reports](https://www.mattilsynet.no/dyr/forsoksdyr/bruk-av-dyr-i-forsok) | Annual reports; ALURES EEA/EU context; underlying reporting system authenticated | No public research-facility master verified. Keep protocols, institutions and locations separate/aggregated. Medium/high effort. | +| Inspections/enforcement | Mattilsynet approved-business and control surfaces | No national row-level inspection/enforcement API verified | Model events separately from approvals; absence is not closure. High effort/fragmented. | +| Environmental permits/releases | [Norwegian Environment Agency PRTR reference](https://www.miljodirektoratet.no/globalassets/publikasjoner/M138/M138.pdf) | Public route, cadence/API/export unresolved | No national permit master or coordinate route verified. High effort; terms, coverage and sensitive-site review required. | +| Corporate identifiers | [Brønnøysundregistrene open-data API](https://data.brreg.no/enhetsregisteret/api/dokumentasjon/en/index.html) and [Nordic lookup](https://www.brreg.no/en/use-of-data-from-the-bronnoysund-register-centre/datasets-and-api/data-about-nordic-businesses/) | REST JSON, CSV/gzip/XLSX; organization number; update feeds; Nordic lookup requires API key | NLOD 2.0 stated. Identity-only crosswalk; suppress person roles, birth numbers, sole-trader/mixed residential addresses. Low/medium effort. | +| Slaughter and animal-use statistics | [SSB meat production](https://www.ssb.no/en/jord-skog-jakt-og-fiskeri/jordbruk/statistikk/kjotproduksjon), [meat tables](https://www.ssb.no/en/statbank1/list/slakt), [SSB APIs](https://www.ssb.no/en/api), [Mattilsynet animal-use reports](https://www.mattilsynet.no/dyr/forsoksdyr/bruk-av-dyr-i-forsok) | PxWeb/StatBank open API; SSB states CC BY 4.0 and daily update schedule; animal-use annual reports | Aggregate only, no facility coordinates. Preserve table IDs, dimensions and revision markers. Low/medium effort. | + +## Automation and release gates + +First implementation order: (1) scheduled Mattilsynet section-CSV retrieval; (2) Fiskeridirektoratet aquaculture API/GeoJSON; (3) Brønnøysundregistrene identity crosswalk; (4) SSB aggregates. Require URL/API contract capture, hash/bytes, timestamps, content type, schema fingerprints, pagination checks, approval-ID validation, category preservation, CRS/precision checks, and quarantine on drift. No geocoding or inferred identity is authorized. + +Blockers: file-specific Mattilsynet reuse terms and complete section coverage; exact CSV URLs/headers; aquaculture API versioning and geometry semantics; public farm/intensive-facility scope; inspection/enforcement and environmental permit exports; and research-facility privacy boundaries. Publication remains blocked pending ethics, rights, privacy, coverage, and maintainer approval. + +Recommended next country after Norway: Finland, using separate food-authority, aquaculture/farm, environmental, corporate, and statistics routes. diff --git a/docs/source-status.json b/docs/source-status.json index 64285fb..105fd6e 100644 --- a/docs/source-status.json +++ b/docs/source-status.json @@ -95,6 +95,14 @@ {"source_id":"se.jordbruksverket.slaughter-stats","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-se.md","pipeline/source_registry.json"],"next_action":"Verify current quarterly/annual XLSX links and preserve consent-limited named versus Other scope."}, {"source_id":"se.jordbruksverket.animal-experiments","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-se.md","pipeline/source_registry.json"],"next_action":"Keep anonymized annual/ALURES summaries separate; no facility master is verified."}, {"source_id":"se.naturvardsverket.prtr","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-se.md","pipeline/source_registry.json"],"next_action":"Verify public export/API, facility key, completeness, terms, and separation from permit evidence."}, - {"source_id":"se.bolagsverket.company-api","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-se.md","pipeline/source_registry.json"],"next_action":"Confirm API access, quotas, terms, and privacy before identity-only crosswalk use."} + {"source_id":"se.bolagsverket.company-api","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-se.md","pipeline/source_registry.json"],"next_action":"Confirm API access, quotas, terms, and privacy before identity-only crosswalk use."}, + {"source_id":"no.mattilsynet.approved-food","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-no.md","pipeline/source_registry.json"],"next_action":"Verify complete section coverage, CSV contracts, terms, privacy and approval-ID semantics."}, + {"source_id":"no.mattilsynet.feed-abp","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-no.md","pipeline/source_registry.json"],"next_action":"Verify direct files, category boundaries, completeness, terms and privacy."}, + {"source_id":"no.fiskeridir.aquaculture","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-no.md","pipeline/source_registry.json"],"next_action":"Capture API metadata and validate pagination, identifiers, geometry and terms."}, + {"source_id":"no.landbruksdirektoratet.farm-register","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-no.md","pipeline/source_registry.json"],"next_action":"Do not acquire property/person/farm rows without an authorized purpose-limited route."}, + {"source_id":"no.mattilsynet.animal-experiments","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-no.md","pipeline/source_registry.json"],"next_action":"Keep annual aggregates separate; no research-facility master is verified."}, + {"source_id":"no.miljodirektoratet.prtr-permits","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-no.md","pipeline/source_registry.json"],"next_action":"Verify current public environmental export/API and stable IDs before acquisition."}, + {"source_id":"no.brreg.organizations","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-no.md","pipeline/source_registry.json"],"next_action":"Verify endpoint versions, rate limits and identity-only privacy policy."}, + {"source_id":"no.ssb.meat-and-animal-use","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-no.md","pipeline/source_registry.json"],"next_action":"Pin SSB table IDs and metadata; preserve aggregate scope and revisions."} ] } diff --git a/pipeline/source_registry.json b/pipeline/source_registry.json index ea6492d..c4090c6 100644 --- a/pipeline/source_registry.json +++ b/pipeline/source_registry.json @@ -727,6 +727,14 @@ {"source_id":"se.jordbruksverket.slaughter-stats","jurisdiction_scope":"Sweden; Jordbruksverket aggregate slaughter and classification statistics","legacy_paths":[],"url":"https://jordbruksverket.se/djur/djurtransportorer-och-slakterier/statistik-om-slaktade-djur-och-klassning","access_method":"official XLSX downloads and explanatory HTML","cadence":"weekly reporting; publication/update quarterly","attribution_licensing_notes":"Official Swedish authority; source-specific reuse terms not verified; preserve consent-limited naming and aggregate scope.","adapter_status":"reference_only","expected_artifact_schema":"Periodized XLSX aggregate species, quantity, weight/classification and named-largest/Other dimensions","blockers":["Do not infer facility completeness from consent-based named subset; verify file URLs, schema, revisions and terms."]}, {"source_id":"se.jordbruksverket.animal-experiments","jurisdiction_scope":"Sweden; experimental-animal permits, approvals and use statistics","legacy_paths":[],"url":"https://jordbruksverket.se/djur/ovriga-djur/forsoksdjur-och-djurforsok/verksamhet-med-forsoksdjur","access_method":"official guidance, annual statistics and ALURES summaries","cadence":"annual statistics; ALURES summaries from 2021","attribution_licensing_notes":"Sensitive animal-use/research evidence; anonymization, purpose limitation and privacy review required; terms not fully verified.","adapter_status":"reference_only","expected_artifact_schema":"Anonymized project/use summaries with year, species, count, purpose and severity; no facility master assumed","blockers":["No public national facility export verified; resolve stable statistics route and keep facility-level research claims out of canonical entities."]}, {"source_id":"se.naturvardsverket.prtr","jurisdiction_scope":"Sweden; Swedish Pollutant Release and Transfer Register and environmental document catalogue","legacy_paths":[],"url":"https://www.naturvardsverket.se/en/services-and-permits/data-databases-and-applications/the-swedish-pollutant-release-and-transfer-register/","access_method":"public search service and browser-filtered document catalogue","cadence":"annual PRTR emissions/transfers; document publication varies","attribution_licensing_notes":"Official Swedish environmental authority; bulk reuse terms/API not verified; location and permit-document privacy review required.","adapter_status":"reference_only","expected_artifact_schema":"Facility/environmental observations with reporting year, emissions/transfers, authority, document IDs and optional reviewed site link","blockers":["Verify export/API, stable facility key, completeness, terms, and separation of PRTR from permit/food approval evidence."]}, - {"source_id":"se.bolagsverket.company-api","jurisdiction_scope":"Sweden; Bolagsverket corporate identity and organization-number API","legacy_paths":[],"url":"https://bolagsverket.se/apierochoppnadata/hamtaforetagsinformation/apiforatthamtaforetagsinformation.3988.html","access_method":"official API/download documentation; access not exercised","cadence":"provider-defined; auth and quota cadence unknown","attribution_licensing_notes":"Identity linkage only; API terms, personal-data handling, and sole-trader/mixed-address suppression required.","adapter_status":"reference_only","expected_artifact_schema":"Organization-number keyed identity/status/business/address response with retrieval timestamp","blockers":["Confirm account/auth, rate limits, terms, field availability and privacy before automated crosswalk use; never treat corporate identity as facility proof."]} + {"source_id":"se.bolagsverket.company-api","jurisdiction_scope":"Sweden; Bolagsverket corporate identity and organization-number API","legacy_paths":[],"url":"https://bolagsverket.se/apierochoppnadata/hamtaforetagsinformation/apiforatthamtaforetagsinformation.3988.html","access_method":"official API/download documentation; access not exercised","cadence":"provider-defined; auth and quota cadence unknown","attribution_licensing_notes":"Identity linkage only; API terms, personal-data handling, and sole-trader/mixed-address suppression required.","adapter_status":"reference_only","expected_artifact_schema":"Organization-number keyed identity/status/business/address response with retrieval timestamp","blockers":["Confirm account/auth, rate limits, terms, field availability and privacy before automated crosswalk use; never treat corporate identity as facility proof."]} ,{"source_id":"no.mattilsynet.approved-food","jurisdiction_scope":"Norway; Mattilsynet approved animal-origin food establishments","legacy_paths":[],"url":"https://www.mattilsynet.no/godkjente-produkter-og-virksomheter","access_method":"official section pages with CSV-backed lists","cadence":"most lists stated daily; timestamp each retrieval","attribution_licensing_notes":"Official Norwegian Food Safety Authority; confirm file-specific reuse terms; names/addresses require privacy review.","adapter_status":"reference_only","expected_artifact_schema":"Sectioned CSV with approval date, organization/town, activities, associated activities, municipality, approval ID, category and list IDs","blockers":["Verify complete section coverage, direct CSV URLs, headers, terms, privacy and approval-ID semantics."]}, + {"source_id":"no.mattilsynet.feed-abp","jurisdiction_scope":"Norway; Mattilsynet approved/registered feed and animal-by-product establishments","legacy_paths":[],"url":"https://www.mattilsynet.no/godkjente-produkter-og-virksomheter/forvarefeed-sector-approved-and-registered-feed-companies-tse","access_method":"official sectioned list pages and CSV/HTML links","cadence":"publisher list cadence; verify per section","attribution_licensing_notes":"Official authority; terms and category-specific privacy review required.","adapter_status":"reference_only","expected_artifact_schema":"Sectioned feed/ABP rows with registration or approval number, site identity, category and activity","blockers":["Verify direct files, category boundaries, completeness, terms and privacy before acquisition."]}, + {"source_id":"no.fiskeridir.aquaculture","jurisdiction_scope":"Norway; Fiskeridirektoratet Aquaculture Register localities and permits","legacy_paths":[],"url":"https://api.fiskeridir.no/catalog/","access_method":"public REST/API catalogue, CSV/XLSX downloads and ArcGIS feature service","cadence":"provider-defined; timestamp API and layer metadata","attribution_licensing_notes":"Official Fiskeridirektoratet; preserve provider attribution, CRS/geometry semantics and verify current API terms.","adapter_status":"reference_only","expected_artifact_schema":"JSON/GeoJSON/PBF or CSV/XLSX locality/permit/species observations with stable locality and permit identifiers; EPSG:25833 point geometry","blockers":["Capture OpenAPI/version metadata, pagination, field semantics, geometry precision and release/privacy terms."]}, + {"source_id":"no.landbruksdirektoratet.farm-register","jurisdiction_scope":"Norway; Landbruksdirektoratet agricultural property/farm register and livestock statistics","legacy_paths":[],"url":"https://www.landbruksdirektoratet.no/nb/jordbruk/kart-og-register/landbruksregisteret","access_method":"official register guidance and aggregate/downloadable reports","cadence":"report/register-specific; exact public export cadence unknown","attribution_licensing_notes":"Official authority; farm/person/property linkage is sensitive and must not be exposed without purpose and review.","adapter_status":"reference_only","expected_artifact_schema":"Aggregate farm/livestock statistics or authorized property/enterprise identifiers; no public facility master assumed","blockers":["No public facility-level export verified; obtain authorization and privacy review before any acquisition."]}, + {"source_id":"no.mattilsynet.animal-experiments","jurisdiction_scope":"Norway; experimental-animal use statistics and reporting","legacy_paths":[],"url":"https://www.mattilsynet.no/dyr/forsoksdyr/bruk-av-dyr-i-forsok","access_method":"annual reports and ALURES statistical database; reporting system authenticated","cadence":"annual","attribution_licensing_notes":"Sensitive research/animal-use evidence; aggregate/anonymize and keep facility claims out of canonical entities.","adapter_status":"reference_only","expected_artifact_schema":"Annual aggregate/report observations by year, species, purpose and severity","blockers":["No public research-facility master verified; resolve stable report links and privacy boundary."]}, + {"source_id":"no.miljodirektoratet.prtr-permits","jurisdiction_scope":"Norway; Norwegian Environment Agency pollutant-release and permit evidence","legacy_paths":[],"url":"https://www.miljodirektoratet.no/globalassets/publikasjoner/M138/M138.pdf","access_method":"official environmental register/document references; public bulk route not verified","cadence":"annual/report-specific; unresolved","attribution_licensing_notes":"Official environmental authority; terms, sensitive sites and document privacy require review.","adapter_status":"reference_only","expected_artifact_schema":"Permit/PRTR observations with facility or permit identifier, reporting year, releases/transfers and authority","blockers":["Verify current public search/export/API, stable identifiers, completeness and coordinate/address exposure."]}, + {"source_id":"no.brreg.organizations","jurisdiction_scope":"Norway; Brønnøysundregistrene Central Coordinating Register open organization data","legacy_paths":[],"url":"https://data.brreg.no/enhetsregisteret/api/dokumentasjon/en/index.html","access_method":"public REST JSON API and CSV/gzip/XLSX downloads","cadence":"API/update-feed specific; timestamp each run","attribution_licensing_notes":"NLOD 2.0 stated by publisher; suppress person roles, birth numbers, sole-trader and mixed residential addresses.","adapter_status":"reference_only","expected_artifact_schema":"Organization/subunit records keyed by nine-digit organization number with name, type, status, address, dates and update events","blockers":["Verify rate limits, endpoint versions, field semantics and identity-only matching policy."]}, + {"source_id":"no.ssb.meat-and-animal-use","jurisdiction_scope":"Norway; Statistics Norway meat production and Mattilsynet animal-use aggregates","legacy_paths":[],"url":"https://www.ssb.no/en/api","access_method":"open SSB PxWeb/StatBank API plus official annual animal-use reports","cadence":"SSB daily 08:00 update; animal-use annual","attribution_licensing_notes":"SSB API states CC BY 4.0; cite table/source and preserve revisions; animal-use reports require separate provenance.","adapter_status":"reference_only","expected_artifact_schema":"Multidimensional aggregate tables with table ID, dimensions, periods, values and revision flags","blockers":["Pin table IDs and metadata contracts; preserve aggregate scope and do not double-count facility sources."]} ] } + diff --git a/pipeline/tests/test_norway_recon_metadata.py b/pipeline/tests/test_norway_recon_metadata.py new file mode 100644 index 0000000..942554d --- /dev/null +++ b/pipeline/tests/test_norway_recon_metadata.py @@ -0,0 +1,29 @@ +import json +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +NORWAY_IDS = { + "no.mattilsynet.approved-food", "no.mattilsynet.feed-abp", "no.fiskeridir.aquaculture", + "no.landbruksdirektoratet.farm-register", "no.mattilsynet.animal-experiments", + "no.miljodirektoratet.prtr-permits", "no.brreg.organizations", "no.ssb.meat-and-animal-use", +} + +class NorwayReconMetadataTests(unittest.TestCase): + def test_sources_are_registered_and_document_is_row_free(self): + registry = json.loads((ROOT / "pipeline/source_registry.json").read_text(encoding="utf-8")) + self.assertTrue(NORWAY_IDS.issubset({s["source_id"] for s in registry["sources"]})) + doc = (ROOT / "docs/country-recon-no.md").read_text(encoding="utf-8") + self.assertIn("row-free", doc) + self.assertIn("fully automated", doc) + self.assertNotIn("Approval ID |", doc) + + def test_sources_are_conservative(self): + status = json.loads((ROOT / "docs/source-status.json").read_text(encoding="utf-8")) + by_id = {s["source_id"]: s for s in status["sources"]} + for source_id in NORWAY_IDS: + self.assertEqual(by_id[source_id]["publication_eligibility"], "blocked") + self.assertIn(by_id[source_id]["acquisition"], {"not_run", "blocked"}) + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/tests/test_source_registry.py b/pipeline/tests/test_source_registry.py index 337c7cf..2961bfa 100644 --- a/pipeline/tests/test_source_registry.py +++ b/pipeline/tests/test_source_registry.py @@ -9,8 +9,8 @@ class SourceRegistryTests(unittest.TestCase): def test_repository_registry_loads_and_references_existing_legacy_paths(self): registry = load_registry() - self.assertEqual(len(registry["sources"]), 87) - self.assertEqual(len({source["source_id"] for source in registry["sources"]}), 87) + self.assertEqual(len(registry["sources"]), 95) + self.assertEqual(len({source["source_id"] for source in registry["sources"]}), 95) def test_unknowns_are_explicit(self): registry = json.loads(REGISTRY_PATH.read_text(encoding="utf-8")) From 4767613a3479f4b72a06486de01efc638b3c0c75 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Wed, 16 Sep 2026 09:38:55 -0700 Subject: [PATCH 145/311] docs: add Finland source reconnaissance --- docs/country-recon-fi.md | 31 +++++++++++++++++++ docs/source-status.json | 10 +++++- pipeline/source_registry.json | 10 +++++- pipeline/tests/test_finland_recon_metadata.py | 29 +++++++++++++++++ pipeline/tests/test_source_registry.py | 4 +-- 5 files changed, 80 insertions(+), 4 deletions(-) create mode 100644 docs/country-recon-fi.md create mode 100644 pipeline/tests/test_finland_recon_metadata.py diff --git a/docs/country-recon-fi.md b/docs/country-recon-fi.md new file mode 100644 index 0000000..e36a7c2 --- /dev/null +++ b/docs/country-recon-fi.md @@ -0,0 +1,31 @@ +# Finland source reconnaissance + +Status: row-free reconnaissance only. No facility rows, raw exports, coordinates, or production ingestion were retained. Official routes checked 2026-09-16. + +## Decision + +Finland is a strong medium-effort candidate. The Finnish Food Authority (Ruokavirasto) publishes approved establishments and animal-sector registers; the Finnish Transport and Communications Agency/Environmental Institute provide environmental and geospatial surfaces; PRH/YTJ provides corporate identifiers; and Statistics Finland exposes PxWeb statistics. Aquaculture, farms, approvals, inspections, permits, and statistics must remain separate evidence products. + +The required pipeline is fully automated end-to-end: scheduled retrieval, hashing, schema validation, provenance, normalization, privacy/review gates, and guarded ingestion. Browser-only searches/manual exports are reconnaissance fallbacks, not production routes. + +## Source inventory + +| Source | Route and findings | Cadence / format / identifiers | Location, terms, privacy, automation | +|---|---|---|---| +| Approved animal-origin establishments/slaughterhouses | [Ruokavirasto approved establishments](https://www.ruokavirasto.fi/en/companies/food-sector/food-establishments/approved-establishments/) and EU approved-establishment context | Authority pages link establishment lists and approval guidance; exact current bulk file/API and section headers require capture | Approval number, operator/site and activity fields expected; address/coordinates unresolved. Verify Finnish/EU reuse terms and mixed-address privacy. Medium effort candidate. | +| Feed and animal-by-products | [Ruokavirasto feed and ABP guidance/register surface](https://www.ruokavirasto.fi/en/companies/feed/) | Register/list formats and cadence require live verification | Keep registration/approval categories separate; no coordinates assumed. Terms and coverage unresolved. Medium/high effort. | +| Farms/intensive agriculture | [Natural Resources Institute Finland (Luke) statistics](https://www.luke.fi/en/statistics) and Finnish agriculture registers | Public statistics are aggregate; a public national intensive-farm facility export was not verified | Farm/property/person data are sensitive. Do not infer facility rows from livestock totals; blocked pending authorized route. | +| Aquaculture | [Luke aquaculture statistics](https://www.luke.fi/en/statistics/aquaculture) and Finnish environmental/geospatial services | Statistics and possible geospatial datasets; current permit/site API not verified | Separate production statistics from licensed sites. Coordinates/terms/API contract unresolved; medium/high effort. | +| Animal experimentation | [Animal Experiment Board / Finnish Food Authority guidance](https://www.ruokavirasto.fi/en/animals/animal-experiments/) | Annual reports/guidance; no public facility master verified | Keep institutions, protocols and locations restricted/aggregated. High privacy and medium/high automation effort. | +| Inspections/enforcement | Ruokavirasto control and food-establishment guidance | Annual/authority reports; no national row-level enforcement API verified | Model observations/events separately from approvals; absence is not closure. High effort/fragmented. | +| Environmental permits/releases | [Finnish Environment Institute](https://www.syke.fi/en-US/Open_information) and environmental permit services | Open environmental datasets/services exist, but a stable national animal-facility permit export was not verified | Verify license, geometry precision, permit identity and sensitive-site exposure. Medium/high effort. | +| Corporate identifiers | [PRH/YTJ open data](https://www.prh.fi/en/uutislistaus/uutiset/2020/P_23520.html) | Organization/business IDs and downloadable/API routes require current access verification | Identity-only crosswalk; suppress personal/sole-trader and residential details. Low/medium effort. | +| Slaughter/animal-use statistics | [Statistics Finland PxWeb](https://stat.fi/en/services/statistical-data-services/statistical-databases) and Luke statistics | Open PxWeb/API or downloads; table IDs, cadence and license need pinning | Aggregate only; preserve dimensions/revisions and do not create facility entities. Low/medium effort. | + +## Automation and release gates + +First implementation order: verify Ruokavirasto approved-establishment exports, then corporate ID crosswalk and statistics; treat aquaculture/environment as separate adapters. Require URL/API contract capture, hash/bytes, timestamps, content type, schema fingerprints, pagination checks, stable IDs, category preservation, CRS/precision checks, and quarantine on drift. No geocoding or inferred identity is authorized. + +Open blockers: current bulk routes and exact schemas for approved food/feed/ABP lists; national farm/intensive-site scope; aquaculture permit/site API; inspection/enforcement and environmental permit exports; research-facility privacy; and source-specific licensing. Publication remains blocked pending ethics, rights, privacy, coverage, and maintainer approval. + +Recommended next country after Finland: Estonia, with separate food, agricultural, environmental, corporate and statistics routes. diff --git a/docs/source-status.json b/docs/source-status.json index 105fd6e..8deea61 100644 --- a/docs/source-status.json +++ b/docs/source-status.json @@ -103,6 +103,14 @@ {"source_id":"no.mattilsynet.animal-experiments","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-no.md","pipeline/source_registry.json"],"next_action":"Keep annual aggregates separate; no research-facility master is verified."}, {"source_id":"no.miljodirektoratet.prtr-permits","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-no.md","pipeline/source_registry.json"],"next_action":"Verify current public environmental export/API and stable IDs before acquisition."}, {"source_id":"no.brreg.organizations","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-no.md","pipeline/source_registry.json"],"next_action":"Verify endpoint versions, rate limits and identity-only privacy policy."}, - {"source_id":"no.ssb.meat-and-animal-use","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-no.md","pipeline/source_registry.json"],"next_action":"Pin SSB table IDs and metadata; preserve aggregate scope and revisions."} + {"source_id":"no.ssb.meat-and-animal-use","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-no.md","pipeline/source_registry.json"],"next_action":"Pin SSB table IDs and metadata; preserve aggregate scope and revisions."}, + {"source_id":"fi.ruokavirasto.approved-food","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-fi.md","pipeline/source_registry.json"],"next_action":"Verify current export/API, section coverage, headers, terms and privacy."}, + {"source_id":"fi.ruokavirasto.feed-abp","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-fi.md","pipeline/source_registry.json"],"next_action":"Verify public list/export, category boundaries and source rights."}, + {"source_id":"fi.luke.agriculture","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-fi.md","pipeline/source_registry.json"],"next_action":"Pin table/API contracts and preserve aggregate scope."}, + {"source_id":"fi.aquaculture","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-fi.md","pipeline/source_registry.json"],"next_action":"Verify current public permit/site API, geometry, terms and privacy."}, + {"source_id":"fi.animal-experiments","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-fi.md","pipeline/source_registry.json"],"next_action":"Keep annual aggregates separate; no research-facility master verified."}, + {"source_id":"fi.syke.environment","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-fi.md","pipeline/source_registry.json"],"next_action":"Verify current environmental export/API, license, geometry and coverage."}, + {"source_id":"fi.prh.ytj.organizations","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-fi.md","pipeline/source_registry.json"],"next_action":"Verify current endpoint, auth, quotas and identity-only privacy policy."}, + {"source_id":"fi.statfin.pxweb","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-fi.md","pipeline/source_registry.json"],"next_action":"Pin current table IDs/API contracts and preserve revisions."} ] } diff --git a/pipeline/source_registry.json b/pipeline/source_registry.json index c4090c6..54bad4d 100644 --- a/pipeline/source_registry.json +++ b/pipeline/source_registry.json @@ -734,7 +734,15 @@ {"source_id":"no.mattilsynet.animal-experiments","jurisdiction_scope":"Norway; experimental-animal use statistics and reporting","legacy_paths":[],"url":"https://www.mattilsynet.no/dyr/forsoksdyr/bruk-av-dyr-i-forsok","access_method":"annual reports and ALURES statistical database; reporting system authenticated","cadence":"annual","attribution_licensing_notes":"Sensitive research/animal-use evidence; aggregate/anonymize and keep facility claims out of canonical entities.","adapter_status":"reference_only","expected_artifact_schema":"Annual aggregate/report observations by year, species, purpose and severity","blockers":["No public research-facility master verified; resolve stable report links and privacy boundary."]}, {"source_id":"no.miljodirektoratet.prtr-permits","jurisdiction_scope":"Norway; Norwegian Environment Agency pollutant-release and permit evidence","legacy_paths":[],"url":"https://www.miljodirektoratet.no/globalassets/publikasjoner/M138/M138.pdf","access_method":"official environmental register/document references; public bulk route not verified","cadence":"annual/report-specific; unresolved","attribution_licensing_notes":"Official environmental authority; terms, sensitive sites and document privacy require review.","adapter_status":"reference_only","expected_artifact_schema":"Permit/PRTR observations with facility or permit identifier, reporting year, releases/transfers and authority","blockers":["Verify current public search/export/API, stable identifiers, completeness and coordinate/address exposure."]}, {"source_id":"no.brreg.organizations","jurisdiction_scope":"Norway; Brønnøysundregistrene Central Coordinating Register open organization data","legacy_paths":[],"url":"https://data.brreg.no/enhetsregisteret/api/dokumentasjon/en/index.html","access_method":"public REST JSON API and CSV/gzip/XLSX downloads","cadence":"API/update-feed specific; timestamp each run","attribution_licensing_notes":"NLOD 2.0 stated by publisher; suppress person roles, birth numbers, sole-trader and mixed residential addresses.","adapter_status":"reference_only","expected_artifact_schema":"Organization/subunit records keyed by nine-digit organization number with name, type, status, address, dates and update events","blockers":["Verify rate limits, endpoint versions, field semantics and identity-only matching policy."]}, - {"source_id":"no.ssb.meat-and-animal-use","jurisdiction_scope":"Norway; Statistics Norway meat production and Mattilsynet animal-use aggregates","legacy_paths":[],"url":"https://www.ssb.no/en/api","access_method":"open SSB PxWeb/StatBank API plus official annual animal-use reports","cadence":"SSB daily 08:00 update; animal-use annual","attribution_licensing_notes":"SSB API states CC BY 4.0; cite table/source and preserve revisions; animal-use reports require separate provenance.","adapter_status":"reference_only","expected_artifact_schema":"Multidimensional aggregate tables with table ID, dimensions, periods, values and revision flags","blockers":["Pin table IDs and metadata contracts; preserve aggregate scope and do not double-count facility sources."]} + {"source_id":"no.ssb.meat-and-animal-use","jurisdiction_scope":"Norway; Statistics Norway meat production and Mattilsynet animal-use aggregates","legacy_paths":[],"url":"https://www.ssb.no/en/api","access_method":"open SSB PxWeb/StatBank API plus official annual animal-use reports","cadence":"SSB daily 08:00 update; animal-use annual","attribution_licensing_notes":"SSB API states CC BY 4.0; cite table/source and preserve revisions; animal-use reports require separate provenance.","adapter_status":"reference_only","expected_artifact_schema":"Multidimensional aggregate tables with table ID, dimensions, periods, values and revision flags","blockers":["Pin table IDs and metadata contracts; preserve aggregate scope and do not double-count facility sources."]} ,{"source_id":"fi.ruokavirasto.approved-food","jurisdiction_scope":"Finland; Ruokavirasto approved animal-origin food establishments","legacy_paths":[],"url":"https://www.ruokavirasto.fi/en/companies/food-sector/food-establishments/approved-establishments/","access_method":"official web pages and linked lists; bulk route unverified","cadence":"publisher-defined; timestamp each retrieval","attribution_licensing_notes":"Official Finnish Food Authority; verify file-specific reuse terms and privacy before acquisition.","adapter_status":"reference_only","expected_artifact_schema":"Approval list with stable approval ID, operator/site, activities, status/dates and address fields; coordinates unknown","blockers":["Verify current export/API, section coverage, headers, terms and mixed-address privacy."]}, + {"source_id":"fi.ruokavirasto.feed-abp","jurisdiction_scope":"Finland; Ruokavirasto feed and animal-by-product establishments","legacy_paths":[],"url":"https://www.ruokavirasto.fi/en/companies/feed/","access_method":"official guidance/register surface; bulk route unverified","cadence":"unknown; timestamp each access","attribution_licensing_notes":"Official authority; category-specific terms, coverage and privacy require review.","adapter_status":"reference_only","expected_artifact_schema":"Feed/ABP registration or approval observations with number, activity, status and site fields","blockers":["Verify public list/export, category boundaries and source rights before acquisition."]}, + {"source_id":"fi.luke.agriculture","jurisdiction_scope":"Finland; Luke aggregate agriculture, livestock and aquaculture statistics","legacy_paths":[],"url":"https://www.luke.fi/en/statistics","access_method":"official statistics pages/downloads/API where available","cadence":"table-specific; timestamp each release","attribution_licensing_notes":"Official statistics; verify dataset license and preserve revisions/aggregate scope.","adapter_status":"reference_only","expected_artifact_schema":"Aggregate time-series tables by species, production, region and period","blockers":["Pin table IDs, API/download contracts and license; do not infer facility rows."]}, + {"source_id":"fi.aquaculture","jurisdiction_scope":"Finland; aquaculture production/sites and environmental evidence","legacy_paths":[],"url":"https://www.luke.fi/en/statistics/aquaculture","access_method":"official statistics and environmental/geospatial services","cadence":"statistics/service-specific; unknown","attribution_licensing_notes":"Verify provider terms, geometry precision and site privacy before use.","adapter_status":"reference_only","expected_artifact_schema":"Aggregate aquaculture statistics or reviewed site/permit records with stable IDs and optional CRS geometry","blockers":["No current national public permit/site API verified."]}, + {"source_id":"fi.animal-experiments","jurisdiction_scope":"Finland; animal experimentation guidance/statistics","legacy_paths":[],"url":"https://www.ruokavirasto.fi/en/animals/animal-experiments/","access_method":"official guidance and annual reports","cadence":"annual/report-specific","attribution_licensing_notes":"Sensitive research evidence; aggregate/anonymize and apply strict privacy review.","adapter_status":"reference_only","expected_artifact_schema":"Annual aggregate use/report observations by species, purpose, severity and year","blockers":["No public research-facility master verified."]}, + {"source_id":"fi.syke.environment","jurisdiction_scope":"Finland; environmental open information and permit evidence","legacy_paths":[],"url":"https://www.syke.fi/en-US/Open_information","access_method":"official open-data/service catalogue; route-specific","cadence":"dataset-specific","attribution_licensing_notes":"Verify license, attribution, sensitive-site handling and geometry semantics.","adapter_status":"reference_only","expected_artifact_schema":"Environmental observations/permit documents with authority, dates, identifiers, emissions and optional geometry","blockers":["No stable national animal-facility permit export verified."]}, + {"source_id":"fi.prh.ytj.organizations","jurisdiction_scope":"Finland; PRH/YTJ corporate and business identifiers","legacy_paths":[],"url":"https://www.prh.fi/en/uutislistaus/uutiset/2020/P_23520.html","access_method":"official open-data/API/download documentation; current route unverified","cadence":"provider-defined","attribution_licensing_notes":"Identity linkage only; suppress personal, sole-trader and residential details; verify terms.","adapter_status":"reference_only","expected_artifact_schema":"Business ID keyed organization/name/status/activity/address response with retrieval timestamp","blockers":["Verify current endpoint, auth, quotas, fields and identity-only policy."]}, + {"source_id":"fi.statfin.pxweb","jurisdiction_scope":"Finland; Statistics Finland aggregate slaughter and animal-use context","legacy_paths":[],"url":"https://stat.fi/en/services/statistical-data-services/statistical-databases","access_method":"official PxWeb/statistical database API or downloads","cadence":"table-specific; timestamp releases/revisions","attribution_licensing_notes":"Official Statistics Finland; verify table license and preserve dimensions/revisions.","adapter_status":"reference_only","expected_artifact_schema":"PxWeb multidimensional aggregate tables with table ID, dimensions, periods, values and flags","blockers":["Pin current table IDs and API contracts; keep aggregate scope separate."]} ] } + diff --git a/pipeline/tests/test_finland_recon_metadata.py b/pipeline/tests/test_finland_recon_metadata.py new file mode 100644 index 0000000..9ebeee2 --- /dev/null +++ b/pipeline/tests/test_finland_recon_metadata.py @@ -0,0 +1,29 @@ +import json +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +FINLAND_IDS = { + "fi.ruokavirasto.approved-food", "fi.ruokavirasto.feed-abp", "fi.luke.agriculture", + "fi.aquaculture", "fi.animal-experiments", "fi.syke.environment", + "fi.prh.ytj.organizations", "fi.statfin.pxweb", +} + +class FinlandReconMetadataTests(unittest.TestCase): + def test_sources_are_registered_and_document_is_row_free(self): + registry = json.loads((ROOT / "pipeline/source_registry.json").read_text(encoding="utf-8")) + self.assertTrue(FINLAND_IDS.issubset({s["source_id"] for s in registry["sources"]})) + doc = (ROOT / "docs/country-recon-fi.md").read_text(encoding="utf-8") + self.assertIn("row-free", doc) + self.assertIn("fully automated", doc) + self.assertNotIn("Approval ID |", doc) + + def test_sources_are_conservative(self): + status = json.loads((ROOT / "docs/source-status.json").read_text(encoding="utf-8")) + by_id = {s["source_id"]: s for s in status["sources"]} + for source_id in FINLAND_IDS: + self.assertEqual(by_id[source_id]["publication_eligibility"], "blocked") + self.assertIn(by_id[source_id]["acquisition"], {"not_run", "blocked"}) + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/tests/test_source_registry.py b/pipeline/tests/test_source_registry.py index 2961bfa..e4ced0b 100644 --- a/pipeline/tests/test_source_registry.py +++ b/pipeline/tests/test_source_registry.py @@ -9,8 +9,8 @@ class SourceRegistryTests(unittest.TestCase): def test_repository_registry_loads_and_references_existing_legacy_paths(self): registry = load_registry() - self.assertEqual(len(registry["sources"]), 95) - self.assertEqual(len({source["source_id"] for source in registry["sources"]}), 95) + self.assertEqual(len(registry["sources"]), 103) + self.assertEqual(len({source["source_id"] for source in registry["sources"]}), 103) def test_unknowns_are_explicit(self): registry = json.loads(REGISTRY_PATH.read_text(encoding="utf-8")) From c1832920320e478f389fcd7f16216525d90d08e3 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Wed, 16 Sep 2026 09:41:47 -0700 Subject: [PATCH 146/311] docs: add Estonia source reconnaissance --- docs/country-recon-ee.md | 30 +++++++++++++++++++ docs/source-status.json | 8 ++++- pipeline/source_registry.json | 8 ++++- pipeline/tests/test_estonia_recon_metadata.py | 13 ++++++++ pipeline/tests/test_source_registry.py | 4 +-- 5 files changed, 59 insertions(+), 4 deletions(-) create mode 100644 docs/country-recon-ee.md create mode 100644 pipeline/tests/test_estonia_recon_metadata.py diff --git a/docs/country-recon-ee.md b/docs/country-recon-ee.md new file mode 100644 index 0000000..b1d4636 --- /dev/null +++ b/docs/country-recon-ee.md @@ -0,0 +1,30 @@ +# Estonia source reconnaissance + +Status: row-free reconnaissance only. No facility rows, raw exports, coordinates, or production ingestion retained. Official routes checked 2026-09-16. + +## Decision + +Estonia is a strong medium-effort candidate. The Agriculture and Food Board (PTA) exposes approved/registered food and animal-sector registers; PRIA publishes public animal-register and geospatial establishment data; Keskkonnaamet exposes environmental permit systems; Äriregister provides corporate identifiers; and Statistics Estonia provides PxWeb tables. Approval, farm location, aquaculture, permits, inspections, corporate identity, and statistics must remain separate evidence products. + +The pipeline must be fully automated from scheduled retrieval through hashing, validation, provenance, normalization, privacy/review gates, and guarded ingestion. Browser-only/manual exports are fallbacks, not production routes. + +## Source inventory + +| Source | Official route and findings | Cadence / format / identifiers | Location, terms, privacy, automation | +|---|---|---|---| +| Approved food/slaughter establishments | [PTA registers and datasets](https://pta.agri.ee/riiklikud-registrid-ja-andmekogud) provides notified/licensed food operators and animal-sector lists; EU approved-establishment links are also surfaced | Current list/download formats require capture; approval/registry numbers, operator, activity and address expected | Verify current CSV/XLSX/API and licence. Medium effort; names/addresses need privacy review. | +| Farms and intensive animal sites | [PRIA public data](https://www.pria.ee/registrid/avalikud-andmed) and [spatial-data documentation](https://www.pria.ee/sites/default/files/2024-06/pindalatoetuste_ja_loomade_registri_tegevuskohade_ruumiandmed_25062024.pdf) | Public establishment data include location ID, activity/status, registration number, county/municipality/address, coordinates and species; prior-day data is described | Strong candidate but animal/property data can expose sensitive locations. Verify current download/service, CRS, terms and field suppression; medium/high effort. | +| Aquaculture | PRIA animal-register public establishment data; EU veterinary registers for approved sites | Same register categories include aquaculture establishments; current aquaculture-specific bulk contract unresolved | Keep aquaculture separate from farms/food approvals; no inferred rows. | +| Animal experimentation | PTA/official animal-welfare and research guidance | No public facility master or stable animal-use statistics route verified | High privacy; keep research institutions/protocols aggregated and separate. | +| Inspections/enforcement | PTA register/control surface and official notices | No national row-level inspection/enforcement export verified | Model events separately; absence is not closure. High effort/fragmented. | +| Environmental permits | [Keskkonnaamet permit notices/KOTKAS](https://keskkonnaamet.ee/keskkonnateadlikkus-avalikustamised/raagi-kaasa/lubade-eelnoude-avalik-valjapanek) | KOTKAS contains post-2017 complex permits and post-2020 environmental permits; searchable documents, cadence route-specific | Strong permit evidence candidate but UI/document extraction is high effort. Verify terms, identifiers, geometry and sensitive-site exposure. | +| Corporate identifiers | Estonian commercial register / Äriregister route; current public API contract not verified in this pass | Business registry code is expected stable identifier; format/auth unresolved | Identity-only crosswalk; suppress personal/sole-trader/residential data. Medium effort. | +| Slaughter/animal-use statistics | [Statistics Estonia PM190](https://andmed.stat.ee/en/stat/majandus__pellumajandus__pellumajandussaaduste-tootmine__loomakasvatussaaduste-tootmine/PM190) and [livestock statistics metadata](https://stat.ee/et/metaandmed/21203) | PM190 is monthly slaughter in approved meat establishments; Statistics Estonia open data states CC BY-SA 4.0; PxWeb route/table IDs require pinning | Aggregate only; preserve dimensions/revisions. Low/medium effort. | + +## Automation and gates + +First implementation order: verify PTA approved-establishment exports, then PRIA establishment geospatial/public data, corporate IDs, and Statistics Estonia aggregates. Require URL/API contract, hash/bytes, timestamps, content type, schema fingerprints, pagination/count checks, stable IDs, CRS/precision checks and quarantine on drift. No geocoding or inferred identity is authorized. + +Blockers: current PTA bulk routes and terms; PRIA current service/download contract and privacy/CRS semantics; aquaculture-specific scope; public research/inspection routes; environmental document extraction; and corporate API access. Publication remains blocked pending ethics, rights, privacy, coverage, and maintainer approval. + +Recommended next country after Estonia: Latvia, using separate food, farm/aquaculture, environmental, corporate and statistics authorities. diff --git a/docs/source-status.json b/docs/source-status.json index 8deea61..d037c28 100644 --- a/docs/source-status.json +++ b/docs/source-status.json @@ -111,6 +111,12 @@ {"source_id":"fi.animal-experiments","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-fi.md","pipeline/source_registry.json"],"next_action":"Keep annual aggregates separate; no research-facility master verified."}, {"source_id":"fi.syke.environment","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-fi.md","pipeline/source_registry.json"],"next_action":"Verify current environmental export/API, license, geometry and coverage."}, {"source_id":"fi.prh.ytj.organizations","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-fi.md","pipeline/source_registry.json"],"next_action":"Verify current endpoint, auth, quotas and identity-only privacy policy."}, - {"source_id":"fi.statfin.pxweb","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-fi.md","pipeline/source_registry.json"],"next_action":"Pin current table IDs/API contracts and preserve revisions."} + {"source_id":"fi.statfin.pxweb","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-fi.md","pipeline/source_registry.json"],"next_action":"Pin current table IDs/API contracts and preserve revisions."}, + {"source_id":"ee.pta.approved-food","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ee.md","pipeline/source_registry.json"],"next_action":"Verify current bulk/API route, completeness, headers, terms and privacy."}, + {"source_id":"ee.pria.animal-register","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ee.md","pipeline/source_registry.json"],"next_action":"Verify current public service/download, CRS, terms and field suppression."}, + {"source_id":"ee.pria.aquaculture","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ee.md","pipeline/source_registry.json"],"next_action":"Verify aquaculture-specific bulk contract and privacy scope."}, + {"source_id":"ee.keskkonnaamet.kotkas","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ee.md","pipeline/source_registry.json"],"next_action":"Verify document extraction, terms, identifiers, geometry and coverage."}, + {"source_id":"ee.ariregister.organizations","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ee.md","pipeline/source_registry.json"],"next_action":"Verify API, auth, quotas, terms and identity-only policy."}, + {"source_id":"ee.stat.slaughter","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ee.md","pipeline/source_registry.json"],"next_action":"Pin current PxWeb table metadata and preserve revisions."} ] } diff --git a/pipeline/source_registry.json b/pipeline/source_registry.json index 54bad4d..8a67fbd 100644 --- a/pipeline/source_registry.json +++ b/pipeline/source_registry.json @@ -741,8 +741,14 @@ {"source_id":"fi.animal-experiments","jurisdiction_scope":"Finland; animal experimentation guidance/statistics","legacy_paths":[],"url":"https://www.ruokavirasto.fi/en/animals/animal-experiments/","access_method":"official guidance and annual reports","cadence":"annual/report-specific","attribution_licensing_notes":"Sensitive research evidence; aggregate/anonymize and apply strict privacy review.","adapter_status":"reference_only","expected_artifact_schema":"Annual aggregate use/report observations by species, purpose, severity and year","blockers":["No public research-facility master verified."]}, {"source_id":"fi.syke.environment","jurisdiction_scope":"Finland; environmental open information and permit evidence","legacy_paths":[],"url":"https://www.syke.fi/en-US/Open_information","access_method":"official open-data/service catalogue; route-specific","cadence":"dataset-specific","attribution_licensing_notes":"Verify license, attribution, sensitive-site handling and geometry semantics.","adapter_status":"reference_only","expected_artifact_schema":"Environmental observations/permit documents with authority, dates, identifiers, emissions and optional geometry","blockers":["No stable national animal-facility permit export verified."]}, {"source_id":"fi.prh.ytj.organizations","jurisdiction_scope":"Finland; PRH/YTJ corporate and business identifiers","legacy_paths":[],"url":"https://www.prh.fi/en/uutislistaus/uutiset/2020/P_23520.html","access_method":"official open-data/API/download documentation; current route unverified","cadence":"provider-defined","attribution_licensing_notes":"Identity linkage only; suppress personal, sole-trader and residential details; verify terms.","adapter_status":"reference_only","expected_artifact_schema":"Business ID keyed organization/name/status/activity/address response with retrieval timestamp","blockers":["Verify current endpoint, auth, quotas, fields and identity-only policy."]}, - {"source_id":"fi.statfin.pxweb","jurisdiction_scope":"Finland; Statistics Finland aggregate slaughter and animal-use context","legacy_paths":[],"url":"https://stat.fi/en/services/statistical-data-services/statistical-databases","access_method":"official PxWeb/statistical database API or downloads","cadence":"table-specific; timestamp releases/revisions","attribution_licensing_notes":"Official Statistics Finland; verify table license and preserve dimensions/revisions.","adapter_status":"reference_only","expected_artifact_schema":"PxWeb multidimensional aggregate tables with table ID, dimensions, periods, values and flags","blockers":["Pin current table IDs and API contracts; keep aggregate scope separate."]} + {"source_id":"fi.statfin.pxweb","jurisdiction_scope":"Finland; Statistics Finland aggregate slaughter and animal-use context","legacy_paths":[],"url":"https://stat.fi/en/services/statistical-data-services/statistical-databases","access_method":"official PxWeb/statistical database API or downloads","cadence":"table-specific; timestamp releases/revisions","attribution_licensing_notes":"Official Statistics Finland; verify table license and preserve dimensions/revisions.","adapter_status":"reference_only","expected_artifact_schema":"PxWeb multidimensional aggregate tables with table ID, dimensions, periods, values and flags","blockers":["Pin current table IDs and API contracts; keep aggregate scope separate."]} ,{"source_id":"ee.pta.approved-food","jurisdiction_scope":"Estonia; PTA approved and registered food/animal establishments","legacy_paths":[],"url":"https://pta.agri.ee/riiklikud-registrid-ja-andmekogud","access_method":"official registers and linked downloads","cadence":"publisher-defined; timestamp retrieval","attribution_licensing_notes":"Official Estonian authority; verify dataset terms and privacy.","adapter_status":"reference_only","expected_artifact_schema":"Approval/registration records with operator, activity, status, approval ID and address","blockers":["Verify current bulk/API route, completeness, headers, terms and privacy."]}, + {"source_id":"ee.pria.animal-register","jurisdiction_scope":"Estonia; PRIA public farm-animal and aquaculture establishment data","legacy_paths":[],"url":"https://www.pria.ee/registrid/avalikud-andmed","access_method":"public query/XML and geospatial web map/service","cadence":"prior-day data described; verify service cadence","attribution_licensing_notes":"Official PRIA; animal/property location data require strict privacy and purpose review.","adapter_status":"reference_only","expected_artifact_schema":"Location ID, establishment number, status, species, address, municipality and coordinate fields","blockers":["Verify current service/download, CRS, terms, field suppression and coverage."]}, + {"source_id":"ee.pria.aquaculture","jurisdiction_scope":"Estonia; PRIA aquaculture establishment register","legacy_paths":[],"url":"https://www.pria.ee/registrid/kalad-ja-vahid","access_method":"public register and animal-register services","cadence":"service-specific","attribution_licensing_notes":"Official authority; separate aquaculture from farm/food entities and review sensitive locations.","adapter_status":"reference_only","expected_artifact_schema":"Aquaculture establishment/permit observations with stable location and status identifiers","blockers":["No aquaculture-specific bulk contract verified."]}, + {"source_id":"ee.keskkonnaamet.kotkas","jurisdiction_scope":"Estonia; Keskkonnaamet environmental permit and public-notice evidence","legacy_paths":[],"url":"https://keskkonnaamet.ee/keskkonnateadlikkus-avalikustamised/raagi-kaasa/lubade-eelnoude-avalik-valjapanek","access_method":"KOTKAS/document register and public notices","cadence":"publication/permit-specific","attribution_licensing_notes":"Official environmental authority; verify terms, geometry and sensitive-site handling.","adapter_status":"reference_only","expected_artifact_schema":"Permit/document records with authority, application/decision IDs, dates, activity and optional reviewed geometry","blockers":["UI/document extraction and complete national coverage unresolved."]}, + {"source_id":"ee.ariregister.organizations","jurisdiction_scope":"Estonia; Äriregister corporate identity","legacy_paths":[],"url":"https://ariregister.rik.ee/eng","access_method":"official register/API route unverified","cadence":"provider-defined","attribution_licensing_notes":"Identity-only linkage; suppress personal, sole-trader and residential details; verify terms.","adapter_status":"reference_only","expected_artifact_schema":"Registry-code keyed organization/name/status/activity/address response","blockers":["Verify current API, auth, quotas and privacy policy."]}, + {"source_id":"ee.stat.slaughter","jurisdiction_scope":"Estonia; Statistics Estonia slaughter and livestock aggregates","legacy_paths":[],"url":"https://andmed.stat.ee/en/stat/majandus__pellumajandus__pellumajandussaaduste-tootmine__loomakasvatussaaduste-tootmine/PM190","access_method":"official PxWeb/statistical database","cadence":"monthly/table-specific","attribution_licensing_notes":"Statistics Estonia open-data route states CC BY-SA 4.0; preserve table IDs and revisions.","adapter_status":"reference_only","expected_artifact_schema":"PxWeb aggregate monthly slaughter observations with dimensions, periods, values and flags","blockers":["Pin API/table metadata; keep aggregate scope separate from facility evidence."]} ] } + diff --git a/pipeline/tests/test_estonia_recon_metadata.py b/pipeline/tests/test_estonia_recon_metadata.py new file mode 100644 index 0000000..d3e2d5f --- /dev/null +++ b/pipeline/tests/test_estonia_recon_metadata.py @@ -0,0 +1,13 @@ +import json +import unittest +from pathlib import Path +ROOT = Path(__file__).resolve().parents[2] +IDS={"ee.pta.approved-food","ee.pria.animal-register","ee.pria.aquaculture","ee.keskkonnaamet.kotkas","ee.ariregister.organizations","ee.stat.slaughter"} +class EstoniaReconMetadataTests(unittest.TestCase): + def test_row_free_registered_sources(self): + r=json.loads((ROOT/"pipeline/source_registry.json").read_text(encoding="utf-8")); self.assertTrue(IDS.issubset({s["source_id"] for s in r["sources"]})) + d=(ROOT/"docs/country-recon-ee.md").read_text(encoding="utf-8"); self.assertIn("row-free",d); self.assertIn("fully automated",d); self.assertNotIn("Approval ID |",d) + def test_status_conservative(self): + s=json.loads((ROOT/"docs/source-status.json").read_text(encoding="utf-8")); by={x["source_id"]:x for x in s["sources"]} + for i in IDS: self.assertEqual(by[i]["publication_eligibility"],"blocked") +if __name__ == "__main__": unittest.main() diff --git a/pipeline/tests/test_source_registry.py b/pipeline/tests/test_source_registry.py index e4ced0b..102430a 100644 --- a/pipeline/tests/test_source_registry.py +++ b/pipeline/tests/test_source_registry.py @@ -9,8 +9,8 @@ class SourceRegistryTests(unittest.TestCase): def test_repository_registry_loads_and_references_existing_legacy_paths(self): registry = load_registry() - self.assertEqual(len(registry["sources"]), 103) - self.assertEqual(len({source["source_id"] for source in registry["sources"]}), 103) + self.assertEqual(len(registry["sources"]), 109) + self.assertEqual(len({source["source_id"] for source in registry["sources"]}), 109) def test_unknowns_are_explicit(self): registry = json.loads(REGISTRY_PATH.read_text(encoding="utf-8")) From 8df1ddc19df3f858ce228dfdd8daa4a5df0d1650 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Wed, 16 Sep 2026 09:43:59 -0700 Subject: [PATCH 147/311] docs: add Latvia source reconnaissance --- docs/country-recon-lv.md | 30 ++++++++++++++++++++ docs/source-status.json | 8 +++++- pipeline/source_registry.json | 8 +++++- pipeline/tests/test_latvia_recon_metadata.py | 11 +++++++ pipeline/tests/test_source_registry.py | 4 +-- 5 files changed, 57 insertions(+), 4 deletions(-) create mode 100644 docs/country-recon-lv.md create mode 100644 pipeline/tests/test_latvia_recon_metadata.py diff --git a/docs/country-recon-lv.md b/docs/country-recon-lv.md new file mode 100644 index 0000000..fd88cd0 --- /dev/null +++ b/docs/country-recon-lv.md @@ -0,0 +1,30 @@ +# Latvia source reconnaissance + +Status: row-free reconnaissance only. No facility rows, raw exports, coordinates, or production ingestion retained. Official routes checked 2026-09-16. + +## Decision + +Latvia is a strong medium-effort candidate. The Food and Veterinary Service (PVD/FVS) publishes machine-readable approved/registered enterprise data and many XLSX lists; the Agricultural Data Centre/LDC exposes slaughterhouse and livestock registers; the environmental authority publishes permit datasets; and the official statistics portal provides a machine-readable API. Farm and aquaculture locations require separate privacy treatment. + +The pipeline must be fully automated from scheduled retrieval through hashing, validation, provenance, normalization, privacy/review gates, and guarded ingestion. Browser-only/manual exports are fallbacks, not production routes. + +## Source inventory + +| Source | Route and findings | Cadence / format / identifiers | Location, terms, privacy, automation | +|---|---|---|---| +| Approved food/slaughter establishments | [PVD/FVS registers](https://registri.pvd.gov.lv/en/cr) and [machine-readable enterprise export](https://pakalpojumi.pvd.gov.lv/en/opendata_files/ipvd_object_opendata) | PVD lists animal-origin Sections 0–XV and ABP/feed categories as XLSX; enterprise register offers `ur-csv.zip`, free and machine-readable | Approval/registration IDs, operator/activity/address expected; coordinates unresolved. Strong candidate, medium effort; verify CC/terms and suppress mixed residential addresses. | +| Slaughterhouses | [LDC public slaughterhouse register](https://registri.ldc.gov.lv/en/slaughterhouses) | Public filtered register; exact export/API/cadence unresolved | Separate from PVD approval lists; verify stable IDs, address and terms. Medium/high effort. | +| Farms/intensive agriculture | [LDC registers](https://registri.ldc.gov.lv/) and agricultural statistics | Herd/location register and livestock statistics are exposed; public facility export scope requires verification | Animal-holder/property data can identify individuals and sensitive sites. Do not ingest rows until authorization/privacy review. | +| Aquaculture | Official aquaculture establishment lists are linked through PVD/LDC registers | XLSX/list routes exist but current direct contract unresolved | Treat locations and permits separately; coordinates/terms require capture. Medium effort after contract verification. | +| Environment/permits | [Environmental permits dataset](https://data.gov.lv/dati/dataset/izsniegtas-atlaujas-un-licences) and [VVD registers](https://www.vvd.gov.lv/lv/registri) | Dataset states CC0 1.0; public online registers include permits, environmental decisions and inspections | Strong permit candidate; verify resource schema, cadence, identifiers and geometry. Medium effort. | +| Animal experimentation/inspections | PVD and official veterinary-control surfaces | No stable national public animal-use facility master or row-level inspection API verified | Keep research and enforcement as dated aggregate/events; high privacy and fragmented route risk. | +| Corporate identifiers | Latvian enterprise register route; current public API contract not verified | Registration number expected stable; format/auth unresolved | Identity-only crosswalk; suppress personal/sole-trader data. Medium effort. | +| Slaughter/statistics | [Official Statistics Portal API v2](https://stat.gov.lv/en/api-un-kodu-vardnicas/api-v2) | API supports XLSX, CSV, JSON, JSON-stat2 and PX; max 10,000 cells/request and 30 requests/10 seconds/IP | [Animal production metadata](https://stat.gov.lv/en/meta/21203) and LDC statistics are aggregate. Low/medium effort; preserve table IDs/revisions. | + +## Gates + +First implementation order: PVD enterprise/approved XLSX/ZIP routes, environmental CC0 datasets, then LDC slaughter and statistics. Require URL/API contract, hash/bytes, timestamps, content type, schema fingerprints, pagination/count checks, stable IDs, category preservation and quarantine on drift. No geocoding or inferred identity is authorized. + +Blockers: PVD list/version semantics and terms, LDC export/API, farm/aquaculture privacy and coverage, research/inspection routes, corporate API access, and environmental resource schemas. Publication remains blocked pending ethics, rights, privacy, coverage and maintainer approval. + +Recommended next country after Latvia: Lithuania, using separate food, farm/aquaculture, environmental, corporate and statistics sources. diff --git a/docs/source-status.json b/docs/source-status.json index d037c28..bde58be 100644 --- a/docs/source-status.json +++ b/docs/source-status.json @@ -117,6 +117,12 @@ {"source_id":"ee.pria.aquaculture","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ee.md","pipeline/source_registry.json"],"next_action":"Verify aquaculture-specific bulk contract and privacy scope."}, {"source_id":"ee.keskkonnaamet.kotkas","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ee.md","pipeline/source_registry.json"],"next_action":"Verify document extraction, terms, identifiers, geometry and coverage."}, {"source_id":"ee.ariregister.organizations","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ee.md","pipeline/source_registry.json"],"next_action":"Verify API, auth, quotas, terms and identity-only policy."}, - {"source_id":"ee.stat.slaughter","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ee.md","pipeline/source_registry.json"],"next_action":"Pin current PxWeb table metadata and preserve revisions."} + {"source_id":"ee.stat.slaughter","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ee.md","pipeline/source_registry.json"],"next_action":"Pin current PxWeb table metadata and preserve revisions."}, + {"source_id":"lv.pvd.approved-food","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-lv.md","pipeline/source_registry.json"],"next_action":"Verify current files, section coverage, IDs, cadence, terms and privacy."}, + {"source_id":"lv.ldc.slaughter-farms","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-lv.md","pipeline/source_registry.json"],"next_action":"Verify bulk/API route, coverage, terms and sensitive-field policy."}, + {"source_id":"lv.environment.permits","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-lv.md","pipeline/source_registry.json"],"next_action":"Verify resource URLs, schema, coverage, geometry and privacy."}, + {"source_id":"lv.stat.api","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-lv.md","pipeline/source_registry.json"],"next_action":"Pin table IDs/metadata and preserve aggregate revisions."}, + {"source_id":"lv.animal-experiments","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-lv.md","pipeline/source_registry.json"],"next_action":"Keep research/inspection evidence aggregate until a public route is verified."}, + {"source_id":"lv.ur.organizations","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-lv.md","pipeline/source_registry.json"],"next_action":"Verify current API/access, terms and privacy."} ] } diff --git a/pipeline/source_registry.json b/pipeline/source_registry.json index 8a67fbd..9b4e82d 100644 --- a/pipeline/source_registry.json +++ b/pipeline/source_registry.json @@ -746,9 +746,15 @@ {"source_id":"ee.pria.aquaculture","jurisdiction_scope":"Estonia; PRIA aquaculture establishment register","legacy_paths":[],"url":"https://www.pria.ee/registrid/kalad-ja-vahid","access_method":"public register and animal-register services","cadence":"service-specific","attribution_licensing_notes":"Official authority; separate aquaculture from farm/food entities and review sensitive locations.","adapter_status":"reference_only","expected_artifact_schema":"Aquaculture establishment/permit observations with stable location and status identifiers","blockers":["No aquaculture-specific bulk contract verified."]}, {"source_id":"ee.keskkonnaamet.kotkas","jurisdiction_scope":"Estonia; Keskkonnaamet environmental permit and public-notice evidence","legacy_paths":[],"url":"https://keskkonnaamet.ee/keskkonnateadlikkus-avalikustamised/raagi-kaasa/lubade-eelnoude-avalik-valjapanek","access_method":"KOTKAS/document register and public notices","cadence":"publication/permit-specific","attribution_licensing_notes":"Official environmental authority; verify terms, geometry and sensitive-site handling.","adapter_status":"reference_only","expected_artifact_schema":"Permit/document records with authority, application/decision IDs, dates, activity and optional reviewed geometry","blockers":["UI/document extraction and complete national coverage unresolved."]}, {"source_id":"ee.ariregister.organizations","jurisdiction_scope":"Estonia; Äriregister corporate identity","legacy_paths":[],"url":"https://ariregister.rik.ee/eng","access_method":"official register/API route unverified","cadence":"provider-defined","attribution_licensing_notes":"Identity-only linkage; suppress personal, sole-trader and residential details; verify terms.","adapter_status":"reference_only","expected_artifact_schema":"Registry-code keyed organization/name/status/activity/address response","blockers":["Verify current API, auth, quotas and privacy policy."]}, - {"source_id":"ee.stat.slaughter","jurisdiction_scope":"Estonia; Statistics Estonia slaughter and livestock aggregates","legacy_paths":[],"url":"https://andmed.stat.ee/en/stat/majandus__pellumajandus__pellumajandussaaduste-tootmine__loomakasvatussaaduste-tootmine/PM190","access_method":"official PxWeb/statistical database","cadence":"monthly/table-specific","attribution_licensing_notes":"Statistics Estonia open-data route states CC BY-SA 4.0; preserve table IDs and revisions.","adapter_status":"reference_only","expected_artifact_schema":"PxWeb aggregate monthly slaughter observations with dimensions, periods, values and flags","blockers":["Pin API/table metadata; keep aggregate scope separate from facility evidence."]} + {"source_id":"ee.stat.slaughter","jurisdiction_scope":"Estonia; Statistics Estonia slaughter and livestock aggregates","legacy_paths":[],"url":"https://andmed.stat.ee/en/stat/majandus__pellumajandus__pellumajandussaaduste-tootmine__loomakasvatussaaduste-tootmine/PM190","access_method":"official PxWeb/statistical database","cadence":"monthly/table-specific","attribution_licensing_notes":"Statistics Estonia open-data route states CC BY-SA 4.0; preserve table IDs and revisions.","adapter_status":"reference_only","expected_artifact_schema":"PxWeb aggregate monthly slaughter observations with dimensions, periods, values and flags","blockers":["Pin API/table metadata; keep aggregate scope separate from facility evidence."]} ,{"source_id":"lv.pvd.approved-food","jurisdiction_scope":"Latvia; PVD/FVS approved and registered food, feed and ABP establishments","legacy_paths":[],"url":"https://pakalpojumi.pvd.gov.lv/en/opendata_files/ipvd_object_opendata","access_method":"machine-readable ZIP/CSV and sectioned XLSX lists","cadence":"publisher-defined; timestamp retrieval","attribution_licensing_notes":"Official PVD/FVS; verify dataset terms and privacy before release.","adapter_status":"reference_only","expected_artifact_schema":"ZIP/CSV/XLSX records with registration/approval number, operator, activities, status and address","blockers":["Verify current files, section coverage, IDs, cadence, terms and privacy."]}, + {"source_id":"lv.ldc.slaughter-farms","jurisdiction_scope":"Latvia; LDC slaughterhouse, herd/location and livestock register evidence","legacy_paths":[],"url":"https://registri.ldc.gov.lv/en/slaughterhouses","access_method":"public filtered register and statistics","cadence":"unknown; timestamp access","attribution_licensing_notes":"Official register; animal-holder/property privacy review required.","adapter_status":"reference_only","expected_artifact_schema":"Slaughterhouse or holding records with stable IDs, activity/status, address and optional coordinate fields","blockers":["Verify bulk/API route, coverage, terms and sensitive-field policy."]}, + {"source_id":"lv.environment.permits","jurisdiction_scope":"Latvia; VVD environmental permits and public registers","legacy_paths":[],"url":"https://data.gov.lv/dati/dataset/izsniegtas-atlaujas-un-licences","access_method":"open-data CSV/resource and public registers","cadence":"dataset-specific","attribution_licensing_notes":"Catalogue states CC0 1.0; preserve source and verify resource scope/geometry.","adapter_status":"reference_only","expected_artifact_schema":"Permit/license records with subject, activity, dates, authority, status and optional site geometry","blockers":["Verify current resource URLs, schema, coverage and privacy."]}, + {"source_id":"lv.stat.api","jurisdiction_scope":"Latvia; official statistics API for slaughter, livestock and animal-use aggregates","legacy_paths":[],"url":"https://stat.gov.lv/en/api-un-kodu-vardnicas/api-v2","access_method":"PxWeb API v2","cadence":"table-specific; 30 requests/10 seconds/IP stated","attribution_licensing_notes":"Official statistics; preserve table IDs, revisions and license/attribution metadata.","adapter_status":"reference_only","expected_artifact_schema":"CSV/XLSX/JSON/JSON-stat2/PX multidimensional tables","blockers":["Pin table IDs and metadata; keep aggregate scope separate."]}, + {"source_id":"lv.animal-experiments","jurisdiction_scope":"Latvia; animal experimentation, inspections and enforcement evidence","legacy_paths":[],"url":"https://registri.pvd.gov.lv/en/cr","access_method":"official guidance/registers; public facility/statistics route unverified","cadence":"unknown","attribution_licensing_notes":"Sensitive research/inspection evidence; aggregate, anonymize and require review.","adapter_status":"reference_only","expected_artifact_schema":"Dated aggregate/event observations with authority, category, outcome and source document","blockers":["No stable public facility-level route verified."]}, + {"source_id":"lv.ur.organizations","jurisdiction_scope":"Latvia; Latvian enterprise registration identifiers","legacy_paths":[],"url":"https://www.ur.gov.lv/en/","access_method":"official register/API route unverified","cadence":"provider-defined","attribution_licensing_notes":"Identity-only linkage; suppress personal/sole-trader/residential details.","adapter_status":"reference_only","expected_artifact_schema":"Registration-number keyed organization/status/activity/address response","blockers":["Verify current API/access, terms and privacy policy."]} ] } + diff --git a/pipeline/tests/test_latvia_recon_metadata.py b/pipeline/tests/test_latvia_recon_metadata.py new file mode 100644 index 0000000..645907f --- /dev/null +++ b/pipeline/tests/test_latvia_recon_metadata.py @@ -0,0 +1,11 @@ +import json, unittest +from pathlib import Path +ROOT=Path(__file__).resolve().parents[2] +IDS={"lv.pvd.approved-food","lv.ldc.slaughter-farms","lv.environment.permits","lv.stat.api","lv.animal-experiments","lv.ur.organizations"} +class LatviaReconMetadataTests(unittest.TestCase): + def test_row_free_sources(self): + r=json.loads((ROOT/"pipeline/source_registry.json").read_text()); self.assertTrue(IDS.issubset({x["source_id"] for x in r["sources"]})); d=(ROOT/"docs/country-recon-lv.md").read_text(); self.assertIn("row-free",d); self.assertIn("fully automated",d); self.assertNotIn("Approval ID |",d) + def test_blocked_status(self): + s=json.loads((ROOT/"docs/source-status.json").read_text()); by={x["source_id"]:x for x in s["sources"]} + for i in IDS: self.assertEqual(by[i]["publication_eligibility"],"blocked") +if __name__=="__main__": unittest.main() diff --git a/pipeline/tests/test_source_registry.py b/pipeline/tests/test_source_registry.py index 102430a..9eb1956 100644 --- a/pipeline/tests/test_source_registry.py +++ b/pipeline/tests/test_source_registry.py @@ -9,8 +9,8 @@ class SourceRegistryTests(unittest.TestCase): def test_repository_registry_loads_and_references_existing_legacy_paths(self): registry = load_registry() - self.assertEqual(len(registry["sources"]), 109) - self.assertEqual(len({source["source_id"] for source in registry["sources"]}), 109) + self.assertEqual(len(registry["sources"]), 115) + self.assertEqual(len({source["source_id"] for source in registry["sources"]}), 115) def test_unknowns_are_explicit(self): registry = json.loads(REGISTRY_PATH.read_text(encoding="utf-8")) From ba203a39a33defd0ab1e12435291b0ab405221e2 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Wed, 16 Sep 2026 09:46:33 -0700 Subject: [PATCH 148/311] docs: add Lithuania source reconnaissance --- docs/country-recon-lt.md | 29 +++++++++++++++++++ docs/source-status.json | 9 +++++- pipeline/source_registry.json | 9 +++++- .../tests/test_lithuania_recon_metadata.py | 11 +++++++ pipeline/tests/test_source_registry.py | 4 +-- 5 files changed, 58 insertions(+), 4 deletions(-) create mode 100644 docs/country-recon-lt.md create mode 100644 pipeline/tests/test_lithuania_recon_metadata.py diff --git a/docs/country-recon-lt.md b/docs/country-recon-lt.md new file mode 100644 index 0000000..ffc2d4a --- /dev/null +++ b/docs/country-recon-lt.md @@ -0,0 +1,29 @@ +# Lithuania source reconnaissance + +Status: row-free reconnaissance only. No facility rows, raw exports, coordinates, or production ingestion retained. Official routes checked 2026-09-16. + +## Decision + +Lithuania is a strong medium-effort candidate. VMVT publishes open machine-readable veterinary-control and food-establishment data; the national open-data portal exposes API/CSV/JSON/JSONL resources; PRIA-equivalent agricultural registers and environmental permit systems provide separate farm/environment evidence; and Statistics Lithuania provides official statistical tables. Approval, farm/aquaculture, permits, inspections, corporate identity, and statistics remain distinct. + +The pipeline must be fully automated from scheduled retrieval through hashing, validation, provenance, normalization, privacy/review gates, and guarded ingestion. Browser/manual exports are fallbacks only. + +## Source inventory + +| Source | Route/findings | Format/cadence/IDs | Privacy/automation | +|---|---|---|---| +| Approved food/slaughter establishments | [VMVT open data/registers](https://vmvt.lrv.lt/lt/atviri-vmvt-duomenys-ir-registrai/atviri-duomenys/) and [open control dataset](https://data.gov.lt/dataset/valstybines-veterinarines-kontroles-subjektai) | Public food/animal-origin registers; dataset offers API, CSV, JSON and JSONL; update cadence varies | Contains subject IDs, approval/registration numbers, activity, address and geolocation. Strong candidate, medium effort; verify current terms and suppress personal/mixed addresses. | +| Farms/intensive agriculture/aquaculture | [PRIA public data](https://www.pria.ee/registrid/avalikud-andmed) is Estonia; Lithuania’s VMVT open animal registers expose herd/control data | Current Lithuanian facility export and farm coordinate contract require capture | Treat animal-holder/property data as sensitive; no inferred farm rows. Medium/high effort. | +| Environmental permits | Lithuanian environmental permit/open-data routes require verification | Format/cadence/IDs unresolved | Keep permits separate; high effort until stable bulk/API route. | +| Animal experimentation | VMVT/official animal-welfare guidance | No public facility master verified | Aggregate and anonymize; high privacy. | +| Inspections/enforcement | VMVT open control dataset includes inspection and subject identifiers, dates, deviations and measures | API/CSV/JSON/JSONL; variable update cadence | Model dated events separately from approvals; medium effort after schema/version validation. | +| Corporate identifiers | Lithuanian JAR/company register route, current public API not verified | Legal-person code expected stable | Identity-only crosswalk; suppress personal/sole-trader/residential data. Medium effort. | +| Slaughter/statistics | Statistics Lithuania official PxWeb tables; VMVT annual control reports | Machine-readable statistical tables; table IDs/cadence require pinning | Aggregate only; preserve revisions/dimensions. Low/medium effort. | + +## Gates + +First implementation order: VMVT approved/control API resources, then statistics and reviewed corporate identity links. Require URL/API contract, hash/bytes, timestamps, content type, schema fingerprints, pagination/count checks, stable IDs, category preservation, coordinate CRS/precision checks and quarantine on drift. No geocoding or inferred identity authorized. + +Blockers: exact VMVT resource contracts/terms, farm/aquaculture coverage, environmental permits, research facilities, corporate API access and publication/privacy review. Publication remains blocked pending ethics, rights, privacy, coverage and maintainer approval. + +Recommended next country after Lithuania: Poland, using separate food, farm, environment, corporate and statistics authorities. diff --git a/docs/source-status.json b/docs/source-status.json index bde58be..dc83066 100644 --- a/docs/source-status.json +++ b/docs/source-status.json @@ -123,6 +123,13 @@ {"source_id":"lv.environment.permits","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-lv.md","pipeline/source_registry.json"],"next_action":"Verify resource URLs, schema, coverage, geometry and privacy."}, {"source_id":"lv.stat.api","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-lv.md","pipeline/source_registry.json"],"next_action":"Pin table IDs/metadata and preserve aggregate revisions."}, {"source_id":"lv.animal-experiments","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-lv.md","pipeline/source_registry.json"],"next_action":"Keep research/inspection evidence aggregate until a public route is verified."}, - {"source_id":"lv.ur.organizations","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-lv.md","pipeline/source_registry.json"],"next_action":"Verify current API/access, terms and privacy."} + {"source_id":"lv.ur.organizations","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-lv.md","pipeline/source_registry.json"],"next_action":"Verify current API/access, terms and privacy."}, + {"source_id":"lt.vmvt.approved-food","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-lt.md","pipeline/source_registry.json"],"next_action":"Verify current resource URLs, schema, completeness, terms and privacy."}, + {"source_id":"lt.vmvt.farm-aquaculture","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-lt.md","pipeline/source_registry.json"],"next_action":"Verify public scope, export/API, CRS, terms and coverage."}, + {"source_id":"lt.environment.permits","metadata":"unknown","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-lt.md","pipeline/source_registry.json"],"next_action":"Locate and verify a stable environmental permit route."}, + {"source_id":"lt.animal-experiments","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-lt.md","pipeline/source_registry.json"],"next_action":"Keep aggregate research evidence separate; no facility master verified."}, + {"source_id":"lt.vmvt.inspections","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-lt.md","pipeline/source_registry.json"],"next_action":"Verify resource version, event semantics and coverage."}, + {"source_id":"lt.statistics.slaughter","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-lt.md","pipeline/source_registry.json"],"next_action":"Pin table IDs/API and preserve aggregate revisions."}, + {"source_id":"lt.jar.organizations","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-lt.md","pipeline/source_registry.json"],"next_action":"Verify public API/access, terms and identity-only privacy policy."} ] } diff --git a/pipeline/source_registry.json b/pipeline/source_registry.json index 9b4e82d..b781620 100644 --- a/pipeline/source_registry.json +++ b/pipeline/source_registry.json @@ -751,10 +751,17 @@ {"source_id":"lv.environment.permits","jurisdiction_scope":"Latvia; VVD environmental permits and public registers","legacy_paths":[],"url":"https://data.gov.lv/dati/dataset/izsniegtas-atlaujas-un-licences","access_method":"open-data CSV/resource and public registers","cadence":"dataset-specific","attribution_licensing_notes":"Catalogue states CC0 1.0; preserve source and verify resource scope/geometry.","adapter_status":"reference_only","expected_artifact_schema":"Permit/license records with subject, activity, dates, authority, status and optional site geometry","blockers":["Verify current resource URLs, schema, coverage and privacy."]}, {"source_id":"lv.stat.api","jurisdiction_scope":"Latvia; official statistics API for slaughter, livestock and animal-use aggregates","legacy_paths":[],"url":"https://stat.gov.lv/en/api-un-kodu-vardnicas/api-v2","access_method":"PxWeb API v2","cadence":"table-specific; 30 requests/10 seconds/IP stated","attribution_licensing_notes":"Official statistics; preserve table IDs, revisions and license/attribution metadata.","adapter_status":"reference_only","expected_artifact_schema":"CSV/XLSX/JSON/JSON-stat2/PX multidimensional tables","blockers":["Pin table IDs and metadata; keep aggregate scope separate."]}, {"source_id":"lv.animal-experiments","jurisdiction_scope":"Latvia; animal experimentation, inspections and enforcement evidence","legacy_paths":[],"url":"https://registri.pvd.gov.lv/en/cr","access_method":"official guidance/registers; public facility/statistics route unverified","cadence":"unknown","attribution_licensing_notes":"Sensitive research/inspection evidence; aggregate, anonymize and require review.","adapter_status":"reference_only","expected_artifact_schema":"Dated aggregate/event observations with authority, category, outcome and source document","blockers":["No stable public facility-level route verified."]}, - {"source_id":"lv.ur.organizations","jurisdiction_scope":"Latvia; Latvian enterprise registration identifiers","legacy_paths":[],"url":"https://www.ur.gov.lv/en/","access_method":"official register/API route unverified","cadence":"provider-defined","attribution_licensing_notes":"Identity-only linkage; suppress personal/sole-trader/residential details.","adapter_status":"reference_only","expected_artifact_schema":"Registration-number keyed organization/status/activity/address response","blockers":["Verify current API/access, terms and privacy policy."]} + {"source_id":"lv.ur.organizations","jurisdiction_scope":"Latvia; Latvian enterprise registration identifiers","legacy_paths":[],"url":"https://www.ur.gov.lv/en/","access_method":"official register/API route unverified","cadence":"provider-defined","attribution_licensing_notes":"Identity-only linkage; suppress personal/sole-trader/residential details.","adapter_status":"reference_only","expected_artifact_schema":"Registration-number keyed organization/status/activity/address response","blockers":["Verify current API/access, terms and privacy policy."]} ,{"source_id":"lt.vmvt.approved-food","jurisdiction_scope":"Lithuania; VMVT approved food and veterinary-control establishments","legacy_paths":[],"url":"https://data.gov.lt/dataset/valstybines-veterinarines-kontroles-subjektai","access_method":"open-data API, CSV, JSON and JSONL resources","cadence":"varies by resource; timestamp retrieval","attribution_licensing_notes":"Official VMVT; catalogue states CC BY 4.0; verify current terms and privacy.","adapter_status":"reference_only","expected_artifact_schema":"Subject ID, name/type, dates, JAR code, activity, approval number, authority, address and geolocation","blockers":["Verify current resource URLs, schema, completeness, terms and sensitive-field suppression."]}, + {"source_id":"lt.vmvt.farm-aquaculture","jurisdiction_scope":"Lithuania; VMVT animal, herd and aquaculture establishment evidence","legacy_paths":[],"url":"https://vmvt.lrv.lt/lt/atviri-vmvt-duomenys-ir-registrai/atviri-duomenys/","access_method":"official registers/open-data routes","cadence":"resource-specific","attribution_licensing_notes":"Official authority; animal-holder/property locations require strict privacy review.","adapter_status":"reference_only","expected_artifact_schema":"Herd/location or aquaculture establishment records with stable registration/status IDs and optional coordinates","blockers":["Verify public facility scope, current export/API, CRS, terms and coverage."]}, + {"source_id":"lt.environment.permits","jurisdiction_scope":"Lithuania; environmental permits and releases","legacy_paths":[],"url":"https://data.gov.lt/","access_method":"national open-data/environmental permit routes; not verified","cadence":"unknown","attribution_licensing_notes":"Verify publisher, license, sensitive-site handling and geometry.","adapter_status":"reference_only","expected_artifact_schema":"Permit/document records with authority, IDs, activity, dates, status and optional geometry","blockers":["No stable national animal-facility permit export verified."]}, + {"source_id":"lt.animal-experiments","jurisdiction_scope":"Lithuania; animal experimentation and welfare evidence","legacy_paths":[],"url":"https://vmvt.lrv.lt/lt/atviri-vmvt-duomenys-ir-registrai/atviri-duomenys/","access_method":"official guidance/reports; facility route unverified","cadence":"annual/report-specific","attribution_licensing_notes":"Sensitive research evidence; aggregate/anonymize and require review.","adapter_status":"reference_only","expected_artifact_schema":"Aggregate annual/event observations by species, purpose, severity and year","blockers":["No public research-facility master verified."]}, + {"source_id":"lt.vmvt.inspections","jurisdiction_scope":"Lithuania; VMVT veterinary-control inspections/enforcement","legacy_paths":[],"url":"https://data.gov.lt/dataset/valstybines-veterinarines-kontroles-subjektai","access_method":"open-data API/CSV/JSON/JSONL","cadence":"varies; portal says not uniform","attribution_licensing_notes":"Official control evidence; preserve event status and privacy; CC BY 4.0 catalogue claim requires confirmation.","adapter_status":"reference_only","expected_artifact_schema":"Inspection/subject IDs, dates, authority, announced flag, deviations and measures","blockers":["Verify current resource/version, event semantics and coverage."]}, + {"source_id":"lt.statistics.slaughter","jurisdiction_scope":"Lithuania; official slaughter, livestock and animal-use statistics","legacy_paths":[],"url":"https://osp.stat.gov.lt/","access_method":"official statistical tables/API; exact table IDs unverified","cadence":"table-specific","attribution_licensing_notes":"Official statistics; preserve dimensions, revisions and license metadata.","adapter_status":"reference_only","expected_artifact_schema":"Multidimensional aggregate tables with periods, species, regions, values and flags","blockers":["Pin table IDs/API contract; keep aggregates separate from facilities."]}, + {"source_id":"lt.jar.organizations","jurisdiction_scope":"Lithuania; legal-entity register identity","legacy_paths":[],"url":"https://www.registrucentras.lt/","access_method":"official register/API route unverified","cadence":"provider-defined","attribution_licensing_notes":"Identity-only linkage; suppress personal, sole-trader and residential details.","adapter_status":"reference_only","expected_artifact_schema":"Legal-person code keyed organization/status/activity/address response","blockers":["Verify public API/access, terms, quotas and privacy policy."]} ] } + diff --git a/pipeline/tests/test_lithuania_recon_metadata.py b/pipeline/tests/test_lithuania_recon_metadata.py new file mode 100644 index 0000000..d8caf4f --- /dev/null +++ b/pipeline/tests/test_lithuania_recon_metadata.py @@ -0,0 +1,11 @@ +import json,unittest +from pathlib import Path +ROOT=Path(__file__).resolve().parents[2] +IDS={"lt.vmvt.approved-food","lt.vmvt.farm-aquaculture","lt.environment.permits","lt.animal-experiments","lt.vmvt.inspections","lt.statistics.slaughter","lt.jar.organizations"} +class LithuaniaReconMetadataTests(unittest.TestCase): + def test_row_free_sources(self): + r=json.loads((ROOT/"pipeline/source_registry.json").read_text()); self.assertTrue(IDS.issubset({x["source_id"] for x in r["sources"]})); d=(ROOT/"docs/country-recon-lt.md").read_text(); self.assertIn("row-free",d); self.assertIn("fully automated",d); self.assertNotIn("Approval ID |",d) + def test_blocked_status(self): + s=json.loads((ROOT/"docs/source-status.json").read_text()); by={x["source_id"]:x for x in s["sources"]} + for i in IDS: self.assertEqual(by[i]["publication_eligibility"],"blocked") +if __name__=="__main__": unittest.main() diff --git a/pipeline/tests/test_source_registry.py b/pipeline/tests/test_source_registry.py index 9eb1956..ec131c8 100644 --- a/pipeline/tests/test_source_registry.py +++ b/pipeline/tests/test_source_registry.py @@ -9,8 +9,8 @@ class SourceRegistryTests(unittest.TestCase): def test_repository_registry_loads_and_references_existing_legacy_paths(self): registry = load_registry() - self.assertEqual(len(registry["sources"]), 115) - self.assertEqual(len({source["source_id"] for source in registry["sources"]}), 115) + self.assertEqual(len(registry["sources"]), 122) + self.assertEqual(len({source["source_id"] for source in registry["sources"]}), 122) def test_unknowns_are_explicit(self): registry = json.loads(REGISTRY_PATH.read_text(encoding="utf-8")) From 6040c36f614f3ee674dd45691b34e82f604ff9be Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Wed, 16 Sep 2026 09:48:27 -0700 Subject: [PATCH 149/311] docs: add Czechia source reconnaissance --- docs/country-recon-cz.md | 27 +++++++++++++++++++ docs/source-status.json | 7 ++++- pipeline/source_registry.json | 7 ++++- pipeline/tests/test_czechia_recon_metadata.py | 11 ++++++++ pipeline/tests/test_source_registry.py | 4 +-- 5 files changed, 52 insertions(+), 4 deletions(-) create mode 100644 docs/country-recon-cz.md create mode 100644 pipeline/tests/test_czechia_recon_metadata.py diff --git a/docs/country-recon-cz.md b/docs/country-recon-cz.md new file mode 100644 index 0000000..4e5cac0 --- /dev/null +++ b/docs/country-recon-cz.md @@ -0,0 +1,27 @@ +# Czechia source reconnaissance + +Status: row-free reconnaissance only. No facility rows, raw exports, coordinates, or production ingestion retained. Official routes checked 2026-09-16. + +## Decision + +Czechia is a medium-effort candidate. The State Veterinary Administration (SVS) publishes registered and approved establishment lists, including EU food, ABP, feed and aquaculture categories; the government and statistical registers provide corporate and aggregate statistics; environmental permits require separate registry work. + +The pipeline must be fully automated from scheduled retrieval through hashing, validation, provenance, normalization, privacy/review gates, and guarded ingestion. Browser-only/manual exports are fallbacks only. + +## Source inventory + +| Source | Route/findings | Format/cadence/IDs | Privacy/automation | +|---|---|---|---| +| Approved food/slaughter establishments | [SVS registered/approved establishments](https://en.svs.gov.cz/registered-subjects/) | Lists cover EU food, conditional approval, ABP, feed-related, transport and aquaculture; filters and update dates are exposed, exact bulk contract requires capture | Approval number, activity, region and address expected; no coordinates verified. Strong candidate, medium effort; confirm terms/privacy. | +| Farms/intensive/aquaculture | SVS animal registers and [aquaculture list category](https://en.svs.gov.cz/registered-subjects/) | Public aquaculture/animal establishment categories; national intensive-farm export not verified | Treat holdings and locations as sensitive; no inferred rows. Medium/high effort. | +| Inspections/enforcement and experiments | SVS control/animal-welfare surfaces | No stable national public facility-level experiment/enforcement dataset verified | Keep events/annual aggregates separate; high privacy. | +| Environmental permits | Czech environmental permit/document routes require separate verification | Format/API/cadence unresolved | High effort; verify identifiers, geometry, licensing and sensitive sites. | +| Corporate/statistics | [gov.cz statistical registers](https://portal.gov.cz/sluzby-vs/ziskani-zverejnenych-informaci-ze-statistickych-registru-S4953) and Czech Statistical Office | Open CSV/company register exports and official aggregate tables; table/API contracts require pinning | Identity-only corporate crosswalk; aggregate slaughter/animal-use statistics separate. Medium effort. | + +## Gates + +First implementation order: SVS approved-establishment exports, then corporate/statistics sources. Require URL/API contract, hash/bytes, timestamps, content type, schema fingerprints, pagination/count checks, stable IDs, category preservation and quarantine on drift. No geocoding or inferred identity authorized. + +Blockers: exact SVS file URLs/headers/terms, farm/aquaculture coverage, public inspections/experiments, environmental permit API, corporate access, and privacy/release review. Publication remains blocked pending ethics, rights, privacy, coverage and maintainer approval. + +Recommended next country after Czechia: Slovakia, using separate veterinary, farm, environmental, corporate and statistics authorities. diff --git a/docs/source-status.json b/docs/source-status.json index dc83066..ab8bbe6 100644 --- a/docs/source-status.json +++ b/docs/source-status.json @@ -130,6 +130,11 @@ {"source_id":"lt.animal-experiments","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-lt.md","pipeline/source_registry.json"],"next_action":"Keep aggregate research evidence separate; no facility master verified."}, {"source_id":"lt.vmvt.inspections","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-lt.md","pipeline/source_registry.json"],"next_action":"Verify resource version, event semantics and coverage."}, {"source_id":"lt.statistics.slaughter","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-lt.md","pipeline/source_registry.json"],"next_action":"Pin table IDs/API and preserve aggregate revisions."}, - {"source_id":"lt.jar.organizations","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-lt.md","pipeline/source_registry.json"],"next_action":"Verify public API/access, terms and identity-only privacy policy."} + {"source_id":"lt.jar.organizations","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-lt.md","pipeline/source_registry.json"],"next_action":"Verify public API/access, terms and identity-only privacy policy."}, + {"source_id":"cz.svs.approved-food","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-cz.md","pipeline/source_registry.json"],"next_action":"Verify bulk/API/file contracts, completeness, terms, IDs and coordinates."}, + {"source_id":"cz.svs.farms-aquaculture","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-cz.md","pipeline/source_registry.json"],"next_action":"Verify national scope, export route, terms and sensitive-field policy."}, + {"source_id":"cz.environment.permits","metadata":"unknown","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-cz.md","pipeline/source_registry.json"],"next_action":"Locate and verify environmental permit data route."}, + {"source_id":"cz.statistics","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-cz.md","pipeline/source_registry.json"],"next_action":"Pin statistics table/API contracts and revisions."}, + {"source_id":"cz.business-register","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-cz.md","pipeline/source_registry.json"],"next_action":"Verify current endpoint, terms and identity-only privacy policy."} ] } diff --git a/pipeline/source_registry.json b/pipeline/source_registry.json index b781620..231d32f 100644 --- a/pipeline/source_registry.json +++ b/pipeline/source_registry.json @@ -757,7 +757,11 @@ {"source_id":"lt.animal-experiments","jurisdiction_scope":"Lithuania; animal experimentation and welfare evidence","legacy_paths":[],"url":"https://vmvt.lrv.lt/lt/atviri-vmvt-duomenys-ir-registrai/atviri-duomenys/","access_method":"official guidance/reports; facility route unverified","cadence":"annual/report-specific","attribution_licensing_notes":"Sensitive research evidence; aggregate/anonymize and require review.","adapter_status":"reference_only","expected_artifact_schema":"Aggregate annual/event observations by species, purpose, severity and year","blockers":["No public research-facility master verified."]}, {"source_id":"lt.vmvt.inspections","jurisdiction_scope":"Lithuania; VMVT veterinary-control inspections/enforcement","legacy_paths":[],"url":"https://data.gov.lt/dataset/valstybines-veterinarines-kontroles-subjektai","access_method":"open-data API/CSV/JSON/JSONL","cadence":"varies; portal says not uniform","attribution_licensing_notes":"Official control evidence; preserve event status and privacy; CC BY 4.0 catalogue claim requires confirmation.","adapter_status":"reference_only","expected_artifact_schema":"Inspection/subject IDs, dates, authority, announced flag, deviations and measures","blockers":["Verify current resource/version, event semantics and coverage."]}, {"source_id":"lt.statistics.slaughter","jurisdiction_scope":"Lithuania; official slaughter, livestock and animal-use statistics","legacy_paths":[],"url":"https://osp.stat.gov.lt/","access_method":"official statistical tables/API; exact table IDs unverified","cadence":"table-specific","attribution_licensing_notes":"Official statistics; preserve dimensions, revisions and license metadata.","adapter_status":"reference_only","expected_artifact_schema":"Multidimensional aggregate tables with periods, species, regions, values and flags","blockers":["Pin table IDs/API contract; keep aggregates separate from facilities."]}, - {"source_id":"lt.jar.organizations","jurisdiction_scope":"Lithuania; legal-entity register identity","legacy_paths":[],"url":"https://www.registrucentras.lt/","access_method":"official register/API route unverified","cadence":"provider-defined","attribution_licensing_notes":"Identity-only linkage; suppress personal, sole-trader and residential details.","adapter_status":"reference_only","expected_artifact_schema":"Legal-person code keyed organization/status/activity/address response","blockers":["Verify public API/access, terms, quotas and privacy policy."]} + {"source_id":"lt.jar.organizations","jurisdiction_scope":"Lithuania; legal-entity register identity","legacy_paths":[],"url":"https://www.registrucentras.lt/","access_method":"official register/API route unverified","cadence":"provider-defined","attribution_licensing_notes":"Identity-only linkage; suppress personal, sole-trader and residential details.","adapter_status":"reference_only","expected_artifact_schema":"Legal-person code keyed organization/status/activity/address response","blockers":["Verify public API/access, terms, quotas and privacy policy."]} ,{"source_id":"cz.svs.approved-food","jurisdiction_scope":"Czechia; SVS approved/registered animal-origin food, ABP, feed and aquaculture establishments","legacy_paths":[],"url":"https://en.svs.gov.cz/registered-subjects/","access_method":"official filtered lists and linked files","cadence":"list-specific; timestamp access","attribution_licensing_notes":"Official Czech veterinary authority; verify terms, fields and privacy.","adapter_status":"reference_only","expected_artifact_schema":"Establishment ID/approval number, type, activities, region, address, status and update date","blockers":["Verify bulk/API/file contracts, completeness, terms, IDs and coordinate availability."]}, + {"source_id":"cz.svs.farms-aquaculture","jurisdiction_scope":"Czechia; SVS farms, aquaculture and animal-sector registers","legacy_paths":[],"url":"https://en.svs.gov.cz/registered-subjects/","access_method":"official lists/filter surface","cadence":"unknown","attribution_licensing_notes":"Animal-holder and site data require privacy review.","adapter_status":"reference_only","expected_artifact_schema":"Holding/site records with registration/status/activity and optional address/coordinates","blockers":["Verify national scope, export route, terms and sensitive-field policy."]}, + {"source_id":"cz.environment.permits","jurisdiction_scope":"Czechia; environmental permits and releases","legacy_paths":[],"url":"https://www.mzp.cz/en","access_method":"official environmental registry routes; not verified","cadence":"unknown","attribution_licensing_notes":"Verify license, geometry and sensitive-site handling.","adapter_status":"reference_only","expected_artifact_schema":"Permit/document observations with authority, IDs, activity, dates and optional geometry","blockers":["No stable national animal-facility permit export verified."]}, + {"source_id":"cz.statistics","jurisdiction_scope":"Czechia; official slaughter, livestock and animal-use aggregates","legacy_paths":[],"url":"https://www.czso.cz/csu/czso/statistics","access_method":"official statistical tables/API; exact route unverified","cadence":"table-specific","attribution_licensing_notes":"Official statistics; preserve table IDs, revisions and license metadata.","adapter_status":"reference_only","expected_artifact_schema":"Aggregate tables by species, period, region and measure","blockers":["Pin current table/API contracts and keep aggregate scope separate."]}, + {"source_id":"cz.business-register","jurisdiction_scope":"Czechia; corporate/statistical register identity","legacy_paths":[],"url":"https://portal.gov.cz/sluzby-vs/ziskani-zverejnenych-informaci-ze-statistickych-registru-S4953","access_method":"public register search and CSV export","cadence":"provider-defined","attribution_licensing_notes":"Identity-only linkage; suppress personal, sole-trader and residential details.","adapter_status":"reference_only","expected_artifact_schema":"IČO keyed organization, legal form, activity, address and status","blockers":["Verify current endpoint, terms, quotas and privacy policy."]} ] } @@ -765,3 +769,4 @@ + diff --git a/pipeline/tests/test_czechia_recon_metadata.py b/pipeline/tests/test_czechia_recon_metadata.py new file mode 100644 index 0000000..80bfcf4 --- /dev/null +++ b/pipeline/tests/test_czechia_recon_metadata.py @@ -0,0 +1,11 @@ +import json,unittest +from pathlib import Path +ROOT=Path(__file__).resolve().parents[2] +IDS={"cz.svs.approved-food","cz.svs.farms-aquaculture","cz.environment.permits","cz.statistics","cz.business-register"} +class CzechiaReconMetadataTests(unittest.TestCase): + def test_row_free_sources(self): + r=json.loads((ROOT/"pipeline/source_registry.json").read_text()); self.assertTrue(IDS.issubset({x["source_id"] for x in r["sources"]})); d=(ROOT/"docs/country-recon-cz.md").read_text(); self.assertIn("row-free",d); self.assertIn("fully automated",d) + def test_blocked_status(self): + s=json.loads((ROOT/"docs/source-status.json").read_text()); by={x["source_id"]:x for x in s["sources"]} + for i in IDS: self.assertEqual(by[i]["publication_eligibility"],"blocked") +if __name__=="__main__": unittest.main() diff --git a/pipeline/tests/test_source_registry.py b/pipeline/tests/test_source_registry.py index ec131c8..df58654 100644 --- a/pipeline/tests/test_source_registry.py +++ b/pipeline/tests/test_source_registry.py @@ -9,8 +9,8 @@ class SourceRegistryTests(unittest.TestCase): def test_repository_registry_loads_and_references_existing_legacy_paths(self): registry = load_registry() - self.assertEqual(len(registry["sources"]), 122) - self.assertEqual(len({source["source_id"] for source in registry["sources"]}), 122) + self.assertEqual(len(registry["sources"]), 127) + self.assertEqual(len({source["source_id"] for source in registry["sources"]}), 127) def test_unknowns_are_explicit(self): registry = json.loads(REGISTRY_PATH.read_text(encoding="utf-8")) From e2c3b87db9d6892b2c1833e68d6013fbcf00a690 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Wed, 16 Sep 2026 09:50:46 -0700 Subject: [PATCH 150/311] docs: add Slovakia source reconnaissance --- docs/country-recon-sk.md | 26 +++++++++++++++++++ docs/source-status.json | 7 ++++- pipeline/source_registry.json | 7 ++++- .../tests/test_slovakia_recon_metadata.py | 11 ++++++++ pipeline/tests/test_source_registry.py | 4 +-- 5 files changed, 51 insertions(+), 4 deletions(-) create mode 100644 docs/country-recon-sk.md create mode 100644 pipeline/tests/test_slovakia_recon_metadata.py diff --git a/docs/country-recon-sk.md b/docs/country-recon-sk.md new file mode 100644 index 0000000..3587943 --- /dev/null +++ b/docs/country-recon-sk.md @@ -0,0 +1,26 @@ +# Slovakia source reconnaissance + +Status: row-free reconnaissance only. No facility rows, raw exports, coordinates, or production ingestion retained. Official routes checked 2026-09-16. + +## Decision + +Slovakia is a strong medium-effort candidate. The State Veterinary and Food Administration (ŠVPS/SVFA) publishes current approved food, ABP, veterinary, aquaculture and farm-related lists with dated updates; datasets and regional registers provide additional machine-readable routes. Corporate, environmental and statistical sources require separate verification. + +The pipeline must be fully automated from scheduled retrieval through hashing, validation, provenance, normalization, privacy/review gates, and guarded ingestion. Browser-only/manual exports are fallbacks only. + +## Source inventory + +| Source | Route/findings | Format/cadence/IDs | Privacy/automation | +|---|---|---|---| +| Approved food/slaughter/ABP | [SVPS approved lists](https://zoznamy.svps.sk/?LANG=EN) and [SVPS datasets](https://svps.sk/datasety/) | Filtered lists expose dated updates; EU food, veterinary, ABP and feed categories; XSL/XLSX/list formats | Approval number, category/activity, town/region and address likely. Strong candidate, medium effort; verify direct downloads, terms and privacy. | +| Farms/aquaculture | SVPS datasets include aquaculture/farm categories and animal registers | Current export/API and coordinate fields unresolved | Treat holdings/owners as sensitive; no inferred rows. Medium/high effort. | +| Inspections/enforcement/experiments | SVPS control and animal-welfare systems | Public route not fully verified | Keep dated events/annual aggregates separate; high privacy. | +| Environment/corporate/statistics | National environmental permit, business-register and statistical portals require route verification | IDs/formats/cadence unresolved | Separate evidence products; identity-only crosswalk and aggregate statistics. | + +## Gates + +First implementation order: SVPS approved-list exports, then farm/aquaculture datasets and official statistics. Require contract capture, hashes/bytes, timestamps, content type, schema fingerprints, pagination/count checks, stable IDs, category preservation and drift quarantine. No geocoding or inferred identity authorized. + +Blockers: direct file/API contracts, licensing, farm/aquaculture coverage, inspections/experimentation, environmental permits, corporate access and privacy/release review. Publication remains blocked pending ethics, rights, privacy, coverage and maintainer approval. + +Recommended next country after Slovakia: Slovenia. diff --git a/docs/source-status.json b/docs/source-status.json index ab8bbe6..36d498d 100644 --- a/docs/source-status.json +++ b/docs/source-status.json @@ -135,6 +135,11 @@ {"source_id":"cz.svs.farms-aquaculture","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-cz.md","pipeline/source_registry.json"],"next_action":"Verify national scope, export route, terms and sensitive-field policy."}, {"source_id":"cz.environment.permits","metadata":"unknown","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-cz.md","pipeline/source_registry.json"],"next_action":"Locate and verify environmental permit data route."}, {"source_id":"cz.statistics","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-cz.md","pipeline/source_registry.json"],"next_action":"Pin statistics table/API contracts and revisions."}, - {"source_id":"cz.business-register","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-cz.md","pipeline/source_registry.json"],"next_action":"Verify current endpoint, terms and identity-only privacy policy."} + {"source_id":"cz.business-register","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-cz.md","pipeline/source_registry.json"],"next_action":"Verify current endpoint, terms and identity-only privacy policy."}, + {"source_id":"sk.svps.approved-food","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-sk.md","pipeline/source_registry.json"],"next_action":"Verify direct files, categories, IDs, terms and address/coordinates."}, + {"source_id":"sk.svps.farms-aquaculture","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-sk.md","pipeline/source_registry.json"],"next_action":"Verify public scope, export/API, terms and sensitive fields."}, + {"source_id":"sk.svps.inspections-experiments","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-sk.md","pipeline/source_registry.json"],"next_action":"Keep aggregate/event evidence separate until a public route is verified."}, + {"source_id":"sk.environment.permits","metadata":"unknown","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-sk.md","pipeline/source_registry.json"],"next_action":"Locate and verify environmental permit data."}, + {"source_id":"sk.statistics-corporate","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-sk.md","pipeline/source_registry.json"],"next_action":"Verify statistics/business-register APIs, terms and privacy."} ] } diff --git a/pipeline/source_registry.json b/pipeline/source_registry.json index 231d32f..e8a0853 100644 --- a/pipeline/source_registry.json +++ b/pipeline/source_registry.json @@ -761,7 +761,11 @@ {"source_id":"cz.svs.farms-aquaculture","jurisdiction_scope":"Czechia; SVS farms, aquaculture and animal-sector registers","legacy_paths":[],"url":"https://en.svs.gov.cz/registered-subjects/","access_method":"official lists/filter surface","cadence":"unknown","attribution_licensing_notes":"Animal-holder and site data require privacy review.","adapter_status":"reference_only","expected_artifact_schema":"Holding/site records with registration/status/activity and optional address/coordinates","blockers":["Verify national scope, export route, terms and sensitive-field policy."]}, {"source_id":"cz.environment.permits","jurisdiction_scope":"Czechia; environmental permits and releases","legacy_paths":[],"url":"https://www.mzp.cz/en","access_method":"official environmental registry routes; not verified","cadence":"unknown","attribution_licensing_notes":"Verify license, geometry and sensitive-site handling.","adapter_status":"reference_only","expected_artifact_schema":"Permit/document observations with authority, IDs, activity, dates and optional geometry","blockers":["No stable national animal-facility permit export verified."]}, {"source_id":"cz.statistics","jurisdiction_scope":"Czechia; official slaughter, livestock and animal-use aggregates","legacy_paths":[],"url":"https://www.czso.cz/csu/czso/statistics","access_method":"official statistical tables/API; exact route unverified","cadence":"table-specific","attribution_licensing_notes":"Official statistics; preserve table IDs, revisions and license metadata.","adapter_status":"reference_only","expected_artifact_schema":"Aggregate tables by species, period, region and measure","blockers":["Pin current table/API contracts and keep aggregate scope separate."]}, - {"source_id":"cz.business-register","jurisdiction_scope":"Czechia; corporate/statistical register identity","legacy_paths":[],"url":"https://portal.gov.cz/sluzby-vs/ziskani-zverejnenych-informaci-ze-statistickych-registru-S4953","access_method":"public register search and CSV export","cadence":"provider-defined","attribution_licensing_notes":"Identity-only linkage; suppress personal, sole-trader and residential details.","adapter_status":"reference_only","expected_artifact_schema":"IČO keyed organization, legal form, activity, address and status","blockers":["Verify current endpoint, terms, quotas and privacy policy."]} + {"source_id":"cz.business-register","jurisdiction_scope":"Czechia; corporate/statistical register identity","legacy_paths":[],"url":"https://portal.gov.cz/sluzby-vs/ziskani-zverejnenych-informaci-ze-statistickych-registru-S4953","access_method":"public register search and CSV export","cadence":"provider-defined","attribution_licensing_notes":"Identity-only linkage; suppress personal, sole-trader and residential details.","adapter_status":"reference_only","expected_artifact_schema":"IČO keyed organization, legal form, activity, address and status","blockers":["Verify current endpoint, terms, quotas and privacy policy."]} ,{"source_id":"sk.svps.approved-food","jurisdiction_scope":"Slovakia; SVPS approved food, slaughter and ABP establishments","legacy_paths":[],"url":"https://zoznamy.svps.sk/?LANG=EN","access_method":"official filtered lists and linked XSL/XLSX datasets","cadence":"list-specific dated updates","attribution_licensing_notes":"Official Slovak authority; verify terms, attribution and privacy.","adapter_status":"reference_only","expected_artifact_schema":"Approval number, name, town/region, category, associated activities, species, remarks and update date","blockers":["Verify direct files, complete categories, stable IDs, terms and address/coordinate fields."]}, + {"source_id":"sk.svps.farms-aquaculture","jurisdiction_scope":"Slovakia; SVPS farm, veterinary and aquaculture registers","legacy_paths":[],"url":"https://svps.sk/datasety/","access_method":"official datasets and register links","cadence":"resource-specific","attribution_licensing_notes":"Animal-holder/site data require privacy review.","adapter_status":"reference_only","expected_artifact_schema":"Holding/aquaculture records with registration/status/activity and optional location","blockers":["Verify public scope, current export/API, terms and sensitive-field policy."]}, + {"source_id":"sk.svps.inspections-experiments","jurisdiction_scope":"Slovakia; SVPS inspections, enforcement and animal-experiment evidence","legacy_paths":[],"url":"https://svps.sk/english/","access_method":"official reports/systems; row-level route unverified","cadence":"annual/report-specific","attribution_licensing_notes":"Sensitive evidence; aggregate/anonymize and require review.","adapter_status":"reference_only","expected_artifact_schema":"Dated event/aggregate observations with authority, category, outcome and source document","blockers":["No stable public facility-level route verified."]}, + {"source_id":"sk.environment.permits","jurisdiction_scope":"Slovakia; environmental permit evidence","legacy_paths":[],"url":"https://www.minzp.sk/en/","access_method":"official environmental routes; not verified","cadence":"unknown","attribution_licensing_notes":"Verify license, geometry and sensitive-site handling.","adapter_status":"reference_only","expected_artifact_schema":"Permit/document records with authority, IDs, activities, dates and optional geometry","blockers":["No stable national animal-facility permit export verified."]}, + {"source_id":"sk.statistics-corporate","jurisdiction_scope":"Slovakia; official slaughter/livestock statistics and business identifiers","legacy_paths":[],"url":"https://slovak.statistics.sk/","access_method":"official statistical/business-register routes; exact APIs unverified","cadence":"table/provider-specific","attribution_licensing_notes":"Identity-only corporate linkage; aggregate statistics preserve revisions and source terms.","adapter_status":"reference_only","expected_artifact_schema":"Aggregate statistical tables and legal-entity records keyed by stable national identifier","blockers":["Verify current table/API contracts, business-register access, terms and privacy."]} ] } @@ -770,3 +774,4 @@ + diff --git a/pipeline/tests/test_slovakia_recon_metadata.py b/pipeline/tests/test_slovakia_recon_metadata.py new file mode 100644 index 0000000..4bf66fc --- /dev/null +++ b/pipeline/tests/test_slovakia_recon_metadata.py @@ -0,0 +1,11 @@ +import json,unittest +from pathlib import Path +ROOT=Path(__file__).resolve().parents[2] +IDS={"sk.svps.approved-food","sk.svps.farms-aquaculture","sk.svps.inspections-experiments","sk.environment.permits","sk.statistics-corporate"} +class SlovakiaReconMetadataTests(unittest.TestCase): + def test_row_free_sources(self): + r=json.loads((ROOT/"pipeline/source_registry.json").read_text()); self.assertTrue(IDS.issubset({x["source_id"] for x in r["sources"]})); d=(ROOT/"docs/country-recon-sk.md").read_text(); self.assertIn("row-free",d); self.assertIn("fully automated",d) + def test_blocked_status(self): + s=json.loads((ROOT/"docs/source-status.json").read_text()); by={x["source_id"]:x for x in s["sources"]} + for i in IDS: self.assertEqual(by[i]["publication_eligibility"],"blocked") +if __name__=="__main__": unittest.main() diff --git a/pipeline/tests/test_source_registry.py b/pipeline/tests/test_source_registry.py index df58654..56ad035 100644 --- a/pipeline/tests/test_source_registry.py +++ b/pipeline/tests/test_source_registry.py @@ -9,8 +9,8 @@ class SourceRegistryTests(unittest.TestCase): def test_repository_registry_loads_and_references_existing_legacy_paths(self): registry = load_registry() - self.assertEqual(len(registry["sources"]), 127) - self.assertEqual(len({source["source_id"] for source in registry["sources"]}), 127) + self.assertEqual(len(registry["sources"]), 132) + self.assertEqual(len({source["source_id"] for source in registry["sources"]}), 132) def test_unknowns_are_explicit(self): registry = json.loads(REGISTRY_PATH.read_text(encoding="utf-8")) From 42f4163f696346747ef10bd1b10afce8463c047f Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Wed, 16 Sep 2026 09:52:49 -0700 Subject: [PATCH 151/311] docs: add Slovenia source reconnaissance --- docs/country-recon-si.md | 27 +++++++++++++++++++ docs/source-status.json | 7 ++++- pipeline/source_registry.json | 7 ++++- .../tests/test_slovenia_recon_metadata.py | 11 ++++++++ pipeline/tests/test_source_registry.py | 4 +-- 5 files changed, 52 insertions(+), 4 deletions(-) create mode 100644 docs/country-recon-si.md create mode 100644 pipeline/tests/test_slovenia_recon_metadata.py diff --git a/docs/country-recon-si.md b/docs/country-recon-si.md new file mode 100644 index 0000000..2050d7d --- /dev/null +++ b/docs/country-recon-si.md @@ -0,0 +1,27 @@ +# Slovenia source reconnaissance + +Status: row-free reconnaissance only. No facility rows, raw exports, coordinates, or production ingestion retained. Official routes checked 2026-09-16. + +## Decision + +Slovenia is a strong medium-effort candidate. UVHVVR publishes approved food/feed registers and animal-sector systems; OPSI exposes agricultural datasets; environmental and statistical portals provide separate permit and slaughter evidence. Farm, aquaculture, experimentation, inspection and corporate data require distinct privacy and access review. + +The pipeline must be fully automated from scheduled retrieval through hashing, validation, provenance, normalization, privacy/review gates, and guarded ingestion. Browser/manual exports are fallbacks only. + +## Source inventory + +| Source | Route/findings | Format/cadence/IDs | Privacy/automation | +|---|---|---|---| +| Approved food/slaughter establishments | [GOV.SI approved food establishment service](https://www.gov.si/zbirke/storitve/odobritev-zivilskega-obrata/) | Official approved-establishment PDF/list routes; current direct file/API and update cadence require capture | Approval numbers, activity and address expected; no coordinates assumed. Medium effort; verify terms/privacy. | +| Feed | [GOV.SI feed business registers](https://www.gov.si/teme/poslovanje-s-krmo/) | Approved/registered lists, current PDF updates | Separate feed categories; PDF extraction brittle; verify reuse and addresses. Medium/high effort. | +| Farms/aquaculture | [OPSI agricultural datasets](https://podatki.gov.si/) and UVHVVR animal registers | Public datasets include animal/holding registers; exact current farm/aquaculture export must be verified | Holding/property/person data sensitive; no facility rows without authorization. Medium/high effort. | +| Inspections/experiments/environment | UVHVVR systems and environmental permit routes | Public control/permit contracts unresolved | Model events/documents separately; high privacy/coverage risk. | +| Statistics/corporate | [Slovenia livestock slaughter PxWeb](https://pxweb.stat.si/SiStatData/pxweb/en/Data/-/H202S.px) and national business register | PxWeb annual slaughter table; corporate identifier route requires verification | Aggregate stats and identity-only crosswalk. Low/medium statistics effort. | + +## Gates + +First implementation order: approved food/feed lists, then PxWeb statistics and reviewed agricultural datasets. Require contract capture, hashes/bytes, timestamps, schema fingerprints, stable IDs, category preservation, CRS/precision checks and drift quarantine. No geocoding or inferred identity authorized. + +Blockers: exact files/API/terms, farm/aquaculture scope, inspections/experiments, environmental permits, corporate access, and privacy/release approval. Publication remains blocked pending ethics, rights, privacy, coverage and maintainer approval. + +Recommended next country after Slovenia: Croatia. diff --git a/docs/source-status.json b/docs/source-status.json index 36d498d..f40b995 100644 --- a/docs/source-status.json +++ b/docs/source-status.json @@ -140,6 +140,11 @@ {"source_id":"sk.svps.farms-aquaculture","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-sk.md","pipeline/source_registry.json"],"next_action":"Verify public scope, export/API, terms and sensitive fields."}, {"source_id":"sk.svps.inspections-experiments","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-sk.md","pipeline/source_registry.json"],"next_action":"Keep aggregate/event evidence separate until a public route is verified."}, {"source_id":"sk.environment.permits","metadata":"unknown","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-sk.md","pipeline/source_registry.json"],"next_action":"Locate and verify environmental permit data."}, - {"source_id":"sk.statistics-corporate","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-sk.md","pipeline/source_registry.json"],"next_action":"Verify statistics/business-register APIs, terms and privacy."} + {"source_id":"sk.statistics-corporate","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-sk.md","pipeline/source_registry.json"],"next_action":"Verify statistics/business-register APIs, terms and privacy."}, + {"source_id":"si.uvhvvr.approved-food-feed","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-si.md","pipeline/source_registry.json"],"next_action":"Verify current files/API, completeness, terms and privacy."}, + {"source_id":"si.uvhvvr.farms-aquaculture","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-si.md","pipeline/source_registry.json"],"next_action":"Verify public scope, export/API, terms and sensitive fields."}, + {"source_id":"si.environment.permits","metadata":"unknown","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-si.md","pipeline/source_registry.json"],"next_action":"Locate and verify environmental permit data."}, + {"source_id":"si.statistics-slaughter","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-si.md","pipeline/source_registry.json"],"next_action":"Pin PxWeb table/API metadata and revisions."}, + {"source_id":"si.corporate-register","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-si.md","pipeline/source_registry.json"],"next_action":"Verify current access, terms and identity-only privacy policy."} ] } diff --git a/pipeline/source_registry.json b/pipeline/source_registry.json index e8a0853..91633d6 100644 --- a/pipeline/source_registry.json +++ b/pipeline/source_registry.json @@ -765,7 +765,11 @@ {"source_id":"sk.svps.farms-aquaculture","jurisdiction_scope":"Slovakia; SVPS farm, veterinary and aquaculture registers","legacy_paths":[],"url":"https://svps.sk/datasety/","access_method":"official datasets and register links","cadence":"resource-specific","attribution_licensing_notes":"Animal-holder/site data require privacy review.","adapter_status":"reference_only","expected_artifact_schema":"Holding/aquaculture records with registration/status/activity and optional location","blockers":["Verify public scope, current export/API, terms and sensitive-field policy."]}, {"source_id":"sk.svps.inspections-experiments","jurisdiction_scope":"Slovakia; SVPS inspections, enforcement and animal-experiment evidence","legacy_paths":[],"url":"https://svps.sk/english/","access_method":"official reports/systems; row-level route unverified","cadence":"annual/report-specific","attribution_licensing_notes":"Sensitive evidence; aggregate/anonymize and require review.","adapter_status":"reference_only","expected_artifact_schema":"Dated event/aggregate observations with authority, category, outcome and source document","blockers":["No stable public facility-level route verified."]}, {"source_id":"sk.environment.permits","jurisdiction_scope":"Slovakia; environmental permit evidence","legacy_paths":[],"url":"https://www.minzp.sk/en/","access_method":"official environmental routes; not verified","cadence":"unknown","attribution_licensing_notes":"Verify license, geometry and sensitive-site handling.","adapter_status":"reference_only","expected_artifact_schema":"Permit/document records with authority, IDs, activities, dates and optional geometry","blockers":["No stable national animal-facility permit export verified."]}, - {"source_id":"sk.statistics-corporate","jurisdiction_scope":"Slovakia; official slaughter/livestock statistics and business identifiers","legacy_paths":[],"url":"https://slovak.statistics.sk/","access_method":"official statistical/business-register routes; exact APIs unverified","cadence":"table/provider-specific","attribution_licensing_notes":"Identity-only corporate linkage; aggregate statistics preserve revisions and source terms.","adapter_status":"reference_only","expected_artifact_schema":"Aggregate statistical tables and legal-entity records keyed by stable national identifier","blockers":["Verify current table/API contracts, business-register access, terms and privacy."]} + {"source_id":"sk.statistics-corporate","jurisdiction_scope":"Slovakia; official slaughter/livestock statistics and business identifiers","legacy_paths":[],"url":"https://slovak.statistics.sk/","access_method":"official statistical/business-register routes; exact APIs unverified","cadence":"table/provider-specific","attribution_licensing_notes":"Identity-only corporate linkage; aggregate statistics preserve revisions and source terms.","adapter_status":"reference_only","expected_artifact_schema":"Aggregate statistical tables and legal-entity records keyed by stable national identifier","blockers":["Verify current table/API contracts, business-register access, terms and privacy."]} ,{"source_id":"si.uvhvvr.approved-food-feed","jurisdiction_scope":"Slovenia; UVHVVR approved food and feed establishments","legacy_paths":[],"url":"https://www.gov.si/zbirke/storitve/odobritev-zivilskega-obrata/","access_method":"official PDF/list registers","cadence":"publisher-defined; timestamp retrieval","attribution_licensing_notes":"Official Slovenian authority; verify terms, attribution and privacy.","adapter_status":"reference_only","expected_artifact_schema":"Approval number, operator/site, activity, status/date and address","blockers":["Verify current files/API, completeness, terms and coordinates/privacy."]}, + {"source_id":"si.uvhvvr.farms-aquaculture","jurisdiction_scope":"Slovenia; UVHVVR/OPSI animal holding and aquaculture data","legacy_paths":[],"url":"https://podatki.gov.si/","access_method":"open-data catalogue and register systems","cadence":"dataset-specific","attribution_licensing_notes":"Animal/property location data require strict privacy review.","adapter_status":"reference_only","expected_artifact_schema":"Holding/aquaculture records with stable IDs, activity/status and optional location","blockers":["Verify public scope, export/API, terms and sensitive fields."]}, + {"source_id":"si.environment.permits","jurisdiction_scope":"Slovenia; environmental permit evidence","legacy_paths":[],"url":"https://www.gov.si/","access_method":"official environmental routes; not verified","cadence":"unknown","attribution_licensing_notes":"Verify license, geometry and sensitive-site handling.","adapter_status":"reference_only","expected_artifact_schema":"Permit/document observations with authority, IDs, activities, dates and optional geometry","blockers":["No stable national animal-facility permit export verified."]}, + {"source_id":"si.statistics-slaughter","jurisdiction_scope":"Slovenia; official livestock slaughter statistics","legacy_paths":[],"url":"https://pxweb.stat.si/SiStatData/pxweb/en/Data/-/H202S.px","access_method":"official PxWeb API/table","cadence":"annual/table-specific","attribution_licensing_notes":"Official statistics; preserve table ID, revisions and license metadata.","adapter_status":"reference_only","expected_artifact_schema":"PxWeb aggregate slaughter by measure, species and year","blockers":["Pin API metadata and preserve aggregate scope."]}, + {"source_id":"si.corporate-register","jurisdiction_scope":"Slovenia; corporate identifiers","legacy_paths":[],"url":"https://www.ajpes.si/","access_method":"official business-register route; API unverified","cadence":"provider-defined","attribution_licensing_notes":"Identity-only linkage; suppress personal, sole-trader and residential details.","adapter_status":"reference_only","expected_artifact_schema":"Legal-entity identifier keyed organization/status/activity/address","blockers":["Verify current access, terms, quotas and privacy policy."]} ] } @@ -775,3 +779,4 @@ + diff --git a/pipeline/tests/test_slovenia_recon_metadata.py b/pipeline/tests/test_slovenia_recon_metadata.py new file mode 100644 index 0000000..a17564a --- /dev/null +++ b/pipeline/tests/test_slovenia_recon_metadata.py @@ -0,0 +1,11 @@ +import json,unittest +from pathlib import Path +ROOT=Path(__file__).resolve().parents[2] +IDS={"si.uvhvvr.approved-food-feed","si.uvhvvr.farms-aquaculture","si.environment.permits","si.statistics-slaughter","si.corporate-register"} +class SloveniaReconMetadataTests(unittest.TestCase): + def test_row_free_sources(self): + r=json.loads((ROOT/"pipeline/source_registry.json").read_text()); self.assertTrue(IDS.issubset({x["source_id"] for x in r["sources"]})); d=(ROOT/"docs/country-recon-si.md").read_text(); self.assertIn("row-free",d); self.assertIn("fully automated",d) + def test_blocked_status(self): + s=json.loads((ROOT/"docs/source-status.json").read_text()); by={x["source_id"]:x for x in s["sources"]} + for i in IDS: self.assertEqual(by[i]["publication_eligibility"],"blocked") +if __name__=="__main__": unittest.main() diff --git a/pipeline/tests/test_source_registry.py b/pipeline/tests/test_source_registry.py index 56ad035..e2de08b 100644 --- a/pipeline/tests/test_source_registry.py +++ b/pipeline/tests/test_source_registry.py @@ -9,8 +9,8 @@ class SourceRegistryTests(unittest.TestCase): def test_repository_registry_loads_and_references_existing_legacy_paths(self): registry = load_registry() - self.assertEqual(len(registry["sources"]), 132) - self.assertEqual(len({source["source_id"] for source in registry["sources"]}), 132) + self.assertEqual(len(registry["sources"]), 137) + self.assertEqual(len({source["source_id"] for source in registry["sources"]}), 137) def test_unknowns_are_explicit(self): registry = json.loads(REGISTRY_PATH.read_text(encoding="utf-8")) From 284974102a7bd5893a07281f3533651405124a74 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Wed, 16 Sep 2026 09:59:15 -0700 Subject: [PATCH 152/311] docs: add Croatia source reconnaissance --- docs/country-recon-hr.md | 42 +++++++++++++++++++ docs/source-status.json | 8 +++- pipeline/source_registry.json | 9 +++- pipeline/tests/test_croatia_recon_metadata.py | 26 ++++++++++++ pipeline/tests/test_source_registry.py | 4 +- 5 files changed, 85 insertions(+), 4 deletions(-) create mode 100644 docs/country-recon-hr.md create mode 100644 pipeline/tests/test_croatia_recon_metadata.py diff --git a/docs/country-recon-hr.md b/docs/country-recon-hr.md new file mode 100644 index 0000000..a4e2c33 --- /dev/null +++ b/docs/country-recon-hr.md @@ -0,0 +1,42 @@ +# Croatia source reconnaissance + +Status: row-free reconnaissance only; no facility rows, raw exports, or production ingestion artifacts are committed. + +## Scope and automation requirement + +Croatia has a useful official open-data surface for animal-origin establishments, aquaculture permits, environmental permits, business identity, and aggregate statistics. The production pipeline must remain fully automated from source discovery/scraping through acquisition, validation, normalization, provenance, quarantine, and ingestion; this reconnaissance does not authorize publication or replace maintainer review. + +## Strongest candidates + +| Source | Verified surface | Automation assessment | Main unresolved items | +| --- | --- | --- | --- | +| `hr.approved-food` | National CKAN catalogue exposes an approved animal-origin food-establishment register as a public WEB resource, and related registered-establishment data as XLS. | Medium: catalogue discovery can be automated; WEB/export contract needs probing. | Stable direct URL, schema, update cadence, approval IDs, address/coordinate semantics, license and privacy. | +| `hr.aquaculture-permits` | CKAN exposes the Ministry aquaculture permit register as XLS and states the ministry maintains and publishes it under the Aquaculture Act. | Low–medium: scheduled XLS retrieval and hash/version tracking are plausible. | Direct resource URL, schema, cadence, identifiers, coordinates and terms. | +| `hr.environment-permits` | National CKAN lists environmental permit/decision registers, including integrated environmental conditions. | Medium–high: CKAN metadata/resources are machine-discoverable, but documents may require extraction. | Current publisher, complete animal-facility coverage, document/API route, geometry, licensing and privacy. | +| `hr.business-register` | Croatian Court Register provides a public API surface with MBS, OIB, status, company, registered office/address and legal form; registration is required. | Medium: REST XML/JSON ingestion is automatable after account credentials are provisioned. | Credentials/quotas, terms, legal-person-only matching and address-use policy. | +| `hr.statistics` | Croatian Bureau of Statistics publishes official statistical registers and data services; CKAN includes business-register time series. | Low–medium for aggregates: table/API identifiers and revision metadata must be pinned. | Slaughter and animal-use table IDs, API contracts, cadence, licensing and aggregate-only handling. | +| `hr.inspections-experiments` | State Inspectorate and Ministry/official statistical surfaces identify control and animal-experimentation responsibilities. | High for reports/aggregates; low for facility-level extraction until a public register is confirmed. | Public facility/event route, stable IDs, publication scope, privacy and retention constraints. | + +## Compliance and ingestion notes + +- Treat catalogue metadata as government-sourced evidence, not project approval or proof of current operation. +- Preserve source URL, retrieval timestamp, content hash, byte size, supplied publication/effective date, adapter/configuration version, and source values. +- Keep raw, parsed, normalized, enriched, reviewed, and released layers separate. Use synthetic fixtures only; do not commit rows or raw artifacts. +- Represent unavailable cadence, coordinates, IDs, licensing, and privacy decisions explicitly. A missing source snapshot means “not observed,” not closure. +- Facility addresses and coordinates require privacy/safety screening; corporate registered offices are not automatically operating sites. +- Candidate adapters should fail closed on changed schemas, missing IDs, suspicious count changes, and inaccessible resources, retaining the previous validated release. + +## Official evidence + +- [Croatian CKAN agriculture organization](https://data.gov.hr/ckan/en/organization/ministarstvo-poljoprivrede?_tags_limit=0&publisher_type=public_sector&tags=poljopriveda) +- [Approved animal-origin food establishments dataset](https://data.gov.hr/ckan/en/dataset/upisnik-odobrenih-objekata-u-poslovanju-s-hranom-za-zivotinje) +- [Aquaculture permits dataset](https://data.gov.hr/ckan/hr/dataset/registar-dozvola-u-akvakulturi) +- [Sanitary food-register guidance](https://inspektorat.gov.hr/ustrojstvo-77/7-sektor-sanitarne-inspekcije/evidentiranje-i-vodjenje-registra-subjekta-i-pripadajucih-objekta-u-poslovanju-s-hranom-iz-nadleznosti-iz-nadleznosti-sanitarne-inspekcije/431) +- [Court Register public API](https://sudreg-data.gov.hr/ords/r/srn_rep/vanjski-srn-rep/home) +- [Croatian Bureau of Statistics publishing programme](https://dzs.gov.hr/usluge/objavljivanje/program-publiciranja-2026/2439) + +## Effort and blockers + +Initial source reconnaissance: approximately 2–4 engineering days to pin the approved-food and aquaculture routes, build deterministic acquisition/validation adapters, and produce sanitized fixtures; 3–7 additional days for environmental-document extraction and inspection/animal-experimentation coverage. The main blockers are unstable or undocumented resource URLs, undefined refresh cadence, possible registration/authentication for the Court Register API, and unresolved privacy/licensing/coordinate semantics. + +Recommended next country: Romania, subject to checking the existing country inventory before delegation. diff --git a/docs/source-status.json b/docs/source-status.json index f40b995..af7c0cf 100644 --- a/docs/source-status.json +++ b/docs/source-status.json @@ -145,6 +145,12 @@ {"source_id":"si.uvhvvr.farms-aquaculture","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-si.md","pipeline/source_registry.json"],"next_action":"Verify public scope, export/API, terms and sensitive fields."}, {"source_id":"si.environment.permits","metadata":"unknown","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-si.md","pipeline/source_registry.json"],"next_action":"Locate and verify environmental permit data."}, {"source_id":"si.statistics-slaughter","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-si.md","pipeline/source_registry.json"],"next_action":"Pin PxWeb table/API metadata and revisions."}, - {"source_id":"si.corporate-register","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-si.md","pipeline/source_registry.json"],"next_action":"Verify current access, terms and identity-only privacy policy."} + {"source_id":"si.corporate-register","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-si.md","pipeline/source_registry.json"],"next_action":"Verify current access, terms and identity-only privacy policy."}, + {"source_id":"hr.approved-food","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-hr.md","pipeline/source_registry.json"],"next_action":"Pin direct resource route, schema, cadence, identifiers, terms and privacy."}, + {"source_id":"hr.aquaculture-permits","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-hr.md","pipeline/source_registry.json"],"next_action":"Pin direct XLS URL, schema, cadence, IDs, coordinates and terms."}, + {"source_id":"hr.environment-permits","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-hr.md","pipeline/source_registry.json"],"next_action":"Verify publisher, resource route, coverage, geometry, licensing and privacy."}, + {"source_id":"hr.business-register","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-hr.md","pipeline/source_registry.json"],"next_action":"Provision and verify API credentials, quotas, contract and identity-only policy."}, + {"source_id":"hr.statistics","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-hr.md","pipeline/source_registry.json"],"next_action":"Pin official slaughter and animal-use table IDs, APIs and revisions."}, + {"source_id":"hr.inspections-experiments","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-hr.md","pipeline/source_registry.json"],"next_action":"Locate a stable public facility/event route; keep sensitive evidence aggregate until verified."} ] } diff --git a/pipeline/source_registry.json b/pipeline/source_registry.json index 91633d6..fbda0f1 100644 --- a/pipeline/source_registry.json +++ b/pipeline/source_registry.json @@ -770,7 +770,12 @@ {"source_id":"si.environment.permits","jurisdiction_scope":"Slovenia; environmental permit evidence","legacy_paths":[],"url":"https://www.gov.si/","access_method":"official environmental routes; not verified","cadence":"unknown","attribution_licensing_notes":"Verify license, geometry and sensitive-site handling.","adapter_status":"reference_only","expected_artifact_schema":"Permit/document observations with authority, IDs, activities, dates and optional geometry","blockers":["No stable national animal-facility permit export verified."]}, {"source_id":"si.statistics-slaughter","jurisdiction_scope":"Slovenia; official livestock slaughter statistics","legacy_paths":[],"url":"https://pxweb.stat.si/SiStatData/pxweb/en/Data/-/H202S.px","access_method":"official PxWeb API/table","cadence":"annual/table-specific","attribution_licensing_notes":"Official statistics; preserve table ID, revisions and license metadata.","adapter_status":"reference_only","expected_artifact_schema":"PxWeb aggregate slaughter by measure, species and year","blockers":["Pin API metadata and preserve aggregate scope."]}, {"source_id":"si.corporate-register","jurisdiction_scope":"Slovenia; corporate identifiers","legacy_paths":[],"url":"https://www.ajpes.si/","access_method":"official business-register route; API unverified","cadence":"provider-defined","attribution_licensing_notes":"Identity-only linkage; suppress personal, sole-trader and residential details.","adapter_status":"reference_only","expected_artifact_schema":"Legal-entity identifier keyed organization/status/activity/address","blockers":["Verify current access, terms, quotas and privacy policy."]} - ] + ,{"source_id":"hr.approved-food","jurisdiction_scope":"Croatia; approved and registered food establishments handling food of animal origin","legacy_paths":[],"url":"https://data.gov.hr/ckan/en/dataset/upisnik-odobrenih-objekata-u-poslovanju-s-hranom-za-zivotinje","access_method":"national CKAN WEB/XLS resources and official register guidance","cadence":"not defined; timestamp retrieval","attribution_licensing_notes":"CKAN lists public/open access; verify exact resource license, privacy and attribution before acquisition.","adapter_status":"reference_only","expected_artifact_schema":"Approval or registration ID, operator/site, activities, status, address and optional coordinates","blockers":["Pin direct resource route, schema, cadence, identifiers, coordinates, terms and privacy before acquisition."]}, + {"source_id":"hr.aquaculture-permits","jurisdiction_scope":"Croatia; Ministry aquaculture permit register","legacy_paths":[],"url":"https://data.gov.hr/ckan/hr/dataset/registar-dozvola-u-akvakulturi","access_method":"CKAN XLS resource","cadence":"not defined; timestamp retrieval","attribution_licensing_notes":"CKAN marks the dataset open; verify current license, sensitive-site handling and attribution.","adapter_status":"reference_only","expected_artifact_schema":"Permit ID, holder, activity/species, water/site, status, dates and optional coordinates","blockers":["Pin direct XLS URL, schema, update cadence, stable IDs, coordinates and terms."]}, + {"source_id":"hr.environment-permits","jurisdiction_scope":"Croatia; environmental permits and integrated environmental conditions","legacy_paths":[],"url":"https://data.gov.hr/ckan/en/dataset/o-evidnik-uporabnih-dozvola-i-rje-enja-o-objedinjenim-uvjetima-za-tite-okoli-a","access_method":"national CKAN resources and environmental registers","cadence":"dataset-specific","attribution_licensing_notes":"Verify current publisher, resource license, document rights, geometry and privacy.","adapter_status":"reference_only","expected_artifact_schema":"Permit/decision ID, authority, subject, activity, dates, status and optional reviewed geometry","blockers":["Complete national animal-facility coverage and machine-readable route are unresolved."]}, + {"source_id":"hr.business-register","jurisdiction_scope":"Croatia; Court Register corporate identity and registered office","legacy_paths":[],"url":"https://sudreg-data.gov.hr/ords/r/srn_rep/vanjski-srn-rep/home","access_method":"public REST API after free registration; XML or JSON","cadence":"daily on working days","attribution_licensing_notes":"Identity linkage only; verify API terms, quotas and suppression of personal/sole-trader/residential details.","adapter_status":"reference_only","expected_artifact_schema":"MBS/OIB keyed legal entity, name, status, legal form and registered office/address","blockers":["Credentials, quota, exact API contract and legal-person-only matching policy require verification."]}, + {"source_id":"hr.statistics","jurisdiction_scope":"Croatia; official slaughter, livestock and animal-use aggregate statistics","legacy_paths":[],"url":"https://dzs.gov.hr/usluge/objavljivanje/program-publiciranja-2026/2439","access_method":"Croatian Bureau of Statistics tables/data services","cadence":"table-specific; preserve revisions","attribution_licensing_notes":"Official statistics; verify table terms, citation requirements and aggregate-only scope.","adapter_status":"reference_only","expected_artifact_schema":"Aggregate multidimensional observations by species, measure, geography and period","blockers":["Pin slaughter and animal-use table IDs, API/download contracts and release cadence."]}, + {"source_id":"hr.inspections-experiments","jurisdiction_scope":"Croatia; food/veterinary inspections, enforcement and animal-experimentation evidence","legacy_paths":[],"url":"https://inspektorat.gov.hr/ustrojstvo-77/7-sektor-sanitarne-inspekcije/evidentiranje-i-vodjenje-registra-subjekta-i-pripadajucih-objekta-u-poslovanju-s-hranom-iz-nadleznosti-iz-nadleznosti-sanitarne-inspekcije/431","access_method":"official register guidance, reports and statistical publications","cadence":"report/register-specific","attribution_licensing_notes":"Sensitive control/research evidence; aggregate or anonymize and require project review.","adapter_status":"reference_only","expected_artifact_schema":"Dated aggregate/event observations with authority, category, outcome, measure and source document","blockers":["No stable public facility-level inspection or animal-experimentation master verified."]} ] } @@ -780,3 +785,5 @@ + + diff --git a/pipeline/tests/test_croatia_recon_metadata.py b/pipeline/tests/test_croatia_recon_metadata.py new file mode 100644 index 0000000..e1393e0 --- /dev/null +++ b/pipeline/tests/test_croatia_recon_metadata.py @@ -0,0 +1,26 @@ +import json +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] + +class CroatiaReconMetadataTests(unittest.TestCase): + def test_row_free_croatia_recon_is_present(self): + text = (ROOT / "docs" / "country-recon-hr.md").read_text(encoding="utf-8") + self.assertIn("row-free", text) + self.assertIn("fully automated", text) + self.assertIn("Recommended next country: Romania", text) + + def test_croatia_sources_have_conservative_status(self): + registry = json.loads((ROOT / "pipeline" / "source_registry.json").read_text(encoding="utf-8")) + status = json.loads((ROOT / "docs" / "source-status.json").read_text(encoding="utf-8")) + ids = {source["source_id"] for source in registry["sources"] if source["source_id"].startswith("hr.")} + self.assertEqual(ids, {source["source_id"] for source in status["sources"] if source["source_id"].startswith("hr.")}) + self.assertEqual(len(ids), 6) + for source in status["sources"]: + if source["source_id"].startswith("hr."): + self.assertEqual(source["publication_eligibility"], "blocked") + self.assertNotEqual(source["runtime_health"], "healthy") + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/tests/test_source_registry.py b/pipeline/tests/test_source_registry.py index e2de08b..9df549a 100644 --- a/pipeline/tests/test_source_registry.py +++ b/pipeline/tests/test_source_registry.py @@ -9,8 +9,8 @@ class SourceRegistryTests(unittest.TestCase): def test_repository_registry_loads_and_references_existing_legacy_paths(self): registry = load_registry() - self.assertEqual(len(registry["sources"]), 137) - self.assertEqual(len({source["source_id"] for source in registry["sources"]}), 137) + self.assertEqual(len(registry["sources"]), 143) + self.assertEqual(len({source["source_id"] for source in registry["sources"]}), 143) def test_unknowns_are_explicit(self): registry = json.loads(REGISTRY_PATH.read_text(encoding="utf-8")) From f486870eed457949ffb53473b29e755b9b1a84aa Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Wed, 16 Sep 2026 10:01:46 -0700 Subject: [PATCH 153/311] docs: add Romania source reconnaissance --- docs/country-recon-ro.md | 42 +++++++++++++++++++ docs/source-status.json | 8 +++- pipeline/source_registry.json | 9 +++- pipeline/tests/test_romania_recon_metadata.py | 26 ++++++++++++ pipeline/tests/test_source_registry.py | 4 +- 5 files changed, 85 insertions(+), 4 deletions(-) create mode 100644 docs/country-recon-ro.md create mode 100644 pipeline/tests/test_romania_recon_metadata.py diff --git a/docs/country-recon-ro.md b/docs/country-recon-ro.md new file mode 100644 index 0000000..4f26c5c --- /dev/null +++ b/docs/country-recon-ro.md @@ -0,0 +1,42 @@ +# Romania source reconnaissance + +Status: row-free reconnaissance only; no facility rows, raw exports, or production ingestion artifacts are committed. + +## Scope and automation requirement + +Romania exposes relevant official surfaces through data.gov.ro, ANSVSA/DSVSA veterinary systems, ANPM environmental systems, ONRC corporate data, and INSSE statistics. The production pipeline must be fully automated from source discovery/scraping through acquisition, validation, normalization, provenance, quarantine, and ingestion. This reconnaissance does not authorize publication or replace maintainer review. + +## Strongest candidates + +| Source | Verified surface | Automation assessment | Main unresolved items | +| --- | --- | --- | --- | +| `ro.ansvsa.approved-food` | ANSVSA is the competent veterinary authority; national open-data/search surfaces and EU approval frameworks are relevant to approved animal-origin establishments and slaughterhouses. | Medium–high if a stable list/export is confirmed; otherwise portal/browser automation may be required. | Current national list/API, approval IDs, activity/status semantics, addresses/coordinates, cadence, license and privacy. | +| `ro.farm-aquaculture` | Romanian agriculture/open-data catalogues expose agriculture records; ANPM IBIS includes official “Crescătorii”/authorizations surfaces, while aquaculture permit coverage needs authority confirmation. | Medium for catalogued files; low–medium for authenticated or legacy portals. | National farm/holding register scope, aquaculture permit export, stable IDs, coordinates, terms and privacy. | +| `ro.environment-permits` | ANPM’s SIM/eFORM is an official environmental authorization system with operator, work-point, coordinates and authorization fields; ANPM currently reports the integrated system as technically nonfunctional. | Medium once public read access is restored; currently blocked for dependable automated acquisition. | Public read/export API, completeness, document route, current availability, license and geometry semantics. | +| `ro.onrc.organizations` | data.gov.ro publishes ONRC company snapshots as CSV, including registered-office, status and authorized-activity information; the catalogue exposes CKAN API metadata. | High for scheduled CSV snapshots and hash/version tracking. | Current snapshot cadence, field dictionary, OIB/CUI linkage, registered-office privacy and license scope. | +| `ro.insse.statistics` | INSSE is the official statistical authority; Romanian official statistics provide aggregate livestock/slaughter and related animal-use context. | Medium–high for table/API downloads after table IDs are pinned. | Exact current table IDs/API, cadence, revisions, licensing and aggregate-only handling. | +| `ro.inspections-experiments` | ANSVSA/DSVSA and ANPM systems support official control/authorization workflows; a public facility-level animal-experimentation master was not verified. | High for published aggregate reports; low for facility-level extraction until a public route is confirmed. | Public event/facility register, stable IDs, inspection outcomes, animal-use categories, privacy and retention. | + +## Compliance and ingestion notes + +- Treat government portals and catalogue records as government-sourced evidence, not proof of current operation or project approval. +- Preserve source URL, retrieval timestamp, content hash, byte size, supplied publication/effective date, adapter/configuration version, and source values. +- Keep raw, parsed, normalized, enriched, reviewed, and released layers separate. Use synthetic fixtures only; do not commit rows or raw artifacts. +- Represent unavailable cadence, IDs, coordinates, licensing, and privacy decisions explicitly. A source outage or disappearance means “not observed,” not closure. +- Registered offices, operator addresses, farms, and work points are not automatically safe public facility locations; apply privacy and safety review before any coordinates or addresses are released. +- Adapters must fail closed on changed schemas, missing identifiers, suspicious count changes, and authentication/service failures, retaining the previous validated release. + +## Official evidence + +- [Romanian national open-data catalogue](https://data.gov.ro/) +- [ONRC company snapshots and CKAN API surface](https://data.gov.ro/dataset?organiza=&organization=onrc&res_format=csv) +- [ANSVSA veterinary portal](https://portal.ansvsa.ro/) +- [ANPM integrated environmental system](https://raportare.anpm.ro/) +- [ANPM system status notice](https://raportare.anpm.ro/irj/servlet/prt/portal/prteventname/Navigate/prtroot/pcd%213aportal_content%212fevery_user%212fgeneral%212fdefaultAjaxframeworkContent) +- [INSSE official statistics portal](https://insse.ro/cms/) + +## Effort and blockers + +Initial reconnaissance: approximately 2–4 engineering days to pin ONRC snapshots and any approved-establishment/aquaculture exports, then build deterministic acquisition and validation adapters; 4–8 additional days for ANPM document/API recovery and inspection/animal-experimentation coverage. Main blockers are the absence of a clearly documented national ANSVSA bulk/API contract, authenticated or legacy environmental systems, ANPM’s reported outage, unresolved update cadence, and privacy/licensing/coordinate semantics. + +Recommended next country: Bulgaria, subject to checking the existing country inventory before delegation. diff --git a/docs/source-status.json b/docs/source-status.json index af7c0cf..ac13a1d 100644 --- a/docs/source-status.json +++ b/docs/source-status.json @@ -151,6 +151,12 @@ {"source_id":"hr.environment-permits","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-hr.md","pipeline/source_registry.json"],"next_action":"Verify publisher, resource route, coverage, geometry, licensing and privacy."}, {"source_id":"hr.business-register","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-hr.md","pipeline/source_registry.json"],"next_action":"Provision and verify API credentials, quotas, contract and identity-only policy."}, {"source_id":"hr.statistics","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-hr.md","pipeline/source_registry.json"],"next_action":"Pin official slaughter and animal-use table IDs, APIs and revisions."}, - {"source_id":"hr.inspections-experiments","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-hr.md","pipeline/source_registry.json"],"next_action":"Locate a stable public facility/event route; keep sensitive evidence aggregate until verified."} + {"source_id":"hr.inspections-experiments","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-hr.md","pipeline/source_registry.json"],"next_action":"Locate a stable public facility/event route; keep sensitive evidence aggregate until verified."}, + {"source_id":"ro.ansvsa.approved-food","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ro.md","pipeline/source_registry.json"],"next_action":"Pin national list/API/export, identifiers, cadence, coordinates, terms and privacy."}, + {"source_id":"ro.farm-aquaculture","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ro.md","pipeline/source_registry.json"],"next_action":"Verify national farm/aquaculture scope, export/API, IDs, coordinates, terms and privacy."}, + {"source_id":"ro.environment-permits","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ro.md","pipeline/source_registry.json"],"next_action":"Recheck ANPM availability and verify public read/API, document route, license, geometry and privacy."}, + {"source_id":"ro.onrc.organizations","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ro.md","pipeline/source_registry.json"],"next_action":"Pin current CSV snapshot, cadence, fields, license and registered-office privacy policy."}, + {"source_id":"ro.insse.statistics","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ro.md","pipeline/source_registry.json"],"next_action":"Pin official slaughter/animal-use table IDs, APIs, cadence and revisions."}, + {"source_id":"ro.inspections-experiments","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ro.md","pipeline/source_registry.json"],"next_action":"Locate a stable public facility/event route; keep sensitive evidence aggregate until verified."} ] } diff --git a/pipeline/source_registry.json b/pipeline/source_registry.json index fbda0f1..3bc66cf 100644 --- a/pipeline/source_registry.json +++ b/pipeline/source_registry.json @@ -775,7 +775,12 @@ {"source_id":"hr.environment-permits","jurisdiction_scope":"Croatia; environmental permits and integrated environmental conditions","legacy_paths":[],"url":"https://data.gov.hr/ckan/en/dataset/o-evidnik-uporabnih-dozvola-i-rje-enja-o-objedinjenim-uvjetima-za-tite-okoli-a","access_method":"national CKAN resources and environmental registers","cadence":"dataset-specific","attribution_licensing_notes":"Verify current publisher, resource license, document rights, geometry and privacy.","adapter_status":"reference_only","expected_artifact_schema":"Permit/decision ID, authority, subject, activity, dates, status and optional reviewed geometry","blockers":["Complete national animal-facility coverage and machine-readable route are unresolved."]}, {"source_id":"hr.business-register","jurisdiction_scope":"Croatia; Court Register corporate identity and registered office","legacy_paths":[],"url":"https://sudreg-data.gov.hr/ords/r/srn_rep/vanjski-srn-rep/home","access_method":"public REST API after free registration; XML or JSON","cadence":"daily on working days","attribution_licensing_notes":"Identity linkage only; verify API terms, quotas and suppression of personal/sole-trader/residential details.","adapter_status":"reference_only","expected_artifact_schema":"MBS/OIB keyed legal entity, name, status, legal form and registered office/address","blockers":["Credentials, quota, exact API contract and legal-person-only matching policy require verification."]}, {"source_id":"hr.statistics","jurisdiction_scope":"Croatia; official slaughter, livestock and animal-use aggregate statistics","legacy_paths":[],"url":"https://dzs.gov.hr/usluge/objavljivanje/program-publiciranja-2026/2439","access_method":"Croatian Bureau of Statistics tables/data services","cadence":"table-specific; preserve revisions","attribution_licensing_notes":"Official statistics; verify table terms, citation requirements and aggregate-only scope.","adapter_status":"reference_only","expected_artifact_schema":"Aggregate multidimensional observations by species, measure, geography and period","blockers":["Pin slaughter and animal-use table IDs, API/download contracts and release cadence."]}, - {"source_id":"hr.inspections-experiments","jurisdiction_scope":"Croatia; food/veterinary inspections, enforcement and animal-experimentation evidence","legacy_paths":[],"url":"https://inspektorat.gov.hr/ustrojstvo-77/7-sektor-sanitarne-inspekcije/evidentiranje-i-vodjenje-registra-subjekta-i-pripadajucih-objekta-u-poslovanju-s-hranom-iz-nadleznosti-iz-nadleznosti-sanitarne-inspekcije/431","access_method":"official register guidance, reports and statistical publications","cadence":"report/register-specific","attribution_licensing_notes":"Sensitive control/research evidence; aggregate or anonymize and require project review.","adapter_status":"reference_only","expected_artifact_schema":"Dated aggregate/event observations with authority, category, outcome, measure and source document","blockers":["No stable public facility-level inspection or animal-experimentation master verified."]} ] + {"source_id":"hr.inspections-experiments","jurisdiction_scope":"Croatia; food/veterinary inspections, enforcement and animal-experimentation evidence","legacy_paths":[],"url":"https://inspektorat.gov.hr/ustrojstvo-77/7-sektor-sanitarne-inspekcije/evidentiranje-i-vodjenje-registra-subjekta-i-pripadajucih-objekta-u-poslovanju-s-hranom-iz-nadleznosti-iz-nadleznosti-sanitarne-inspekcije/431","access_method":"official register guidance, reports and statistical publications","cadence":"report/register-specific","attribution_licensing_notes":"Sensitive control/research evidence; aggregate or anonymize and require project review.","adapter_status":"reference_only","expected_artifact_schema":"Dated aggregate/event observations with authority, category, outcome, measure and source document","blockers":["No stable public facility-level inspection or animal-experimentation master verified."]} ,{"source_id":"ro.ansvsa.approved-food","jurisdiction_scope":"Romania; ANSVSA approved and registered food establishments handling food of animal origin","legacy_paths":[],"url":"https://portal.ansvsa.ro/","access_method":"official ANSVSA/DSVSA registers and relevant EU approval surfaces; national bulk route unverified","cadence":"unknown; timestamp retrieval","attribution_licensing_notes":"Official authority; verify national list license, privacy, attribution and source terms before acquisition.","adapter_status":"reference_only","expected_artifact_schema":"Approval/registration ID, operator/site, activities, status, address and optional coordinates","blockers":["Pin current national list/API/export, stable identifiers, cadence, coordinates, terms and privacy."]}, + {"source_id":"ro.farm-aquaculture","jurisdiction_scope":"Romania; farm, holding and aquaculture establishment evidence","legacy_paths":[],"url":"https://data.gov.ro/","access_method":"national open-data catalogue, agriculture records and official environmental/biodiversity systems","cadence":"resource-specific","attribution_licensing_notes":"Verify publisher, license, animal-holder/property privacy and coordinate precision.","adapter_status":"reference_only","expected_artifact_schema":"Holding/site or aquaculture permit records with stable IDs, activity/status, address and optional coordinates","blockers":["No complete national farm/aquaculture public export contract verified."]}, + {"source_id":"ro.environment-permits","jurisdiction_scope":"Romania; ANPM environmental authorizations and integrated environmental system","legacy_paths":[],"url":"https://raportare.anpm.ro/","access_method":"SIM/eFORM official web system and document/publication routes","cadence":"unknown; system currently reports technical unavailability","attribution_licensing_notes":"Official environmental authority; verify public-read rights, document license, geometry and privacy.","adapter_status":"reference_only","expected_artifact_schema":"Permit/decision ID, operator/work point, authority, coordinates, activity, dates and status","blockers":["Public read/API route and dependable availability are unresolved; ANPM reports SIM technical outage."]}, + {"source_id":"ro.onrc.organizations","jurisdiction_scope":"Romania; ONRC legal-entity and authorized-activity snapshots","legacy_paths":[],"url":"https://data.gov.ro/dataset?organiza=&organization=onrc&res_format=csv","access_method":"data.gov.ro CKAN CSV snapshots and API metadata","cadence":"snapshot-specific; timestamp each release","attribution_licensing_notes":"Catalogue snapshots are public and some are CC BY 4.0; verify current file license, field privacy and attribution.","adapter_status":"reference_only","expected_artifact_schema":"CUI/OIB keyed organization, registered office, status and authorized activity fields","blockers":["Pin current snapshot, cadence, field dictionary and registered-office publication policy."]}, + {"source_id":"ro.insse.statistics","jurisdiction_scope":"Romania; INSSE aggregate slaughter, livestock and animal-use statistics","legacy_paths":[],"url":"https://insse.ro/cms/","access_method":"official statistical tables/data services; exact table/API route unverified","cadence":"table-specific; preserve revisions","attribution_licensing_notes":"Official statistics; verify table terms, citation requirements and aggregate-only scope.","adapter_status":"reference_only","expected_artifact_schema":"Aggregate multidimensional observations by species, measure, geography and period","blockers":["Pin current slaughter and animal-use table IDs, API/download contracts and cadence."]}, + {"source_id":"ro.inspections-experiments","jurisdiction_scope":"Romania; ANSVSA/DSVSA inspections, enforcement and animal-experimentation evidence","legacy_paths":[],"url":"https://portal.ansvsa.ro/","access_method":"official portals, reports and statistical publications","cadence":"report/register-specific","attribution_licensing_notes":"Sensitive control/research evidence; aggregate or anonymize and require project review.","adapter_status":"reference_only","expected_artifact_schema":"Dated aggregate/event observations with authority, category, outcome, measure and source document","blockers":["No stable public facility-level inspection or animal-experimentation master verified."]} ] } @@ -787,3 +792,5 @@ + + diff --git a/pipeline/tests/test_romania_recon_metadata.py b/pipeline/tests/test_romania_recon_metadata.py new file mode 100644 index 0000000..88b60bb --- /dev/null +++ b/pipeline/tests/test_romania_recon_metadata.py @@ -0,0 +1,26 @@ +import json +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] + +class RomaniaReconMetadataTests(unittest.TestCase): + def test_row_free_romania_recon_is_present(self): + text = (ROOT / "docs" / "country-recon-ro.md").read_text(encoding="utf-8") + self.assertIn("row-free", text) + self.assertIn("fully automated", text) + self.assertIn("Recommended next country: Bulgaria", text) + + def test_romania_sources_have_conservative_status(self): + registry=json.loads((ROOT/"pipeline"/"source_registry.json").read_text(encoding="utf-8")) + status=json.loads((ROOT/"docs"/"source-status.json").read_text(encoding="utf-8")) + ids={s["source_id"] for s in registry["sources"] if s["source_id"].startswith("ro.")} + self.assertEqual(ids,{s["source_id"] for s in status["sources"] if s["source_id"].startswith("ro.")}) + self.assertEqual(len(ids),6) + for s in status["sources"]: + if s["source_id"].startswith("ro."): + self.assertEqual(s["publication_eligibility"],"blocked") + self.assertNotEqual(s["runtime_health"],"healthy") + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/tests/test_source_registry.py b/pipeline/tests/test_source_registry.py index 9df549a..e7b5519 100644 --- a/pipeline/tests/test_source_registry.py +++ b/pipeline/tests/test_source_registry.py @@ -9,8 +9,8 @@ class SourceRegistryTests(unittest.TestCase): def test_repository_registry_loads_and_references_existing_legacy_paths(self): registry = load_registry() - self.assertEqual(len(registry["sources"]), 143) - self.assertEqual(len({source["source_id"] for source in registry["sources"]}), 143) + self.assertEqual(len(registry["sources"]), 149) + self.assertEqual(len({source["source_id"] for source in registry["sources"]}), 149) def test_unknowns_are_explicit(self): registry = json.loads(REGISTRY_PATH.read_text(encoding="utf-8")) From c8e29724da97e3c164f0013e2fff543209b5f127 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Wed, 16 Sep 2026 10:03:13 -0700 Subject: [PATCH 154/311] Add operations scale and E2E hardening --- .github/workflows/tests.yml | 10 +- docs/performance/v2-discovery-100k.md | 38 ++- .../scripts/benchmarks/run_discovery_scale.py | 235 ++++++++++++++++++ pipeline/tests/e2e/README.md | 12 +- pipeline/tests/e2e/run-suite.ps1 | 36 +++ .../tests/test_discovery_scale_benchmark.py | 60 +++++ 6 files changed, 372 insertions(+), 19 deletions(-) create mode 100644 pipeline/scripts/benchmarks/run_discovery_scale.py create mode 100644 pipeline/tests/e2e/run-suite.ps1 create mode 100644 pipeline/tests/test_discovery_scale_benchmark.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 2977a62..6049bba 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -28,14 +28,8 @@ jobs: python-version: '3.11' - run: pip install psycopg[binary] - name: Run Docker-backed API E2E tests - env: - UEC_RUN_E2E: '1' - run: | - python -m unittest pipeline.tests.e2e.test_public_api -v - python -m unittest pipeline.tests.e2e.test_community_api -v - python -m unittest pipeline.tests.e2e.test_seeded_api -v - python -m unittest pipeline.tests.e2e.test_public_surface_safety -v - python -m unittest pipeline.tests.e2e.test_candidate_import -v + shell: pwsh + run: ./pipeline/tests/e2e/run-suite.ps1 backup-restore: # The PostGIS image is Linux-only; Ubuntu includes PowerShell Core for the drill. diff --git a/docs/performance/v2-discovery-100k.md b/docs/performance/v2-discovery-100k.md index ff4e9e2..94a97f8 100644 --- a/docs/performance/v2-discovery-100k.md +++ b/docs/performance/v2-discovery-100k.md @@ -14,13 +14,28 @@ retained source records, names, addresses, coordinates, or geocoder responses. ## Reproduction -Run `pipeline/tests/benchmarks/discovery_100k.sql` against a disposable PostGIS -database after migrations through `025_discovery_query_indexes.sql` have been -applied. The script creates only `bench_discovery_100k`, fills deterministic -synthetic rows, runs `ANALYZE`, and emits `EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON)` -for the list, cursor, text, bbox, radius, and detail shapes used by the API. -Drop the benchmark table after capture. The benchmark is evidence about query -shape and budget only; it is not publication or release evidence. +The reproducible scale runner requires only PostGIS and the pinned Python +dependencies; it does not require application migrations because all benchmark +tables are temporary and synthetic: + +```powershell +$env:UEC_DATABASE_URL = "postgresql://uec:uec-local-development-only@localhost:55434/uec?sslmode=disable" +python pipeline/scripts/benchmarks/run_discovery_scale.py --json-output .tmp/discovery-scale.json +``` + +It creates deterministic 100k and 1m observation sets and runs aggregate +`EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON)` checks for list, cursor pagination, +category filters, text filters, bbox, radius, detail, and release/privacy/ +suppression-aware graph joins. The plan checker requires the expected index +family for each shape and bounds every result page at 50 rows. The JSON report +contains timings, aggregate buffer counts, plan node types, and index names; +it never contains query rows or source/private values. Closing the connection +automatically drops every temporary table. The benchmark is evidence about +query shape and budget only; it is not publication, release, or production +load evidence. + +The original `pipeline/tests/benchmarks/discovery_100k.sql` remains available +as a minimal SQL-only query-shape sample. ## Local disposable capture (2026-09-15) @@ -30,3 +45,12 @@ search 3.206 ms, bbox 1.578 ms, radius 0.057 ms, and detail lookup 5.848 ms. The list/cursor/search/spatial plans used the expected B-tree, trigram GIN, or geography GiST indexes. These are cold/warm local database plan samples, not a production load test or an end-to-end p95 claim. + +## Operational interpretation + +Capture reports on the same PostGIS image and representative hardware when +comparing revisions. A plan regression, increasing shared reads, or a query +that fails its expected-index check is a release-review input. The measurements +do not establish capacity, cloud cost, or public-source completeness; those +require a separate load test with an approved traffic model and deployment +configuration. diff --git a/pipeline/scripts/benchmarks/run_discovery_scale.py b/pipeline/scripts/benchmarks/run_discovery_scale.py new file mode 100644 index 0000000..dd0eb73 --- /dev/null +++ b/pipeline/scripts/benchmarks/run_discovery_scale.py @@ -0,0 +1,235 @@ +#!/usr/bin/env python3 +"""Run deterministic, synthetic V2 discovery query-plan and scale checks. + +The benchmark uses only PostgreSQL TEMP tables. Closing the connection drops +all generated data, so this script cannot contaminate the application's ``uec`` +schema. Output is an aggregate report: it never prints query rows or plan +constants. +""" + +from __future__ import annotations + +import argparse +import json +import os +import sys +from pathlib import Path +from typing import Any + +DEFAULT_SCALES = (100_000, 1_000_000) +MAX_SCALE = 2_000_000 + +GRAPH_INDEXES = { + "bench_graph_edge_public_idx", + "bench_graph_claim_public_idx", +} + +QUERY_SPECS = ( + {"name": "list", "sql": """ + SELECT ordinal FROM bench_discovery_scale + WHERE country_code = 'DK' ORDER BY facility_id LIMIT 50 + """, "params": (), "expected_indexes": ("bench_discovery_country_cursor_idx", "bench_discovery_facility_pk")}, + {"name": "pagination", "sql": """ + SELECT ordinal FROM bench_discovery_scale + WHERE country_code = 'DK' + AND facility_id > '00000000-0000-4000-8000-000000050000'::uuid + ORDER BY facility_id LIMIT 50 + """, "params": (), "expected_indexes": ("bench_discovery_country_cursor_idx", "bench_discovery_facility_pk")}, + {"name": "filters", "sql": """ + SELECT ordinal FROM bench_discovery_scale + WHERE country_code = 'DK' AND category = 'slaughter' + ORDER BY facility_id LIMIT 50 + """, "params": (), "expected_indexes": ("bench_discovery_country_category_cursor_idx", "bench_discovery_facility_pk")}, + {"name": "text_filter", "sql": """ + SELECT ordinal FROM bench_discovery_scale + WHERE lower(canonical_name) LIKE 'synthetic facility 999%%' + ORDER BY facility_id LIMIT 50 + """, "params": (), "expected_indexes": ("bench_discovery_name_idx",)}, + {"name": "bbox", "sql": """ + SELECT ordinal FROM bench_discovery_scale + WHERE location && ST_MakeEnvelope(-10, 45, -9.99, 45.01, 4326)::geography + LIMIT 50 + """, "params": (), "expected_indexes": ("bench_discovery_location_idx",)}, + {"name": "radius", "sql": """ + SELECT ordinal FROM bench_discovery_scale + WHERE ST_DWithin(location, ST_SetSRID(ST_Point(-5, 50), 4326)::geography, 50000) + ORDER BY facility_id LIMIT 50 + """, "params": (), "expected_indexes": ("bench_discovery_location_idx",)}, + {"name": "detail", "sql": """ + SELECT ordinal FROM bench_discovery_scale + WHERE facility_id = '00000000-0000-4000-8000-000000050000'::uuid + """, "params": (), "expected_indexes": ("bench_discovery_facility_pk",)}, + {"name": "graph_ready", "sql": """ + SELECT edge.ordinal + FROM bench_graph_edges_scale edge + JOIN bench_graph_claims_scale claim + ON claim.edge_ordinal = edge.ordinal AND claim.release_ordinal = edge.release_ordinal + WHERE edge.release_ordinal = 1 AND edge.release_status = 'promoted' + AND edge.publication_status = 'released' AND edge.storage_state = 'released' + AND edge.review_state = 'accepted' AND edge.privacy_status = 'passed' + AND edge.source_restricted = false + AND claim.publication_status = 'released' AND claim.storage_state = 'released' + AND claim.review_state = 'accepted' AND claim.privacy_status = 'passed' + AND claim.source_restricted = false + ORDER BY edge.ordinal LIMIT 50 + """, "params": (), "expected_indexes": GRAPH_INDEXES}, +) + + +def _validate_scales(scales: list[int]) -> tuple[int, ...]: + if not scales: + raise ValueError("at least one scale is required") + if any(scale < 1 or scale > MAX_SCALE for scale in scales): + raise ValueError(f"scales must be between 1 and {MAX_SCALE:,}") + if len(set(scales)) != len(scales): + raise ValueError("scales must be unique") + return tuple(scales) + + +def _json_value(value: Any) -> Any: + return json.loads(value) if isinstance(value, str) else value + + +def summarize_plan(payload: Any) -> dict[str, Any]: + """Reduce EXPLAIN JSON to row-free operational measurements.""" + document = _json_value(payload) + if not isinstance(document, list) or not document or not isinstance(document[0], dict): + raise ValueError("unexpected EXPLAIN JSON envelope") + root = document[0].get("Plan") + if not isinstance(root, dict): + raise ValueError("EXPLAIN JSON did not contain a plan") + node_types: set[str] = set() + index_names: set[str] = set() + sequential_scans = 0 + shared_hit_blocks = 0 + shared_read_blocks = 0 + actual_rows = 0 + + def visit(node: dict[str, Any]) -> None: + nonlocal sequential_scans, shared_hit_blocks, shared_read_blocks, actual_rows + node_type = node.get("Node Type") + if isinstance(node_type, str): + node_types.add(node_type) + sequential_scans += node_type == "Seq Scan" + if isinstance(node.get("Index Name"), str): + index_names.add(node["Index Name"]) + shared_hit_blocks += int(node.get("Shared Hit Blocks", 0) or 0) + shared_read_blocks += int(node.get("Shared Read Blocks", 0) or 0) + if isinstance(node.get("Actual Rows"), (int, float)): + actual_rows = int(node["Actual Rows"]) + for child in node.get("Plans", []): + if isinstance(child, dict): + visit(child) + + visit(root) + actual_rows = int(root.get("Actual Rows", 0) or 0) + report = { + "planning_ms": float(document[0].get("Planning Time", 0.0)), + "execution_ms": float(document[0].get("Execution Time", 0.0)), + "actual_rows": actual_rows, + "node_types": sorted(node_types), + "index_names": sorted(index_names), + "sequential_scan_nodes": sequential_scans, + "shared_hit_blocks": shared_hit_blocks, + "shared_read_blocks": shared_read_blocks, + } + forbidden = ("facility_id", "canonical_name", "source_record", "address", "coordinate") + if any(term in json.dumps(report).lower() for term in forbidden): + raise ValueError("row-bearing or private field leaked into plan summary") + return report + + +def _create_tables(connection: Any, scale: int) -> None: + connection.execute(""" + CREATE TEMP TABLE bench_discovery_scale AS + SELECT n AS ordinal, + format('00000000-0000-4000-8000-%%s', lpad(n::text, 12, '0'))::uuid AS facility_id, + format('Synthetic facility %%s', n) AS canonical_name, + CASE WHEN n %% 2 = 0 THEN 'DK' ELSE 'SE' END AS country_code, + CASE n %% 4 WHEN 0 THEN 'slaughter' WHEN 1 THEN 'fish_processing' + WHEN 2 THEN 'logistics_and_storage' ELSE 'retail_and_prepared_food' END AS category, + ST_SetSRID(ST_Point(-10 + (n %% 2000) / 100.0, 45 + (n %% 1000) / 100.0), 4326)::geography AS location + FROM generate_series(1, %s) AS n + """, (scale,)) + connection.execute("ALTER TABLE bench_discovery_scale ADD CONSTRAINT bench_discovery_facility_pk PRIMARY KEY (facility_id)") + connection.execute("CREATE INDEX bench_discovery_country_cursor_idx ON bench_discovery_scale (country_code, facility_id)") + connection.execute("CREATE INDEX bench_discovery_country_category_cursor_idx ON bench_discovery_scale (country_code, category, facility_id)") + connection.execute("CREATE INDEX bench_discovery_name_idx ON bench_discovery_scale (lower(canonical_name) text_pattern_ops)") + connection.execute("CREATE INDEX bench_discovery_location_idx ON bench_discovery_scale USING GIST (location)") + connection.execute(""" + CREATE TEMP TABLE bench_graph_edges_scale AS + SELECT n AS ordinal, 1 AS release_ordinal, 'promoted'::text AS release_status, + 'released'::text AS publication_status, 'released'::text AS storage_state, + 'accepted'::text AS review_state, 'passed'::text AS privacy_status, + (n %% 17 = 0) AS source_restricted + FROM generate_series(1, %s) AS n + """, (scale,)) + connection.execute(""" + CREATE TEMP TABLE bench_graph_claims_scale AS + SELECT n AS edge_ordinal, 1 AS release_ordinal, + 'released'::text AS publication_status, 'released'::text AS storage_state, + 'accepted'::text AS review_state, 'passed'::text AS privacy_status, + (n %% 17 = 0) AS source_restricted + FROM generate_series(1, %s) AS n + """, (scale,)) + connection.execute("CREATE INDEX bench_graph_edge_public_idx ON bench_graph_edges_scale (release_ordinal, publication_status, storage_state, review_state, privacy_status, source_restricted, ordinal)") + connection.execute("CREATE INDEX bench_graph_claim_public_idx ON bench_graph_claims_scale (release_ordinal, publication_status, storage_state, review_state, privacy_status, source_restricted, edge_ordinal)") + connection.execute("ANALYZE bench_discovery_scale") + connection.execute("ANALYZE bench_graph_edges_scale") + connection.execute("ANALYZE bench_graph_claims_scale") + connection.commit() + + +def _run_scale(connection: Any, scale: int, check_plans: bool) -> dict[str, Any]: + _create_tables(connection, scale) + queries: list[dict[str, Any]] = [] + for spec in QUERY_SPECS: + row = connection.execute("EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) " + spec["sql"], spec["params"]).fetchone() + summary = summarize_plan(row[0]) + expected = set(spec["expected_indexes"]) + used = set(summary["index_names"]) + matching_indexes = sorted(expected & used) + plan_check = {"expected_index_family": sorted(expected), "matching_indexes": matching_indexes, "passed": bool(matching_indexes)} + if check_plans and not plan_check["passed"]: + raise RuntimeError(f"{spec['name']} plan did not use an expected index; used={sorted(used)}") + if summary["actual_rows"] > 50: + raise RuntimeError(f"{spec['name']} exceeded the bounded page size") + queries.append({"name": spec["name"], **summary, "plan_check": plan_check}) + return {"observations": scale, "queries": queries} + + +def run(database_url: str, scales: tuple[int, ...], check_plans: bool = True) -> dict[str, Any]: + try: + import psycopg + except ImportError as exc: # pragma: no cover - depends on local environment + raise RuntimeError("install pipeline/requirements.txt before running the benchmark") from exc + reports = [] + for scale in scales: + with psycopg.connect(database_url) as connection: + reports.append(_run_scale(connection, scale, check_plans)) + return {"schema_version": 1, "synthetic_only": True, "temporary_tables": True, "query_count_per_scale": len(QUERY_SPECS), "scales": reports} + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--database-url", default=os.environ.get("UEC_DATABASE_URL")) + parser.add_argument("--scales", nargs="+", type=int, default=list(DEFAULT_SCALES)) + parser.add_argument("--json-output", type=Path) + parser.add_argument("--skip-plan-checks", action="store_true") + args = parser.parse_args(argv) + if not args.database_url: + parser.error("--database-url or UEC_DATABASE_URL is required") + try: + report = run(args.database_url, _validate_scales(args.scales), not args.skip_plan_checks) + except (RuntimeError, ValueError) as exc: + parser.error(str(exc)) + serialized = json.dumps(report, indent=2, sort_keys=True) + "\n" + if args.json_output: + args.json_output.parent.mkdir(parents=True, exist_ok=True) + args.json_output.write_text(serialized, encoding="utf-8") + sys.stdout.write(serialized) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/pipeline/tests/e2e/README.md b/pipeline/tests/e2e/README.md index ba11888..d390297 100644 --- a/pipeline/tests/e2e/README.md +++ b/pipeline/tests/e2e/README.md @@ -2,16 +2,20 @@ These tests exercise the compiled Rust service over real HTTP while it uses a disposable PostGIS container. They never use the Denmark database or real personal data. -Run them from the repository root: +Run the isolated core suite from the repository root: ```powershell -$env:UEC_RUN_E2E = "1" -python -m unittest discover -s pipeline/tests/e2e -p "test_*.py" -v +pwsh -NoProfile -ExecutionPolicy Bypass -File pipeline/tests/e2e/run-suite.ps1 ``` +Use `-Suite full` to include the suppression-lifecycle and country handoff +modules. Each module owns a fresh database and is run in sequence; a failure +stops the suite and the fixture's `finally` cleanup removes its container, +volume, backend process, and temporary build directory. + This requires Docker Desktop, Cargo, and the pinned Python dependencies. The fixture uses isolated random ports and tears down its Compose project even after setup failures. Fast non-Docker checks remain available with `python -m unittest discover -s pipeline/tests -p "test_*.py" -v`. -`fixture.py` owns the environment lifecycle: it selects isolated ports, starts Docker Compose, applies migrations as UTF-8, builds and starts the backend from a per-run temporary Cargo target directory, waits for readiness, and tears everything down. The isolated target prevents E2E builds from contending with a developer's running backend binary. Setup failures also trigger cleanup. Run the API modules sequentially (`test_public_api`, `test_community_api`, `test_seeded_api`, and `test_candidate_import`) because each module owns a disposable PostGIS environment; running all classes in one discovery process can create avoidable Docker resource/lifecycle contention. +`fixture.py` owns the environment lifecycle: it selects isolated ports, starts Docker Compose, applies migrations as UTF-8, builds and starts the backend from a per-run temporary Cargo target directory, waits for readiness, and tears everything down. The isolated target prevents E2E builds from contending with a developer's running backend binary. Setup failures also trigger cleanup. Run API modules through `run-suite.ps1` because each module owns a disposable PostGIS environment; running all classes in one discovery process can create avoidable Docker resource/lifecycle contention. `test_public_api.py` verifies the publication boundary with an empty database: candidate data and filters remain unavailable, and malformed pagination is rejected. diff --git a/pipeline/tests/e2e/run-suite.ps1 b/pipeline/tests/e2e/run-suite.ps1 new file mode 100644 index 0000000..e44785d --- /dev/null +++ b/pipeline/tests/e2e/run-suite.ps1 @@ -0,0 +1,36 @@ +param( + [ValidateSet('core', 'full')] + [string]$Suite = 'core' +) + +$ErrorActionPreference = 'Stop' +$root = (Resolve-Path (Join-Path $PSScriptRoot '..\..\..')).Path +$core = @( + 'pipeline.tests.e2e.test_public_api', + 'pipeline.tests.e2e.test_community_api', + 'pipeline.tests.e2e.test_seeded_api', + 'pipeline.tests.e2e.test_public_surface_safety', + 'pipeline.tests.e2e.test_candidate_import', + 'pipeline.tests.e2e.test_readiness' +) +$extended = @( + 'pipeline.tests.e2e.test_suppression_lifecycle', + 'pipeline.tests.e2e.test_italy_candidate_import', + 'pipeline.tests.e2e.test_germany_belgium_candidate_import' +) +$tests = if ($Suite -eq 'full') { $core + $extended } else { $core } + +Push-Location $root +try { + $env:UEC_RUN_E2E = '1' + foreach ($test in $tests) { + Write-Host "[e2e] running $test" + & python -m unittest $test -v + if ($LASTEXITCODE -ne 0) { throw "E2E module failed: $test (exit $LASTEXITCODE)" } + } + Write-Host "[e2e] $Suite suite passed ($($tests.Count) isolated modules)" +} +finally { + Remove-Item Env:UEC_RUN_E2E -ErrorAction SilentlyContinue + Pop-Location +} diff --git a/pipeline/tests/test_discovery_scale_benchmark.py b/pipeline/tests/test_discovery_scale_benchmark.py new file mode 100644 index 0000000..8a174be --- /dev/null +++ b/pipeline/tests/test_discovery_scale_benchmark.py @@ -0,0 +1,60 @@ +"""Unit contracts for the synthetic scale benchmark.""" + +import importlib.util +import json +import unittest +from pathlib import Path + +ROOT = Path(__file__).parents[1] +SCRIPT = ROOT / "scripts" / "benchmarks" / "run_discovery_scale.py" +SPEC = importlib.util.spec_from_file_location("run_discovery_scale", SCRIPT) +MODULE = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(MODULE) + + +class DiscoveryScaleBenchmarkTests(unittest.TestCase): + def test_default_scales_and_query_catalog_are_bounded(self): + self.assertEqual(MODULE._validate_scales(list(MODULE.DEFAULT_SCALES)), (100_000, 1_000_000)) + self.assertEqual(len(MODULE.QUERY_SPECS), 8) + self.assertEqual( + {spec["name"] for spec in MODULE.QUERY_SPECS}, + {"list", "pagination", "filters", "text_filter", "bbox", "radius", "detail", "graph_ready"}, + ) + for spec in MODULE.QUERY_SPECS: + self.assertNotIn("uec.", spec["sql"].lower()) + if spec["name"] != "detail": + self.assertIn("limit 50", spec["sql"].lower()) + + def test_scale_validation_rejects_unsafe_or_ambiguous_inputs(self): + for scales in ([], [0], [-1], [MODULE.MAX_SCALE + 1], [100_000, 100_000]): + with self.subTest(scales=scales), self.assertRaises(ValueError): + MODULE._validate_scales(scales) + + def test_plan_summary_is_row_free_and_aggregated(self): + payload = [{ + "Planning Time": 0.12, + "Execution Time": 1.23, + "Plan": { + "Node Type": "Limit", + "Actual Rows": 50, + "Shared Hit Blocks": 3, + "Plans": [{ + "Node Type": "Index Scan", + "Index Name": "synthetic_index", + "Actual Rows": 50, + "Shared Read Blocks": 2, + }], + }, + }] + report = MODULE.summarize_plan(json.dumps(payload)) + self.assertEqual(report["actual_rows"], 50) + self.assertEqual(report["index_names"], ["synthetic_index"]) + self.assertEqual(report["shared_hit_blocks"], 3) + self.assertEqual(report["shared_read_blocks"], 2) + self.assertNotIn("Plan", report) + self.assertNotIn("canonical_name", json.dumps(report).lower()) + self.assertNotIn("source_record", json.dumps(report).lower()) + + +if __name__ == "__main__": + unittest.main() From 82244a2c9547d274438b0f553718e3933d630251 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Wed, 16 Sep 2026 10:04:05 -0700 Subject: [PATCH 155/311] docs: add Bulgaria source reconnaissance --- docs/country-recon-bg.md | 43 +++++++++++++++++++ docs/source-status.json | 8 +++- pipeline/source_registry.json | 9 +++- .../tests/test_bulgaria_recon_metadata.py | 15 +++++++ pipeline/tests/test_source_registry.py | 4 +- 5 files changed, 75 insertions(+), 4 deletions(-) create mode 100644 docs/country-recon-bg.md create mode 100644 pipeline/tests/test_bulgaria_recon_metadata.py diff --git a/docs/country-recon-bg.md b/docs/country-recon-bg.md new file mode 100644 index 0000000..e149865 --- /dev/null +++ b/docs/country-recon-bg.md @@ -0,0 +1,43 @@ +# Bulgaria source reconnaissance + +Status: row-free reconnaissance only; no facility rows, raw exports, or production ingestion artifacts are committed. + +## Scope and automation requirement + +Bulgaria has authoritative surfaces through the Bulgarian Food Safety Agency (BFSA/BАБХ), the national open-data catalogue, environmental systems, the Registry Agency, and the National Statistical Institute. The production pipeline must be fully automated from source discovery/scraping through acquisition, validation, normalization, provenance, quarantine, and ingestion. This reconnaissance does not authorize publication or replace maintainer review. + +## Strongest candidates + +| Source | Verified surface | Automation assessment | Main unresolved items | +| --- | --- | --- | --- | +| `bg.bfsa.approved-food` | BFSA maintains public national registers for approved/registered food and feed establishments; its official material links feed approvals to the European Commission list. | Medium: public register discovery is clear, but direct bulk/API format needs probing. | Current establishment list/API, slaughter categories, stable approval IDs, status, addresses/coordinates, cadence, license and privacy. | +| `bg.farm-aquaculture` | BFSA materials identify official registration/control of livestock holdings; agriculture and fisheries authorities are the likely source for aquaculture permits. | Medium for public files; low–medium where portals are interactive or undocumented. | National farm/holding export, aquaculture permit register, identifiers, coordinates, cadence, terms and privacy. | +| `bg.environment-permits` | Ministry/Executive Environment Agency systems provide environmental authorization and integrated-control workflows; public machine-readable read access is not yet pinned. | Medium after a public API/export is confirmed; otherwise document/browser extraction. | Permit/decision API, national coverage, current public availability, geometry, licensing and privacy. | +| `bg.registry-agency.organizations` | Bulgarian Registry Agency’s Commercial Register exposes EIK-keyed company information and public service surfaces; automated bulk/API contract is not verified. | Medium: likely automatable after endpoint/terms are pinned, but CAPTCHAs/auth may constrain it. | Stable API/query route, fields, rate limits, terms, update cadence and registered-office privacy. | +| `bg.nsi.statistics` | National Statistical Institute publishes official agriculture, livestock and slaughter aggregates and statistical data services. | Medium–high for aggregate tables once table IDs/API contracts are pinned. | Exact slaughter/animal-use tables, API/download route, cadence, revisions, license and aggregate-only handling. | +| `bg.inspections-experiments` | BFSA official-control plans/reports document inspection and enforcement responsibilities; a stable public animal-experimentation facility master was not verified. | High for aggregate reports; low for facility/event extraction until a public route is confirmed. | Public inspection/event data, stable IDs, outcomes, animal-use categories, privacy and retention. | + +## Compliance and ingestion notes + +- Treat BFSA, registry, environmental and statistical sources as government-sourced evidence, not proof of current operation or project approval. +- Preserve source URL, retrieval timestamp, content hash, byte size, supplied publication/effective date, adapter/configuration version, and source values. +- Keep raw, parsed, normalized, enriched, reviewed, and released layers separate. Use synthetic fixtures only; do not commit rows or raw artifacts. +- Represent unavailable cadence, IDs, coordinates, licensing, and privacy decisions explicitly. A source outage or disappearance means “not observed,” not closure. +- Registered offices, farm addresses, and environmental work points are not automatically safe public facility locations; apply privacy and safety review before releasing addresses or coordinates. +- Adapters must fail closed on changed schemas, missing identifiers, suspicious count changes, CAPTCHAs/authentication, and service failures, retaining the previous validated release. + +## Official evidence + +- [BFSA official registers](https://bfsa.egov.bg/wps/portal/bfsa-web/registers) +- [BFSA official-control programme/report evidence](https://bfsa.egov.bg/) +- [European Commission approved feed establishments](https://food.ec.europa.eu/safety/animal-feed/feed-hygiene/approved-establishments_en) +- [Bulgarian national open-data catalogue](https://data.egov.bg/) +- [Executive Environment Agency / environmental system](https://eea.government.bg/) +- [Registry Agency](https://www.registryagency.bg/) +- [National Statistical Institute](https://www.nsi.bg/) + +## Effort and blockers + +Initial reconnaissance: approximately 2–4 engineering days to pin BFSA registers and any agriculture/aquaculture exports, then build deterministic acquisition and validation adapters; 4–8 additional days for environmental documents, registry integration, and inspection/animal-experimentation coverage. Main blockers are undocumented or interactive BFSA routes, incomplete public farm/aquaculture export verification, uncertain environmental read/API access, possible CAPTCHA/authentication on corporate services, and unresolved licensing/privacy/coordinate semantics. + +Recommended next country: Serbia, subject to checking the existing country inventory before delegation. diff --git a/docs/source-status.json b/docs/source-status.json index ac13a1d..7328103 100644 --- a/docs/source-status.json +++ b/docs/source-status.json @@ -157,6 +157,12 @@ {"source_id":"ro.environment-permits","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ro.md","pipeline/source_registry.json"],"next_action":"Recheck ANPM availability and verify public read/API, document route, license, geometry and privacy."}, {"source_id":"ro.onrc.organizations","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ro.md","pipeline/source_registry.json"],"next_action":"Pin current CSV snapshot, cadence, fields, license and registered-office privacy policy."}, {"source_id":"ro.insse.statistics","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ro.md","pipeline/source_registry.json"],"next_action":"Pin official slaughter/animal-use table IDs, APIs, cadence and revisions."}, - {"source_id":"ro.inspections-experiments","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ro.md","pipeline/source_registry.json"],"next_action":"Locate a stable public facility/event route; keep sensitive evidence aggregate until verified."} + {"source_id":"ro.inspections-experiments","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ro.md","pipeline/source_registry.json"],"next_action":"Locate a stable public facility/event route; keep sensitive evidence aggregate until verified."}, + {"source_id":"bg.bfsa.approved-food","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-bg.md","pipeline/source_registry.json"],"next_action":"Pin BFSA list/API/export, categories, IDs, cadence, terms and privacy."}, + {"source_id":"bg.farm-aquaculture","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-bg.md","pipeline/source_registry.json"],"next_action":"Verify national farm/aquaculture scope, export/API, IDs, coordinates, terms and privacy."}, + {"source_id":"bg.environment-permits","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-bg.md","pipeline/source_registry.json"],"next_action":"Verify public permit API/export, coverage, documents, geometry, license and privacy."}, + {"source_id":"bg.registry-agency.organizations","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-bg.md","pipeline/source_registry.json"],"next_action":"Pin corporate query/API route, auth/CAPTCHA, fields, cadence, terms and privacy."}, + {"source_id":"bg.nsi.statistics","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-bg.md","pipeline/source_registry.json"],"next_action":"Pin official slaughter/animal-use table IDs, APIs, cadence and revisions."}, + {"source_id":"bg.inspections-experiments","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-bg.md","pipeline/source_registry.json"],"next_action":"Locate a stable public facility/event route; keep sensitive evidence aggregate until verified."} ] } diff --git a/pipeline/source_registry.json b/pipeline/source_registry.json index 3bc66cf..eb9ed5b 100644 --- a/pipeline/source_registry.json +++ b/pipeline/source_registry.json @@ -780,7 +780,12 @@ {"source_id":"ro.environment-permits","jurisdiction_scope":"Romania; ANPM environmental authorizations and integrated environmental system","legacy_paths":[],"url":"https://raportare.anpm.ro/","access_method":"SIM/eFORM official web system and document/publication routes","cadence":"unknown; system currently reports technical unavailability","attribution_licensing_notes":"Official environmental authority; verify public-read rights, document license, geometry and privacy.","adapter_status":"reference_only","expected_artifact_schema":"Permit/decision ID, operator/work point, authority, coordinates, activity, dates and status","blockers":["Public read/API route and dependable availability are unresolved; ANPM reports SIM technical outage."]}, {"source_id":"ro.onrc.organizations","jurisdiction_scope":"Romania; ONRC legal-entity and authorized-activity snapshots","legacy_paths":[],"url":"https://data.gov.ro/dataset?organiza=&organization=onrc&res_format=csv","access_method":"data.gov.ro CKAN CSV snapshots and API metadata","cadence":"snapshot-specific; timestamp each release","attribution_licensing_notes":"Catalogue snapshots are public and some are CC BY 4.0; verify current file license, field privacy and attribution.","adapter_status":"reference_only","expected_artifact_schema":"CUI/OIB keyed organization, registered office, status and authorized activity fields","blockers":["Pin current snapshot, cadence, field dictionary and registered-office publication policy."]}, {"source_id":"ro.insse.statistics","jurisdiction_scope":"Romania; INSSE aggregate slaughter, livestock and animal-use statistics","legacy_paths":[],"url":"https://insse.ro/cms/","access_method":"official statistical tables/data services; exact table/API route unverified","cadence":"table-specific; preserve revisions","attribution_licensing_notes":"Official statistics; verify table terms, citation requirements and aggregate-only scope.","adapter_status":"reference_only","expected_artifact_schema":"Aggregate multidimensional observations by species, measure, geography and period","blockers":["Pin current slaughter and animal-use table IDs, API/download contracts and cadence."]}, - {"source_id":"ro.inspections-experiments","jurisdiction_scope":"Romania; ANSVSA/DSVSA inspections, enforcement and animal-experimentation evidence","legacy_paths":[],"url":"https://portal.ansvsa.ro/","access_method":"official portals, reports and statistical publications","cadence":"report/register-specific","attribution_licensing_notes":"Sensitive control/research evidence; aggregate or anonymize and require project review.","adapter_status":"reference_only","expected_artifact_schema":"Dated aggregate/event observations with authority, category, outcome, measure and source document","blockers":["No stable public facility-level inspection or animal-experimentation master verified."]} ] + {"source_id":"ro.inspections-experiments","jurisdiction_scope":"Romania; ANSVSA/DSVSA inspections, enforcement and animal-experimentation evidence","legacy_paths":[],"url":"https://portal.ansvsa.ro/","access_method":"official portals, reports and statistical publications","cadence":"report/register-specific","attribution_licensing_notes":"Sensitive control/research evidence; aggregate or anonymize and require project review.","adapter_status":"reference_only","expected_artifact_schema":"Dated aggregate/event observations with authority, category, outcome, measure and source document","blockers":["No stable public facility-level inspection or animal-experimentation master verified."]} ,{"source_id":"bg.bfsa.approved-food","jurisdiction_scope":"Bulgaria; BFSA approved and registered food/feed establishments handling animal-origin products","legacy_paths":[],"url":"https://bfsa.egov.bg/wps/portal/bfsa-web/registers","access_method":"official BFSA public registers and linked EU feed-establishment list","cadence":"publisher-defined; timestamp retrieval","attribution_licensing_notes":"Official authority; verify register terms, attribution, privacy and coordinates before acquisition.","adapter_status":"reference_only","expected_artifact_schema":"Approval/registration ID, operator/site, activities, status, address and optional coordinates","blockers":["Pin current BFSA list/API/export, slaughter categories, IDs, cadence, terms and privacy."]}, + {"source_id":"bg.farm-aquaculture","jurisdiction_scope":"Bulgaria; livestock holdings and aquaculture permit/establishment evidence","legacy_paths":[],"url":"https://bfsa.egov.bg/","access_method":"official BFSA/agriculture/fisheries registers and open-data resources","cadence":"resource-specific","attribution_licensing_notes":"Verify publisher, license, animal-holder/property privacy and coordinate precision.","adapter_status":"reference_only","expected_artifact_schema":"Holding/site or aquaculture permit records with stable IDs, activity/status, address and optional coordinates","blockers":["No complete national farm/aquaculture public export contract verified."]}, + {"source_id":"bg.environment-permits","jurisdiction_scope":"Bulgaria; environmental permits and integrated-control authorizations","legacy_paths":[],"url":"https://eea.government.bg/","access_method":"Executive Environment Agency systems and public notices/documents","cadence":"permit/document-specific","attribution_licensing_notes":"Verify public-read rights, document license, geometry and privacy.","adapter_status":"reference_only","expected_artifact_schema":"Permit/decision ID, authority, operator/work point, activity, dates, status and optional reviewed geometry","blockers":["Stable public read/API/export route and complete national coverage are unresolved."]}, + {"source_id":"bg.registry-agency.organizations","jurisdiction_scope":"Bulgaria; Registry Agency Commercial Register corporate identifiers","legacy_paths":[],"url":"https://www.registryagency.bg/","access_method":"official public register/service surface; API/bulk route unverified","cadence":"provider-defined","attribution_licensing_notes":"Identity linkage only; verify terms, fields, rate limits and registered-office privacy.","adapter_status":"reference_only","expected_artifact_schema":"EIK keyed organization, name, status, legal form, activity and registered office/address","blockers":["Pin stable query/API route, authentication/CAPTCHA, cadence, terms and privacy policy."]}, + {"source_id":"bg.nsi.statistics","jurisdiction_scope":"Bulgaria; NSI aggregate slaughter, livestock and animal-use statistics","legacy_paths":[],"url":"https://www.nsi.bg/","access_method":"official statistical tables/data services; exact table/API route unverified","cadence":"table-specific; preserve revisions","attribution_licensing_notes":"Official statistics; verify table terms, citation requirements and aggregate-only scope.","adapter_status":"reference_only","expected_artifact_schema":"Aggregate multidimensional observations by species, measure, geography and period","blockers":["Pin current slaughter and animal-use table IDs, API/download contracts and cadence."]}, + {"source_id":"bg.inspections-experiments","jurisdiction_scope":"Bulgaria; BFSA inspections/enforcement and animal-experimentation evidence","legacy_paths":[],"url":"https://bfsa.egov.bg/","access_method":"official control plans, reports and statistical publications","cadence":"report/register-specific","attribution_licensing_notes":"Sensitive control/research evidence; aggregate or anonymize and require project review.","adapter_status":"reference_only","expected_artifact_schema":"Dated aggregate/event observations with authority, category, outcome, measure and source document","blockers":["No stable public facility-level inspection or animal-experimentation master verified."]} ] } @@ -794,3 +799,5 @@ + + diff --git a/pipeline/tests/test_bulgaria_recon_metadata.py b/pipeline/tests/test_bulgaria_recon_metadata.py new file mode 100644 index 0000000..6e610a5 --- /dev/null +++ b/pipeline/tests/test_bulgaria_recon_metadata.py @@ -0,0 +1,15 @@ +import json +import unittest +from pathlib import Path +ROOT=Path(__file__).resolve().parents[2] +class BulgariaReconMetadataTests(unittest.TestCase): + def test_row_free_bulgaria_recon_is_present(self): + text=(ROOT/"docs"/"country-recon-bg.md").read_text(encoding="utf-8") + self.assertIn("row-free",text); self.assertIn("fully automated",text); self.assertIn("Recommended next country: Serbia",text) + def test_bulgaria_sources_have_conservative_status(self): + registry=json.loads((ROOT/"pipeline"/"source_registry.json").read_text(encoding="utf-8")); status=json.loads((ROOT/"docs"/"source-status.json").read_text(encoding="utf-8")) + ids={s["source_id"] for s in registry["sources"] if s["source_id"].startswith("bg.")} + self.assertEqual(ids,{s["source_id"] for s in status["sources"] if s["source_id"].startswith("bg.")}); self.assertEqual(len(ids),6) + for s in status["sources"]: + if s["source_id"].startswith("bg."): self.assertEqual(s["publication_eligibility"],"blocked"); self.assertNotEqual(s["runtime_health"],"healthy") +if __name__ == "__main__": unittest.main() diff --git a/pipeline/tests/test_source_registry.py b/pipeline/tests/test_source_registry.py index e7b5519..b688abb 100644 --- a/pipeline/tests/test_source_registry.py +++ b/pipeline/tests/test_source_registry.py @@ -9,8 +9,8 @@ class SourceRegistryTests(unittest.TestCase): def test_repository_registry_loads_and_references_existing_legacy_paths(self): registry = load_registry() - self.assertEqual(len(registry["sources"]), 149) - self.assertEqual(len({source["source_id"] for source in registry["sources"]}), 149) + self.assertEqual(len(registry["sources"]), 155) + self.assertEqual(len({source["source_id"] for source in registry["sources"]}), 155) def test_unknowns_are_explicit(self): registry = json.loads(REGISTRY_PATH.read_text(encoding="utf-8")) From e6512f2826ab03b4cd8e276f78e1f579a7a13217 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Wed, 16 Sep 2026 10:06:51 -0700 Subject: [PATCH 156/311] docs: add Serbia source reconnaissance --- docs/country-recon-rs.md | 43 ++++++++++++++++++++ docs/source-status.json | 8 +++- pipeline/source_registry.json | 9 +++- pipeline/tests/test_serbia_recon_metadata.py | 15 +++++++ pipeline/tests/test_source_registry.py | 4 +- 5 files changed, 75 insertions(+), 4 deletions(-) create mode 100644 docs/country-recon-rs.md create mode 100644 pipeline/tests/test_serbia_recon_metadata.py diff --git a/docs/country-recon-rs.md b/docs/country-recon-rs.md new file mode 100644 index 0000000..d98bd88 --- /dev/null +++ b/docs/country-recon-rs.md @@ -0,0 +1,43 @@ +# Serbia source reconnaissance + +Status: row-free reconnaissance only; no facility rows, raw exports, or production ingestion artifacts are committed. + +## Scope and automation requirement + +Serbia exposes relevant official surfaces through the Ministry of Agriculture/veterinary authority, the national open-data portal, the Statistical Office, environmental authorities, and the Serbian Business Registers Agency (APR). The production pipeline must be fully automated from source discovery/scraping through acquisition, validation, normalization, provenance, quarantine, and ingestion. This reconnaissance does not authorize publication or replace maintainer review. + +## Strongest candidates + +| Source | Verified surface | Automation assessment | Main unresolved items | +| --- | --- | --- | --- | +| `rs.veterinary.approved-food` | Ministry/veterinary authority and EU/official control frameworks cover approved food establishments and slaughterhouses; a stable national bulk/API route was not pinned. | Medium if an official export is found; otherwise interactive/browser acquisition. | Current national list/API, stable approval IDs, status/categories, addresses/coordinates, cadence, license and privacy. | +| `rs.farm-aquaculture` | Serbia’s open-data portal publishes farm aggregates and agriculture resources; aquaculture permit/site coverage needs authority-specific confirmation. | Medium for catalogued CSV/XLS/JSON; low–medium for interactive registers. | Establishment-level farm scope, aquaculture permits, IDs, coordinates, cadence, terms and privacy. | +| `rs.environment-permits` | Environmental permitting is handled through official environmental systems and agencies; a complete public read/export contract was not verified. | Medium after a stable public route is confirmed; otherwise document extraction. | Permit/API route, national coverage, availability, geometry, licensing and privacy. | +| `rs.apr.organizations` | APR provides public browser/web-service access to centralized electronic business registers; bulk/automated access is restricted and some data services are fee-based. | Low–medium: automation must use authorized web services and respect anti-automated-download terms. | API contract, fees, quotas, allowed automation, fields, cadence and registered-office privacy. | +| `rs.stat.statistics` | Statistical Office open-data resources provide machine-readable JSON/CSV APIs with dataset IDs, including livestock/farm aggregates; slaughter table IDs need pinning. | High for official aggregate APIs after table discovery. | Exact slaughter/animal-use tables, cadence, revisions, license and aggregate-only handling. | +| `rs.inspections-experiments` | Official veterinary control plans/reports and relevant authorities provide inspection/enforcement evidence; public animal-experimentation facility master was not verified. | High for published aggregates; low for facility/event extraction until a public route is confirmed. | Public event/facility route, stable IDs, outcomes, animal-use categories, privacy and retention. | + +## Compliance and ingestion notes + +- Treat government and open-data records as government-sourced evidence, not proof of current operation or project approval. +- Preserve source URL, retrieval timestamp, content hash, byte size, supplied publication/effective date, adapter/configuration version, and source values. +- Keep raw, parsed, normalized, enriched, reviewed, and released layers separate. Use synthetic fixtures only; do not commit rows or raw artifacts. +- Represent unavailable cadence, IDs, coordinates, licensing, and privacy decisions explicitly. A source outage or disappearance means “not observed,” not closure. +- Registered offices, farm addresses and permit locations are not automatically safe public facility locations; apply privacy and safety review before releasing addresses or coordinates. +- Adapters must fail closed on changed schemas, missing identifiers, suspicious count changes, unauthorized automation, and service failures, retaining the previous validated release. + +## Official evidence + +- [Serbian open-data portal](https://data.gov.rs/) +- [Machine-readable Statistical Office farm dataset](https://data.gov.rs/sr/datasets/broj-gazdinstava-i-grla-stoke-po-vrstama-i-broj-uslovnih-grla-prema-tipu-proizvodnje/) +- [APR data-search terms and access](https://www.apr.gov.rs/registers/media/data-search.1728.html) +- [APR electronic data services](https://www.apr.gov.rs/services/e-data-on-request/status-and-other-business-data.4270.html) +- [Ministry of Agriculture](https://www.minpolj.gov.rs/) +- [Environmental Protection Agency](https://www.sepa.gov.rs/) +- [Statistical Office of the Republic of Serbia](https://www.stat.gov.rs/) + +## Effort and blockers + +Initial reconnaissance: approximately 2–4 engineering days to pin Statistical Office APIs and any veterinary/agriculture exports, then build deterministic acquisition and validation adapters; 4–8 additional days for environmental documents, APR integration, and inspection/animal-experimentation coverage. Main blockers are the undocumented national veterinary establishment route, incomplete farm/aquaculture export verification, environmental read/API uncertainty, APR restrictions/fees on automated access, and unresolved licensing/privacy/coordinate semantics. + +Recommended next country: Bosnia and Herzegovina, subject to checking the existing country inventory before delegation. diff --git a/docs/source-status.json b/docs/source-status.json index 7328103..ee06a4b 100644 --- a/docs/source-status.json +++ b/docs/source-status.json @@ -163,6 +163,12 @@ {"source_id":"bg.environment-permits","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-bg.md","pipeline/source_registry.json"],"next_action":"Verify public permit API/export, coverage, documents, geometry, license and privacy."}, {"source_id":"bg.registry-agency.organizations","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-bg.md","pipeline/source_registry.json"],"next_action":"Pin corporate query/API route, auth/CAPTCHA, fields, cadence, terms and privacy."}, {"source_id":"bg.nsi.statistics","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-bg.md","pipeline/source_registry.json"],"next_action":"Pin official slaughter/animal-use table IDs, APIs, cadence and revisions."}, - {"source_id":"bg.inspections-experiments","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-bg.md","pipeline/source_registry.json"],"next_action":"Locate a stable public facility/event route; keep sensitive evidence aggregate until verified."} + {"source_id":"bg.inspections-experiments","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-bg.md","pipeline/source_registry.json"],"next_action":"Locate a stable public facility/event route; keep sensitive evidence aggregate until verified."}, + {"source_id":"rs.veterinary.approved-food","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-rs.md","pipeline/source_registry.json"],"next_action":"Pin veterinary list/API/export, categories, IDs, cadence, terms and privacy."}, + {"source_id":"rs.farm-aquaculture","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-rs.md","pipeline/source_registry.json"],"next_action":"Verify farm/aquaculture scope, export/API, IDs, coordinates, terms and privacy."}, + {"source_id":"rs.environment-permits","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-rs.md","pipeline/source_registry.json"],"next_action":"Verify public environmental permit route, coverage, documents, geometry, license and privacy."}, + {"source_id":"rs.apr.organizations","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-rs.md","pipeline/source_registry.json"],"next_action":"Pin authorized APR service, fees, quotas, cadence, fields and automation permissions."}, + {"source_id":"rs.stat.statistics","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-rs.md","pipeline/source_registry.json"],"next_action":"Pin official slaughter/animal-use dataset IDs, APIs, cadence and revisions."}, + {"source_id":"rs.inspections-experiments","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-rs.md","pipeline/source_registry.json"],"next_action":"Locate a stable public facility/event route; keep sensitive evidence aggregate until verified."} ] } diff --git a/pipeline/source_registry.json b/pipeline/source_registry.json index eb9ed5b..9943867 100644 --- a/pipeline/source_registry.json +++ b/pipeline/source_registry.json @@ -785,7 +785,12 @@ {"source_id":"bg.environment-permits","jurisdiction_scope":"Bulgaria; environmental permits and integrated-control authorizations","legacy_paths":[],"url":"https://eea.government.bg/","access_method":"Executive Environment Agency systems and public notices/documents","cadence":"permit/document-specific","attribution_licensing_notes":"Verify public-read rights, document license, geometry and privacy.","adapter_status":"reference_only","expected_artifact_schema":"Permit/decision ID, authority, operator/work point, activity, dates, status and optional reviewed geometry","blockers":["Stable public read/API/export route and complete national coverage are unresolved."]}, {"source_id":"bg.registry-agency.organizations","jurisdiction_scope":"Bulgaria; Registry Agency Commercial Register corporate identifiers","legacy_paths":[],"url":"https://www.registryagency.bg/","access_method":"official public register/service surface; API/bulk route unverified","cadence":"provider-defined","attribution_licensing_notes":"Identity linkage only; verify terms, fields, rate limits and registered-office privacy.","adapter_status":"reference_only","expected_artifact_schema":"EIK keyed organization, name, status, legal form, activity and registered office/address","blockers":["Pin stable query/API route, authentication/CAPTCHA, cadence, terms and privacy policy."]}, {"source_id":"bg.nsi.statistics","jurisdiction_scope":"Bulgaria; NSI aggregate slaughter, livestock and animal-use statistics","legacy_paths":[],"url":"https://www.nsi.bg/","access_method":"official statistical tables/data services; exact table/API route unverified","cadence":"table-specific; preserve revisions","attribution_licensing_notes":"Official statistics; verify table terms, citation requirements and aggregate-only scope.","adapter_status":"reference_only","expected_artifact_schema":"Aggregate multidimensional observations by species, measure, geography and period","blockers":["Pin current slaughter and animal-use table IDs, API/download contracts and cadence."]}, - {"source_id":"bg.inspections-experiments","jurisdiction_scope":"Bulgaria; BFSA inspections/enforcement and animal-experimentation evidence","legacy_paths":[],"url":"https://bfsa.egov.bg/","access_method":"official control plans, reports and statistical publications","cadence":"report/register-specific","attribution_licensing_notes":"Sensitive control/research evidence; aggregate or anonymize and require project review.","adapter_status":"reference_only","expected_artifact_schema":"Dated aggregate/event observations with authority, category, outcome, measure and source document","blockers":["No stable public facility-level inspection or animal-experimentation master verified."]} ] + {"source_id":"bg.inspections-experiments","jurisdiction_scope":"Bulgaria; BFSA inspections/enforcement and animal-experimentation evidence","legacy_paths":[],"url":"https://bfsa.egov.bg/","access_method":"official control plans, reports and statistical publications","cadence":"report/register-specific","attribution_licensing_notes":"Sensitive control/research evidence; aggregate or anonymize and require project review.","adapter_status":"reference_only","expected_artifact_schema":"Dated aggregate/event observations with authority, category, outcome, measure and source document","blockers":["No stable public facility-level inspection or animal-experimentation master verified."]} ,{"source_id":"rs.veterinary.approved-food","jurisdiction_scope":"Serbia; approved and registered food establishments handling food of animal origin","legacy_paths":[],"url":"https://www.minpolj.gov.rs/","access_method":"official ministry/veterinary registers and relevant EU/control surfaces; national bulk route unverified","cadence":"unknown; timestamp retrieval","attribution_licensing_notes":"Official authority; verify list license, attribution, privacy and coordinates before acquisition.","adapter_status":"reference_only","expected_artifact_schema":"Approval/registration ID, operator/site, activities, status, address and optional coordinates","blockers":["Pin current national list/API/export, categories, IDs, cadence, terms and privacy."]}, + {"source_id":"rs.farm-aquaculture","jurisdiction_scope":"Serbia; livestock holdings and aquaculture permit/establishment evidence","legacy_paths":[],"url":"https://data.gov.rs/","access_method":"open-data portal, agriculture resources and official fisheries/veterinary registers","cadence":"resource-specific","attribution_licensing_notes":"Verify publisher, license, animal-holder/property privacy and coordinate precision.","adapter_status":"reference_only","expected_artifact_schema":"Holding/site or aquaculture permit records with stable IDs, activity/status, address and optional coordinates","blockers":["No complete national farm/aquaculture establishment export contract verified."]}, + {"source_id":"rs.environment-permits","jurisdiction_scope":"Serbia; environmental permits and environmental authorization evidence","legacy_paths":[],"url":"https://www.sepa.gov.rs/","access_method":"official environmental agency systems, public notices and documents","cadence":"permit/document-specific","attribution_licensing_notes":"Verify public-read rights, document license, geometry and privacy.","adapter_status":"reference_only","expected_artifact_schema":"Permit/decision ID, authority, operator/site, activity, dates, status and optional reviewed geometry","blockers":["Stable public read/API/export route and complete national coverage are unresolved."]}, + {"source_id":"rs.apr.organizations","jurisdiction_scope":"Serbia; APR centralized business-entity registers","legacy_paths":[],"url":"https://www.apr.gov.rs/registers/media/data-search.1728.html","access_method":"official browser and authorized web services; automated downloading restricted","cadence":"provider-defined","attribution_licensing_notes":"Identity linkage only; respect APR terms, fees, access restrictions and registered-office privacy.","adapter_status":"reference_only","expected_artifact_schema":"Registration ID, organization name, status, legal form, activity and registered office/address","blockers":["Pin authorized API/service, fees, quotas, cadence, fields and automation permissions."]}, + {"source_id":"rs.stat.statistics","jurisdiction_scope":"Serbia; Statistical Office aggregate slaughter, livestock and animal-use statistics","legacy_paths":[],"url":"https://www.stat.gov.rs/","access_method":"official JSON/CSV statistical APIs and data portal","cadence":"table-specific; preserve revisions","attribution_licensing_notes":"Official statistics; verify table terms, citation requirements and aggregate-only scope.","adapter_status":"reference_only","expected_artifact_schema":"Aggregate multidimensional observations by species, measure, geography and period","blockers":["Pin current slaughter and animal-use dataset IDs, API contracts and cadence."]}, + {"source_id":"rs.inspections-experiments","jurisdiction_scope":"Serbia; veterinary inspections/enforcement and animal-experimentation evidence","legacy_paths":[],"url":"https://www.minpolj.gov.rs/","access_method":"official control plans, reports and statistical publications","cadence":"report/register-specific","attribution_licensing_notes":"Sensitive control/research evidence; aggregate or anonymize and require project review.","adapter_status":"reference_only","expected_artifact_schema":"Dated aggregate/event observations with authority, category, outcome, measure and source document","blockers":["No stable public facility-level inspection or animal-experimentation master verified."]} ] } @@ -801,3 +806,5 @@ + + diff --git a/pipeline/tests/test_serbia_recon_metadata.py b/pipeline/tests/test_serbia_recon_metadata.py new file mode 100644 index 0000000..af0d97e --- /dev/null +++ b/pipeline/tests/test_serbia_recon_metadata.py @@ -0,0 +1,15 @@ +import json +import unittest +from pathlib import Path +ROOT=Path(__file__).resolve().parents[2] +class SerbiaReconMetadataTests(unittest.TestCase): + def test_row_free_serbia_recon_is_present(self): + text=(ROOT/"docs"/"country-recon-rs.md").read_text(encoding="utf-8") + self.assertIn("row-free",text); self.assertIn("fully automated",text); self.assertIn("Recommended next country: Bosnia and Herzegovina",text) + def test_serbia_sources_have_conservative_status(self): + registry=json.loads((ROOT/"pipeline"/"source_registry.json").read_text(encoding="utf-8")); status=json.loads((ROOT/"docs"/"source-status.json").read_text(encoding="utf-8")) + ids={s["source_id"] for s in registry["sources"] if s["source_id"].startswith("rs.")} + self.assertEqual(ids,{s["source_id"] for s in status["sources"] if s["source_id"].startswith("rs.")}); self.assertEqual(len(ids),6) + for s in status["sources"]: + if s["source_id"].startswith("rs."): self.assertEqual(s["publication_eligibility"],"blocked"); self.assertNotEqual(s["runtime_health"],"healthy") +if __name__ == "__main__": unittest.main() diff --git a/pipeline/tests/test_source_registry.py b/pipeline/tests/test_source_registry.py index b688abb..dbc8264 100644 --- a/pipeline/tests/test_source_registry.py +++ b/pipeline/tests/test_source_registry.py @@ -9,8 +9,8 @@ class SourceRegistryTests(unittest.TestCase): def test_repository_registry_loads_and_references_existing_legacy_paths(self): registry = load_registry() - self.assertEqual(len(registry["sources"]), 155) - self.assertEqual(len({source["source_id"] for source in registry["sources"]}), 155) + self.assertEqual(len(registry["sources"]), 161) + self.assertEqual(len({source["source_id"] for source in registry["sources"]}), 161) def test_unknowns_are_explicit(self): registry = json.loads(REGISTRY_PATH.read_text(encoding="utf-8")) From b4aca9cf9d46faebc733b6724191b19191d06f44 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Wed, 16 Sep 2026 10:09:23 -0700 Subject: [PATCH 157/311] docs: add Bosnia and Herzegovina source reconnaissance --- docs/country-recon-ba.md | 43 ++++++++++++++++++++ docs/source-status.json | 8 +++- pipeline/source_registry.json | 9 +++- pipeline/tests/test_bosnia_recon_metadata.py | 16 ++++++++ pipeline/tests/test_source_registry.py | 4 +- 5 files changed, 76 insertions(+), 4 deletions(-) create mode 100644 docs/country-recon-ba.md create mode 100644 pipeline/tests/test_bosnia_recon_metadata.py diff --git a/docs/country-recon-ba.md b/docs/country-recon-ba.md new file mode 100644 index 0000000..451c361 --- /dev/null +++ b/docs/country-recon-ba.md @@ -0,0 +1,43 @@ +# Bosnia and Herzegovina source reconnaissance + +Status: row-free reconnaissance only; no facility rows, raw exports, or production ingestion artifacts are committed. + +## Scope and automation requirement + +Bosnia and Herzegovina is structurally fragmented: state-level coordination coexists with the Federation of BiH, its cantons, Republika Srpska, and Brčko District. The pipeline must model source authority and coverage by administrative entity, not assume one national register. Production acquisition must be fully automated from source discovery/scraping through validation, normalization, provenance, quarantine, and ingestion; this reconnaissance does not authorize publication or replace maintainer review. + +## Strongest candidates + +| Source | Verified surface | Automation assessment | Main unresolved items | +| --- | --- | --- | --- | +| `ba.veterinary.approved-food` | State/entity and FBiH veterinary/food-control authorities publish approval and inspection materials; FBiH’s veterinary inspectorate documents food/feed/farm controls. | Medium–low: national completeness requires federated adapters and entity-level discovery. | State list, entity/cantonal coverage, slaughter categories, stable IDs, addresses/coordinates, cadence, license and privacy. | +| `ba.farm-aquaculture` | FBiH official inspection mandate explicitly covers primary animal production, livestock registrations, feed, aquaculture, fishponds and fisheries; RS/Brčko counterparts must be mapped. | Low–medium: entity portals and documents may be interactive or inconsistent. | All-authority holding/aquaculture exports, identifiers, coordinates, cadence, terms and privacy. | +| `ba.environment-permits` | Environmental permitting is divided among state/entity authorities; FBiH and RS environmental ministries/agencies are relevant, but a unified machine-readable permit register was not verified. | Low–medium until entity-specific public APIs/exports are confirmed. | Complete coverage, document/API routes, permit IDs, geometry, licensing and privacy. | +| `ba.bizreg.organizations` | The state judicial BIZREG portal searches separate registers for Brčko, Federation BiH and Republika Srpska; RNS is described as unique, permanent and unrepeatable. | Medium for browser/query automation; API/bulk contract and rate limits are unverified. | Entity-specific fields, stable endpoints, automation permissions, cadence, terms and registered-office privacy. | +| `ba.bhas.statistics` | Agency for Statistics of BiH and entity statistical offices provide official agriculture/livestock/slaughter aggregates; national/entity dimensions must be preserved. | Medium–high for downloadable statistical tables after table IDs are pinned. | Exact slaughter/animal-use tables, entity coverage, API/download route, revisions, license and aggregate-only handling. | +| `ba.inspections-experiments` | FBiH veterinary/food inspectorates publish control plans and responsibilities; animal-experimentation facility data were not found as a stable public national register. | High for published reports/aggregates; low for facility/event extraction. | Entity/cantonal control events, stable IDs, outcomes, animal-use categories, privacy and retention. | + +## Compliance and ingestion notes + +- Record `state`, `entity`, `canton`, and `Brčko District` authority scope for every source; never merge records solely by name/address. +- Treat government and entity-level records as government-sourced evidence, not proof of current operation or project approval. +- Preserve source URL, retrieval timestamp, content hash, byte size, supplied publication/effective date, adapter/configuration version, and source values. +- Keep raw, parsed, normalized, enriched, reviewed, and released layers separate. Use synthetic fixtures only; do not commit rows or raw artifacts. +- Represent unavailable cadence, IDs, coordinates, licensing, and privacy decisions explicitly. A source outage or disappearance means “not observed,” not closure. +- Registered offices, farm addresses, permit locations and cantonal records are not automatically safe public facility locations; apply privacy and safety review before releasing addresses or coordinates. +- Adapters must fail closed on changed schemas, missing identifiers, duplicate cross-authority entities, suspicious count changes, and service failures, retaining the previous validated release. + +## Official evidence + +- [FBiH Federal Veterinary Inspectorate](https://fuzip.gov.ba/federalni-veterinarski-inspektorat/) +- [FBiH inspection mandate for agriculture, livestock and aquaculture](https://fuzip.gov.ba/unutrasnja-organizacija/federalni-poljoprivredni-inspektorat/) +- [FBiH veterinary/food control plans](https://fuzip.gov.ba/plan-sluzbenih-kontrola-farmi-imanja-goveda-ovaca-koza-i-dr-u-fbih-za-2025-godinu/) +- [BiH BIZREG federated business-register portal](https://bizreg.pravosudje.ba/pls/apex/f?p=186%3A%3A2313976059615753%3A%3ANO%3A%3A) +- [BIZREG explanation of entity/Brčko registers](https://bizreg.pravosudje.ba/pls/apex/f?p=186%3A20%3A4513352517506%3A%3ANO%3A%3AP20_SEKCIJA_TIP%3AKAKO_RADI) +- [Agency for Statistics of BiH](https://bhas.gov.ba/) + +## Effort and blockers + +Initial reconnaissance: approximately 4–7 engineering days to inventory state/entity/cantonal authorities and build source-discovery plus deterministic adapters for the strongest veterinary/statistical routes; 7–14 additional days for environmental permits, BIZREG integration and cross-authority reconciliation. Main blockers are fragmented authority, absence of a verified unified facility register, inconsistent entity/cantonal publishing, unverified public APIs, possible document-only routes, and unresolved licensing/privacy/coordinate semantics. + +Recommended next country: North Macedonia, subject to checking the existing country inventory before delegation. diff --git a/docs/source-status.json b/docs/source-status.json index ee06a4b..42285f4 100644 --- a/docs/source-status.json +++ b/docs/source-status.json @@ -169,6 +169,12 @@ {"source_id":"rs.environment-permits","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-rs.md","pipeline/source_registry.json"],"next_action":"Verify public environmental permit route, coverage, documents, geometry, license and privacy."}, {"source_id":"rs.apr.organizations","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-rs.md","pipeline/source_registry.json"],"next_action":"Pin authorized APR service, fees, quotas, cadence, fields and automation permissions."}, {"source_id":"rs.stat.statistics","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-rs.md","pipeline/source_registry.json"],"next_action":"Pin official slaughter/animal-use dataset IDs, APIs, cadence and revisions."}, - {"source_id":"rs.inspections-experiments","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-rs.md","pipeline/source_registry.json"],"next_action":"Locate a stable public facility/event route; keep sensitive evidence aggregate until verified."} + {"source_id":"rs.inspections-experiments","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-rs.md","pipeline/source_registry.json"],"next_action":"Locate a stable public facility/event route; keep sensitive evidence aggregate until verified."}, + {"source_id":"ba.veterinary.approved-food","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ba.md","pipeline/source_registry.json"],"next_action":"Map state/entity/cantonal coverage and pin establishment routes, IDs, cadence, terms and privacy."}, + {"source_id":"ba.farm-aquaculture","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ba.md","pipeline/source_registry.json"],"next_action":"Verify all-authority farm/aquaculture scope, exports, IDs, coordinates, terms and privacy."}, + {"source_id":"ba.environment-permits","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ba.md","pipeline/source_registry.json"],"next_action":"Verify entity environmental permit routes, coverage, documents, geometry, license and privacy."}, + {"source_id":"ba.bizreg.organizations","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ba.md","pipeline/source_registry.json"],"next_action":"Pin BIZREG endpoints, entity fields, rate limits, cadence, terms and automation permissions."}, + {"source_id":"ba.bhas.statistics","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ba.md","pipeline/source_registry.json"],"next_action":"Pin state/entity slaughter and animal-use table IDs, APIs, cadence and revisions."}, + {"source_id":"ba.inspections-experiments","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ba.md","pipeline/source_registry.json"],"next_action":"Locate cross-entity inspection/event routes; keep sensitive evidence aggregate until verified."} ] } diff --git a/pipeline/source_registry.json b/pipeline/source_registry.json index 9943867..0adf884 100644 --- a/pipeline/source_registry.json +++ b/pipeline/source_registry.json @@ -790,7 +790,12 @@ {"source_id":"rs.environment-permits","jurisdiction_scope":"Serbia; environmental permits and environmental authorization evidence","legacy_paths":[],"url":"https://www.sepa.gov.rs/","access_method":"official environmental agency systems, public notices and documents","cadence":"permit/document-specific","attribution_licensing_notes":"Verify public-read rights, document license, geometry and privacy.","adapter_status":"reference_only","expected_artifact_schema":"Permit/decision ID, authority, operator/site, activity, dates, status and optional reviewed geometry","blockers":["Stable public read/API/export route and complete national coverage are unresolved."]}, {"source_id":"rs.apr.organizations","jurisdiction_scope":"Serbia; APR centralized business-entity registers","legacy_paths":[],"url":"https://www.apr.gov.rs/registers/media/data-search.1728.html","access_method":"official browser and authorized web services; automated downloading restricted","cadence":"provider-defined","attribution_licensing_notes":"Identity linkage only; respect APR terms, fees, access restrictions and registered-office privacy.","adapter_status":"reference_only","expected_artifact_schema":"Registration ID, organization name, status, legal form, activity and registered office/address","blockers":["Pin authorized API/service, fees, quotas, cadence, fields and automation permissions."]}, {"source_id":"rs.stat.statistics","jurisdiction_scope":"Serbia; Statistical Office aggregate slaughter, livestock and animal-use statistics","legacy_paths":[],"url":"https://www.stat.gov.rs/","access_method":"official JSON/CSV statistical APIs and data portal","cadence":"table-specific; preserve revisions","attribution_licensing_notes":"Official statistics; verify table terms, citation requirements and aggregate-only scope.","adapter_status":"reference_only","expected_artifact_schema":"Aggregate multidimensional observations by species, measure, geography and period","blockers":["Pin current slaughter and animal-use dataset IDs, API contracts and cadence."]}, - {"source_id":"rs.inspections-experiments","jurisdiction_scope":"Serbia; veterinary inspections/enforcement and animal-experimentation evidence","legacy_paths":[],"url":"https://www.minpolj.gov.rs/","access_method":"official control plans, reports and statistical publications","cadence":"report/register-specific","attribution_licensing_notes":"Sensitive control/research evidence; aggregate or anonymize and require project review.","adapter_status":"reference_only","expected_artifact_schema":"Dated aggregate/event observations with authority, category, outcome, measure and source document","blockers":["No stable public facility-level inspection or animal-experimentation master verified."]} ] + {"source_id":"rs.inspections-experiments","jurisdiction_scope":"Serbia; veterinary inspections/enforcement and animal-experimentation evidence","legacy_paths":[],"url":"https://www.minpolj.gov.rs/","access_method":"official control plans, reports and statistical publications","cadence":"report/register-specific","attribution_licensing_notes":"Sensitive control/research evidence; aggregate or anonymize and require project review.","adapter_status":"reference_only","expected_artifact_schema":"Dated aggregate/event observations with authority, category, outcome, measure and source document","blockers":["No stable public facility-level inspection or animal-experimentation master verified."]} ,{"source_id":"ba.veterinary.approved-food","jurisdiction_scope":"Bosnia and Herzegovina; state/entity veterinary and food authorities, including FBiH and cantonal controls","legacy_paths":[],"url":"https://fuzip.gov.ba/federalni-veterinarski-inspektorat/","access_method":"state/entity veterinary registers, inspection portals and official documents","cadence":"authority-specific; timestamp retrieval","attribution_licensing_notes":"Official but fragmented authorities; verify each source license, attribution, privacy and coordinates.","adapter_status":"reference_only","expected_artifact_schema":"Authority-scope, approval/registration ID, operator/site, activities, status, address and optional coordinates","blockers":["No unified national establishment export verified; map FBiH, RS, Brčko and cantonal coverage first."]}, + {"source_id":"ba.farm-aquaculture","jurisdiction_scope":"Bosnia and Herzegovina; livestock holdings, feed and aquaculture across state/entity/cantonal authorities","legacy_paths":[],"url":"https://fuzip.gov.ba/unutrasnja-organizacija/federalni-poljoprivredni-inspektorat/","access_method":"entity/cantonal agriculture, veterinary, fisheries and official open-data/document resources","cadence":"authority/resource-specific","attribution_licensing_notes":"Verify authority, license, animal-holder/property privacy and coordinate precision by entity.","adapter_status":"reference_only","expected_artifact_schema":"Authority-scope, holding/site or aquaculture permit record with stable ID, activity/status and optional location","blockers":["All-authority farm/aquaculture export and cross-entity identifiers are unresolved."]}, + {"source_id":"ba.environment-permits","jurisdiction_scope":"Bosnia and Herzegovina; state/entity environmental permits and integrated controls","legacy_paths":[],"url":"https://fmoit.gov.ba/","access_method":"entity environmental ministries/agencies, public notices and documents","cadence":"permit/document-specific","attribution_licensing_notes":"Verify public-read rights, document license, geometry, sensitive sites and entity coverage.","adapter_status":"reference_only","expected_artifact_schema":"Authority-scope, permit/decision ID, operator/site, activity, dates, status and optional reviewed geometry","blockers":["No unified public read/API/export route or complete cross-entity coverage verified."]}, + {"source_id":"ba.bizreg.organizations","jurisdiction_scope":"Bosnia and Herzegovina; BIZREG registers for Federation BiH, Republika Srpska and Brčko District","legacy_paths":[],"url":"https://bizreg.pravosudje.ba/pls/apex/f?p=186%3A%3A2313976059615753%3A%3ANO%3A%3A","access_method":"official searchable web portal; API/bulk route unverified","cadence":"portal-defined; last-update field displayed","attribution_licensing_notes":"Identity linkage only; verify terms, automation permissions, fields and registered-office privacy.","adapter_status":"reference_only","expected_artifact_schema":"Authority/register scope, RNS, name, status, legal form, activity and registered office/address","blockers":["Pin stable endpoints, query contract, rate limits, cadence and entity-specific field semantics."]}, + {"source_id":"ba.bhas.statistics","jurisdiction_scope":"Bosnia and Herzegovina; state/entity official slaughter, livestock and animal-use aggregates","legacy_paths":[],"url":"https://bhas.gov.ba/","access_method":"Agency for Statistics of BiH and entity statistical tables/data services","cadence":"table/authority-specific; preserve revisions","attribution_licensing_notes":"Official statistics; verify table terms, citation, entity coverage and aggregate-only scope.","adapter_status":"reference_only","expected_artifact_schema":"Authority/entity, aggregate multidimensional observations by species, measure, geography and period","blockers":["Pin state/entity slaughter and animal-use table IDs, APIs/downloads and cadence."]}, + {"source_id":"ba.inspections-experiments","jurisdiction_scope":"Bosnia and Herzegovina; veterinary/food inspections, enforcement and animal-experimentation evidence","legacy_paths":[],"url":"https://fuzip.gov.ba/kontrolne-liste/","access_method":"state/entity/cantonal control plans, inspection reports and statistical publications","cadence":"report/authority-specific","attribution_licensing_notes":"Sensitive control/research evidence; aggregate or anonymize and require project review.","adapter_status":"reference_only","expected_artifact_schema":"Authority-scope, dated aggregate/event observation, category, outcome, measure and source document","blockers":["No stable public cross-entity facility/event or animal-experimentation master verified."]} ] } @@ -808,3 +813,5 @@ + + diff --git a/pipeline/tests/test_bosnia_recon_metadata.py b/pipeline/tests/test_bosnia_recon_metadata.py new file mode 100644 index 0000000..c2ba2df --- /dev/null +++ b/pipeline/tests/test_bosnia_recon_metadata.py @@ -0,0 +1,16 @@ +import json +import unittest +from pathlib import Path +ROOT=Path(__file__).resolve().parents[2] +class BosniaReconMetadataTests(unittest.TestCase): + def test_row_free_bosnia_recon_is_present(self): + text=(ROOT/"docs"/"country-recon-ba.md").read_text(encoding="utf-8") + self.assertIn("row-free",text); self.assertIn("fully automated",text); self.assertIn("state/entity/cantonal",text) + self.assertIn("Recommended next country: North Macedonia",text) + def test_bosnia_sources_have_conservative_status(self): + registry=json.loads((ROOT/"pipeline"/"source_registry.json").read_text(encoding="utf-8")); status=json.loads((ROOT/"docs"/"source-status.json").read_text(encoding="utf-8")) + ids={s["source_id"] for s in registry["sources"] if s["source_id"].startswith("ba.")} + self.assertEqual(ids,{s["source_id"] for s in status["sources"] if s["source_id"].startswith("ba.")}); self.assertEqual(len(ids),6) + for s in status["sources"]: + if s["source_id"].startswith("ba."): self.assertEqual(s["publication_eligibility"],"blocked"); self.assertNotEqual(s["runtime_health"],"healthy") +if __name__ == "__main__": unittest.main() diff --git a/pipeline/tests/test_source_registry.py b/pipeline/tests/test_source_registry.py index dbc8264..67fce41 100644 --- a/pipeline/tests/test_source_registry.py +++ b/pipeline/tests/test_source_registry.py @@ -9,8 +9,8 @@ class SourceRegistryTests(unittest.TestCase): def test_repository_registry_loads_and_references_existing_legacy_paths(self): registry = load_registry() - self.assertEqual(len(registry["sources"]), 161) - self.assertEqual(len({source["source_id"] for source in registry["sources"]}), 161) + self.assertEqual(len(registry["sources"]), 167) + self.assertEqual(len({source["source_id"] for source in registry["sources"]}), 167) def test_unknowns_are_explicit(self): registry = json.loads(REGISTRY_PATH.read_text(encoding="utf-8")) From f356d6d436a7794de0e210c0753dfa43f6765c35 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Wed, 16 Sep 2026 10:11:37 -0700 Subject: [PATCH 158/311] docs: add North Macedonia source reconnaissance --- docs/country-recon-mk.md | 43 +++++++++++++++++++ docs/source-status.json | 8 +++- pipeline/source_registry.json | 9 +++- .../test_north_macedonia_recon_metadata.py | 15 +++++++ pipeline/tests/test_source_registry.py | 4 +- 5 files changed, 75 insertions(+), 4 deletions(-) create mode 100644 docs/country-recon-mk.md create mode 100644 pipeline/tests/test_north_macedonia_recon_metadata.py diff --git a/docs/country-recon-mk.md b/docs/country-recon-mk.md new file mode 100644 index 0000000..408ac7c --- /dev/null +++ b/docs/country-recon-mk.md @@ -0,0 +1,43 @@ +# North Macedonia source reconnaissance + +Status: row-free reconnaissance only; no facility rows, raw exports, or production ingestion artifacts are committed. + +## Scope and automation requirement + +North Macedonia has a comparatively clear national Food and Veterinary Agency (FVA) register surface, alongside environmental, statistical, open-data and Central Registry systems. The production pipeline must be fully automated from source discovery/scraping through acquisition, validation, normalization, provenance, quarantine, and ingestion. This reconnaissance does not authorize publication or replace maintainer review. + +## Strongest candidates + +| Source | Verified surface | Automation assessment | Main unresolved items | +| --- | --- | --- | --- | +| `mk.fva.approved-food` | FVA lists approved animal-origin establishments under Regulation 853/2004, including meat, fish, dairy, egg, honey and other categories, plus export-approved and registered operators. | Medium: register pages link documents hosted through Google Drive; scheduled discovery/download is plausible once stable file IDs are pinned. | Direct file IDs, schema, update cadence, stable approval IDs, addresses/coordinates, license and privacy. | +| `mk.fva.farms-aquaculture` | FVA animal-health page lists slaughterhouses, livestock markets, controlled pig holdings and related animal registers; fisheries/aquaculture coverage needs route confirmation. | Medium for linked files; low–medium for interactive or incomplete registers. | Farm/aquaculture export, IDs, coordinates, cadence, terms and privacy. | +| `mk.environment-permits` | Ministry of Environment and Physical Planning is the relevant national authority for environmental permissions and public notices; machine-readable permit route was not pinned. | Medium after API/export confirmation; otherwise document extraction. | Permit API/export, facility coverage, IDs, geometry, license and privacy. | +| `mk.crm.organizations` | Central Registry provides an online distribution system for legal-entity current/historical status and electronic confirmations; terms restrict commercial reproduction without consent. | Medium for authorized paid service; do not scrape or republish outside permitted use. | API/service contract, fees, commercial permission, fields, cadence and registered-office privacy. | +| `mk.stat.statistics` | State Statistical Office publishes official agriculture/livestock and slaughter aggregates and data services. | Medium–high for aggregate tables after table IDs and formats are pinned. | Exact slaughter/animal-use tables, API/download route, cadence, revisions and license. | +| `mk.fva.inspections-experiments` | FVA explicitly lists a register of institutions breeding/supplying experimental animals and user institutions conducting experiments, alongside official control registers. | Medium if linked register files are stable; high privacy sensitivity. | Current file/API, fields, stable IDs, inspection outcomes, animal-use categories, privacy and retention. | + +## Compliance and ingestion notes + +- Treat FVA and ministry data as government-sourced evidence, not proof of current operation or project approval. +- Preserve source URL, retrieval timestamp, content hash, byte size, supplied publication/effective date, adapter/configuration version, and source values. +- Keep raw, parsed, normalized, enriched, reviewed, and released layers separate. Use synthetic fixtures only; do not commit rows or raw artifacts. +- Represent unavailable cadence, IDs, coordinates, licensing, and privacy decisions explicitly. A source outage or disappearance means “not observed,” not closure. +- Registered offices, farm addresses and permit locations are not automatically safe public facility locations; apply privacy and safety review before releasing addresses or coordinates. +- The Central Registry’s commercial-use restriction is a hard acquisition/publication gate; adapters must fail closed if authorization or terms are unclear. + +## Official evidence + +- [FVA animal-origin food registers](https://fva.gov.mk/mk/registri-hrana-zivotinsko-poteklo) +- [FVA approved establishments register](https://fva.gov.mk/mk/registar-odobreni-objekti) +- [FVA animal-health and welfare registers](https://fva.gov.mk/mk/zdravstvena-zastita-blagosostojba-zivotni-1) +- [Central Registry online distribution system](https://www.crm.com.mk/en/professional-users/lessors/access-to-data-via-the-online-distribution-system) +- [Central Registry commercial-use terms](https://www.crm.com.mk/en/professional-users/accountants) +- [State Statistical Office](https://www.stat.gov.mk/) +- [Ministry of Environment and Physical Planning](https://www.moepp.gov.mk/) + +## Effort and blockers + +Initial reconnaissance: approximately 2–4 engineering days to pin FVA document IDs and build deterministic register adapters; 4–8 additional days for environmental permits, animal-experimentation fields, statistical table mapping and authorized Central Registry integration. Main blockers are Google Drive-backed register links, undefined refresh cadence, unverified farm/aquaculture exports, environmental API uncertainty, Central Registry commercial-use restrictions/fees, and privacy/coordinate semantics. + +Recommended next country: Albania, subject to checking the existing country inventory before delegation. diff --git a/docs/source-status.json b/docs/source-status.json index 42285f4..bc32150 100644 --- a/docs/source-status.json +++ b/docs/source-status.json @@ -175,6 +175,12 @@ {"source_id":"ba.environment-permits","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ba.md","pipeline/source_registry.json"],"next_action":"Verify entity environmental permit routes, coverage, documents, geometry, license and privacy."}, {"source_id":"ba.bizreg.organizations","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ba.md","pipeline/source_registry.json"],"next_action":"Pin BIZREG endpoints, entity fields, rate limits, cadence, terms and automation permissions."}, {"source_id":"ba.bhas.statistics","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ba.md","pipeline/source_registry.json"],"next_action":"Pin state/entity slaughter and animal-use table IDs, APIs, cadence and revisions."}, - {"source_id":"ba.inspections-experiments","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ba.md","pipeline/source_registry.json"],"next_action":"Locate cross-entity inspection/event routes; keep sensitive evidence aggregate until verified."} + {"source_id":"ba.inspections-experiments","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ba.md","pipeline/source_registry.json"],"next_action":"Locate cross-entity inspection/event routes; keep sensitive evidence aggregate until verified."}, + {"source_id":"mk.fva.approved-food","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-mk.md","pipeline/source_registry.json"],"next_action":"Pin stable FVA/Drive files, schemas, cadence, IDs, terms and privacy."}, + {"source_id":"mk.fva.farms-aquaculture","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-mk.md","pipeline/source_registry.json"],"next_action":"Verify farm/aquaculture files, IDs, cadence, coordinates, terms and privacy."}, + {"source_id":"mk.environment-permits","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-mk.md","pipeline/source_registry.json"],"next_action":"Verify public environmental permit API/export, coverage, documents, geometry and privacy."}, + {"source_id":"mk.crm.organizations","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-mk.md","pipeline/source_registry.json"],"next_action":"Confirm authorized commercial access, fees, service contract, fields and privacy."}, + {"source_id":"mk.stat.statistics","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-mk.md","pipeline/source_registry.json"],"next_action":"Pin official slaughter/animal-use table IDs, APIs, cadence and revisions."}, + {"source_id":"mk.fva.inspections-experiments","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-mk.md","pipeline/source_registry.json"],"next_action":"Pin experimentation register/API, fields, IDs, privacy and retention policy."} ] } diff --git a/pipeline/source_registry.json b/pipeline/source_registry.json index 0adf884..8c1cfd2 100644 --- a/pipeline/source_registry.json +++ b/pipeline/source_registry.json @@ -795,7 +795,12 @@ {"source_id":"ba.environment-permits","jurisdiction_scope":"Bosnia and Herzegovina; state/entity environmental permits and integrated controls","legacy_paths":[],"url":"https://fmoit.gov.ba/","access_method":"entity environmental ministries/agencies, public notices and documents","cadence":"permit/document-specific","attribution_licensing_notes":"Verify public-read rights, document license, geometry, sensitive sites and entity coverage.","adapter_status":"reference_only","expected_artifact_schema":"Authority-scope, permit/decision ID, operator/site, activity, dates, status and optional reviewed geometry","blockers":["No unified public read/API/export route or complete cross-entity coverage verified."]}, {"source_id":"ba.bizreg.organizations","jurisdiction_scope":"Bosnia and Herzegovina; BIZREG registers for Federation BiH, Republika Srpska and Brčko District","legacy_paths":[],"url":"https://bizreg.pravosudje.ba/pls/apex/f?p=186%3A%3A2313976059615753%3A%3ANO%3A%3A","access_method":"official searchable web portal; API/bulk route unverified","cadence":"portal-defined; last-update field displayed","attribution_licensing_notes":"Identity linkage only; verify terms, automation permissions, fields and registered-office privacy.","adapter_status":"reference_only","expected_artifact_schema":"Authority/register scope, RNS, name, status, legal form, activity and registered office/address","blockers":["Pin stable endpoints, query contract, rate limits, cadence and entity-specific field semantics."]}, {"source_id":"ba.bhas.statistics","jurisdiction_scope":"Bosnia and Herzegovina; state/entity official slaughter, livestock and animal-use aggregates","legacy_paths":[],"url":"https://bhas.gov.ba/","access_method":"Agency for Statistics of BiH and entity statistical tables/data services","cadence":"table/authority-specific; preserve revisions","attribution_licensing_notes":"Official statistics; verify table terms, citation, entity coverage and aggregate-only scope.","adapter_status":"reference_only","expected_artifact_schema":"Authority/entity, aggregate multidimensional observations by species, measure, geography and period","blockers":["Pin state/entity slaughter and animal-use table IDs, APIs/downloads and cadence."]}, - {"source_id":"ba.inspections-experiments","jurisdiction_scope":"Bosnia and Herzegovina; veterinary/food inspections, enforcement and animal-experimentation evidence","legacy_paths":[],"url":"https://fuzip.gov.ba/kontrolne-liste/","access_method":"state/entity/cantonal control plans, inspection reports and statistical publications","cadence":"report/authority-specific","attribution_licensing_notes":"Sensitive control/research evidence; aggregate or anonymize and require project review.","adapter_status":"reference_only","expected_artifact_schema":"Authority-scope, dated aggregate/event observation, category, outcome, measure and source document","blockers":["No stable public cross-entity facility/event or animal-experimentation master verified."]} ] + {"source_id":"ba.inspections-experiments","jurisdiction_scope":"Bosnia and Herzegovina; veterinary/food inspections, enforcement and animal-experimentation evidence","legacy_paths":[],"url":"https://fuzip.gov.ba/kontrolne-liste/","access_method":"state/entity/cantonal control plans, inspection reports and statistical publications","cadence":"report/authority-specific","attribution_licensing_notes":"Sensitive control/research evidence; aggregate or anonymize and require project review.","adapter_status":"reference_only","expected_artifact_schema":"Authority-scope, dated aggregate/event observation, category, outcome, measure and source document","blockers":["No stable public cross-entity facility/event or animal-experimentation master verified."]} ,{"source_id":"mk.fva.approved-food","jurisdiction_scope":"North Macedonia; FVA approved and registered food establishments handling animal-origin products","legacy_paths":[],"url":"https://fva.gov.mk/mk/registri-hrana-zivotinsko-poteklo","access_method":"official FVA register pages with linked Google Drive documents","cadence":"unknown; timestamp retrieval","attribution_licensing_notes":"Official authority; verify file terms, attribution, privacy and coordinates before acquisition.","adapter_status":"reference_only","expected_artifact_schema":"Approval/registration ID, operator/site, category, status, address and optional coordinates","blockers":["Pin stable Drive file IDs, schemas, cadence, IDs, terms and privacy."]}, + {"source_id":"mk.fva.farms-aquaculture","jurisdiction_scope":"North Macedonia; FVA livestock, slaughterhouse, holding and aquaculture-related evidence","legacy_paths":[],"url":"https://fva.gov.mk/mk/zdravstvena-zastita-blagosostojba-zivotni-1","access_method":"official FVA registers and linked documents","cadence":"resource-specific","attribution_licensing_notes":"Verify publisher, license, animal-holder/property privacy and coordinate precision.","adapter_status":"reference_only","expected_artifact_schema":"Holding/site or aquaculture/slaughter record with stable ID, activity/status and optional location","blockers":["Verify current farm/aquaculture files, IDs, cadence, terms and privacy."]}, + {"source_id":"mk.environment-permits","jurisdiction_scope":"North Macedonia; environmental permits and physical-planning authorizations","legacy_paths":[],"url":"https://www.moepp.gov.mk/","access_method":"official ministry portals, public notices and documents","cadence":"permit/document-specific","attribution_licensing_notes":"Verify public-read rights, document license, geometry and privacy.","adapter_status":"reference_only","expected_artifact_schema":"Permit/decision ID, operator/site, activity, dates, status and optional reviewed geometry","blockers":["Stable public read/API/export route and complete facility coverage are unresolved."]}, + {"source_id":"mk.crm.organizations","jurisdiction_scope":"North Macedonia; Central Registry legal-entity records","legacy_paths":[],"url":"https://www.crm.com.mk/en/professional-users/lessors/access-to-data-via-the-online-distribution-system","access_method":"authorized online distribution system; paid/prepaid services","cadence":"service-defined; current/historical products","attribution_licensing_notes":"Central Registry terms restrict commercial reproduction/modification without prior consent; identity linkage only.","adapter_status":"reference_only","expected_artifact_schema":"Entity identifier, legal name, current/historical status, activity and registered office/address","blockers":["Confirm commercial permission, fees, API/service contract, quotas, fields and privacy."]}, + {"source_id":"mk.stat.statistics","jurisdiction_scope":"North Macedonia; official aggregate slaughter, livestock and animal-use statistics","legacy_paths":[],"url":"https://www.stat.gov.mk/","access_method":"State Statistical Office tables/data services; exact route unverified","cadence":"table-specific; preserve revisions","attribution_licensing_notes":"Official statistics; verify table terms, citation requirements and aggregate-only scope.","adapter_status":"reference_only","expected_artifact_schema":"Aggregate multidimensional observations by species, measure, geography and period","blockers":["Pin current slaughter and animal-use table IDs, API/download contracts and cadence."]}, + {"source_id":"mk.fva.inspections-experiments","jurisdiction_scope":"North Macedonia; FVA inspections/enforcement and animal-experimentation institutions","legacy_paths":[],"url":"https://fva.gov.mk/mk/zdravstvena-zastita-blagosostojba-zivotni-1","access_method":"official FVA registers and linked documents","cadence":"resource-specific","attribution_licensing_notes":"Sensitive control/research evidence; aggregate or anonymize and require project review.","adapter_status":"reference_only","expected_artifact_schema":"Dated institution/aggregate/event observation with authority, category, outcome and source document","blockers":["Pin current register file/API, fields, IDs, privacy and retention policy."]} ] } @@ -815,3 +820,5 @@ + + diff --git a/pipeline/tests/test_north_macedonia_recon_metadata.py b/pipeline/tests/test_north_macedonia_recon_metadata.py new file mode 100644 index 0000000..5e34cce --- /dev/null +++ b/pipeline/tests/test_north_macedonia_recon_metadata.py @@ -0,0 +1,15 @@ +import json +import unittest +from pathlib import Path +ROOT=Path(__file__).resolve().parents[2] +class NorthMacedoniaReconMetadataTests(unittest.TestCase): + def test_row_free_north_macedonia_recon_is_present(self): + text=(ROOT/"docs"/"country-recon-mk.md").read_text(encoding="utf-8") + self.assertIn("row-free",text); self.assertIn("fully automated",text); self.assertIn("Google Drive",text); self.assertIn("Recommended next country: Albania",text) + def test_north_macedonia_sources_have_conservative_status(self): + registry=json.loads((ROOT/"pipeline"/"source_registry.json").read_text(encoding="utf-8")); status=json.loads((ROOT/"docs"/"source-status.json").read_text(encoding="utf-8")) + ids={s["source_id"] for s in registry["sources"] if s["source_id"].startswith("mk.")} + self.assertEqual(ids,{s["source_id"] for s in status["sources"] if s["source_id"].startswith("mk.")}); self.assertEqual(len(ids),6) + for s in status["sources"]: + if s["source_id"].startswith("mk."): self.assertEqual(s["publication_eligibility"],"blocked"); self.assertNotEqual(s["runtime_health"],"healthy") +if __name__ == "__main__": unittest.main() diff --git a/pipeline/tests/test_source_registry.py b/pipeline/tests/test_source_registry.py index 67fce41..ed22863 100644 --- a/pipeline/tests/test_source_registry.py +++ b/pipeline/tests/test_source_registry.py @@ -9,8 +9,8 @@ class SourceRegistryTests(unittest.TestCase): def test_repository_registry_loads_and_references_existing_legacy_paths(self): registry = load_registry() - self.assertEqual(len(registry["sources"]), 167) - self.assertEqual(len({source["source_id"] for source in registry["sources"]}), 167) + self.assertEqual(len(registry["sources"]), 173) + self.assertEqual(len({source["source_id"] for source in registry["sources"]}), 173) def test_unknowns_are_explicit(self): registry = json.loads(REGISTRY_PATH.read_text(encoding="utf-8")) From 9c6c613d933e8e84776cca7b27cee5812799c175 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Wed, 16 Sep 2026 10:14:07 -0700 Subject: [PATCH 159/311] docs: add Albania source reconnaissance --- docs/country-recon-al.md | 43 +++++++++++++++++++ docs/source-status.json | 8 +++- pipeline/source_registry.json | 9 +++- pipeline/tests/test_albania_recon_metadata.py | 15 +++++++ pipeline/tests/test_source_registry.py | 4 +- 5 files changed, 75 insertions(+), 4 deletions(-) create mode 100644 docs/country-recon-al.md create mode 100644 pipeline/tests/test_albania_recon_metadata.py diff --git a/docs/country-recon-al.md b/docs/country-recon-al.md new file mode 100644 index 0000000..05889fe --- /dev/null +++ b/docs/country-recon-al.md @@ -0,0 +1,43 @@ +# Albania source reconnaissance + +Status: row-free reconnaissance only; no facility rows, raw exports, or production ingestion artifacts are committed. + +## Scope and automation requirement + +Albania has national authority surfaces through the National Food Authority (AKU), National Business Center (QKB), environmental authority, INSTAT, and the government open-data ecosystem. The production pipeline must be fully automated from source discovery/scraping through acquisition, validation, normalization, provenance, quarantine, and ingestion. This reconnaissance does not authorize publication or replace maintainer review. + +## Strongest candidates + +| Source | Verified surface | Automation assessment | Main unresolved items | +| --- | --- | --- | --- | +| `al.aku.approved-food` | AKU publishes registers for approved animal-origin food establishments, EU-export establishments, registered retail, primary producers and dairy-farm risk categorization. | Medium: official pages expose downloadable/linked content, but direct file routes and formats need pinning. | Stable file IDs, schema, update cadence, approval IDs, addresses/coordinates, license and privacy. | +| `al.aku.farms-aquaculture` | AKU animal and veterinary surfaces cover primary producers and animal establishments; agriculture/fisheries authority routes are needed for aquaculture permits. | Medium for linked files; low–medium for undocumented interactive registers. | Complete farm/aquaculture coverage, identifiers, coordinates, cadence, terms and privacy. | +| `al.environment-permits` | National environmental permit/licence publication is integrated with the QKB permits/licences/authorizations surface; environmental authority routes remain relevant for permits outside QKB. | Medium after direct query/export contract is verified. | Complete facility coverage, API/export, permit IDs, documents, geometry, license and privacy. | +| `al.qkb.organizations` | QKB exposes Business Register and permits/licences/authorizations search; official guidance says registered data are publicly accessible except restricted personal data, including individuals’ addresses. | Medium–high for authorized public search/API if available; avoid personal-address fields. | API/bulk route, rate limits, cadence, stable identifiers, terms and legal-entity/site matching. | +| `al.instat.statistics` | INSTAT publishes official agriculture, livestock and slaughter statistics and statistical data services. | Medium–high for aggregate tables after table IDs/API routes are pinned. | Exact slaughter/animal-use tables, formats, cadence, revisions, license and aggregate-only handling. | +| `al.aku.inspections-experiments` | AKU animal-welfare page lists a register of institutions breeding/supplying experimental animals and institutions conducting experiments, alongside slaughterhouse and veterinary registers. | Medium if linked files are stable; high privacy sensitivity. | Current file/API, fields, IDs, inspection outcomes, animal-use categories, privacy and retention. | + +## Compliance and ingestion notes + +- Treat AKU, QKB, environmental and statistical sources as government-sourced evidence, not proof of current operation or project approval. +- Preserve source URL, retrieval timestamp, content hash, byte size, supplied publication/effective date, adapter/configuration version, and source values. +- Keep raw, parsed, normalized, enriched, reviewed, and released layers separate. Use synthetic fixtures only; do not commit rows or raw artifacts. +- Represent unavailable cadence, IDs, coordinates, licensing, and privacy decisions explicitly. A source outage or disappearance means “not observed,” not closure. +- Registered offices, farm addresses and permit locations are not automatically safe public facility locations; remove or restrict personal-address material and apply privacy/safety review. +- Adapters must fail closed on changed schemas, missing identifiers, personal-address exposure, suspicious count changes, authentication/captcha, and service failures. + +## Official evidence + +- [AKU animal-origin food registers](https://aku.gov.al/) +- [AKU animal health and welfare registers](https://aku.gov.al/) +- [QKB Business Register and services](https://qkb.gov.al/en/home-3/) +- [QKB permits, licences and authorizations register](https://qkb.gov.al/en/permits-licenses-authorizations/) +- [QKB privacy policy](https://qkb.gov.al/en/privacy-policy/) +- [INSTAT](https://www.instat.gov.al/en/) +- [Albanian environmental authority](https://akm.gov.al/) + +## Effort and blockers + +Initial reconnaissance: approximately 2–4 engineering days to pin AKU file routes and QKB public/API access, then build deterministic acquisition and validation adapters; 4–8 additional days for environmental permits, aquaculture, experimentation registers and statistical table mapping. Main blockers are undocumented AKU download routes, incomplete aquaculture coverage, uncertain environmental API/export, QKB automation/rate limits, exact INSTAT table IDs, and licensing/privacy/coordinate semantics. + +Recommended next country: Kosovo, subject to checking the existing country inventory before delegation. diff --git a/docs/source-status.json b/docs/source-status.json index bc32150..cfae25d 100644 --- a/docs/source-status.json +++ b/docs/source-status.json @@ -181,6 +181,12 @@ {"source_id":"mk.environment-permits","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-mk.md","pipeline/source_registry.json"],"next_action":"Verify public environmental permit API/export, coverage, documents, geometry and privacy."}, {"source_id":"mk.crm.organizations","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-mk.md","pipeline/source_registry.json"],"next_action":"Confirm authorized commercial access, fees, service contract, fields and privacy."}, {"source_id":"mk.stat.statistics","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-mk.md","pipeline/source_registry.json"],"next_action":"Pin official slaughter/animal-use table IDs, APIs, cadence and revisions."}, - {"source_id":"mk.fva.inspections-experiments","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-mk.md","pipeline/source_registry.json"],"next_action":"Pin experimentation register/API, fields, IDs, privacy and retention policy."} + {"source_id":"mk.fva.inspections-experiments","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-mk.md","pipeline/source_registry.json"],"next_action":"Pin experimentation register/API, fields, IDs, privacy and retention policy."}, + {"source_id":"al.aku.approved-food","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-al.md","pipeline/source_registry.json"],"next_action":"Pin stable AKU files, schemas, cadence, IDs, terms and privacy."}, + {"source_id":"al.aku.farms-aquaculture","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-al.md","pipeline/source_registry.json"],"next_action":"Verify farm/aquaculture coverage, exports, IDs, coordinates, terms and privacy."}, + {"source_id":"al.environment-permits","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-al.md","pipeline/source_registry.json"],"next_action":"Verify public environmental permit route, coverage, documents, geometry and privacy."}, + {"source_id":"al.qkb.organizations","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-al.md","pipeline/source_registry.json"],"next_action":"Pin QKB API/search route, rate limits, fields, terms and personal-address suppression."}, + {"source_id":"al.instat.statistics","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-al.md","pipeline/source_registry.json"],"next_action":"Pin official slaughter/animal-use table IDs, APIs, cadence and revisions."}, + {"source_id":"al.aku.inspections-experiments","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-al.md","pipeline/source_registry.json"],"next_action":"Pin experimentation register/API, fields, IDs, privacy and retention policy."} ] } diff --git a/pipeline/source_registry.json b/pipeline/source_registry.json index 8c1cfd2..c13fd95 100644 --- a/pipeline/source_registry.json +++ b/pipeline/source_registry.json @@ -800,7 +800,12 @@ {"source_id":"mk.environment-permits","jurisdiction_scope":"North Macedonia; environmental permits and physical-planning authorizations","legacy_paths":[],"url":"https://www.moepp.gov.mk/","access_method":"official ministry portals, public notices and documents","cadence":"permit/document-specific","attribution_licensing_notes":"Verify public-read rights, document license, geometry and privacy.","adapter_status":"reference_only","expected_artifact_schema":"Permit/decision ID, operator/site, activity, dates, status and optional reviewed geometry","blockers":["Stable public read/API/export route and complete facility coverage are unresolved."]}, {"source_id":"mk.crm.organizations","jurisdiction_scope":"North Macedonia; Central Registry legal-entity records","legacy_paths":[],"url":"https://www.crm.com.mk/en/professional-users/lessors/access-to-data-via-the-online-distribution-system","access_method":"authorized online distribution system; paid/prepaid services","cadence":"service-defined; current/historical products","attribution_licensing_notes":"Central Registry terms restrict commercial reproduction/modification without prior consent; identity linkage only.","adapter_status":"reference_only","expected_artifact_schema":"Entity identifier, legal name, current/historical status, activity and registered office/address","blockers":["Confirm commercial permission, fees, API/service contract, quotas, fields and privacy."]}, {"source_id":"mk.stat.statistics","jurisdiction_scope":"North Macedonia; official aggregate slaughter, livestock and animal-use statistics","legacy_paths":[],"url":"https://www.stat.gov.mk/","access_method":"State Statistical Office tables/data services; exact route unverified","cadence":"table-specific; preserve revisions","attribution_licensing_notes":"Official statistics; verify table terms, citation requirements and aggregate-only scope.","adapter_status":"reference_only","expected_artifact_schema":"Aggregate multidimensional observations by species, measure, geography and period","blockers":["Pin current slaughter and animal-use table IDs, API/download contracts and cadence."]}, - {"source_id":"mk.fva.inspections-experiments","jurisdiction_scope":"North Macedonia; FVA inspections/enforcement and animal-experimentation institutions","legacy_paths":[],"url":"https://fva.gov.mk/mk/zdravstvena-zastita-blagosostojba-zivotni-1","access_method":"official FVA registers and linked documents","cadence":"resource-specific","attribution_licensing_notes":"Sensitive control/research evidence; aggregate or anonymize and require project review.","adapter_status":"reference_only","expected_artifact_schema":"Dated institution/aggregate/event observation with authority, category, outcome and source document","blockers":["Pin current register file/API, fields, IDs, privacy and retention policy."]} ] + {"source_id":"mk.fva.inspections-experiments","jurisdiction_scope":"North Macedonia; FVA inspections/enforcement and animal-experimentation institutions","legacy_paths":[],"url":"https://fva.gov.mk/mk/zdravstvena-zastita-blagosostojba-zivotni-1","access_method":"official FVA registers and linked documents","cadence":"resource-specific","attribution_licensing_notes":"Sensitive control/research evidence; aggregate or anonymize and require project review.","adapter_status":"reference_only","expected_artifact_schema":"Dated institution/aggregate/event observation with authority, category, outcome and source document","blockers":["Pin current register file/API, fields, IDs, privacy and retention policy."]} ,{"source_id":"al.aku.approved-food","jurisdiction_scope":"Albania; National Food Authority approved and registered animal-origin food establishments","legacy_paths":[],"url":"https://aku.gov.al/","access_method":"official AKU register pages and linked downloads","cadence":"unknown; timestamp retrieval","attribution_licensing_notes":"Official authority; verify file terms, attribution, privacy and coordinates before acquisition.","adapter_status":"reference_only","expected_artifact_schema":"Approval/registration ID, operator/site, category, status, address and optional coordinates","blockers":["Pin stable AKU file routes, schemas, cadence, IDs, terms and privacy."]}, + {"source_id":"al.aku.farms-aquaculture","jurisdiction_scope":"Albania; AKU primary producers, livestock establishments and agriculture/fisheries aquaculture evidence","legacy_paths":[],"url":"https://aku.gov.al/","access_method":"official AKU/agriculture/fisheries registers and linked documents","cadence":"resource-specific","attribution_licensing_notes":"Verify publisher, license, animal-holder/property privacy and coordinate precision.","adapter_status":"reference_only","expected_artifact_schema":"Holding/site or aquaculture record with stable ID, activity/status and optional location","blockers":["Verify complete farm/aquaculture coverage, exports, IDs, cadence, terms and privacy."]}, + {"source_id":"al.environment-permits","jurisdiction_scope":"Albania; environmental permits and authorizations","legacy_paths":[],"url":"https://akm.gov.al/","access_method":"environmental authority routes and QKB permit/licence register","cadence":"permit/document-specific","attribution_licensing_notes":"Verify public-read rights, document license, geometry and privacy.","adapter_status":"reference_only","expected_artifact_schema":"Permit/decision ID, authority, operator/site, activity, dates, status and optional reviewed geometry","blockers":["Stable public read/API/export and complete facility coverage are unresolved."]}, + {"source_id":"al.qkb.organizations","jurisdiction_scope":"Albania; National Business Center commercial and permit/licence registers","legacy_paths":[],"url":"https://qkb.gov.al/en/home-3/","access_method":"official public search/services; API/bulk route unverified","cadence":"provider-defined","attribution_licensing_notes":"Identity linkage only; QKB privacy policy limits personal/address disclosure and terms require verification.","adapter_status":"reference_only","expected_artifact_schema":"NIPT/entity ID, legal name, status, activity, registered office and permit/licence records","blockers":["Pin API/bulk route, rate limits, cadence, fields, terms and personal-address suppression."]}, + {"source_id":"al.instat.statistics","jurisdiction_scope":"Albania; INSTAT aggregate slaughter, livestock and animal-use statistics","legacy_paths":[],"url":"https://www.instat.gov.al/en/","access_method":"official statistical tables/data services; exact route unverified","cadence":"table-specific; preserve revisions","attribution_licensing_notes":"Official statistics; verify table terms, citation requirements and aggregate-only scope.","adapter_status":"reference_only","expected_artifact_schema":"Aggregate multidimensional observations by species, measure, geography and period","blockers":["Pin current slaughter and animal-use table IDs, API/download contracts and cadence."]}, + {"source_id":"al.aku.inspections-experiments","jurisdiction_scope":"Albania; AKU inspections/enforcement and experimental-animal institutions","legacy_paths":[],"url":"https://aku.gov.al/","access_method":"official AKU registers, control reports and linked documents","cadence":"resource/report-specific","attribution_licensing_notes":"Sensitive control/research evidence; aggregate or anonymize and require project review.","adapter_status":"reference_only","expected_artifact_schema":"Dated institution/aggregate/event observation with authority, category, outcome and source document","blockers":["Pin experimentation register/API, fields, IDs, privacy and retention policy."]} ] } @@ -820,5 +825,7 @@ + + diff --git a/pipeline/tests/test_albania_recon_metadata.py b/pipeline/tests/test_albania_recon_metadata.py new file mode 100644 index 0000000..58aeacc --- /dev/null +++ b/pipeline/tests/test_albania_recon_metadata.py @@ -0,0 +1,15 @@ +import json +import unittest +from pathlib import Path +ROOT=Path(__file__).resolve().parents[2] +class AlbaniaReconMetadataTests(unittest.TestCase): + def test_row_free_albania_recon_is_present(self): + text=(ROOT/"docs"/"country-recon-al.md").read_text(encoding="utf-8") + self.assertIn("row-free",text); self.assertIn("fully automated",text); self.assertIn("personal-address",text); self.assertIn("Recommended next country: Kosovo",text) + def test_albania_sources_have_conservative_status(self): + registry=json.loads((ROOT/"pipeline"/"source_registry.json").read_text(encoding="utf-8")); status=json.loads((ROOT/"docs"/"source-status.json").read_text(encoding="utf-8")) + ids={s["source_id"] for s in registry["sources"] if s["source_id"].startswith("al.")} + self.assertEqual(ids,{s["source_id"] for s in status["sources"] if s["source_id"].startswith("al.")}); self.assertEqual(len(ids),6) + for s in status["sources"]: + if s["source_id"].startswith("al."): self.assertEqual(s["publication_eligibility"],"blocked"); self.assertNotEqual(s["runtime_health"],"healthy") +if __name__ == "__main__": unittest.main() diff --git a/pipeline/tests/test_source_registry.py b/pipeline/tests/test_source_registry.py index ed22863..bba7118 100644 --- a/pipeline/tests/test_source_registry.py +++ b/pipeline/tests/test_source_registry.py @@ -9,8 +9,8 @@ class SourceRegistryTests(unittest.TestCase): def test_repository_registry_loads_and_references_existing_legacy_paths(self): registry = load_registry() - self.assertEqual(len(registry["sources"]), 173) - self.assertEqual(len({source["source_id"] for source in registry["sources"]}), 173) + self.assertEqual(len(registry["sources"]), 179) + self.assertEqual(len({source["source_id"] for source in registry["sources"]}), 179) def test_unknowns_are_explicit(self): registry = json.loads(REGISTRY_PATH.read_text(encoding="utf-8")) From ebee719d733eddde8a5bcff0e42146264c8db75c Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Wed, 16 Sep 2026 10:19:17 -0700 Subject: [PATCH 160/311] test: read recon metadata as UTF-8 --- pipeline/tests/test_czechia_recon_metadata.py | 4 ++-- pipeline/tests/test_latvia_recon_metadata.py | 4 ++-- pipeline/tests/test_lithuania_recon_metadata.py | 4 ++-- pipeline/tests/test_slovakia_recon_metadata.py | 4 ++-- pipeline/tests/test_slovenia_recon_metadata.py | 4 ++-- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/pipeline/tests/test_czechia_recon_metadata.py b/pipeline/tests/test_czechia_recon_metadata.py index 80bfcf4..97bb523 100644 --- a/pipeline/tests/test_czechia_recon_metadata.py +++ b/pipeline/tests/test_czechia_recon_metadata.py @@ -4,8 +4,8 @@ IDS={"cz.svs.approved-food","cz.svs.farms-aquaculture","cz.environment.permits","cz.statistics","cz.business-register"} class CzechiaReconMetadataTests(unittest.TestCase): def test_row_free_sources(self): - r=json.loads((ROOT/"pipeline/source_registry.json").read_text()); self.assertTrue(IDS.issubset({x["source_id"] for x in r["sources"]})); d=(ROOT/"docs/country-recon-cz.md").read_text(); self.assertIn("row-free",d); self.assertIn("fully automated",d) + r=json.loads((ROOT/"pipeline/source_registry.json").read_text(encoding="utf-8")); self.assertTrue(IDS.issubset({x["source_id"] for x in r["sources"]})); d=(ROOT/"docs/country-recon-cz.md").read_text(encoding="utf-8"); self.assertIn("row-free",d); self.assertIn("fully automated",d) def test_blocked_status(self): - s=json.loads((ROOT/"docs/source-status.json").read_text()); by={x["source_id"]:x for x in s["sources"]} + s=json.loads((ROOT/"docs/source-status.json").read_text(encoding="utf-8")); by={x["source_id"]:x for x in s["sources"]} for i in IDS: self.assertEqual(by[i]["publication_eligibility"],"blocked") if __name__=="__main__": unittest.main() diff --git a/pipeline/tests/test_latvia_recon_metadata.py b/pipeline/tests/test_latvia_recon_metadata.py index 645907f..c72fdab 100644 --- a/pipeline/tests/test_latvia_recon_metadata.py +++ b/pipeline/tests/test_latvia_recon_metadata.py @@ -4,8 +4,8 @@ IDS={"lv.pvd.approved-food","lv.ldc.slaughter-farms","lv.environment.permits","lv.stat.api","lv.animal-experiments","lv.ur.organizations"} class LatviaReconMetadataTests(unittest.TestCase): def test_row_free_sources(self): - r=json.loads((ROOT/"pipeline/source_registry.json").read_text()); self.assertTrue(IDS.issubset({x["source_id"] for x in r["sources"]})); d=(ROOT/"docs/country-recon-lv.md").read_text(); self.assertIn("row-free",d); self.assertIn("fully automated",d); self.assertNotIn("Approval ID |",d) + r=json.loads((ROOT/"pipeline/source_registry.json").read_text(encoding="utf-8")); self.assertTrue(IDS.issubset({x["source_id"] for x in r["sources"]})); d=(ROOT/"docs/country-recon-lv.md").read_text(encoding="utf-8"); self.assertIn("row-free",d); self.assertIn("fully automated",d); self.assertNotIn("Approval ID |",d) def test_blocked_status(self): - s=json.loads((ROOT/"docs/source-status.json").read_text()); by={x["source_id"]:x for x in s["sources"]} + s=json.loads((ROOT/"docs/source-status.json").read_text(encoding="utf-8")); by={x["source_id"]:x for x in s["sources"]} for i in IDS: self.assertEqual(by[i]["publication_eligibility"],"blocked") if __name__=="__main__": unittest.main() diff --git a/pipeline/tests/test_lithuania_recon_metadata.py b/pipeline/tests/test_lithuania_recon_metadata.py index d8caf4f..6c43dad 100644 --- a/pipeline/tests/test_lithuania_recon_metadata.py +++ b/pipeline/tests/test_lithuania_recon_metadata.py @@ -4,8 +4,8 @@ IDS={"lt.vmvt.approved-food","lt.vmvt.farm-aquaculture","lt.environment.permits","lt.animal-experiments","lt.vmvt.inspections","lt.statistics.slaughter","lt.jar.organizations"} class LithuaniaReconMetadataTests(unittest.TestCase): def test_row_free_sources(self): - r=json.loads((ROOT/"pipeline/source_registry.json").read_text()); self.assertTrue(IDS.issubset({x["source_id"] for x in r["sources"]})); d=(ROOT/"docs/country-recon-lt.md").read_text(); self.assertIn("row-free",d); self.assertIn("fully automated",d); self.assertNotIn("Approval ID |",d) + r=json.loads((ROOT/"pipeline/source_registry.json").read_text(encoding="utf-8")); self.assertTrue(IDS.issubset({x["source_id"] for x in r["sources"]})); d=(ROOT/"docs/country-recon-lt.md").read_text(encoding="utf-8"); self.assertIn("row-free",d); self.assertIn("fully automated",d); self.assertNotIn("Approval ID |",d) def test_blocked_status(self): - s=json.loads((ROOT/"docs/source-status.json").read_text()); by={x["source_id"]:x for x in s["sources"]} + s=json.loads((ROOT/"docs/source-status.json").read_text(encoding="utf-8")); by={x["source_id"]:x for x in s["sources"]} for i in IDS: self.assertEqual(by[i]["publication_eligibility"],"blocked") if __name__=="__main__": unittest.main() diff --git a/pipeline/tests/test_slovakia_recon_metadata.py b/pipeline/tests/test_slovakia_recon_metadata.py index 4bf66fc..537f18e 100644 --- a/pipeline/tests/test_slovakia_recon_metadata.py +++ b/pipeline/tests/test_slovakia_recon_metadata.py @@ -4,8 +4,8 @@ IDS={"sk.svps.approved-food","sk.svps.farms-aquaculture","sk.svps.inspections-experiments","sk.environment.permits","sk.statistics-corporate"} class SlovakiaReconMetadataTests(unittest.TestCase): def test_row_free_sources(self): - r=json.loads((ROOT/"pipeline/source_registry.json").read_text()); self.assertTrue(IDS.issubset({x["source_id"] for x in r["sources"]})); d=(ROOT/"docs/country-recon-sk.md").read_text(); self.assertIn("row-free",d); self.assertIn("fully automated",d) + r=json.loads((ROOT/"pipeline/source_registry.json").read_text(encoding="utf-8")); self.assertTrue(IDS.issubset({x["source_id"] for x in r["sources"]})); d=(ROOT/"docs/country-recon-sk.md").read_text(encoding="utf-8"); self.assertIn("row-free",d); self.assertIn("fully automated",d) def test_blocked_status(self): - s=json.loads((ROOT/"docs/source-status.json").read_text()); by={x["source_id"]:x for x in s["sources"]} + s=json.loads((ROOT/"docs/source-status.json").read_text(encoding="utf-8")); by={x["source_id"]:x for x in s["sources"]} for i in IDS: self.assertEqual(by[i]["publication_eligibility"],"blocked") if __name__=="__main__": unittest.main() diff --git a/pipeline/tests/test_slovenia_recon_metadata.py b/pipeline/tests/test_slovenia_recon_metadata.py index a17564a..08940ff 100644 --- a/pipeline/tests/test_slovenia_recon_metadata.py +++ b/pipeline/tests/test_slovenia_recon_metadata.py @@ -4,8 +4,8 @@ IDS={"si.uvhvvr.approved-food-feed","si.uvhvvr.farms-aquaculture","si.environment.permits","si.statistics-slaughter","si.corporate-register"} class SloveniaReconMetadataTests(unittest.TestCase): def test_row_free_sources(self): - r=json.loads((ROOT/"pipeline/source_registry.json").read_text()); self.assertTrue(IDS.issubset({x["source_id"] for x in r["sources"]})); d=(ROOT/"docs/country-recon-si.md").read_text(); self.assertIn("row-free",d); self.assertIn("fully automated",d) + r=json.loads((ROOT/"pipeline/source_registry.json").read_text(encoding="utf-8")); self.assertTrue(IDS.issubset({x["source_id"] for x in r["sources"]})); d=(ROOT/"docs/country-recon-si.md").read_text(encoding="utf-8"); self.assertIn("row-free",d); self.assertIn("fully automated",d) def test_blocked_status(self): - s=json.loads((ROOT/"docs/source-status.json").read_text()); by={x["source_id"]:x for x in s["sources"]} + s=json.loads((ROOT/"docs/source-status.json").read_text(encoding="utf-8")); by={x["source_id"]:x for x in s["sources"]} for i in IDS: self.assertEqual(by[i]["publication_eligibility"],"blocked") if __name__=="__main__": unittest.main() From a06ac179d46f5eb8abcb4e0328c48a1c319c64d7 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Wed, 16 Sep 2026 10:16:27 -0700 Subject: [PATCH 161/311] docs: add Kosovo source reconnaissance --- docs/country-recon-xk.md | 43 ++++++++++++++++++++ docs/source-status.json | 8 +++- pipeline/source_registry.json | 9 +++- pipeline/tests/test_kosovo_recon_metadata.py | 15 +++++++ pipeline/tests/test_source_registry.py | 4 +- 5 files changed, 75 insertions(+), 4 deletions(-) create mode 100644 docs/country-recon-xk.md create mode 100644 pipeline/tests/test_kosovo_recon_metadata.py diff --git a/docs/country-recon-xk.md b/docs/country-recon-xk.md new file mode 100644 index 0000000..2c9317f --- /dev/null +++ b/docs/country-recon-xk.md @@ -0,0 +1,43 @@ +# Kosovo source reconnaissance + +Status: row-free reconnaissance only; no facility rows, raw exports, or production ingestion artifacts are committed. + +## Scope and automation requirement + +Kosovo’s Food and Veterinary Agency (AUVK) provides the clearest national source surface in this round, with approved animal-origin businesses, slaughterhouses, fishponds, feed establishments, animal registration and experimental-animal resources. The production pipeline must be fully automated from source discovery/scraping through acquisition, validation, normalization, provenance, quarantine, and ingestion. This reconnaissance does not authorize publication or replace maintainer review. + +## Strongest candidates + +| Source | Verified surface | Automation assessment | Main unresolved items | +| --- | --- | --- | --- | +| `xk.auvk.approved-food` | AUVK publishes category-specific approved-business registers for meat processing/slaughter, cold stores, fish processing, dairy, eggs, honey and other animal-origin foods. | Medium: WordPress download pages are discoverable and can be scheduled, but direct files and freshness need pinning. | Stable file URLs, schema, update cadence, approval IDs, addresses/coordinates, license and privacy. | +| `xk.auvk.farms-aquaculture` | AUVK lists fishpond register resources and explains nationwide animal/holding registration and traceability; livestock sources are distributed through veterinary practices. | Medium for linked files; low–medium for incomplete or field/portal-driven holding data. | Current holding export, aquaculture file, IDs, coordinates, cadence, terms and privacy. | +| `xk.environment-permits` | Kosovo environmental authority/ministry publishes permitting and environmental documents; a stable machine-readable national permit API was not verified. | Medium after route confirmation; otherwise document extraction. | Public API/export, complete facility coverage, permit IDs, geometry, license and privacy. | +| `xk.arbk.organizations` | Kosovo Business Registration Agency (ARBK) maintains the Business Organizations Registry and has online/admin surfaces; current public bulk/API access was not verified. | Low–medium pending an authorized service contract; avoid scripted access to login-only surfaces. | Public API/query route, rate limits, cadence, identifiers, terms and registered-office privacy. | +| `xk.ask.statistics` | Kosovo Agency of Statistics publishes official agriculture/livestock/slaughter aggregates and data services. | Medium–high for aggregate tables after table IDs/routes are pinned. | Exact slaughter/animal-use tables, formats, cadence, revisions, license and aggregate-only handling. | +| `xk.auvk.inspections-experiments` | AUVK publishes official-control summaries and lists experimental-animal institutions among animal-health/welfare registers. | Medium for documents; high privacy sensitivity. | Current experimentation register, stable fields/IDs, inspection outcomes, privacy and retention. | + +## Compliance and ingestion notes + +- Treat AUVK, ARBK, environmental and statistical records as government-sourced evidence, not proof of current operation or project approval. +- Preserve source URL, retrieval timestamp, content hash, byte size, supplied publication/effective date, adapter/configuration version, and source values. +- Keep raw, parsed, normalized, enriched, reviewed, and released layers separate. Use synthetic fixtures only; do not commit rows or raw artifacts. +- Represent unavailable cadence, IDs, coordinates, licensing, and privacy decisions explicitly. A source outage or disappearance means “not observed,” not closure. +- Registered offices, farm addresses, permit locations and fishpond sites are not automatically safe public facility locations; apply privacy and safety review before releasing addresses or coordinates. +- Adapters must fail closed on changed schemas, stale files, missing identifiers, personal-address exposure, suspicious count changes, authentication, and service failures. + +## Official evidence + +- [AUVK approved animal-origin businesses](https://auvk.rks-gov.net/en/approved-businesses-for-food-of-animal-origin/) +- [AUVK animal health registers](https://auvk.rks-gov.net/shendeti-i-kafsheve/) +- [AUVK veterinary inspection responsibilities](https://auvk.rks-gov.net/kontrolli-i-brendshem/veterinar/) +- [AUVK business information/downloads](https://auvk.rks-gov.net/en/business-information/) +- [Kosovo Business Registration Agency](https://arbk.rks-gov.net/) +- [Kosovo Agency of Statistics](https://ask.rks-gov.net/) +- [Kosovo environmental authority](https://mmphi.rks-gov.net/) + +## Effort and blockers + +Initial reconnaissance: approximately 2–4 engineering days to pin AUVK download files and build deterministic approved-food/fishpond adapters; 4–8 additional days for holding data, environmental permits, ARBK access, experimentation registers and statistical table mapping. Main blockers are stale or undocumented AUVK file routes, incomplete holding exports, uncertain environmental/API coverage, ARBK access restrictions, exact ASK table IDs, and licensing/privacy/coordinate semantics. + +Recommended next country: Moldova, subject to checking the existing country inventory before delegation. diff --git a/docs/source-status.json b/docs/source-status.json index cfae25d..2caa10e 100644 --- a/docs/source-status.json +++ b/docs/source-status.json @@ -187,6 +187,12 @@ {"source_id":"al.environment-permits","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-al.md","pipeline/source_registry.json"],"next_action":"Verify public environmental permit route, coverage, documents, geometry and privacy."}, {"source_id":"al.qkb.organizations","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-al.md","pipeline/source_registry.json"],"next_action":"Pin QKB API/search route, rate limits, fields, terms and personal-address suppression."}, {"source_id":"al.instat.statistics","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-al.md","pipeline/source_registry.json"],"next_action":"Pin official slaughter/animal-use table IDs, APIs, cadence and revisions."}, - {"source_id":"al.aku.inspections-experiments","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-al.md","pipeline/source_registry.json"],"next_action":"Pin experimentation register/API, fields, IDs, privacy and retention policy."} + {"source_id":"al.aku.inspections-experiments","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-al.md","pipeline/source_registry.json"],"next_action":"Pin experimentation register/API, fields, IDs, privacy and retention policy."}, + {"source_id":"xk.auvk.approved-food","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-xk.md","pipeline/source_registry.json"],"next_action":"Pin current AUVK files, schemas, freshness, IDs, terms and privacy."}, + {"source_id":"xk.auvk.farms-aquaculture","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-xk.md","pipeline/source_registry.json"],"next_action":"Verify holding/fishpond files, IDs, cadence, coordinates, terms and privacy."}, + {"source_id":"xk.environment-permits","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-xk.md","pipeline/source_registry.json"],"next_action":"Verify public environmental permit route, coverage, documents, geometry and privacy."}, + {"source_id":"xk.arbk.organizations","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-xk.md","pipeline/source_registry.json"],"next_action":"Pin ARBK API/search route, auth, rate limits, cadence, terms and privacy."}, + {"source_id":"xk.ask.statistics","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-xk.md","pipeline/source_registry.json"],"next_action":"Pin official slaughter/animal-use table IDs, APIs, cadence and revisions."}, + {"source_id":"xk.auvk.inspections-experiments","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-xk.md","pipeline/source_registry.json"],"next_action":"Pin experimentation register/API, fields, IDs, privacy and retention policy."} ] } diff --git a/pipeline/source_registry.json b/pipeline/source_registry.json index c13fd95..f90b5b6 100644 --- a/pipeline/source_registry.json +++ b/pipeline/source_registry.json @@ -805,7 +805,12 @@ {"source_id":"al.environment-permits","jurisdiction_scope":"Albania; environmental permits and authorizations","legacy_paths":[],"url":"https://akm.gov.al/","access_method":"environmental authority routes and QKB permit/licence register","cadence":"permit/document-specific","attribution_licensing_notes":"Verify public-read rights, document license, geometry and privacy.","adapter_status":"reference_only","expected_artifact_schema":"Permit/decision ID, authority, operator/site, activity, dates, status and optional reviewed geometry","blockers":["Stable public read/API/export and complete facility coverage are unresolved."]}, {"source_id":"al.qkb.organizations","jurisdiction_scope":"Albania; National Business Center commercial and permit/licence registers","legacy_paths":[],"url":"https://qkb.gov.al/en/home-3/","access_method":"official public search/services; API/bulk route unverified","cadence":"provider-defined","attribution_licensing_notes":"Identity linkage only; QKB privacy policy limits personal/address disclosure and terms require verification.","adapter_status":"reference_only","expected_artifact_schema":"NIPT/entity ID, legal name, status, activity, registered office and permit/licence records","blockers":["Pin API/bulk route, rate limits, cadence, fields, terms and personal-address suppression."]}, {"source_id":"al.instat.statistics","jurisdiction_scope":"Albania; INSTAT aggregate slaughter, livestock and animal-use statistics","legacy_paths":[],"url":"https://www.instat.gov.al/en/","access_method":"official statistical tables/data services; exact route unverified","cadence":"table-specific; preserve revisions","attribution_licensing_notes":"Official statistics; verify table terms, citation requirements and aggregate-only scope.","adapter_status":"reference_only","expected_artifact_schema":"Aggregate multidimensional observations by species, measure, geography and period","blockers":["Pin current slaughter and animal-use table IDs, API/download contracts and cadence."]}, - {"source_id":"al.aku.inspections-experiments","jurisdiction_scope":"Albania; AKU inspections/enforcement and experimental-animal institutions","legacy_paths":[],"url":"https://aku.gov.al/","access_method":"official AKU registers, control reports and linked documents","cadence":"resource/report-specific","attribution_licensing_notes":"Sensitive control/research evidence; aggregate or anonymize and require project review.","adapter_status":"reference_only","expected_artifact_schema":"Dated institution/aggregate/event observation with authority, category, outcome and source document","blockers":["Pin experimentation register/API, fields, IDs, privacy and retention policy."]} ] + {"source_id":"al.aku.inspections-experiments","jurisdiction_scope":"Albania; AKU inspections/enforcement and experimental-animal institutions","legacy_paths":[],"url":"https://aku.gov.al/","access_method":"official AKU registers, control reports and linked documents","cadence":"resource/report-specific","attribution_licensing_notes":"Sensitive control/research evidence; aggregate or anonymize and require project review.","adapter_status":"reference_only","expected_artifact_schema":"Dated institution/aggregate/event observation with authority, category, outcome and source document","blockers":["Pin experimentation register/API, fields, IDs, privacy and retention policy."]} ,{"source_id":"xk.auvk.approved-food","jurisdiction_scope":"Kosovo; AUVK approved and registered animal-origin food establishments","legacy_paths":[],"url":"https://auvk.rks-gov.net/en/approved-businesses-for-food-of-animal-origin/","access_method":"official AUVK category pages and linked downloads","cadence":"publisher-defined; timestamp retrieval","attribution_licensing_notes":"Official authority; verify file terms, attribution, privacy and coordinates before acquisition.","adapter_status":"reference_only","expected_artifact_schema":"Approval/registration ID, operator/site, category, status, address and optional coordinates","blockers":["Pin current AUVK files, schemas, freshness, IDs, terms and privacy."]}, + {"source_id":"xk.auvk.farms-aquaculture","jurisdiction_scope":"Kosovo; AUVK livestock holdings, fishponds and aquaculture-related evidence","legacy_paths":[],"url":"https://auvk.rks-gov.net/shendeti-i-kafsheve/","access_method":"official AUVK registers, linked files and animal-registration guidance","cadence":"resource-specific","attribution_licensing_notes":"Verify publisher, license, animal-holder/property privacy and coordinate precision.","adapter_status":"reference_only","expected_artifact_schema":"Holding/site or fishpond/aquaculture record with stable ID, activity/status and optional location","blockers":["Verify current holding/fishpond files, IDs, cadence, terms and privacy."]}, + {"source_id":"xk.environment-permits","jurisdiction_scope":"Kosovo; environmental permits and authorizations","legacy_paths":[],"url":"https://mmphi.rks-gov.net/","access_method":"official ministry/environmental authority documents and public notices","cadence":"permit/document-specific","attribution_licensing_notes":"Verify public-read rights, document license, geometry and privacy.","adapter_status":"reference_only","expected_artifact_schema":"Permit/decision ID, authority, operator/site, activity, dates, status and optional reviewed geometry","blockers":["Stable public read/API/export and complete facility coverage are unresolved."]}, + {"source_id":"xk.arbk.organizations","jurisdiction_scope":"Kosovo; ARBK business organizations and corporate identifiers","legacy_paths":[],"url":"https://arbk.rks-gov.net/","access_method":"official business-register web/admin surface; public API/bulk route unverified","cadence":"provider-defined","attribution_licensing_notes":"Identity linkage only; verify automation permissions, terms, fields and registered-office privacy.","adapter_status":"reference_only","expected_artifact_schema":"Registration/fiscal ID, legal name, status, legal form, activity and registered office/address","blockers":["Pin public query/API, authentication, rate limits, cadence, terms and privacy."]}, + {"source_id":"xk.ask.statistics","jurisdiction_scope":"Kosovo; official aggregate slaughter, livestock and animal-use statistics","legacy_paths":[],"url":"https://ask.rks-gov.net/","access_method":"Kosovo Agency of Statistics tables/data services; exact route unverified","cadence":"table-specific; preserve revisions","attribution_licensing_notes":"Official statistics; verify table terms, citation requirements and aggregate-only scope.","adapter_status":"reference_only","expected_artifact_schema":"Aggregate multidimensional observations by species, measure, geography and period","blockers":["Pin current slaughter and animal-use table IDs, API/download contracts and cadence."]}, + {"source_id":"xk.auvk.inspections-experiments","jurisdiction_scope":"Kosovo; AUVK inspections/enforcement and experimental-animal institutions","legacy_paths":[],"url":"https://auvk.rks-gov.net/kontrolli-i-brendshem/veterinar/","access_method":"official AUVK control summaries, registers and linked documents","cadence":"resource/report-specific","attribution_licensing_notes":"Sensitive control/research evidence; aggregate or anonymize and require project review.","adapter_status":"reference_only","expected_artifact_schema":"Dated institution/aggregate/event observation with authority, category, outcome and source document","blockers":["Pin current experimentation register/API, fields, IDs, privacy and retention policy."]} ] } @@ -826,6 +831,8 @@ + + diff --git a/pipeline/tests/test_kosovo_recon_metadata.py b/pipeline/tests/test_kosovo_recon_metadata.py new file mode 100644 index 0000000..47fb34e --- /dev/null +++ b/pipeline/tests/test_kosovo_recon_metadata.py @@ -0,0 +1,15 @@ +import json +import unittest +from pathlib import Path +ROOT=Path(__file__).resolve().parents[2] +class KosovoReconMetadataTests(unittest.TestCase): + def test_row_free_kosovo_recon_is_present(self): + text=(ROOT/"docs"/"country-recon-xk.md").read_text(encoding="utf-8") + self.assertIn("row-free",text); self.assertIn("fully automated",text); self.assertIn("Recommended next country: Moldova",text) + def test_kosovo_sources_have_conservative_status(self): + registry=json.loads((ROOT/"pipeline"/"source_registry.json").read_text(encoding="utf-8")); status=json.loads((ROOT/"docs"/"source-status.json").read_text(encoding="utf-8")) + ids={s["source_id"] for s in registry["sources"] if s["source_id"].startswith("xk.")} + self.assertEqual(ids,{s["source_id"] for s in status["sources"] if s["source_id"].startswith("xk.")}); self.assertEqual(len(ids),6) + for s in status["sources"]: + if s["source_id"].startswith("xk."): self.assertEqual(s["publication_eligibility"],"blocked"); self.assertNotEqual(s["runtime_health"],"healthy") +if __name__ == "__main__": unittest.main() diff --git a/pipeline/tests/test_source_registry.py b/pipeline/tests/test_source_registry.py index bba7118..ebcf320 100644 --- a/pipeline/tests/test_source_registry.py +++ b/pipeline/tests/test_source_registry.py @@ -9,8 +9,8 @@ class SourceRegistryTests(unittest.TestCase): def test_repository_registry_loads_and_references_existing_legacy_paths(self): registry = load_registry() - self.assertEqual(len(registry["sources"]), 179) - self.assertEqual(len({source["source_id"] for source in registry["sources"]}), 179) + self.assertEqual(len(registry["sources"]), 185) + self.assertEqual(len({source["source_id"] for source in registry["sources"]}), 185) def test_unknowns_are_explicit(self): registry = json.loads(REGISTRY_PATH.read_text(encoding="utf-8")) From 56b752771a1b1feda73c00a7337d00f817a47b85 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Wed, 16 Sep 2026 10:19:00 -0700 Subject: [PATCH 162/311] docs: add Moldova source reconnaissance --- docs/country-recon-md.md | 43 +++++++++++++++++++ docs/source-status.json | 8 +++- pipeline/source_registry.json | 9 +++- pipeline/tests/test_moldova_recon_metadata.py | 15 +++++++ pipeline/tests/test_source_registry.py | 4 +- 5 files changed, 75 insertions(+), 4 deletions(-) create mode 100644 docs/country-recon-md.md create mode 100644 pipeline/tests/test_moldova_recon_metadata.py diff --git a/docs/country-recon-md.md b/docs/country-recon-md.md new file mode 100644 index 0000000..1c6a9f4 --- /dev/null +++ b/docs/country-recon-md.md @@ -0,0 +1,43 @@ +# Moldova source reconnaissance + +Status: row-free reconnaissance only; no facility rows, raw exports, or production ingestion artifacts are committed. + +## Scope and automation requirement + +Moldova has a strong national food-safety authority surface through ANSA, supported by environmental, statistical and Public Services Agency systems. The production pipeline must be fully automated from source discovery/scraping through acquisition, validation, normalization, provenance, quarantine, and ingestion. This reconnaissance does not authorize publication or replace maintainer review. + +## Strongest candidates + +| Source | Verified surface | Automation assessment | Main unresolved items | +| --- | --- | --- | --- | +| `md.ansa.approved-food` | ANSA publishes authorized animal-origin food units by categories including meat, fish, dairy, eggs, honey, small producers and slaughter-related units. | Medium: official pages expose list links, but direct files/formats and freshness need pinning. | Stable file URLs, schema, cadence, approval IDs, addresses/coordinates, license and privacy. | +| `md.ansa.farms-aquaculture` | ANSA animal-health materials cover authorized veterinary units and animal establishments; official checklists explicitly cover fish farms, cattle, laying hens, broilers and pigs. | Medium for linked files; low–medium for register routes not yet pinned. | Complete holding/aquaculture exports, IDs, coordinates, cadence, terms and privacy. | +| `md.environment-permits` | Moldovan environmental authority and permitting systems publish environmental information and authorization routes; public machine-readable permit export was not verified. | Medium after API/export confirmation; otherwise document extraction. | Public read/API, facility coverage, permit IDs, geometry, licensing and privacy. | +| `md.asp.organizations` | Public Services Agency provides state legal-entity register extracts and contracted Web/ACCES-Web/statistical access; IDNO is the core identifier. | Medium for authorized service integration; fees/contracts and personal data require strict controls. | API/service contract, fees, fields, cadence, legal-entity/site matching and privacy. | +| `md.stat.statistics` | National Bureau of Statistics publishes official agriculture/livestock/slaughter aggregates and statistical data services. | Medium–high for aggregate tables after table IDs/routes are pinned. | Exact slaughter/animal-use tables, formats, cadence, revisions, license and aggregate-only handling. | +| `md.ansa.inspections-experiments` | ANSA publishes risk-based official-control checklists and animal-health materials; public experimental-animal facility master was not verified. | High for published checklists/aggregates; low for facility/event extraction. | Public experimentation register, stable IDs, inspection outcomes, animal-use categories, privacy and retention. | + +## Compliance and ingestion notes + +- Treat ANSA, ASP, environmental and statistical records as government-sourced evidence, not proof of current operation or project approval. +- Preserve source URL, retrieval timestamp, content hash, byte size, supplied publication/effective date, adapter/configuration version, and source values. +- Keep raw, parsed, normalized, enriched, reviewed, and released layers separate. Use synthetic fixtures only; do not commit rows or raw artifacts. +- Represent unavailable cadence, IDs, coordinates, licensing, and privacy decisions explicitly. A source outage or disappearance means “not observed,” not closure. +- Registered offices, farmer-household addresses, permit sites and farm locations are not automatically safe public facility locations; suppress personal/household data and apply privacy review. +- ASP’s contracted information services and beneficial-owner fields are not unrestricted ingestion sources; adapters must fail closed unless authorized and privacy-eligible. + +## Official evidence + +- [ANSA animal-origin food safety](https://www.ansa.gov.md/siguranta-alimentelor.html) +- [ANSA animal health and welfare](https://www.ansa.gov.md/sanatatea-si-bunastarea-animalelor.html) +- [ANSA control checklists](https://ansa.gov.md/conducerea/liste-de-verificare.html) +- [Public Services Agency business information](https://asp.gov.md/en/servicii/persoane-juridice/informatii-afaceri) +- [ASP electronic information services](https://www.asp.gov.md/ro/servicii/alte-servicii/servicii-informationale-electronice/611) +- [National Bureau of Statistics](https://statistica.gov.md/en) +- [Environmental authority](https://www.mediu.gov.md/) + +## Effort and blockers + +Initial reconnaissance: approximately 2–4 engineering days to pin ANSA lists and build deterministic approved-food adapters; 4–8 additional days for holding/aquaculture, environmental permits, ASP authorization, experimentation registers and statistical table mapping. Main blockers are undocumented ANSA file routes, incomplete aquaculture exports, uncertain environmental API/export, ASP contracts/fees and personal-data fields, exact statistical table IDs, and licensing/privacy/coordinate semantics. + +Recommended next country: Ukraine, subject to checking the existing country inventory before delegation. diff --git a/docs/source-status.json b/docs/source-status.json index 2caa10e..1a908ac 100644 --- a/docs/source-status.json +++ b/docs/source-status.json @@ -193,6 +193,12 @@ {"source_id":"xk.environment-permits","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-xk.md","pipeline/source_registry.json"],"next_action":"Verify public environmental permit route, coverage, documents, geometry and privacy."}, {"source_id":"xk.arbk.organizations","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-xk.md","pipeline/source_registry.json"],"next_action":"Pin ARBK API/search route, auth, rate limits, cadence, terms and privacy."}, {"source_id":"xk.ask.statistics","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-xk.md","pipeline/source_registry.json"],"next_action":"Pin official slaughter/animal-use table IDs, APIs, cadence and revisions."}, - {"source_id":"xk.auvk.inspections-experiments","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-xk.md","pipeline/source_registry.json"],"next_action":"Pin experimentation register/API, fields, IDs, privacy and retention policy."} + {"source_id":"xk.auvk.inspections-experiments","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-xk.md","pipeline/source_registry.json"],"next_action":"Pin experimentation register/API, fields, IDs, privacy and retention policy."}, + {"source_id":"md.ansa.approved-food","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-md.md","pipeline/source_registry.json"],"next_action":"Pin current ANSA files, schemas, cadence, IDs, terms and privacy."}, + {"source_id":"md.ansa.farms-aquaculture","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-md.md","pipeline/source_registry.json"],"next_action":"Verify holding/fish-farm exports, IDs, cadence, coordinates, terms and privacy."}, + {"source_id":"md.environment-permits","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-md.md","pipeline/source_registry.json"],"next_action":"Verify public environmental permit route, coverage, documents, geometry and privacy."}, + {"source_id":"md.asp.organizations","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-md.md","pipeline/source_registry.json"],"next_action":"Pin authorized ASP service/API, fees, fields, cadence, matching and privacy."}, + {"source_id":"md.stat.statistics","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-md.md","pipeline/source_registry.json"],"next_action":"Pin official slaughter/animal-use table IDs, APIs, cadence and revisions."}, + {"source_id":"md.ansa.inspections-experiments","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-md.md","pipeline/source_registry.json"],"next_action":"Locate experimentation facility route; keep sensitive evidence aggregate until verified."} ] } diff --git a/pipeline/source_registry.json b/pipeline/source_registry.json index f90b5b6..cc1b8df 100644 --- a/pipeline/source_registry.json +++ b/pipeline/source_registry.json @@ -810,7 +810,12 @@ {"source_id":"xk.environment-permits","jurisdiction_scope":"Kosovo; environmental permits and authorizations","legacy_paths":[],"url":"https://mmphi.rks-gov.net/","access_method":"official ministry/environmental authority documents and public notices","cadence":"permit/document-specific","attribution_licensing_notes":"Verify public-read rights, document license, geometry and privacy.","adapter_status":"reference_only","expected_artifact_schema":"Permit/decision ID, authority, operator/site, activity, dates, status and optional reviewed geometry","blockers":["Stable public read/API/export and complete facility coverage are unresolved."]}, {"source_id":"xk.arbk.organizations","jurisdiction_scope":"Kosovo; ARBK business organizations and corporate identifiers","legacy_paths":[],"url":"https://arbk.rks-gov.net/","access_method":"official business-register web/admin surface; public API/bulk route unverified","cadence":"provider-defined","attribution_licensing_notes":"Identity linkage only; verify automation permissions, terms, fields and registered-office privacy.","adapter_status":"reference_only","expected_artifact_schema":"Registration/fiscal ID, legal name, status, legal form, activity and registered office/address","blockers":["Pin public query/API, authentication, rate limits, cadence, terms and privacy."]}, {"source_id":"xk.ask.statistics","jurisdiction_scope":"Kosovo; official aggregate slaughter, livestock and animal-use statistics","legacy_paths":[],"url":"https://ask.rks-gov.net/","access_method":"Kosovo Agency of Statistics tables/data services; exact route unverified","cadence":"table-specific; preserve revisions","attribution_licensing_notes":"Official statistics; verify table terms, citation requirements and aggregate-only scope.","adapter_status":"reference_only","expected_artifact_schema":"Aggregate multidimensional observations by species, measure, geography and period","blockers":["Pin current slaughter and animal-use table IDs, API/download contracts and cadence."]}, - {"source_id":"xk.auvk.inspections-experiments","jurisdiction_scope":"Kosovo; AUVK inspections/enforcement and experimental-animal institutions","legacy_paths":[],"url":"https://auvk.rks-gov.net/kontrolli-i-brendshem/veterinar/","access_method":"official AUVK control summaries, registers and linked documents","cadence":"resource/report-specific","attribution_licensing_notes":"Sensitive control/research evidence; aggregate or anonymize and require project review.","adapter_status":"reference_only","expected_artifact_schema":"Dated institution/aggregate/event observation with authority, category, outcome and source document","blockers":["Pin current experimentation register/API, fields, IDs, privacy and retention policy."]} ] + {"source_id":"xk.auvk.inspections-experiments","jurisdiction_scope":"Kosovo; AUVK inspections/enforcement and experimental-animal institutions","legacy_paths":[],"url":"https://auvk.rks-gov.net/kontrolli-i-brendshem/veterinar/","access_method":"official AUVK control summaries, registers and linked documents","cadence":"resource/report-specific","attribution_licensing_notes":"Sensitive control/research evidence; aggregate or anonymize and require project review.","adapter_status":"reference_only","expected_artifact_schema":"Dated institution/aggregate/event observation with authority, category, outcome and source document","blockers":["Pin current experimentation register/API, fields, IDs, privacy and retention policy."]} ,{"source_id":"md.ansa.approved-food","jurisdiction_scope":"Moldova; ANSA authorized and registered animal-origin food establishments","legacy_paths":[],"url":"https://www.ansa.gov.md/siguranta-alimentelor.html","access_method":"official ANSA category pages and linked lists/documents","cadence":"unknown; timestamp retrieval","attribution_licensing_notes":"Official authority; verify file terms, attribution, privacy and coordinates before acquisition.","adapter_status":"reference_only","expected_artifact_schema":"Approval/registration ID, operator/site, category, status, address and optional coordinates","blockers":["Pin stable ANSA file routes, schemas, cadence, IDs, terms and privacy."]}, + {"source_id":"md.ansa.farms-aquaculture","jurisdiction_scope":"Moldova; ANSA livestock holdings, fish farms and aquaculture-related evidence","legacy_paths":[],"url":"https://www.ansa.gov.md/sanatatea-si-bunastarea-animalelor.html","access_method":"official ANSA registers, checklists and linked documents","cadence":"resource-specific","attribution_licensing_notes":"Verify publisher, license, animal-holder/property privacy and coordinate precision.","adapter_status":"reference_only","expected_artifact_schema":"Holding/site or fish-farm/aquaculture record with stable ID, activity/status and optional location","blockers":["Verify complete holding/aquaculture exports, IDs, cadence, terms and privacy."]}, + {"source_id":"md.environment-permits","jurisdiction_scope":"Moldova; environmental permits and authorizations","legacy_paths":[],"url":"https://www.mediu.gov.md/","access_method":"official environmental authority portals, public notices and documents","cadence":"permit/document-specific","attribution_licensing_notes":"Verify public-read rights, document license, geometry and privacy.","adapter_status":"reference_only","expected_artifact_schema":"Permit/decision ID, authority, operator/site, activity, dates, status and optional reviewed geometry","blockers":["Stable public read/API/export and complete facility coverage are unresolved."]}, + {"source_id":"md.asp.organizations","jurisdiction_scope":"Moldova; Public Services Agency State Register of Legal Entities","legacy_paths":[],"url":"https://asp.gov.md/en/servicii/persoane-juridice/informatii-afaceri","access_method":"official online extracts and contracted ACCES-Web/statistical services","cadence":"service-defined; online/non-stop services described","attribution_licensing_notes":"Identity linkage only; contracts, fees and personal/beneficial-owner privacy apply.","adapter_status":"reference_only","expected_artifact_schema":"IDNO keyed legal entity, name, status, activity, registered office/address and permitted public fields","blockers":["Pin authorized service/API, fees, fields, cadence, site matching and privacy policy."]}, + {"source_id":"md.stat.statistics","jurisdiction_scope":"Moldova; official aggregate slaughter, livestock and animal-use statistics","legacy_paths":[],"url":"https://statistica.gov.md/en","access_method":"National Bureau of Statistics tables/data services; exact route unverified","cadence":"table-specific; preserve revisions","attribution_licensing_notes":"Official statistics; verify table terms, citation requirements and aggregate-only scope.","adapter_status":"reference_only","expected_artifact_schema":"Aggregate multidimensional observations by species, measure, geography and period","blockers":["Pin current slaughter and animal-use table IDs, API/download contracts and cadence."]}, + {"source_id":"md.ansa.inspections-experiments","jurisdiction_scope":"Moldova; ANSA inspections/enforcement and animal-experimentation evidence","legacy_paths":[],"url":"https://ansa.gov.md/conducerea/liste-de-verificare.html","access_method":"official control checklists, reports and linked registers","cadence":"resource/report-specific","attribution_licensing_notes":"Sensitive control/research evidence; aggregate or anonymize and require project review.","adapter_status":"reference_only","expected_artifact_schema":"Dated institution/aggregate/event observation with authority, category, outcome and source document","blockers":["No stable public experimental-animal facility master verified; pin current fields and privacy policy."]} ] } @@ -833,6 +838,8 @@ + + diff --git a/pipeline/tests/test_moldova_recon_metadata.py b/pipeline/tests/test_moldova_recon_metadata.py new file mode 100644 index 0000000..19f5a41 --- /dev/null +++ b/pipeline/tests/test_moldova_recon_metadata.py @@ -0,0 +1,15 @@ +import json +import unittest +from pathlib import Path +ROOT=Path(__file__).resolve().parents[2] +class MoldovaReconMetadataTests(unittest.TestCase): + def test_row_free_moldova_recon_is_present(self): + text=(ROOT/"docs"/"country-recon-md.md").read_text(encoding="utf-8") + self.assertIn("row-free",text); self.assertIn("fully automated",text); self.assertIn("Recommended next country: Ukraine",text) + def test_moldova_sources_have_conservative_status(self): + registry=json.loads((ROOT/"pipeline"/"source_registry.json").read_text(encoding="utf-8")); status=json.loads((ROOT/"docs"/"source-status.json").read_text(encoding="utf-8")) + ids={s["source_id"] for s in registry["sources"] if s["source_id"].startswith("md.")} + self.assertEqual(ids,{s["source_id"] for s in status["sources"] if s["source_id"].startswith("md.")}); self.assertEqual(len(ids),6) + for s in status["sources"]: + if s["source_id"].startswith("md."): self.assertEqual(s["publication_eligibility"],"blocked"); self.assertNotEqual(s["runtime_health"],"healthy") +if __name__ == "__main__": unittest.main() diff --git a/pipeline/tests/test_source_registry.py b/pipeline/tests/test_source_registry.py index ebcf320..70d1cfa 100644 --- a/pipeline/tests/test_source_registry.py +++ b/pipeline/tests/test_source_registry.py @@ -9,8 +9,8 @@ class SourceRegistryTests(unittest.TestCase): def test_repository_registry_loads_and_references_existing_legacy_paths(self): registry = load_registry() - self.assertEqual(len(registry["sources"]), 185) - self.assertEqual(len({source["source_id"] for source in registry["sources"]}), 185) + self.assertEqual(len(registry["sources"]), 191) + self.assertEqual(len({source["source_id"] for source in registry["sources"]}), 191) def test_unknowns_are_explicit(self): registry = json.loads(REGISTRY_PATH.read_text(encoding="utf-8")) From 4bb4affa7cb1947440db3ed86c12d3814b04a57a Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Wed, 16 Sep 2026 10:19:04 -0700 Subject: [PATCH 163/311] Harden spatial projection and request observability --- docs/performance/v2-observability.md | 59 +++++++++++++ ...0_discovery_projection_support_indexes.sql | 13 +++ .../tests/test_discovery_scale_benchmark.py | 8 ++ src/main.rs | 84 ++++++++++++++++++- 4 files changed, 163 insertions(+), 1 deletion(-) create mode 100644 docs/performance/v2-observability.md create mode 100644 pipeline/migrations/030_discovery_projection_support_indexes.sql diff --git a/docs/performance/v2-observability.md b/docs/performance/v2-observability.md new file mode 100644 index 0000000..24ed098 --- /dev/null +++ b/docs/performance/v2-observability.md @@ -0,0 +1,59 @@ +# V2 backend observability and spatial operations + +The API emits one bounded JSON event after each HTTP response. The event uses +an allowlisted route class rather than the request path, so facility IDs, +coordinates, query strings, and arbitrary paths are never logged: + +```json +{"event":"http_request","method":"GET","route_class":"v2_locations_list","status":200,"outcome":"success","latency_ms":12} +``` + +The fields are intentionally limited to method, route class, status, outcome, +and latency. Latency is capped at 60 seconds. There is no request ID, client +address, forwarded address, query text, response body, secret, or source-row +field. Health requests are classified separately so operators can exclude +probe traffic from user-facing latency summaries. `client_error` and +`server_error` outcomes make bounded error-rate aggregation possible from the +existing process logs without a telemetry service. + +## Spatial investigation + +The synthetic scale runner covers the same bounded radius predicate and checks +that the geography GiST index is selected. On the local PostGIS 16 / PostGIS +3.4 disposable environment, the 1m radius sample was 238.354 ms in the +baseline run and 201.824 ms in a later run with the projection-support +migration applied. Because the runner uses temporary benchmark tables, this +delta is not attributable to migration 030; it is retained only as directional +before/after aggregate evidence. These are single EXPLAIN samples, not p95 or +capacity measurements; cache state and planner variation can move them +materially. + +The public display projection computes a latest geocode and, for approximate +records, a city reference point per candidate row. Migration 030 adds +supporting indexes for both append-only lookup paths: + +- `geocode_results_discovery_latest_idx` preserves newest-result ordering, + including unresolved/no-point results. +- `city_reference_points_discovery_lookup_idx` supports the country, + case-insensitive city, and postal lookup. + +No radius predicate, ordering rule, privacy filter, or release gate was +weakened. If a deployment still approaches the database budget, capture a +representative approved load test before changing pool sizes or introducing a +materialized projection. + +## Operator procedure + +1. Run the synthetic scale benchmark against the pinned PostGIS image and + compare aggregate reports on the same class of hardware. +2. Treat a missing expected index, any sequential scan, a page over 50 rows, + or a radius result above the documented database budget as a review signal. +3. Use the JSON request logs to aggregate latency and status by route class. + Do not add raw request paths, query parameters, headers, IP addresses, or + response payloads to the log pipeline. +4. Keep the service stopped during backup restore until the independent + restriction-ledger gate and current replay succeed. + +The benchmark and logs provide operational evidence only. They do not establish +source completeness, publication eligibility, production capacity, cloud +cost, or a guarantee for a particular traffic pattern. diff --git a/pipeline/migrations/030_discovery_projection_support_indexes.sql b/pipeline/migrations/030_discovery_projection_support_indexes.sql new file mode 100644 index 0000000..801ee0e --- /dev/null +++ b/pipeline/migrations/030_discovery_projection_support_indexes.sql @@ -0,0 +1,13 @@ +-- Additive support for the public display projection's per-record lookups. +-- These indexes preserve latest-row semantics: the geocoder lookup must still +-- see the newest result even when that result is unresolved or has no point. +CREATE INDEX IF NOT EXISTS geocode_results_discovery_latest_idx + ON uec.geocode_results (source_record_id, queried_at DESC, geocode_result_id DESC); + +CREATE INDEX IF NOT EXISTS city_reference_points_discovery_lookup_idx + ON uec.city_reference_points (country_code, lower(city_name), postal_code); + +COMMENT ON INDEX uec.geocode_results_discovery_latest_idx IS + 'Supports the latest append-only geocode lookup used by the release display projection.'; +COMMENT ON INDEX uec.city_reference_points_discovery_lookup_idx IS + 'Supports country/case-insensitive-city/postal lookup used for coarse public display locations.'; diff --git a/pipeline/tests/test_discovery_scale_benchmark.py b/pipeline/tests/test_discovery_scale_benchmark.py index 8a174be..8f626d9 100644 --- a/pipeline/tests/test_discovery_scale_benchmark.py +++ b/pipeline/tests/test_discovery_scale_benchmark.py @@ -55,6 +55,14 @@ def test_plan_summary_is_row_free_and_aggregated(self): self.assertNotIn("canonical_name", json.dumps(report).lower()) self.assertNotIn("source_record", json.dumps(report).lower()) + def test_projection_support_migration_is_additive_and_latest_safe(self): + migration = (ROOT / "migrations" / "030_discovery_projection_support_indexes.sql").read_text(encoding="utf-8").lower() + self.assertIn("geocode_results_discovery_latest_idx", migration) + self.assertIn("city_reference_points_discovery_lookup_idx", migration) + self.assertIn("queried_at desc, geocode_result_id desc", migration) + self.assertNotIn("drop table", migration) + self.assertNotIn("drop index", migration) + if __name__ == "__main__": unittest.main() diff --git a/src/main.rs b/src/main.rs index 224657a..0afd12c 100644 --- a/src/main.rs +++ b/src/main.rs @@ -100,6 +100,7 @@ pub fn app(state: uec_api::ApiState, proxy: private_environment::ProxyConfig) -> }, rate_limit, )) + .layer(axum::middleware::from_fn(request_observability)) .layer(cors) .with_state(state) } @@ -291,6 +292,58 @@ async fn rate_limit( next.run(request).await } +fn request_route_class(path: &str) -> &'static str { + match path { + "/health/live" | "/health/ready" | "/health/diagnostics" => "health", + "/api/v2/locations" => "v2_locations_list", + "/api/v2/locations.csv" => "v2_locations_export", + "/api/v2/discovery/filters" => "v2_discovery_filters", + "/api/v2/discovery/facets" => "v2_discovery_facets", + "/api/v2/releases/manifest" => "v2_release_manifest", + path if path.starts_with("/api/v2/locations/") => "v2_location_detail", + path if path.starts_with("/api/v2/") => "v2_other", + path if path.starts_with("/api/") => "api_other", + _ => "other", + } +} + +fn request_log_payload( + method: &Method, + path: &str, + status: StatusCode, + elapsed: Duration, +) -> serde_json::Value { + let elapsed_ms = elapsed.as_millis().min(60_000) as u64; + let outcome = match status.as_u16() { + 200..=399 => "success", + 400..=499 => "client_error", + _ => "server_error", + }; + serde_json::json!({ + "event": "http_request", + "method": method.as_str(), + "route_class": request_route_class(path), + "status": status.as_u16(), + "outcome": outcome, + "latency_ms": elapsed_ms, + }) +} + +async fn request_observability( + request: Request, + next: axum::middleware::Next, +) -> Response { + let method = request.method().clone(); + let path = request.uri().path().to_owned(); + let started = Instant::now(); + let response = next.run(request).await; + println!( + "{}", + request_log_payload(&method, &path, response.status(), started.elapsed()) + ); + response +} + async fn liveness() -> impl IntoResponse { Json(serde_json::json!({"status": "ok", "service": "uec-api"})) } @@ -568,7 +621,13 @@ async fn main() { #[cfg(test)] mod config_tests { - use super::{parse_cors_origins, preview_config, validate_runtime}; + use axum::http::{Method, StatusCode}; + use std::time::Duration; + + use super::{ + parse_cors_origins, preview_config, request_log_payload, request_route_class, + validate_runtime, + }; #[test] fn development_allows_local_defaults() { assert_eq!(validate_runtime("development", None, "8000"), Ok(8000)); @@ -667,6 +726,29 @@ mod config_tests { None ); } + + #[test] + fn request_observability_uses_safe_bounded_route_fields() { + let payload = request_log_payload( + &Method::GET, + "/api/v2/locations/synthetic-facility-id?latitude=51.5&longitude=-0.1", + StatusCode::TOO_MANY_REQUESTS, + Duration::from_secs(90_000), + ); + assert_eq!( + request_route_class("/api/v2/locations/synthetic-id"), + "v2_location_detail" + ); + assert_eq!(payload["route_class"], "v2_location_detail"); + assert_eq!(payload["status"], 429); + assert_eq!(payload["outcome"], "client_error"); + assert_eq!(payload["latency_ms"], 60_000); + let serialized = payload.to_string(); + assert!(!serialized.contains("synthetic-facility-id")); + assert!(!serialized.contains("latitude")); + assert!(!serialized.contains("longitude")); + assert!(!serialized.contains("query")); + } } #[cfg(test)] From c0f444e89503558a1b867d10c18c29eb137e9660 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Wed, 16 Sep 2026 10:23:07 -0700 Subject: [PATCH 164/311] test: include migration 030 in graph ordering contract --- pipeline/tests/test_graph_migrations.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pipeline/tests/test_graph_migrations.py b/pipeline/tests/test_graph_migrations.py index 3d56b73..2249859 100644 --- a/pipeline/tests/test_graph_migrations.py +++ b/pipeline/tests/test_graph_migrations.py @@ -11,11 +11,13 @@ def read(self, name): def test_reserved_migrations_are_present_and_ordered(self): migrations = sorted(path.name for path in (ROOT / "migrations").glob("*.sql")) - self.assertEqual(migrations[-4:], [ + self.assertEqual(migrations[-5:], [ + "025_discovery_query_indexes.sql", "026_graph_entities_crosswalks.sql", "027_graph_relationship_observations.sql", "028_graph_claims_support.sql", "029_graph_publication_projections.sql", + "030_discovery_projection_support_indexes.sql", ]) def test_entities_are_distinct_and_crosswalk_is_scoped(self): From 737ddcea30ba9b39f33eb59a98ad2df8337d31d7 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Wed, 16 Sep 2026 10:23:39 -0700 Subject: [PATCH 165/311] test: correct final migration window --- pipeline/tests/test_graph_migrations.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pipeline/tests/test_graph_migrations.py b/pipeline/tests/test_graph_migrations.py index 2249859..1e9afee 100644 --- a/pipeline/tests/test_graph_migrations.py +++ b/pipeline/tests/test_graph_migrations.py @@ -12,7 +12,6 @@ def read(self, name): def test_reserved_migrations_are_present_and_ordered(self): migrations = sorted(path.name for path in (ROOT / "migrations").glob("*.sql")) self.assertEqual(migrations[-5:], [ - "025_discovery_query_indexes.sql", "026_graph_entities_crosswalks.sql", "027_graph_relationship_observations.sql", "028_graph_claims_support.sql", From c9d8def6394b8e8ebf11137c3e8f4283be59577a Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Wed, 16 Sep 2026 10:23:41 -0700 Subject: [PATCH 166/311] Add Ukraine source reconnaissance metadata --- docs/country-recon-ua.md | 26 ++++++++++++++ docs/source-status.json | 8 ++++- pipeline/source_registry.json | 8 ++++- pipeline/tests/test_source_registry.py | 4 +-- pipeline/tests/test_ukraine_recon_metadata.py | 35 +++++++++++++++++++ 5 files changed, 77 insertions(+), 4 deletions(-) create mode 100644 docs/country-recon-ua.md create mode 100644 pipeline/tests/test_ukraine_recon_metadata.py diff --git a/docs/country-recon-ua.md b/docs/country-recon-ua.md new file mode 100644 index 0000000..d9c6d18 --- /dev/null +++ b/docs/country-recon-ua.md @@ -0,0 +1,26 @@ +# Ukraine source reconnaissance + +Status: metadata-only reconnaissance; no facility rows, raw exports, coordinates, or operationally sensitive records were retained. + +## Safety and scope + +Ukraine requires heightened review because public registries can expose operational locations during an active war, records can be stale or incomplete because of displacement and temporarily occupied territories, and a missing record does not prove closure. The pipeline must remain fully automated from source discovery/scraping through validation, quarantine, transformation, and ingestion, but automation must stop before acquisition when a source's publication, security, privacy, or terms are unresolved. Do not geocode, publish addresses, retain facility rows, or infer current operational status from an unavailable or stale source. Escalate immediately if a source exposes sensitive facility-level data or if access controls, wartime restrictions, or lawful-use terms are unclear. + +## Candidate inventory + +| ID | Authority / scope | Verified route | Format/API and cadence | Conservative disposition | +|---|---|---|---|---| +| `ua.dpss.approved-food` | State Production and Consumer Service (Держпродспоживслужба); registered food-market operators and facilities, including animal-origin food | [Registry and registration guidance](https://dpss.gov.ua/diyalnist/bezpechnist-harchovih-produktiv-ta-veterinarna-medicina/reyestri) and [registration route](https://dpss.gov.ua/diyalnist/bezpechnist-harchovih-produktiv-ta-veterinarna-medicina/dozvoly-ta-reiestratsiia-dlia-biznesu-u-sferakh-veterynarnoi-medytsyny-bezpechnosti-kharchovykh-produktiv-ta-kormiv/derzhavna-reiestratsiia-potuzhnostei) | Interactive search is discoverable; bulk route, schema, stable identifier, update cadence, terms, and privacy controls not pinned | Partial; blocked pending authorized, safety-reviewed export/API contract | +| `ua.farm-aquaculture` | DPSS livestock facilities/operators, including the stated aquaculture registration scope | [Livestock facilities and operators](https://dpss.gov.ua/diyalnist/bezpechnist-harchovih-produktiv-ta-veterinarna-medicina/dozvoly-ta-reiestratsiia-dlia-biznesu-u-sferakh-veterynarnoi-medytsyny-bezpechnosti-kharchovykh-produktiv-ta-kormiv/tvarynnytski-potuzhnosti/derzhavna-reiestratsiia-tvarynnytskykh-potuzhnostei-ta-operatoriv-rynku) | Electronic registration is described; no safe public bulk export/API, cadence, stable ID, coordinate policy, or reuse terms verified | Partial; blocked | +| `ua.environment-permits` | Ministry of Environmental Protection / EcoSystem environmental registers and permits | [Ministry environmental monitoring/open-register background](https://mepr.gov.ua/) | Public web platform and registers are discoverable; exact permit API/export, geometry, cadence, license, and security policy not pinned | Partial; blocked | +| `ua.edr.organizations` | Unified State Register of legal entities and organizations | [National open-data portal](https://data.gov.ua/) | Official catalog is the discovery point; current authorized API/download route, field-level personal-address policy, cadence, and rate limits not verified | Partial; blocked | +| `ua.ukrstat.statistics` | State Statistics Service aggregate livestock, animal-production, and slaughter indicators | [Official livestock series example](https://www.vn.ukrstat.gov.ua/index.php/component/content/article/741/7954--1995-2024.html) and [Ukrstat publications](https://ukrstat.gov.ua/) | Published tables/PDF/HTML; exact current table IDs, machine API, revision/cadence contract, and regional suppression rules not pinned | Partial; not run; aggregate-only candidate | +| `ua.inspections-experiments` | DPSS controls/enforcement and any public animal-experimentation evidence | [DPSS registers and services](https://dpss.gov.ua/diyalnist/bezpechnist-harchovih-produktiv-ta-veterinarna-medicina/reyestri) | No safe, stable public facility/event master verified; reports and control evidence are resource-specific | Partial; blocked; retain only aggregate evidence until authorized | + +## Automation acceptance gates + +Before any live run, the orchestrator must pin a stable source URL/API, response format and schema fingerprint, pagination/query behavior, stable identifiers and lifecycle semantics, observed cadence/freshness, attribution/license, privacy and retention rules, and a documented prohibition on sensitive geolocation. A compliant job would acquire only an authorized public snapshot, hash and quarantine it, validate schema and freshness, normalize without exposing rows, and ingest only after the safety and publication gates pass. Failed access, disappearance, or stale data must be recorded as an observation—not converted into a closure or status change. + +## Recommendation + +Ukraine is not ingestion-ready. Keep all six candidates publication-blocked and do not create a facility dataset from the interactive DPSS search. A future pass should begin with aggregate Ukrstat tables, then request an authorized, safety-reviewed DPSS metadata/export contract. Any facility-level acquisition should require explicit human approval and a wartime security review. diff --git a/docs/source-status.json b/docs/source-status.json index 1a908ac..7649798 100644 --- a/docs/source-status.json +++ b/docs/source-status.json @@ -199,6 +199,12 @@ {"source_id":"md.environment-permits","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-md.md","pipeline/source_registry.json"],"next_action":"Verify public environmental permit route, coverage, documents, geometry and privacy."}, {"source_id":"md.asp.organizations","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-md.md","pipeline/source_registry.json"],"next_action":"Pin authorized ASP service/API, fees, fields, cadence, matching and privacy."}, {"source_id":"md.stat.statistics","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-md.md","pipeline/source_registry.json"],"next_action":"Pin official slaughter/animal-use table IDs, APIs, cadence and revisions."}, - {"source_id":"md.ansa.inspections-experiments","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-md.md","pipeline/source_registry.json"],"next_action":"Locate experimentation facility route; keep sensitive evidence aggregate until verified."} + {"source_id":"md.ansa.inspections-experiments","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-md.md","pipeline/source_registry.json"],"next_action":"Locate experimentation facility route; keep sensitive evidence aggregate until verified."}, + {"source_id":"ua.dpss.approved-food","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ua.md","pipeline/source_registry.json"],"next_action":"Pin authorized route, schema, cadence, IDs, terms, privacy, and wartime safety controls."}, + {"source_id":"ua.farm-aquaculture","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ua.md","pipeline/source_registry.json"],"next_action":"Pin authorized route, schema, cadence, IDs, terms, privacy, and wartime safety controls."}, + {"source_id":"ua.environment-permits","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ua.md","pipeline/source_registry.json"],"next_action":"Pin authorized route, schema, cadence, IDs, terms, privacy, and wartime safety controls."}, + {"source_id":"ua.edr.organizations","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ua.md","pipeline/source_registry.json"],"next_action":"Pin authorized route, schema, cadence, IDs, terms, privacy, and wartime safety controls."}, + {"source_id":"ua.ukrstat.statistics","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ua.md","pipeline/source_registry.json"],"next_action":"Pin authorized route, schema, cadence, IDs, terms, privacy, and wartime safety controls."}, + {"source_id":"ua.inspections-experiments","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ua.md","pipeline/source_registry.json"],"next_action":"Pin authorized route, schema, cadence, IDs, terms, privacy, and wartime safety controls."} ] } diff --git a/pipeline/source_registry.json b/pipeline/source_registry.json index cc1b8df..d4f8b49 100644 --- a/pipeline/source_registry.json +++ b/pipeline/source_registry.json @@ -815,7 +815,13 @@ {"source_id":"md.environment-permits","jurisdiction_scope":"Moldova; environmental permits and authorizations","legacy_paths":[],"url":"https://www.mediu.gov.md/","access_method":"official environmental authority portals, public notices and documents","cadence":"permit/document-specific","attribution_licensing_notes":"Verify public-read rights, document license, geometry and privacy.","adapter_status":"reference_only","expected_artifact_schema":"Permit/decision ID, authority, operator/site, activity, dates, status and optional reviewed geometry","blockers":["Stable public read/API/export and complete facility coverage are unresolved."]}, {"source_id":"md.asp.organizations","jurisdiction_scope":"Moldova; Public Services Agency State Register of Legal Entities","legacy_paths":[],"url":"https://asp.gov.md/en/servicii/persoane-juridice/informatii-afaceri","access_method":"official online extracts and contracted ACCES-Web/statistical services","cadence":"service-defined; online/non-stop services described","attribution_licensing_notes":"Identity linkage only; contracts, fees and personal/beneficial-owner privacy apply.","adapter_status":"reference_only","expected_artifact_schema":"IDNO keyed legal entity, name, status, activity, registered office/address and permitted public fields","blockers":["Pin authorized service/API, fees, fields, cadence, site matching and privacy policy."]}, {"source_id":"md.stat.statistics","jurisdiction_scope":"Moldova; official aggregate slaughter, livestock and animal-use statistics","legacy_paths":[],"url":"https://statistica.gov.md/en","access_method":"National Bureau of Statistics tables/data services; exact route unverified","cadence":"table-specific; preserve revisions","attribution_licensing_notes":"Official statistics; verify table terms, citation requirements and aggregate-only scope.","adapter_status":"reference_only","expected_artifact_schema":"Aggregate multidimensional observations by species, measure, geography and period","blockers":["Pin current slaughter and animal-use table IDs, API/download contracts and cadence."]}, - {"source_id":"md.ansa.inspections-experiments","jurisdiction_scope":"Moldova; ANSA inspections/enforcement and animal-experimentation evidence","legacy_paths":[],"url":"https://ansa.gov.md/conducerea/liste-de-verificare.html","access_method":"official control checklists, reports and linked registers","cadence":"resource/report-specific","attribution_licensing_notes":"Sensitive control/research evidence; aggregate or anonymize and require project review.","adapter_status":"reference_only","expected_artifact_schema":"Dated institution/aggregate/event observation with authority, category, outcome and source document","blockers":["No stable public experimental-animal facility master verified; pin current fields and privacy policy."]} ] + {"source_id":"md.ansa.inspections-experiments","jurisdiction_scope":"Moldova; ANSA inspections/enforcement and animal-experimentation evidence","legacy_paths":[],"url":"https://ansa.gov.md/conducerea/liste-de-verificare.html","access_method":"official control checklists, reports and linked registers","cadence":"resource/report-specific","attribution_licensing_notes":"Sensitive control/research evidence; aggregate or anonymize and require project review.","adapter_status":"reference_only","expected_artifact_schema":"Dated institution/aggregate/event observation with authority, category, outcome and source document","blockers":["No stable public experimental-animal facility master verified; pin current fields and privacy policy."]}, + {"source_id":"ua.dpss.approved-food","jurisdiction_scope":"Ukraine; DPSS registered food-market operators and facilities, including animal-origin food","legacy_paths":[],"url":"https://dpss.gov.ua/diyalnist/bezpechnist-harchovih-produktiv-ta-veterinarna-medicina/reyestri","access_method":"official registry/search route; authorized bounded export or API only","cadence":"unknown; verify from authorized metadata","attribution_licensing_notes":"Official Ukrainian government source; terms, privacy, wartime safety, and attribution require review","adapter_status":"reference_only","expected_artifact_schema":"Metadata-only contract pending; do not retain facility rows, addresses, coordinates, or operational status","blockers":["Bulk route, schema, stable ID, cadence, terms, privacy, and wartime safety controls are not pinned."]}, + {"source_id":"ua.farm-aquaculture","jurisdiction_scope":"Ukraine; DPSS livestock facilities and operators, including aquaculture scope","legacy_paths":[],"url":"https://dpss.gov.ua/diyalnist/bezpechnist-harchovih-produktiv-ta-veterinarna-medicina/dozvoly-ta-reiestratsiia-dlia-biznesu-u-sferakh-veterynarnoi-medytsyny-bezpechnosti-harchovykh-produktiv-ta-kormiv/tvarynnytski-potuzhnosti/derzhavna-reiestratsiia-tvarynnytskykh-potuzhnostei-ta-operatoriv-rynku","access_method":"official registration guidance; no facility acquisition until safety-reviewed export/API exists","cadence":"unknown","attribution_licensing_notes":"Official government source; coordinates, privacy, reuse terms, and wartime exposure require review","adapter_status":"reference_only","expected_artifact_schema":"Metadata-only livestock/aquaculture source contract; row-free reconnaissance","blockers":["No safe public bulk route, stable ID, cadence, coordinate policy, or reuse terms verified."]}, + {"source_id":"ua.environment-permits","jurisdiction_scope":"Ukraine; Ministry environmental registers and permits","legacy_paths":[],"url":"https://mepr.gov.ua/","access_method":"official EcoSystem/register route; bounded authorized API/export only","cadence":"resource-specific; unknown","attribution_licensing_notes":"Ministry source; license, geometry, privacy, and security review required","adapter_status":"reference_only","expected_artifact_schema":"Permit metadata and aggregate status contract; no facility rows or sensitive geometry retained","blockers":["Exact public permit API/export, coverage, cadence, geometry policy, and wartime security controls not pinned."]}, + {"source_id":"ua.edr.organizations","jurisdiction_scope":"Ukraine; Unified State Register of legal entities and organizations","legacy_paths":[],"url":"https://data.gov.ua/","access_method":"official open-data catalog discovery; authorized API/download only","cadence":"unknown","attribution_licensing_notes":"Government open-data source; field-level personal-address, terms, and rate-limit review required","adapter_status":"reference_only","expected_artifact_schema":"Organization metadata keyed by official identifier; suppress personal addresses and retain no rows in reconnaissance","blockers":["Current authorized route, fields, cadence, licensing, privacy policy, and automation permissions not verified."]}, + {"source_id":"ua.ukrstat.statistics","jurisdiction_scope":"Ukraine; aggregate livestock, animal-production, and slaughter statistics","legacy_paths":[],"url":"https://ukrstat.gov.ua/","access_method":"official statistical tables/publications; API or bounded download to be pinned","cadence":"table-specific; unknown","attribution_licensing_notes":"State Statistics Service source; publication terms, revisions, and regional suppression require review","adapter_status":"reference_only","expected_artifact_schema":"Aggregate time-series/table metadata; no establishment rows","blockers":["Current table IDs, machine API, cadence, revision semantics, and suppression rules not pinned."]}, + {"source_id":"ua.inspections-experiments","jurisdiction_scope":"Ukraine; DPSS inspections/enforcement and public animal-experimentation evidence","legacy_paths":[],"url":"https://dpss.gov.ua/diyalnist/bezpechnist-harchovih-produktiv-ta-veterinarna-medicina/reyestri","access_method":"official reports/registers; aggregate-only until a safe public route is authorized","cadence":"resource-specific; unknown","attribution_licensing_notes":"Sensitive control/research evidence; privacy, retention, and wartime safety review required","adapter_status":"reference_only","expected_artifact_schema":"Aggregate institution/event evidence with source document; no facility rows or operational locations","blockers":["No safe stable public facility/event master verified; stop and escalate on sensitive exposure."]} ] } diff --git a/pipeline/tests/test_source_registry.py b/pipeline/tests/test_source_registry.py index 70d1cfa..c84a883 100644 --- a/pipeline/tests/test_source_registry.py +++ b/pipeline/tests/test_source_registry.py @@ -9,8 +9,8 @@ class SourceRegistryTests(unittest.TestCase): def test_repository_registry_loads_and_references_existing_legacy_paths(self): registry = load_registry() - self.assertEqual(len(registry["sources"]), 191) - self.assertEqual(len({source["source_id"] for source in registry["sources"]}), 191) + self.assertEqual(len(registry["sources"]), 197) + self.assertEqual(len({source["source_id"] for source in registry["sources"]}), 197) def test_unknowns_are_explicit(self): registry = json.loads(REGISTRY_PATH.read_text(encoding="utf-8")) diff --git a/pipeline/tests/test_ukraine_recon_metadata.py b/pipeline/tests/test_ukraine_recon_metadata.py new file mode 100644 index 0000000..60e24d3 --- /dev/null +++ b/pipeline/tests/test_ukraine_recon_metadata.py @@ -0,0 +1,35 @@ +import json +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +UKRAINE_IDS = { + "ua.dpss.approved-food", + "ua.farm-aquaculture", + "ua.environment-permits", + "ua.edr.organizations", + "ua.ukrstat.statistics", + "ua.inspections-experiments", +} + + +class UkraineReconMetadataTests(unittest.TestCase): + def test_ukraine_recon_is_row_free_and_automated(self): + registry = json.loads((ROOT / "pipeline/source_registry.json").read_text(encoding="utf-8-sig")) + self.assertTrue(UKRAINE_IDS.issubset({s["source_id"] for s in registry["sources"]})) + recon = (ROOT / "docs/country-recon-ua.md").read_text(encoding="utf-8") + self.assertIn("no facility rows", recon.lower()) + self.assertIn("fully automated", recon.lower()) + self.assertIn("wartime", recon.lower()) + self.assertIn("operationally sensitive", recon.lower()) + + def test_ukraine_sources_are_blocked_or_not_run(self): + status = json.loads((ROOT / "docs/source-status.json").read_text(encoding="utf-8-sig")) + by_id = {s["source_id"]: s for s in status["sources"]} + for source_id in UKRAINE_IDS: + self.assertEqual(by_id[source_id]["publication_eligibility"], "blocked") + self.assertIn(by_id[source_id]["acquisition"], {"blocked", "not_run"}) + + +if __name__ == "__main__": + unittest.main() From b466bf833b11c4e189207ea89e66fb492bd96c91 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Wed, 16 Sep 2026 10:26:31 -0700 Subject: [PATCH 167/311] Document map visualization platform decision --- .../adr-map-visualization-platform.md | 146 ++++++++++++++++++ 1 file changed, 146 insertions(+) create mode 100644 docs/architecture/adr-map-visualization-platform.md diff --git a/docs/architecture/adr-map-visualization-platform.md b/docs/architecture/adr-map-visualization-platform.md new file mode 100644 index 0000000..8e3511e --- /dev/null +++ b/docs/architecture/adr-map-visualization-platform.md @@ -0,0 +1,146 @@ +# ADR: map and visualization platform + +Status: proposed for maintainer review. Date: 2026-09-16. This is a technology decision for future V2 platform work, not authorization to publish data, enable a live map, acquire third-party services, or imply that present safeguards are deployed. + +## Decision + +Keep the existing Svelte 5 application's **flat 2D investigative map** as the primary experience. Replace the future production Leaflet implementation with a `MapLibre GL JS` adapter that consumes release-scoped vector tiles and is loaded only when the visitor opens the map. Use `deck.gl` as a lazy GPU overlay inside the same adapter for aggregation, density, relationship arcs, and temporal rendering. Store basemap and project vector tiles in `PMTiles` archives and serve them from project-controlled object storage/CDN; begin with a deliberately plain, self-hosted/open-data basemap. Use PostGIS for private/release generation and a tile service or generated PMTiles for public viewport queries. + +Treat a 3D globe as a separate, optional **exploration view** after the 2D map, access-control, accessibility, and performance gates have passed. Start it with MapLibre's globe projection where a globe adds orientation value. Evaluate CesiumJS only when the product has a reviewed 3D Tiles, terrain, or genuinely three-dimensional data requirement. Do not make globe mode the default, and never use 3D terrain/buildings to make a sensitive point easier to identify. + +This preserves the product's primary job: inspect place-based evidence with clear uncertainty. A globe is good at global orientation and broad patterns; it is a worse default for filtering, comparing rows, keyboard operation, coarse-location explanation, and careful evidence reading. The story lane likewise says a facility row is not an animal and a map pin is not an animal count; visual scale, map points, and relationship lines must remain distinct claims. + +## Why this fits the project + +The current Svelte preview has a local blank Leaflet map, fixture-first repository boundary, explicit publication states, and no tile provider. It is a sound Phase 2 safety boundary, but it cannot scale from the browser's loaded page to a 100k–500k worldwide collection. Phase 4 already calls for viewport loading, clustering, cursor traversal, and mobile benchmarks. The accountability-graph foundation keeps facilities, organizations, source identifiers, relationships, claims, source evidence, review state, privacy status, and publication state separate. The visualization layer must preserve those distinctions instead of converting them into a single confidence color or an unqualified network graph. + +ETHICS.md requires restrictions to apply to maps, APIs, downloads, embeds, previews, caches, historical releases, reimports, and restores. It also requires exact/city/unmapped uncertainty, explicit community context, no targeting of workers/residents, no inferred closure, and no invented coordinates. Consequently, the public map is a **derived release artifact** rather than a direct renderer for evidence tables or graph tables. + +## Decision matrix + +Scores are project fit, not generic library quality: 5 is strongest. “No live decision” means the capability should remain absent until its data, privacy, and operational gate exists. + +| Candidate | 2D vector performance | 3D/globe | Aggregation / arcs / time | Open licensing / self-hosting | Svelte fit | Decision | +| --- | ---: | ---: | ---: | ---: | ---: | --- | +| MapLibre GL JS | 5 | 3 | 3 | 5 | 5 | Primary map renderer and globe experiment | +| deck.gl | 5 | 3 | 5 | 5 | 4 | Optional GPU overlay, not application map owner | +| PMTiles + Protomaps-compatible tooling | 5 | — | — | 5 | 5 | Primary portable tile artifact and static/offline delivery option | +| CesiumJS | 3 | 5 | 4 | 4 | 3 | Deferred 3D specialist option | +| OpenLayers | 4 | 1 | 3 | 5 | 4 | Fallback if WebGL/vector-style requirements conflict with MapLibre | +| Current Leaflet | 2 | 1 | 1 | 5 | 5 | Keep only for current fixture boundary; do not scale it to production | +| Mapbox GL JS/services | 5 | 3 | 4 | 2 | 4 | Optional paid comparator, not default | +| Google Maps Platform | 4 | 3 | 2 | 1 | 4 | Do not use for primary map/geocoding | + +MapLibre GL JS is TypeScript/WebGL and renders browser vector tiles; it has a globe-capable style/projection model and published guidance for vector tiling and clustering large datasets. deck.gl is an MIT-licensed WebGL2 visualization framework with layers suited to hex bins, heatmaps, arcs, paths, tiled vectors, and animation. MapLibre is BSD-3-Clause, deck.gl is MIT, CesiumJS is Apache-2.0, and OpenLayers is BSD-2-Clause; each dependency still requires a pinned version, license inventory, and attribution review. PMTiles is an archive format, not a map renderer; it makes a tile set a versionable, immutable release object. + +## Architecture + +```text +private evidence / graph observations / restrictions + │ release validation + human authority + ▼ + promoted public display projection (per profile/release) + │ + tile/export build ──► manifest, hashes, attribution, policy revision + │ │ + PostGIS/tiles service immutable PMTiles archive + │ │ + └──► Map repository ◄─────┘ + │ + Svelte map adapter: MapLibre base + optional deck.gl overlays + │ + accessible list/detail/provenance and non-map equivalent +``` + +The browser receives only a public projection whose tile attributes are a minimal allowlist. A point feature needs, at most: stable public facility ID, display geometry, `display_precision`, category, profile, public source-origin/review/approval labels, release ID/ruleset, and a token for a detail fetch. It must not contain street address, source payload, source-record ID, geocoding query, reviewer identity, restriction reason, internal relationship IDs, or raw graph evidence. A map click resolves through the same profile- and release-scoped public detail route as the list; it never fetches private data or uses the browser tile cache as an evidence store. + +### Geometry and visual semantics + +| State | Map behavior | Required text/list equivalent | +| --- | --- | --- | +| Exact public point | Eligible marker only after privacy/release projection; no address inferred from it | “Exact display point,” source/retrieval context, and limitations | +| City precision | Distinct coarse marker/symbol and clustering membership only at city level | “Approximate city location — not the facility site” | +| Unmapped | No marker, no cluster contribution, no guessed point | Eligible record remains discoverable as “No publishable map location” | +| Restricted/unscreened/rejected | Absent from tiles, map, aggregates, exports, caches, and direct detail | Ambiguous unavailable state; never confirm why a record is absent | +| Community unreviewed | Separate selected profile and persistent canonical warning in controls, map legend, list, details, exports | Warning before access and on every result; never merge into curated totals | +| `not_seen_recently` | Existing eligible geometry may remain with lifecycle styling | “Not observed recently,” never “closed” | + +Do not encode credibility only with hue. Show source origin, factual review, privacy eligibility, project approval, publication profile, precision, lifecycle, release, and provenance as separately named controls or panels. A relationship arc represents a source-backed, time-bounded assertion, with its observation date, relation type, uncertainty/review state, and source context. It is never a force-directed “ownership truth” visualization. Contradictions and unknowns need an accessible table; contested, sensitive, or non-public edges produce no public arc. + +### Rendering modes + +1. **Low zoom:** server-built country/region aggregates or H3/hex bins with explicit unit, period, profile, method, release, and uncertainty. They are counts of eligible records, never animal totals or proof of activity. Avoid client aggregation over arbitrary partial pages. +2. **Mid zoom:** vector-tile cluster circles or deck.gl `HexagonLayer`/`ScreenGridLayer`; the legend must say whether a symbol is a cluster, bin, or facility. Each aggregate excludes unmapped/restricted rows and carries the same profile separation. +3. **High zoom:** individual public exact points and intentionally coarse city points, with a bounded visible feature count. A list remains primary for keyboard/screen-reader use. +4. **Time:** a scrubber only when observations or claims have a defined time basis. Default to static release view. Never animate source disappearance into a closure claim; retain prior data only while it remains eligible under current restrictions. +5. **Network/overlay:** deferred graph layer for reviewed, public relationships and separately governed environmental/permit layers. Polygon/raster/vector overlays require their own source terms, effective date, resolution, attribution, publication approval, and suppression assessment. Do not spatially join an environmental/permit layer to claim a facility caused an effect without a reviewed causal methodology. + +## Data delivery and costs + +### Primary: generated PMTiles plus controlled object storage + +For low/moderate traffic, generate a small number of profile- and release-specific PMTiles artifacts (for example, curated official, secondary, and community only when eligible) and host them as immutable objects. A generated archive is auditable, supports static preview/offline use, gives a stable checksum/manifest target, and does not require a permanently exposed tile database. Tile URLs and archives must be removed or denied as part of suppression propagation; immutable caching is safe only after a release/public-revision design gives revocation a bounded response path. + +Use a service worker **only after** a suppression/cache audit. The current V1 worker is not a safe V2 cache policy. No static artifact, browser cache, CDN cache, preview, or offline bundle may retain a later-restricted payload. Offline packaging is a future explicitly authorized release artifact with a manifest and recall/restriction procedure, not a default browser feature. + +Cloudflare R2 is a cost comparator, not a selected vendor: its current published Standard tier has 10 GB-month and 10 million Class B reads free, then $0.015/GB-month storage and $0.36/million reads, with no R2 internet-egress charge. Thus a 1–5 GB low-traffic tile archive is commonly within the free storage tier; even 50 GB stored is roughly $0.60/month before request charges. At a moderate 20 million tile/archive reads beyond the free tier, the listed R2 read rate implies about $3.60/month, excluding domain, Workers, logging, WAF, build, and application/database costs. These are arithmetic examples from the published rate, not a quote or performance prediction. Recheck pricing and provider logging before procurement. AWS CloudFront is a viable alternative, but its current pricing varies by data transfer, request region, and features, making R2's simple no-egress model more predictable for this narrow artifact workload. + +### Scale-up: dynamic vector tiles + +When release artifacts become too large or updates too frequent, introduce a read-only tile service backed by a public PostGIS projection. It must accept only allowlisted profile/release/filter/viewport inputs, enforce restriction state in the query, emit bounded simplified vector tiles, and retain the release/revision in cache keys and headers. Possible implementation paths include PostgreSQL vector-tile functions or a dedicated open-source tile server; choose after a deployment threat model and benchmark, not as part of this ADR. The app API still owns search, detail, and export semantics. + +Avoid browser-delivering 100k–500k raw GeoJSON. MapLibre's own guidance recommends vector tiles for larger data and warns that styling/overlap calculations matter. The current user-facing requirement is viewport-based access, not a benchmark contest. Use a 2D map's low-zoom aggregate mode before individual features. + +## Privacy, accessibility, and failure behavior + +Map tiles expose viewport/tile requests to their host and any third-party provider. Device geolocation is optional and stays off by default. Town/postcode search must work without it; location input is not stored in shared URLs, analytics, or application logs. Do not use browser geocoding for visitor input until a provider inventory, disclosure, retention review, and explicit design are approved. Do not silently transmit a query to Mapbox, Google, or another third-party geocoder. The existing visitor-privacy inventory already identifies V1 tile/directions disclosures as unverified; V2 must not inherit them. + +The map itself is progressive enhancement. On no WebGL/WebGL2, a context-loss event, `prefers-reduced-motion`, a low-power/mobile decision, a failed tile request, or a user selecting “list view,” render the same release-scoped list, filters, detail/provenance, and download limits without a map. Do not auto-fallback to V1. Disable globe, animations, extrusions, continuous fly-to, and animated arcs under reduced motion; allow an explicit non-animated retry. Screen readers get semantic filter/list/detail controls and concise aggregate descriptions, not canvas-derived labels. Keyboard users can select a result and request a map focus, but no map interaction is required to inspect evidence. + +Use terrain/buildings only if a future source and privacy review approves them. They increase transfer/GPU cost and can make coarse/residential context more identifiable. Do not draw route/directions, “nearby home,” or targetable travel paths. Rate-limit/bound tile and detail requests, protect public endpoints from arbitrary expensive queries, and do not return a special status that distinguishes a restricted facility from an unknown ID. + +## Why not the alternatives + +**Leaflet:** excellent for the existing fixture map and simple page-scale point sets, but DOM markers and client-side clustering are not the target architecture for 100k–500k global features, GPU bins, vector tiles, globe, and time/arc overlays. Keeping it would create a second later rewrite. + +**CesiumJS as the default:** CesiumJS is a strong Apache-2.0 WGS84 globe/3D Tiles engine, but defaulting to globe/3D would overinvest in the least important interaction. Cesium ion's free Community plan is personal/non-commercial; its published commercial pricing begins at $149/month for an individual and hosted/self-hosted ion adds service/licensing/operations complexity. Use it only when reviewed terrain, 3D Tiles, or high-accuracy globe analysis creates an actual user benefit. CesiumJS alone is open source, but Cesium ion content/services are a separate cost and policy choice. + +**OpenLayers as the primary:** it is mature, open, and has WebGL point/vector support, making it the credible fallback. It lacks the selected stack's cohesive Mapbox-style vector/globe path and requires more custom visualization composition for the planned GPU layers. Use it if MapLibre's style/projection implementation, browser support, or attribution integration fails acceptance benchmarks. + +**Mapbox or Google as primary:** their services are capable, but mapping and geocoding introduce vendor lock-in, account keys, usage billing, and visitor-query disclosure. Mapbox bills map loads and separately bills hosted tileset processing/storage; Google likewise requires a billing/service relationship. They may be evaluated for a specifically approved feature with a published cost cap and privacy review, not adopted by convenience. No need exists to send visitor interests or private candidate data to either service. + +**A graph database/force graph:** the accepted graph foundation intentionally uses PostgreSQL and public read-only projections. A graph database or visual force graph would not resolve source-scoped identity, conflicting observations, or publication policy; it would make false relationships visually persuasive. Defer graph visualization until the relationship data meets its own evidentiary and safety gates. + +## Staged plan and gates + +| Stage | Deliverable | Gate before advancing | +| --- | --- | --- | +| 0: retain | Current blank local Leaflet fixture adapter | Existing fixture safety/boundary tests remain passing; no V2 publication claim | +| 1: interface | Framework-neutral `MapAdapter`, `TileSource`, `AggregateModel`, and accessibility/list contracts; MapLibre spike using synthetic tiles only | Type/boundary tests; no external tile/geocoder requests; no V1 changes | +| 2: 2D release map | MapLibre 2D with self-hosted/open-data basemap, synthetic/publicly eligible vector tiles, exact/city/unmapped semantics, list parity | Manual privacy-provider inventory; keyboard, screen-reader, 320px/200% zoom, reduced-motion, WebGL-fallback, and cross-browser checks | +| 3: scale | Generated PMTiles pipeline and manifest/revocation design; aggregate/cluster modes | Synthetic 100k/500k benchmarks, profile separation, suppression/reimport/restore/cache tests, and operator review | +| 4: overlays | Reviewed source-backed time, relation, environmental, or permit overlays | Overlay-specific provenance/terms/privacy/publication review; causal-language review; mobile budget passes | +| 5: globe | MapLibre globe experiment, behind explicit toggle | Demonstrated orientation benefit over 2D; a11y/list parity; no increased coarse/sensitive disclosure; mobile GPU budget passes | +| 6: Cesium decision | A bounded CesiumJS/3D Tiles proof only if required | Written reviewed 3D data need, source/license/provider review, operational cost approval, and equivalent 2D fallback | + +## Benchmark and acceptance plan + +Benchmark only synthetic/sanitized public-shaped data. Use three distributions: global uniform, dense urban/city clusters, and a mix of exact/city/unmapped. At 100k and 500k records, measure cold and warm map open, first usable filter/list response, pan/zoom frame time, peak JS heap/GPU memory where observable, tile bytes/requests, selection latency, map context loss/recovery, and battery/thermal observations on representative mobile hardware. Test low zoom aggregation, mid zoom clusters, and high zoom points separately. Do not report a single FPS score as a safety or usability result. + +Set provisional acceptance targets before implementation: no raw worldwide collection download; an interactive map first becomes usable within 3 seconds on the defined mid-tier desktop and 5 seconds on the defined mid-tier mobile under a throttled representative connection; map pan stays responsive without long tasks over 200 ms in the tested view; all list/detail/filter functions remain usable with WebGL disabled. Revise targets only with recorded device/network/data assumptions. Bundle budgets should report base application, map renderer, deck overlay, and tiles separately; globe/Cesium must be separate lazy chunks. + +Regression tests cover suppression after tile build, cache invalidation/revocation, profile isolation, direct links, city/unmapped absence from exact point layers, aggregate scope/legend correctness, stale requests, map context loss, and a no-WebGL list-only route. Browser tests block unapproved network hosts. Accessibility tests include automated checks plus manual screen-reader, focus, reflow, reduced-motion, touch-target, and mobile orientation review. A performance pass does not replace human review or release authority. + +## Primary sources reviewed + +- [MapLibre GL JS documentation](https://maplibre.org/maplibre-gl-js/docs/) and [large-data guidance](https://maplibre.org/maplibre-gl-js/docs/guides/large-data/) +- [MapLibre GL JS BSD-3-Clause license](https://github.com/maplibre/maplibre-gl-js/blob/main/LICENSE.txt) +- [deck.gl project and MIT license](https://github.com/visgl/deck.gl) and [performance documentation](https://deck.gl/docs/developer-guide/performance) +- [CesiumJS Apache-2.0 license](https://github.com/CesiumGS/cesium/blob/main/LICENSE.md), [Cesium platform](https://cesium.com/platform/), [ion pricing](https://cesium.com/platform/cesium-ion/pricing/), and [ion self-hosted](https://cesium.com/platform/cesium-ion/cesium-ion-self-hosted/) +- [OpenLayers WebGL points example](https://openlayers.org/en/latest/examples/webgl-points-layer.html) and [WebGL workshop](https://openlayers.org/workshop/en/webgl/points.html) +- [Protomaps basemap license/attribution guidance](https://github.com/protomaps/basemaps) +- [Cloudflare R2 pricing](https://developers.cloudflare.com/r2/pricing/) +- [Mapbox GL JS guide](https://docs.mapbox.com/mapbox-gl-js/guides/), [usage billing explanation](https://docs.mapbox.com/playground/gl-js-usage/), and [geocoding behavior](https://docs.mapbox.com/help/dive-deeper/geocoding/) +- [AWS CloudFront pricing](https://aws.amazon.com/cloudfront/pricing/) + +All pricing and service terms are time-sensitive and must be rechecked at procurement or public deployment. This ADR does not endorse the content licenses of any basemap, imagery, terrain, environmental, permit, or geocoding source; each remains a separate source-terms, attribution, privacy, and publication decision. From 083967c00622c723de110e70a9c7b110be5e9fda Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Wed, 16 Sep 2026 10:56:44 -0700 Subject: [PATCH 168/311] Add bounded concurrent API load rehearsal --- docs/performance/v2-observability.md | 29 ++ .../031_public_release_read_path_indexes.sql | 12 + .../benchmarks/run_api_load_rehearsal.py | 367 ++++++++++++++++++ pipeline/tests/test_api_load_rehearsal.py | 71 ++++ .../tests/test_discovery_scale_benchmark.py | 9 + pipeline/tests/test_graph_migrations.py | 12 +- .../tests/test_publication_scoped_stages.py | 5 + src/lib.rs | 49 +-- 8 files changed, 521 insertions(+), 33 deletions(-) create mode 100644 pipeline/migrations/031_public_release_read_path_indexes.sql create mode 100644 pipeline/scripts/benchmarks/run_api_load_rehearsal.py create mode 100644 pipeline/tests/test_api_load_rehearsal.py diff --git a/docs/performance/v2-observability.md b/docs/performance/v2-observability.md index 24ed098..3469a7e 100644 --- a/docs/performance/v2-observability.md +++ b/docs/performance/v2-observability.md @@ -54,6 +54,35 @@ materialized projection. 4. Keep the service stopped during backup restore until the independent restriction-ledger gate and current replay succeed. +## Concurrent-load rehearsal + +For a local disposable rehearsal against the actual populated API projection: + +```powershell +python pipeline/scripts/benchmarks/run_api_load_rehearsal.py ` + --observations 5000 --concurrency 1,4,8,16 ` + --requests-per-level 40 --timeout-ms 2000 ` + --json-output .tmp/api-load.json +``` + +The harness starts its own E2E PostGIS/API environment, seeds deterministic +synthetic released rows and graph projections, runs a fixed mix of list, +filters, bbox, radius, facets, detail, and graph-ready reads, then destroys +the environment. It refuses non-loopback targets and bounds observations, +concurrency, request count, and timeout. It reports aggregate p50/p95/p99, +throughput, status/error/timeouts, response byte totals, observed database +active/waiting sessions, and pool-pressure signals only. It does not retain +request paths, query values, coordinates, IDs, client data, or response rows. + +The default rehearsal currently establishes no clean concurrency level: the +1,000- and 5,000-observation runs show timeout pressure across the tested +levels, with facets a repeatable hotspot. Therefore it must not be used to +choose a production pool size or to claim capacity. Keep production launch +and pool sizing blocked pending an approved representative traffic test on the +deployment topology. The 2-second request timeout and 350 ms database +radius-query budget remain review thresholds for fail-safe behavior, not +performance guarantees. + The benchmark and logs provide operational evidence only. They do not establish source completeness, publication eligibility, production capacity, cloud cost, or a guarantee for a particular traffic pattern. diff --git a/pipeline/migrations/031_public_release_read_path_indexes.sql b/pipeline/migrations/031_public_release_read_path_indexes.sql new file mode 100644 index 0000000..3583a00 --- /dev/null +++ b/pipeline/migrations/031_public_release_read_path_indexes.sql @@ -0,0 +1,12 @@ +-- Additive indexes for the release-scoped public read path. These support +-- existing joins only; they do not alter eligibility, ordering, or suppression. +CREATE INDEX IF NOT EXISTS release_members_public_discovery_idx + ON uec.release_members (release_id, default_visible, observation_id, facility_id); + +CREATE INDEX IF NOT EXISTS publication_review_scopes_release_event_idx + ON uec.publication_review_release_scopes (release_id, publication_review_event_id); + +COMMENT ON INDEX uec.release_members_public_discovery_idx IS + 'Supports release-scoped public discovery membership checks without changing visibility semantics.'; +COMMENT ON INDEX uec.publication_review_scopes_release_event_idx IS + 'Supports release-scoped publication review joins without changing the current-decision ordering.'; diff --git a/pipeline/scripts/benchmarks/run_api_load_rehearsal.py b/pipeline/scripts/benchmarks/run_api_load_rehearsal.py new file mode 100644 index 0000000..a03d679 --- /dev/null +++ b/pipeline/scripts/benchmarks/run_api_load_rehearsal.py @@ -0,0 +1,367 @@ +#!/usr/bin/env python3 +"""Run a bounded synthetic concurrent-load rehearsal against the real API. + +The harness owns its E2E environment, seeds only synthetic public data, and +never accepts a non-loopback target. Response bodies and graph rows are read to +completion and immediately discarded; the report contains aggregate metrics. +""" + +from __future__ import annotations + +import argparse +import hashlib +import http.client +import ipaddress +import json +import os +import sys +import threading +import time +import urllib.parse +import uuid +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +ROOT = Path(__file__).resolve().parents[3] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) +MAX_SEED = 20_000 +MAX_CONCURRENCY = 16 +MAX_REQUESTS_PER_LEVEL = 80 +QUERY_MIX = ( + ("list", "http", "/api/v2/locations?profile=official&limit=50"), + ("filters", "http", "/api/v2/locations?profile=official&country_code=DK&category=slaughter&limit=50"), + ("bbox", "http", "/api/v2/locations?profile=official&min_lon=-10&min_lat=45&max_lon=-9.99&max_lat=45.01&limit=50"), + ("radius", "http", "/api/v2/locations?profile=official&longitude=-5&latitude=50&radius_km=50&limit=50"), + ("facets", "http", "/api/v2/discovery/facets?profile=official"), + ("detail", "detail", None), + ("graph_ready", "graph", None), +) + + +def deterministic_uuid(prefix: str, ordinal: int) -> str: + return str(uuid.UUID(hex=hashlib.md5(f"{prefix}-{ordinal}".encode()).hexdigest())) + + +def validate_levels(levels: list[int]) -> tuple[int, ...]: + if not levels or any(level < 1 or level > MAX_CONCURRENCY for level in levels): + raise ValueError(f"concurrency levels must be between 1 and {MAX_CONCURRENCY}") + if len(set(levels)) != len(levels): + raise ValueError("concurrency levels must be unique") + return tuple(levels) + + +def validate_loopback_url(base_url: str) -> urllib.parse.SplitResult: + parsed = urllib.parse.urlsplit(base_url) + if parsed.scheme != "http" or parsed.username or parsed.password or parsed.query or parsed.fragment: + raise ValueError("base URL must be an http loopback origin without credentials or query text") + if not parsed.hostname: + raise ValueError("base URL must include a loopback host") + try: + loopback = ipaddress.ip_address(parsed.hostname).is_loopback + except ValueError: + loopback = parsed.hostname.lower() == "localhost" + if not loopback: + raise ValueError("load rehearsal refuses non-loopback targets") + if parsed.path not in ("", "/"): + raise ValueError("base URL must not include a path prefix") + return parsed + + +def seed_public_projection(connection: Any, count: int) -> str: + """Create a deterministic, promoted, synthetic projection in the E2E DB.""" + release_id = "load-promoted" + connection.execute("INSERT INTO uec.sources (source_id,country_code,name,official_url,access_method) VALUES ('load.synthetic','DK','Synthetic load source','https://example.invalid/load','fixture') ON CONFLICT DO NOTHING") + connection.execute("INSERT INTO uec.releases (release_id,status,ruleset_version,profile,test_only,summary) VALUES ('load-promoted','promoted','load-v1','official',false,'{}') ON CONFLICT DO NOTHING") + connection.execute("INSERT INTO uec.release_manifests (release_id,manifest,manifest_sha256) VALUES ('load-promoted',jsonb_build_object('manifest_version','load-v1','profile','official','release_id','load-promoted','ruleset_version','load-v1','eligible_record_count',%s),repeat('a',64)) ON CONFLICT DO NOTHING", (count,)) + connection.execute("INSERT INTO uec.city_reference_points (country_code,city_name,reference_location,reference_source,source_retrieved_at,source_reference_id) VALUES ('DK','Loadville',ST_SetSRID(ST_Point(-5,50),4326)::geography,'https://example.invalid/load-city',TIMESTAMPTZ '2026-01-01 00:00:00+00','load-city') ON CONFLICT DO NOTHING") + connection.execute(""" + INSERT INTO uec.raw_artifacts (artifact_id,storage_key,sha256,byte_size,retrieved_at) + SELECT md5('load-artifact-' || n::text)::uuid, 'synthetic/load/' || n::text, + repeat(md5('load-sha-' || n::text),2), 1, TIMESTAMPTZ '2026-01-01 00:00:00+00' + FROM generate_series(1,%s) n ON CONFLICT DO NOTHING + """, (count,)) + connection.execute(""" + INSERT INTO uec.source_records (source_record_id,source_id,source_record_key,artifact_id,raw_fields,parsed_at) + SELECT md5('load-record-' || n::text)::uuid, 'load.synthetic', 'load-' || n::text, + md5('load-artifact-' || n::text)::uuid, '{}'::jsonb, + TIMESTAMPTZ '2026-01-01 00:00:00+00' + n * interval '1 second' + FROM generate_series(1,%s) n ON CONFLICT DO NOTHING + """, (count,)) + connection.execute(""" + INSERT INTO uec.facilities (facility_id,canonical_name,country_code,city) + SELECT md5('load-facility-' || n::text)::uuid, 'Synthetic load facility ' || n, + CASE WHEN n %% 2 = 0 THEN 'DK' ELSE 'SE' END, 'Loadville' + FROM generate_series(1,%s) n ON CONFLICT DO NOTHING + """, (count,)) + connection.execute(""" + INSERT INTO uec.observations (observation_id,facility_id,source_record_id,observed_at,observation,classification,ruleset_id,rule_id,classification_category,classification_review_status,default_visible,coordinate_review_status,first_observed_at) + SELECT md5('load-observation-' || n::text)::uuid, md5('load-facility-' || n::text)::uuid, + md5('load-record-' || n::text)::uuid, + TIMESTAMPTZ '2026-01-01 00:00:00+00' + n * interval '1 second', '{}'::jsonb, '{}'::jsonb, + 'load-v1','synthetic', + CASE n %% 4 WHEN 0 THEN 'slaughter' WHEN 1 THEN 'fish_processing' WHEN 2 THEN 'logistics_and_storage' ELSE 'retail_and_prepared_food' END, + 'approved',true,'approved',TIMESTAMPTZ '2026-01-01 00:00:00+00' + n * interval '1 second' + FROM generate_series(1,%s) n ON CONFLICT DO NOTHING + """, (count,)) + connection.execute(""" + INSERT INTO uec.release_members (release_id,facility_id,observation_id,default_visible) + SELECT 'load-promoted',md5('load-facility-' || n::text)::uuid,md5('load-observation-' || n::text)::uuid,true + FROM generate_series(1,%s) n ON CONFLICT DO NOTHING + """, (count,)) + connection.execute(""" + INSERT INTO uec.geocode_results (source_record_id,provider_id,query,match_method,status,attempt_number,result,queried_at) + SELECT md5('load-record-' || n::text)::uuid,'synthetic-fixture','synthetic load query','fixture','accepted',1, + ST_SetSRID(ST_Point(-10 + (n %% 2000) / 100.0,45 + (n %% 1000) / 100.0),4326)::geography, + TIMESTAMPTZ '2026-01-01 00:00:00+00' + n * interval '1 second' + FROM generate_series(1,%s) n ON CONFLICT DO NOTHING + """, (count,)) + connection.execute(""" + INSERT INTO uec.publication_review_events (source_record_id,release_id,factual_review_status,privacy_screening_status,maintainer_approval,publication_eligible,reviewer_role,reviewed_at) + SELECT md5('load-record-' || n::text)::uuid,'load-promoted','reviewed','passed','approved',true,'synthetic-reviewer', + TIMESTAMPTZ '2026-01-02 00:00:00+00' + n * interval '1 second' + FROM generate_series(1,%s) n ON CONFLICT DO NOTHING + """, (count,)) + connection.execute(""" + INSERT INTO uec.organizations (organization_id,canonical_name,country_code,organization_type) + SELECT md5('load-organization-' || n::text)::uuid,'Synthetic load organization ' || n,'DK','company' + FROM generate_series(1,%s) n ON CONFLICT DO NOTHING + """, (count,)) + connection.execute(""" + INSERT INTO uec.organization_relationship_observations + (relationship_observation_id,source_id,source_record_id,from_organization_id,target_facility_id,relationship_type,observed_at,confidence,review_state,storage_state,privacy_status,publication_status,release_id) + SELECT md5('load-relationship-' || n::text)::uuid,'load.synthetic',md5('load-record-' || n::text)::uuid, + md5('load-organization-' || n::text)::uuid,md5('load-facility-' || n::text)::uuid,'operator', + TIMESTAMPTZ '2026-01-02 00:00:00+00' + n * interval '1 second',0.9,'accepted','released','passed','released','load-promoted' + FROM generate_series(1,%s) n ON CONFLICT DO NOTHING + """, (count,)) + connection.execute(""" + INSERT INTO uec.claims + (claim_id,source_id,source_record_id,facility_id,claim_domain,claim_kind,value_state,claim_value,observed_at,confidence,review_state,storage_state,privacy_status,publication_status,release_id) + SELECT md5('load-claim-' || n::text)::uuid,'load.synthetic',md5('load-record-' || n::text)::uuid, + md5('load-facility-' || n::text)::uuid,'operation','synthetic_status','known','{}'::jsonb, + TIMESTAMPTZ '2026-01-02 00:00:00+00' + n * interval '1 second',0.9,'accepted','released','passed','released','load-promoted' + FROM generate_series(1,%s) n ON CONFLICT DO NOTHING + """, (count,)) + connection.commit() + for table in ("uec.raw_artifacts", "uec.source_records", "uec.facilities", "uec.observations", "uec.release_members", "uec.geocode_results", "uec.publication_review_events", "uec.organization_relationship_observations", "uec.claims"): + connection.execute(f"ANALYZE {table}") + connection.commit() + return deterministic_uuid("load-facility", 1) + + +@dataclass(frozen=True) +class Sample: + route: str + elapsed_ms: float + status: int + timed_out: bool + error_kind: str | None + response_bytes: int + + +def _http_request(parsed: urllib.parse.SplitResult, route: str, path: str, worker: int, timeout: float) -> Sample: + started = time.perf_counter() + try: + source_ip = f"127.0.0.{10 + (worker % 200)}" + connection = http.client.HTTPConnection(parsed.hostname, parsed.port or 80, timeout=timeout, source_address=(source_ip, 0)) + connection.request("GET", path, headers={"Accept": "application/json"}) + response = connection.getresponse() + body_size = len(response.read(2_000_001)) + status = response.status + connection.close() + return Sample(route, (time.perf_counter() - started) * 1000, status, False, None, body_size) + except TimeoutError: + return Sample(route, (time.perf_counter() - started) * 1000, 0, True, "timeout", 0) + except OSError: + return Sample(route, (time.perf_counter() - started) * 1000, 0, False, "connection_error", 0) + + +def _graph_request(database_url: str, route: str) -> Sample: + started = time.perf_counter() + try: + import psycopg + with psycopg.connect(database_url, connect_timeout=2) as connection: + connection.execute(""" + SELECT count(*) FROM ( + SELECT relationship.relationship_type + FROM uec.graph_public_relationships relationship + JOIN uec.graph_public_claims claim + ON claim.release_id = relationship.release_id + AND claim.facility_id = relationship.target_facility_id + WHERE relationship.release_id = 'load-promoted' + ORDER BY relationship.relationship_observation_id LIMIT 50 + ) bounded + """).fetchone()[0] + return Sample(route, (time.perf_counter() - started) * 1000, 200, False, None, 0) + except TimeoutError: + return Sample(route, (time.perf_counter() - started) * 1000, 0, True, "timeout", 0) + except OSError: + return Sample(route, (time.perf_counter() - started) * 1000, 0, False, "connection_error", 0) + + +def _run_sample(base: str, database_url: str, detail_id: str, job: tuple[str, str, str | None], worker: int, timeout: float) -> Sample: + route, kind, path = job + if kind == "detail": + path = f"/api/v2/locations/{detail_id}?profile=official" + if kind == "graph": + return _graph_request(database_url, route) + return _http_request(urllib.parse.urlsplit(base), route, path or "/", worker, timeout) + + +class DbSampler: + def __init__(self, database_url: str): + self.database_url = database_url + self.stop = threading.Event() + self.max_active = 0 + self.max_waiting = 0 + self.max_connections = None + self.thread = threading.Thread(target=self._sample, daemon=True) + + def _sample(self) -> None: + try: + import psycopg + with psycopg.connect(self.database_url, connect_timeout=2) as connection: + self.max_connections = int(connection.execute("SHOW max_connections").fetchone()[0]) + while not self.stop.is_set(): + row = connection.execute(""" + SELECT count(*) FILTER (WHERE datname=current_database() AND state='active'), + count(*) FILTER (WHERE datname=current_database() AND wait_event_type IS NOT NULL) + FROM pg_stat_activity + """).fetchone() + self.max_active = max(self.max_active, int(row[0])) + self.max_waiting = max(self.max_waiting, int(row[1])) + self.stop.wait(0.02) + except Exception: + return + + def __enter__(self) -> "DbSampler": + self.thread.start() + return self + + def __exit__(self, *_: Any) -> None: + self.stop.set() + self.thread.join(timeout=3) + + +def percentile(values: list[float], fraction: float) -> float: + if not values: + return 0.0 + ordered = sorted(values) + index = min(len(ordered) - 1, max(0, int((len(ordered) - 1) * fraction))) + return round(ordered[index], 3) + + +def summarize(samples: list[Sample], elapsed_s: float, sampler: DbSampler) -> dict[str, Any]: + latencies = [sample.elapsed_ms for sample in samples] + by_route = {} + for route in sorted({sample.route for sample in samples}): + route_samples = [sample for sample in samples if sample.route == route] + by_route[route] = { + "requests": len(route_samples), + "successes": sum(sample.status == 200 for sample in route_samples), + "client_errors": sum(400 <= sample.status < 500 for sample in route_samples), + "server_errors": sum(sample.status >= 500 for sample in route_samples), + "timeouts": sum(sample.timed_out for sample in route_samples), + "latency_ms": {"p50": percentile([s.elapsed_ms for s in route_samples], .50), "p95": percentile([s.elapsed_ms for s in route_samples], .95), "p99": percentile([s.elapsed_ms for s in route_samples], .99)}, + } + return { + "requests": len(samples), + "successes": sum(sample.status == 200 for sample in samples), + "client_errors": sum(400 <= sample.status < 500 for sample in samples), + "server_errors": sum(sample.status >= 500 for sample in samples), + "timeouts": sum(sample.timed_out for sample in samples), + "connection_errors": sum(sample.error_kind == "connection_error" for sample in samples), + "throughput_rps": round(len(samples) / elapsed_s, 3) if elapsed_s else 0.0, + "latency_ms": {"p50": percentile(latencies, .50), "p95": percentile(latencies, .95), "p99": percentile(latencies, .99), "max": round(max(latencies), 3) if latencies else 0.0}, + "response_bytes_total": sum(sample.response_bytes for sample in samples), + "by_route": by_route, + "database_signals": {"max_active_sessions": sampler.max_active, "max_waiting_sessions": sampler.max_waiting, "max_connections": sampler.max_connections}, + "pool_pressure_signals": {"http_503_or_higher": sum(sample.status >= 503 for sample in samples), "timeouts_or_connection_errors": sum(sample.timed_out or sample.error_kind == "connection_error" for sample in samples)}, + } + + +def build_recommendations(results: list[dict[str, Any]], timeout_ms: int) -> dict[str, Any]: + clean_levels = [ + result["concurrency"] + for result in results + if result["timeouts"] == 0 + and result["server_errors"] == 0 + and result["connection_errors"] == 0 + ] + if clean_levels: + pool = max(clean_levels) + basis = "highest tested level without timeout, server, or connection errors; not production capacity evidence" + else: + pool = None + basis = "no clean tested concurrency level; keep production capacity and pool sizing blocked pending an approved representative load test" + return { + "initial_api_pool_per_process": pool, + "clean_tested_concurrency_levels": clean_levels, + "request_timeout_ms": timeout_ms, + "radius_query_budget_ms": 350, + "basis": basis, + } + + +def run_rehearsal(env: Any, observations: int, levels: tuple[int, ...], requests_per_level: int, timeout_ms: int) -> dict[str, Any]: + import psycopg + with psycopg.connect(env.database_url) as connection: + detail_id = seed_public_projection(connection, observations) + base = f"http://127.0.0.1:{env.api_port}" + results = [] + for concurrency in levels: + jobs = [QUERY_MIX[index % len(QUERY_MIX)] for index in range(requests_per_level)] + started = time.perf_counter() + with DbSampler(env.database_url) as sampler: + with ThreadPoolExecutor(max_workers=concurrency) as executor: + futures = [executor.submit(_run_sample, base, env.database_url, detail_id, job, index % concurrency, timeout_ms / 1000) for index, job in enumerate(jobs)] + samples = [future.result() for future in futures] + results.append({"concurrency": concurrency, **summarize(samples, time.perf_counter() - started, sampler)}) + return { + "schema_version": 1, + "synthetic_only": True, + "observations": observations, + "requests_per_level": requests_per_level, + "timeout_ms": timeout_ms, + "levels": results, + "recommendations": build_recommendations(results, timeout_ms), + } + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--observations", type=int, default=5_000) + parser.add_argument("--concurrency", default="1,4,8,16") + parser.add_argument("--requests-per-level", type=int, default=40) + parser.add_argument("--timeout-ms", type=int, default=2_000) + parser.add_argument("--json-output", type=Path) + args = parser.parse_args(argv) + if not 1 <= args.observations <= MAX_SEED: + parser.error(f"observations must be between 1 and {MAX_SEED:,}") + if not 1 <= args.requests_per_level <= MAX_REQUESTS_PER_LEVEL: + parser.error(f"requests-per-level must be between 1 and {MAX_REQUESTS_PER_LEVEL}") + if not 100 <= args.timeout_ms <= 5_000: + parser.error("timeout-ms must be between 100 and 5000") + try: + levels = validate_levels([int(value) for value in args.concurrency.split(",")]) + from pipeline.tests.e2e.fixture import E2EEnvironment + env = E2EEnvironment().start() + try: + report = run_rehearsal(env, args.observations, levels, args.requests_per_level, args.timeout_ms) + finally: + env.stop() + except (ValueError, RuntimeError) as exc: + parser.error(str(exc)) + serialized = json.dumps(report, indent=2, sort_keys=True) + "\n" + if args.json_output: + args.json_output.parent.mkdir(parents=True, exist_ok=True) + args.json_output.write_text(serialized, encoding="utf-8") + print(serialized, end="") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/pipeline/tests/test_api_load_rehearsal.py b/pipeline/tests/test_api_load_rehearsal.py new file mode 100644 index 0000000..dca5d5d --- /dev/null +++ b/pipeline/tests/test_api_load_rehearsal.py @@ -0,0 +1,71 @@ +"""Unit contracts for the bounded local API load rehearsal.""" + +import importlib.util +import sys +import unittest +from pathlib import Path + +ROOT = Path(__file__).parents[1] +SCRIPT = ROOT / "scripts" / "benchmarks" / "run_api_load_rehearsal.py" +SPEC = importlib.util.spec_from_file_location("run_api_load_rehearsal", SCRIPT) +MODULE = importlib.util.module_from_spec(SPEC) +sys.modules[SPEC.name] = MODULE +SPEC.loader.exec_module(MODULE) + + +class ApiLoadRehearsalTests(unittest.TestCase): + def test_levels_and_targets_are_bounded(self): + self.assertEqual(MODULE.validate_levels([1, 4, 8, 16]), (1, 4, 8, 16)) + for levels in ([], [0], [17], [1, 1]): + with self.subTest(levels=levels), self.assertRaises(ValueError): + MODULE.validate_levels(levels) + self.assertEqual(MODULE.validate_loopback_url("http://127.0.0.1:8000").hostname, "127.0.0.1") + self.assertEqual(MODULE.validate_loopback_url("http://localhost:8000/").hostname, "localhost") + for target in ("https://127.0.0.1:8000", "http://203.0.113.5:8000", "http://127.0.0.1:8000/api", "http://user:pass@127.0.0.1:8000"): + with self.subTest(target=target), self.assertRaises(ValueError): + MODULE.validate_loopback_url(target) + + def test_summary_is_aggregate_only_and_preserves_error_signals(self): + samples = [ + MODULE.Sample("list", 10.0, 200, False, None, 40), + MODULE.Sample("radius", 20.0, 503, False, None, 0), + MODULE.Sample("detail", 30.0, 0, True, "timeout", 0), + ] + sampler = type("Sampler", (), {"max_active": 4, "max_waiting": 1, "max_connections": 20})() + report = MODULE.summarize(samples, 1.0, sampler) + self.assertEqual(report["requests"], 3) + self.assertEqual(report["successes"], 1) + self.assertEqual(report["server_errors"], 1) + self.assertEqual(report["timeouts"], 1) + self.assertEqual(report["pool_pressure_signals"]["http_503_or_higher"], 1) + self.assertEqual(report["database_signals"]["max_waiting_sessions"], 1) + self.assertNotIn("/api/", str(report)) + self.assertNotIn("latitude", str(report)) + self.assertNotIn("facility", str(report).lower()) + + def test_deterministic_fixture_identifiers_are_stable(self): + self.assertEqual(MODULE.deterministic_uuid("load-facility", 1), MODULE.deterministic_uuid("load-facility", 1)) + self.assertNotEqual(MODULE.deterministic_uuid("load-facility", 1), MODULE.deterministic_uuid("load-facility", 2)) + + def test_recommendations_fail_safe_when_every_level_has_pressure(self): + results = [ + {"concurrency": 1, "timeouts": 1, "server_errors": 0, "connection_errors": 0}, + {"concurrency": 4, "timeouts": 0, "server_errors": 0, "connection_errors": 1}, + ] + recommendations = MODULE.build_recommendations(results, 2000) + self.assertIsNone(recommendations["initial_api_pool_per_process"]) + self.assertEqual(recommendations["clean_tested_concurrency_levels"], []) + self.assertIn("blocked", recommendations["basis"]) + + def test_recommendations_choose_only_clean_levels(self): + results = [ + {"concurrency": 1, "timeouts": 0, "server_errors": 0, "connection_errors": 0}, + {"concurrency": 4, "timeouts": 1, "server_errors": 0, "connection_errors": 0}, + ] + recommendations = MODULE.build_recommendations(results, 2000) + self.assertEqual(recommendations["initial_api_pool_per_process"], 1) + self.assertEqual(recommendations["clean_tested_concurrency_levels"], [1]) + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/tests/test_discovery_scale_benchmark.py b/pipeline/tests/test_discovery_scale_benchmark.py index 8f626d9..2632dca 100644 --- a/pipeline/tests/test_discovery_scale_benchmark.py +++ b/pipeline/tests/test_discovery_scale_benchmark.py @@ -63,6 +63,15 @@ def test_projection_support_migration_is_additive_and_latest_safe(self): self.assertNotIn("drop table", migration) self.assertNotIn("drop index", migration) + def test_public_read_path_migration_is_additive_and_release_scoped(self): + migration = (ROOT / "migrations" / "031_public_release_read_path_indexes.sql").read_text(encoding="utf-8").lower() + self.assertIn("release_members_public_discovery_idx", migration) + self.assertIn("publication_review_scopes_release_event_idx", migration) + self.assertIn("on uec.release_members (release_id, default_visible", migration) + self.assertIn("on uec.publication_review_release_scopes (release_id, publication_review_event_id)", migration) + self.assertNotIn("drop table", migration) + self.assertNotIn("drop index", migration) + if __name__ == "__main__": unittest.main() diff --git a/pipeline/tests/test_graph_migrations.py b/pipeline/tests/test_graph_migrations.py index 1e9afee..e61236a 100644 --- a/pipeline/tests/test_graph_migrations.py +++ b/pipeline/tests/test_graph_migrations.py @@ -11,12 +11,22 @@ def read(self, name): def test_reserved_migrations_are_present_and_ordered(self): migrations = sorted(path.name for path in (ROOT / "migrations").glob("*.sql")) - self.assertEqual(migrations[-5:], [ + graph_migrations = [ + "026_graph_entities_crosswalks.sql", + "027_graph_relationship_observations.sql", + "028_graph_claims_support.sql", + "029_graph_publication_projections.sql", + ] + positions = [migrations.index(name) for name in graph_migrations] + self.assertEqual(positions, sorted(positions)) + self.assertEqual([migrations[position] for position in positions], graph_migrations) + self.assertEqual(migrations[-6:], [ "026_graph_entities_crosswalks.sql", "027_graph_relationship_observations.sql", "028_graph_claims_support.sql", "029_graph_publication_projections.sql", "030_discovery_projection_support_indexes.sql", + "031_public_release_read_path_indexes.sql", ]) def test_entities_are_distinct_and_crosswalk_is_scoped(self): diff --git a/pipeline/tests/test_publication_scoped_stages.py b/pipeline/tests/test_publication_scoped_stages.py index 678b27b..2455b1f 100644 --- a/pipeline/tests/test_publication_scoped_stages.py +++ b/pipeline/tests/test_publication_scoped_stages.py @@ -45,6 +45,11 @@ def test_legacy_ambiguity_and_explicit_scope(self): except psycopg.Error as error: self.skipTest(f"PostGIS is unavailable: {error}") try: + has_test_only = db.execute( + "SELECT 1 FROM information_schema.columns WHERE table_schema='uec' AND table_name='releases' AND column_name='test_only'" + ).fetchone() + if not has_test_only: + self.skipTest("database migrations are incomplete: releases.test_only is unavailable") prefix = f"test.stage.{uuid.uuid4().hex}" release_a, release_b = prefix + ".a", prefix + ".b" db.execute("INSERT INTO uec.sources (source_id,country_code,name,official_url,access_method) VALUES (%s,'DK','Synthetic','https://example.invalid','test')", (prefix,)) diff --git a/src/lib.rs b/src/lib.rs index dee9a85..dce14d6 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -988,41 +988,26 @@ pub async fn get_v2_facets_handler( let release_id: String = release.get(0); let ruleset_version: String = release.get(1); let release_created_at: chrono::DateTime = release.get(2); - let rows = match client.query("SELECT country_code, classification_category, display_precision, lifecycle_status, provenance_origin_type, city FROM uec.map_facilities_display_history WHERE release_id=$1 AND ($2::text IS NULL OR country_code=$2) AND ($3::text IS NULL OR classification_category=$3) AND ($4::text IS NULL OR provenance_origin_type=$4) AND ($5::text IS NULL OR display_precision=$5) AND ($6::text IS NULL OR lifecycle_status=$6) AND ($7::text IS NULL OR city=$7)", &[&release_id, ¶ms.country_code, ¶ms.category, ¶ms.source_type, ¶ms.display_precision, ¶ms.lifecycle_status, ¶ms.region]).await { Ok(rows) => rows, Err(_) => return v2_error(StatusCode::SERVICE_UNAVAILABLE, "facets_query_failed", "public facets unavailable") }; + let rows = match client.query("SELECT country_code, classification_category, display_precision, lifecycle_status, provenance_origin_type, city, count(*)::bigint FROM uec.map_facilities_display_history WHERE release_id=$1 AND ($2::text IS NULL OR country_code=$2) AND ($3::text IS NULL OR classification_category=$3) AND ($4::text IS NULL OR provenance_origin_type=$4) AND ($5::text IS NULL OR display_precision=$5) AND ($6::text IS NULL OR lifecycle_status=$6) AND ($7::text IS NULL OR city=$7) GROUP BY country_code, classification_category, display_precision, lifecycle_status, provenance_origin_type, city", &[&release_id, ¶ms.country_code, ¶ms.category, ¶ms.source_type, ¶ms.display_precision, ¶ms.lifecycle_status, ¶ms.region]).await { Ok(rows) => rows, Err(_) => return v2_error(StatusCode::SERVICE_UNAVAILABLE, "facets_query_failed", "public facets unavailable") }; let mut dimensions = serde_json::Map::new(); - for (name, values) in [ - ( - "country_code", - rows.iter() - .map(|r| r.get::<_, String>(0)) - .collect::>(), - ), - ( - "category", - rows.iter().map(|r| r.get::<_, String>(1)).collect(), - ), - ( - "display_precision", - rows.iter().map(|r| r.get::<_, String>(2)).collect(), - ), - ( - "lifecycle_status", - rows.iter().map(|r| r.get::<_, String>(3)).collect(), - ), - ( - "source_type", - rows.iter().map(|r| r.get::<_, String>(4)).collect(), - ), - ( - "region", - rows.iter() - .filter_map(|r| r.get::<_, Option>(5)) - .collect(), - ), + for (name, column) in [ + ("country_code", 0), + ("category", 1), + ("display_precision", 2), + ("lifecycle_status", 3), + ("source_type", 4), + ("region", 5), ] { let mut counts = std::collections::BTreeMap::::new(); - for value in values { - *counts.entry(value).or_default() += 1; + for row in &rows { + let value = if column == 5 { + row.get::<_, Option>(column) + } else { + Some(row.get::<_, String>(column)) + }; + if let Some(value) = value { + *counts.entry(value).or_default() += row.get::<_, i64>(6) as usize; + } } dimensions.insert( name.into(), From 408ff2058387c9fa91891cc3b10ac6f3c77b0c38 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Wed, 16 Sep 2026 11:14:25 -0700 Subject: [PATCH 169/311] Add Georgia source reconnaissance metadata --- docs/country-recon-ge.md | 26 +++++++++++++++ docs/source-status.json | 8 ++++- pipeline/source_registry.json | 8 ++++- pipeline/tests/test_georgia_recon_metadata.py | 32 +++++++++++++++++++ pipeline/tests/test_source_registry.py | 4 +-- 5 files changed, 74 insertions(+), 4 deletions(-) create mode 100644 docs/country-recon-ge.md create mode 100644 pipeline/tests/test_georgia_recon_metadata.py diff --git a/docs/country-recon-ge.md b/docs/country-recon-ge.md new file mode 100644 index 0000000..eae16c7 --- /dev/null +++ b/docs/country-recon-ge.md @@ -0,0 +1,26 @@ +# Georgia source reconnaissance + +Status: metadata-only reconnaissance; no facility rows, raw exports, coordinates, or personal data were retained. + +## Scope and safety + +Georgia is not yet in the source registry. This pass covers official food-safety, farm/livestock, environmental, corporate, and statistics discovery only. Fully automated acquisition through validation, quarantine, transformation, and ingestion remains the eventual requirement; no automation may acquire or publish facility-level data until the source contract, terms, privacy, and publication review are complete. A public list is not proof of current operation or completeness. Missing or stale records are recorded as not observed, never inferred as closure. Do not geocode or retain addresses/coordinates during reconnaissance. + +## Candidate inventory + +| ID | Authority / scope | Verified route | Format/cadence | Disposition | +|---|---|---|---|---| +| `ge.nfa.approved-food` | National Food Agency (NFA); registered slaughterhouses and recognized animal-origin food operators | [Registered slaughterhouses](https://nfa.gov.ge/Ge/Page/List%20of%20Slaughterhouses%20Registered%20in%20Georgia), [recognition guidance](https://nfa.gov.ge/Ge/Page/Guidelines%20for%20Recognition), [veterinary-control registers](https://www.nfa.gov.ge/Ge/Page/Veterinary%20Control) | Periodically updated downloadable files are advertised; exact URL, format, schema, stable ID, cadence, terms, and privacy policy not pinned | Partial; blocked | +| `ge.nfa.farms-livestock` | NFA primary-production controls, livestock identification/registration, feed and recognized primary-production operators | [Primary production control](https://nfa.gov.ge/Ge/Page/Primary%20production%20control) and [NFA English portal](https://www.nfa.gov.ge/en) | Linked lists/registers and electronic services; bulk/API contract, scope, cadence, IDs, and location-sensitivity not verified | Partial; blocked | +| `ge.nea.environment-permits` | National Environment Agency; environmental permits and related public environmental information | [NEA official site](https://nea.gov.ge/) | Public services/register routes require endpoint and export verification; format, cadence, geometry, license, and privacy not pinned | Partial; blocked | +| `ge.napr.organizations` | National Agency of Public Registry; entrepreneur and legal-entity identifiers | [NAPR](https://www.napr.gov.ge/) | Online extracts/services are discoverable; authorized bulk/API access, fields, fees, rate limits, cadence, and personal-address policy not verified | Partial; blocked | +| `ge.geostat.slaughter-statistics` | National Statistics Office of Georgia; aggregate livestock slaughterhouse, meat-production, and related agricultural indicators | [Geostat agriculture](https://www.geostat.ge/en/modules/categories/755/section-5-livestock-poultry-and-beehives), [slaughterhouse survey](https://www.geostat.ge/en/single-news/3781/survey-results-for-livestock-slaughterhouses-elevators-and-cold-storage-facilities-2025) | Recurring quarterly/annual publications, often PDF/XLS; table IDs, machine API, revision policy, and suppression rules not pinned | Partial; not run; aggregate-only | +| `ge.nfa.inspections-experiments` | NFA veterinary/food-control findings and any public animal-experimentation evidence | [Veterinary control](https://www.nfa.gov.ge/Ge/Page/Veterinary%20Control) | Dated control results and registers are published as resources; stable event API, retention, privacy, and experimentation coverage not verified | Partial; blocked | + +## Automation and release gates + +Before live acquisition, pin the exact official download/API route, schema fingerprint, pagination, stable identifiers and lifecycle semantics, observed cadence/freshness, attribution/license, rate limits, privacy/retention terms, and whether facility locations may be retained. A compliant job would fetch only an authorized public snapshot, hash and quarantine it, validate schema/freshness, preserve source provenance privately, and ingest only a reviewed safe projection. No raw artifacts or facility rows belong in this repository. Any sensitive location, personal contact, or ambiguous operational record stops the run and is escalated. + +## Recommendation + +Georgia is not ingestion-ready. The lowest-risk next step is an aggregate-only Geostat adapter contract; NFA facility lists should remain blocked until the download links and publication terms are captured and safety-reviewed. NAPR and NEA should be treated as separate services, not silently joined to NFA records. Recommend the next unreconned nearby country only after checking the orchestrator inventory. diff --git a/docs/source-status.json b/docs/source-status.json index 7649798..7a43149 100644 --- a/docs/source-status.json +++ b/docs/source-status.json @@ -205,6 +205,12 @@ {"source_id":"ua.environment-permits","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ua.md","pipeline/source_registry.json"],"next_action":"Pin authorized route, schema, cadence, IDs, terms, privacy, and wartime safety controls."}, {"source_id":"ua.edr.organizations","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ua.md","pipeline/source_registry.json"],"next_action":"Pin authorized route, schema, cadence, IDs, terms, privacy, and wartime safety controls."}, {"source_id":"ua.ukrstat.statistics","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ua.md","pipeline/source_registry.json"],"next_action":"Pin authorized route, schema, cadence, IDs, terms, privacy, and wartime safety controls."}, - {"source_id":"ua.inspections-experiments","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ua.md","pipeline/source_registry.json"],"next_action":"Pin authorized route, schema, cadence, IDs, terms, privacy, and wartime safety controls."} + {"source_id":"ua.inspections-experiments","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ua.md","pipeline/source_registry.json"],"next_action":"Pin authorized route, schema, cadence, IDs, terms, privacy, and wartime safety controls."}, + {"source_id":"ge.nfa.approved-food","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ge.md","pipeline/source_registry.json"],"next_action":"Pin authorized route, schema, cadence, IDs, terms, privacy, and automation permissions."}, + {"source_id":"ge.nfa.farms-livestock","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ge.md","pipeline/source_registry.json"],"next_action":"Pin authorized route, schema, cadence, IDs, terms, privacy, and automation permissions."}, + {"source_id":"ge.nea.environment-permits","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ge.md","pipeline/source_registry.json"],"next_action":"Pin authorized route, schema, cadence, IDs, terms, privacy, and automation permissions."}, + {"source_id":"ge.napr.organizations","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ge.md","pipeline/source_registry.json"],"next_action":"Pin authorized route, schema, cadence, IDs, terms, privacy, and automation permissions."}, + {"source_id":"ge.geostat.slaughter-statistics","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ge.md","pipeline/source_registry.json"],"next_action":"Pin authorized route, schema, cadence, IDs, terms, privacy, and automation permissions."}, + {"source_id":"ge.nfa.inspections-experiments","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ge.md","pipeline/source_registry.json"],"next_action":"Pin authorized route, schema, cadence, IDs, terms, privacy, and automation permissions."} ] } diff --git a/pipeline/source_registry.json b/pipeline/source_registry.json index d4f8b49..3ca9f51 100644 --- a/pipeline/source_registry.json +++ b/pipeline/source_registry.json @@ -821,7 +821,13 @@ {"source_id":"ua.environment-permits","jurisdiction_scope":"Ukraine; Ministry environmental registers and permits","legacy_paths":[],"url":"https://mepr.gov.ua/","access_method":"official EcoSystem/register route; bounded authorized API/export only","cadence":"resource-specific; unknown","attribution_licensing_notes":"Ministry source; license, geometry, privacy, and security review required","adapter_status":"reference_only","expected_artifact_schema":"Permit metadata and aggregate status contract; no facility rows or sensitive geometry retained","blockers":["Exact public permit API/export, coverage, cadence, geometry policy, and wartime security controls not pinned."]}, {"source_id":"ua.edr.organizations","jurisdiction_scope":"Ukraine; Unified State Register of legal entities and organizations","legacy_paths":[],"url":"https://data.gov.ua/","access_method":"official open-data catalog discovery; authorized API/download only","cadence":"unknown","attribution_licensing_notes":"Government open-data source; field-level personal-address, terms, and rate-limit review required","adapter_status":"reference_only","expected_artifact_schema":"Organization metadata keyed by official identifier; suppress personal addresses and retain no rows in reconnaissance","blockers":["Current authorized route, fields, cadence, licensing, privacy policy, and automation permissions not verified."]}, {"source_id":"ua.ukrstat.statistics","jurisdiction_scope":"Ukraine; aggregate livestock, animal-production, and slaughter statistics","legacy_paths":[],"url":"https://ukrstat.gov.ua/","access_method":"official statistical tables/publications; API or bounded download to be pinned","cadence":"table-specific; unknown","attribution_licensing_notes":"State Statistics Service source; publication terms, revisions, and regional suppression require review","adapter_status":"reference_only","expected_artifact_schema":"Aggregate time-series/table metadata; no establishment rows","blockers":["Current table IDs, machine API, cadence, revision semantics, and suppression rules not pinned."]}, - {"source_id":"ua.inspections-experiments","jurisdiction_scope":"Ukraine; DPSS inspections/enforcement and public animal-experimentation evidence","legacy_paths":[],"url":"https://dpss.gov.ua/diyalnist/bezpechnist-harchovih-produktiv-ta-veterinarna-medicina/reyestri","access_method":"official reports/registers; aggregate-only until a safe public route is authorized","cadence":"resource-specific; unknown","attribution_licensing_notes":"Sensitive control/research evidence; privacy, retention, and wartime safety review required","adapter_status":"reference_only","expected_artifact_schema":"Aggregate institution/event evidence with source document; no facility rows or operational locations","blockers":["No safe stable public facility/event master verified; stop and escalate on sensitive exposure."]} ] + {"source_id":"ua.inspections-experiments","jurisdiction_scope":"Ukraine; DPSS inspections/enforcement and public animal-experimentation evidence","legacy_paths":[],"url":"https://dpss.gov.ua/diyalnist/bezpechnist-harchovih-produktiv-ta-veterinarna-medicina/reyestri","access_method":"official reports/registers; aggregate-only until a safe public route is authorized","cadence":"resource-specific; unknown","attribution_licensing_notes":"Sensitive control/research evidence; privacy, retention, and wartime safety review required","adapter_status":"reference_only","expected_artifact_schema":"Aggregate institution/event evidence with source document; no facility rows or operational locations","blockers":["No safe stable public facility/event master verified; stop and escalate on sensitive exposure."]}, + {"source_id":"ge.nfa.approved-food","jurisdiction_scope":"Georgia; NFA registered slaughterhouses and recognized animal-origin food operators","legacy_paths":[],"url":"https://nfa.gov.ge/Ge/Page/List%20of%20Slaughterhouses%20Registered%20in%20Georgia","access_method":"official NFA page with advertised download; authorized bounded capture only","cadence":"periodic; exact cadence unknown","attribution_licensing_notes":"Official Georgian government source; terms, privacy, and location-safety review required","adapter_status":"reference_only","expected_artifact_schema":"Metadata-only contract pending; do not retain facility rows or coordinates","blockers":["Exact download URL, format, schema, stable ID, cadence, terms, and privacy policy not pinned."]}, + {"source_id":"ge.nfa.farms-livestock","jurisdiction_scope":"Georgia; NFA primary-production, livestock identification/registration, feed, and recognized operators","legacy_paths":[],"url":"https://nfa.gov.ge/Ge/Page/Primary%20production%20control","access_method":"official NFA service/list route; no facility acquisition until contract review","cadence":"unknown","attribution_licensing_notes":"Official government source; scope, personal data, coordinates, and reuse terms require review","adapter_status":"reference_only","expected_artifact_schema":"Metadata-only farm/livestock source contract; row-free reconnaissance","blockers":["Bulk/API route, stable IDs, cadence, coverage, privacy, and location policy not verified."]}, + {"source_id":"ge.nea.environment-permits","jurisdiction_scope":"Georgia; National Environment Agency environmental permits and related registers","legacy_paths":[],"url":"https://nea.gov.ge/","access_method":"official NEA service/register route; authorized API/export only","cadence":"resource-specific; unknown","attribution_licensing_notes":"Agency source; license, geometry, privacy, and terms require review","adapter_status":"reference_only","expected_artifact_schema":"Permit metadata/aggregate status contract; no facility rows or sensitive geometry","blockers":["Exact permit route, export/API, coverage, cadence, license, geometry, and privacy not pinned."]}, + {"source_id":"ge.napr.organizations","jurisdiction_scope":"Georgia; NAPR entrepreneur and legal-entity identifiers","legacy_paths":[],"url":"https://www.napr.gov.ge/","access_method":"official NAPR online extract/service; authorized API/bulk access only","cadence":"unknown","attribution_licensing_notes":"Official registry service; fees, rate limits, terms, and personal-address policy require review","adapter_status":"reference_only","expected_artifact_schema":"Organization metadata keyed by official identifier; suppress personal addresses","blockers":["Authorized machine route, fields, cadence, fees, limits, and privacy policy not verified."]}, + {"source_id":"ge.geostat.slaughter-statistics","jurisdiction_scope":"Georgia; aggregate livestock slaughterhouse and animal-production statistics","legacy_paths":[],"url":"https://www.geostat.ge/en/modules/categories/755/section-5-livestock-poultry-and-beehives","access_method":"official Geostat publications and tables; machine API or bounded download to be pinned","cadence":"quarterly/annual; publication-specific","attribution_licensing_notes":"National Statistics Office source; publication terms, revisions, and suppression rules require review","adapter_status":"reference_only","expected_artifact_schema":"Aggregate time-series/table metadata; no establishment rows","blockers":["Current table IDs, machine API, revision semantics, and suppression rules not pinned."]}, + {"source_id":"ge.nfa.inspections-experiments","jurisdiction_scope":"Georgia; NFA veterinary/food-control findings and public animal-experimentation evidence","legacy_paths":[],"url":"https://www.nfa.gov.ge/Ge/Page/Veterinary%20Control","access_method":"official dated reports/registers; aggregate-only until safe event route is verified","cadence":"resource-specific; unknown","attribution_licensing_notes":"Sensitive control/research evidence; privacy, retention, and publication review required","adapter_status":"reference_only","expected_artifact_schema":"Aggregate event/institution evidence with source document; no facility rows","blockers":["No stable public event/experimentation master verified; stop and escalate on sensitive exposure."]} ] } diff --git a/pipeline/tests/test_georgia_recon_metadata.py b/pipeline/tests/test_georgia_recon_metadata.py new file mode 100644 index 0000000..dbd8b8e --- /dev/null +++ b/pipeline/tests/test_georgia_recon_metadata.py @@ -0,0 +1,32 @@ +import json +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +GEORGIA_IDS = { + "ge.nfa.approved-food", + "ge.nfa.farms-livestock", + "ge.nea.environment-permits", + "ge.napr.organizations", + "ge.geostat.slaughter-statistics", + "ge.nfa.inspections-experiments", +} + +class GeorgiaReconMetadataTests(unittest.TestCase): + def test_georgia_recon_is_row_free_and_automated(self): + registry = json.loads((ROOT / "pipeline/source_registry.json").read_text(encoding="utf-8-sig")) + self.assertTrue(GEORGIA_IDS.issubset({s["source_id"] for s in registry["sources"]})) + recon = (ROOT / "docs/country-recon-ge.md").read_text(encoding="utf-8") + self.assertIn("no facility rows", recon.lower()) + self.assertIn("fully automated", recon.lower()) + self.assertIn("not ingestion-ready", recon.lower()) + + def test_georgia_sources_are_blocked_or_not_run(self): + status = json.loads((ROOT / "docs/source-status.json").read_text(encoding="utf-8-sig")) + by_id = {s["source_id"]: s for s in status["sources"]} + for source_id in GEORGIA_IDS: + self.assertEqual(by_id[source_id]["publication_eligibility"], "blocked") + self.assertIn(by_id[source_id]["acquisition"], {"blocked", "not_run"}) + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/tests/test_source_registry.py b/pipeline/tests/test_source_registry.py index c84a883..949bfdf 100644 --- a/pipeline/tests/test_source_registry.py +++ b/pipeline/tests/test_source_registry.py @@ -9,8 +9,8 @@ class SourceRegistryTests(unittest.TestCase): def test_repository_registry_loads_and_references_existing_legacy_paths(self): registry = load_registry() - self.assertEqual(len(registry["sources"]), 197) - self.assertEqual(len({source["source_id"] for source in registry["sources"]}), 197) + self.assertEqual(len(registry["sources"]), 203) + self.assertEqual(len({source["source_id"] for source in registry["sources"]}), 203) def test_unknowns_are_explicit(self): registry = json.loads(REGISTRY_PATH.read_text(encoding="utf-8")) From 2764672a4ed7996f90baa1ebc829c1fe767822f4 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Wed, 16 Sep 2026 11:17:10 -0700 Subject: [PATCH 170/311] Add Armenia source reconnaissance metadata --- docs/country-recon-am.md | 26 +++++++++++++++ docs/source-status.json | 8 ++++- pipeline/source_registry.json | 8 ++++- pipeline/tests/test_armenia_recon_metadata.py | 32 +++++++++++++++++++ pipeline/tests/test_source_registry.py | 4 +-- 5 files changed, 74 insertions(+), 4 deletions(-) create mode 100644 docs/country-recon-am.md create mode 100644 pipeline/tests/test_armenia_recon_metadata.py diff --git a/docs/country-recon-am.md b/docs/country-recon-am.md new file mode 100644 index 0000000..44df455 --- /dev/null +++ b/docs/country-recon-am.md @@ -0,0 +1,26 @@ +# Armenia source reconnaissance + +Status: metadata-only reconnaissance; no facility rows, raw exports, coordinates, or personal data were retained. + +## Scope and safety + +Armenia is not present in the source registry. This pass covers official food-safety, livestock/farm, environmental, corporate, and statistics discovery only. Fully automated acquisition through validation, quarantine, transformation, and ingestion remains the eventual requirement, but no live acquisition or publication occurs until source contracts, terms, privacy, and safety are verified. Public availability does not establish completeness, current operation, or project approval. Missing or stale records mean not observed, never closure. Do not geocode or retain facility addresses/coordinates during reconnaissance. + +## Candidate inventory + +| ID | Authority / scope | Verified route | Format/cadence | Disposition | +|---|---|---|---|---| +| `am.snund.approved-food` | Food Safety Inspection Body; slaughterhouses and food-chain operators, including animal-origin food | [Slaughterhouses](https://snund.am/en/page/operating-slaughterhouses/106), [registry](https://www.snund.am/en/page/registry/109), [requirements](https://www.snund.am/en/page/requirements-for-slaughterhouses/168) | Web pages and advertised registry/download resources; exact file/API, schema, IDs, cadence, terms, and privacy not pinned | Partial; blocked | +| `am.snund.farms-livestock` | Food Safety Inspection Body; food-chain business registration, veterinary controls, livestock-related operators | [FSIB portal](https://www.snund.am/en) and [business registration](https://snund.am/hy/business-registration) | Online services/registry routes; public bulk contract, stable IDs, cadence, location policy, and licensing not verified | Partial; blocked | +| `am.environment-permits` | Armenia environmental authority and permit/register services | [Ministry of Environment](https://env.am/) | Public service discovery only; exact permit API/export, geometry, cadence, license, and privacy not pinned | Partial; blocked | +| `am.e-register.organizations` | Government electronic register of Armenian legal entities | [Electronic Register](https://www.e-register.am/en/) | Search/extract service; full records may require sign-in/payment; authorized API/bulk route, fields, limits, cadence, and personal-address policy unknown | Partial; blocked | +| `am.armstat.livestock-statistics` | Statistical Committee (Armstat); livestock by marz/species/year and agriculture indicators | [PxWeb livestock table](https://statbank.armstat.am/pxweb/en/ArmStatBank/ArmStatBank__6%20Agriculture%2C%20forestry%20and%20fishing/AF-1-2024.px/), [agriculture tables](https://statbank.armstat.am/pxweb/en/ArmStatBank/ArmStatBank__6%20Agriculture%2C%20forestry%20and%20fishing/) | PxWeb supports table selection/query and machine-readable output; exact API contract, revision/cadence, licensing, and suppression rules require pinning | Partial; not run; aggregate-only | +| `am.snund.inspections-experiments` | Food-safety/veterinary inspections and any public animal-experimentation evidence | [FSIB inspection body](https://www.snund.am/en/page/inspection-body/50) | Dated plans/reports and service pages; no stable public experimentation/event master verified | Partial; blocked | + +## Automation and release gates + +Before live acquisition, pin the exact official endpoint/download, response format and schema fingerprint, pagination, stable identifiers and lifecycle semantics, freshness/cadence, attribution/license, rate limits, privacy/retention, and location-safety rules. A compliant job would fetch only an authorized snapshot, hash and quarantine it, validate schema/freshness, preserve provenance privately, and ingest only a reviewed safe projection. No raw artifacts or rows belong in this repository. Sensitive location, personal contact, or ambiguous operational data stops the run and is escalated. + +## Recommendation + +Armenia is not ingestion-ready. Armstat PxWeb is the lowest-risk next contract because it is aggregate and query-oriented. SNUND facility registers remain blocked until download/API details and terms are confirmed. The electronic legal-entity register may involve authentication or payment and must not be scraped around access controls. Recommend the next unreconned nearby country after inventory check: Azerbaijan. diff --git a/docs/source-status.json b/docs/source-status.json index 7a43149..06c0db1 100644 --- a/docs/source-status.json +++ b/docs/source-status.json @@ -211,6 +211,12 @@ {"source_id":"ge.nea.environment-permits","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ge.md","pipeline/source_registry.json"],"next_action":"Pin authorized route, schema, cadence, IDs, terms, privacy, and automation permissions."}, {"source_id":"ge.napr.organizations","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ge.md","pipeline/source_registry.json"],"next_action":"Pin authorized route, schema, cadence, IDs, terms, privacy, and automation permissions."}, {"source_id":"ge.geostat.slaughter-statistics","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ge.md","pipeline/source_registry.json"],"next_action":"Pin authorized route, schema, cadence, IDs, terms, privacy, and automation permissions."}, - {"source_id":"ge.nfa.inspections-experiments","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ge.md","pipeline/source_registry.json"],"next_action":"Pin authorized route, schema, cadence, IDs, terms, privacy, and automation permissions."} + {"source_id":"ge.nfa.inspections-experiments","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ge.md","pipeline/source_registry.json"],"next_action":"Pin authorized route, schema, cadence, IDs, terms, privacy, and automation permissions."}, + {"source_id":"am.snund.approved-food","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-am.md","pipeline/source_registry.json"],"next_action":"Pin authorized route, schema, cadence, IDs, terms, privacy, and automation permissions."}, + {"source_id":"am.snund.farms-livestock","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-am.md","pipeline/source_registry.json"],"next_action":"Pin authorized route, schema, cadence, IDs, terms, privacy, and automation permissions."}, + {"source_id":"am.environment-permits","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-am.md","pipeline/source_registry.json"],"next_action":"Pin authorized route, schema, cadence, IDs, terms, privacy, and automation permissions."}, + {"source_id":"am.e-register.organizations","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-am.md","pipeline/source_registry.json"],"next_action":"Pin authorized route, schema, cadence, IDs, terms, privacy, and automation permissions."}, + {"source_id":"am.armstat.livestock-statistics","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-am.md","pipeline/source_registry.json"],"next_action":"Pin authorized route, schema, cadence, IDs, terms, privacy, and automation permissions."}, + {"source_id":"am.snund.inspections-experiments","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-am.md","pipeline/source_registry.json"],"next_action":"Pin authorized route, schema, cadence, IDs, terms, privacy, and automation permissions."} ] } diff --git a/pipeline/source_registry.json b/pipeline/source_registry.json index 3ca9f51..7f69120 100644 --- a/pipeline/source_registry.json +++ b/pipeline/source_registry.json @@ -827,7 +827,13 @@ {"source_id":"ge.nea.environment-permits","jurisdiction_scope":"Georgia; National Environment Agency environmental permits and related registers","legacy_paths":[],"url":"https://nea.gov.ge/","access_method":"official NEA service/register route; authorized API/export only","cadence":"resource-specific; unknown","attribution_licensing_notes":"Agency source; license, geometry, privacy, and terms require review","adapter_status":"reference_only","expected_artifact_schema":"Permit metadata/aggregate status contract; no facility rows or sensitive geometry","blockers":["Exact permit route, export/API, coverage, cadence, license, geometry, and privacy not pinned."]}, {"source_id":"ge.napr.organizations","jurisdiction_scope":"Georgia; NAPR entrepreneur and legal-entity identifiers","legacy_paths":[],"url":"https://www.napr.gov.ge/","access_method":"official NAPR online extract/service; authorized API/bulk access only","cadence":"unknown","attribution_licensing_notes":"Official registry service; fees, rate limits, terms, and personal-address policy require review","adapter_status":"reference_only","expected_artifact_schema":"Organization metadata keyed by official identifier; suppress personal addresses","blockers":["Authorized machine route, fields, cadence, fees, limits, and privacy policy not verified."]}, {"source_id":"ge.geostat.slaughter-statistics","jurisdiction_scope":"Georgia; aggregate livestock slaughterhouse and animal-production statistics","legacy_paths":[],"url":"https://www.geostat.ge/en/modules/categories/755/section-5-livestock-poultry-and-beehives","access_method":"official Geostat publications and tables; machine API or bounded download to be pinned","cadence":"quarterly/annual; publication-specific","attribution_licensing_notes":"National Statistics Office source; publication terms, revisions, and suppression rules require review","adapter_status":"reference_only","expected_artifact_schema":"Aggregate time-series/table metadata; no establishment rows","blockers":["Current table IDs, machine API, revision semantics, and suppression rules not pinned."]}, - {"source_id":"ge.nfa.inspections-experiments","jurisdiction_scope":"Georgia; NFA veterinary/food-control findings and public animal-experimentation evidence","legacy_paths":[],"url":"https://www.nfa.gov.ge/Ge/Page/Veterinary%20Control","access_method":"official dated reports/registers; aggregate-only until safe event route is verified","cadence":"resource-specific; unknown","attribution_licensing_notes":"Sensitive control/research evidence; privacy, retention, and publication review required","adapter_status":"reference_only","expected_artifact_schema":"Aggregate event/institution evidence with source document; no facility rows","blockers":["No stable public event/experimentation master verified; stop and escalate on sensitive exposure."]} ] + {"source_id":"ge.nfa.inspections-experiments","jurisdiction_scope":"Georgia; NFA veterinary/food-control findings and public animal-experimentation evidence","legacy_paths":[],"url":"https://www.nfa.gov.ge/Ge/Page/Veterinary%20Control","access_method":"official dated reports/registers; aggregate-only until safe event route is verified","cadence":"resource-specific; unknown","attribution_licensing_notes":"Sensitive control/research evidence; privacy, retention, and publication review required","adapter_status":"reference_only","expected_artifact_schema":"Aggregate event/institution evidence with source document; no facility rows","blockers":["No stable public event/experimentation master verified; stop and escalate on sensitive exposure."]}, + {"source_id":"am.snund.approved-food","jurisdiction_scope":"Armenia; Food Safety Inspection Body slaughterhouses and animal-origin food-chain operators","legacy_paths":[],"url":"https://snund.am/en/page/operating-slaughterhouses/106","access_method":"official FSIB registry/page; authorized bounded export or API only","cadence":"unknown; verify from source metadata","attribution_licensing_notes":"Official Armenian government source; terms, privacy, and location-safety review required","adapter_status":"reference_only","expected_artifact_schema":"Metadata-only contract pending; no facility rows or coordinates retained","blockers":["Exact file/API, schema, stable IDs, cadence, terms, and privacy controls not pinned."]}, + {"source_id":"am.snund.farms-livestock","jurisdiction_scope":"Armenia; FSIB food-chain registration and veterinary/livestock-related operators","legacy_paths":[],"url":"https://www.snund.am/en","access_method":"official FSIB service/registry route; no facility acquisition until contract review","cadence":"unknown","attribution_licensing_notes":"Official government source; scope, personal data, locations, and reuse terms require review","adapter_status":"reference_only","expected_artifact_schema":"Metadata-only farm/livestock source contract; row-free reconnaissance","blockers":["Public bulk route, stable IDs, cadence, location policy, and licensing not verified."]}, + {"source_id":"am.environment-permits","jurisdiction_scope":"Armenia; environmental permits and registers","legacy_paths":[],"url":"https://env.am/","access_method":"official environmental service/register route; authorized API/export only","cadence":"resource-specific; unknown","attribution_licensing_notes":"Ministry source; license, geometry, privacy, and terms require review","adapter_status":"reference_only","expected_artifact_schema":"Permit metadata/aggregate status contract; no facility rows or sensitive geometry","blockers":["Exact public permit route, API/export, coverage, cadence, license, and privacy not pinned."]}, + {"source_id":"am.e-register.organizations","jurisdiction_scope":"Armenia; electronic register of legal entities","legacy_paths":[],"url":"https://www.e-register.am/en/","access_method":"official search/extract service; do not bypass authentication or payment","cadence":"unknown","attribution_licensing_notes":"Government registry; access fees, terms, privacy, and personal-address policy require review","adapter_status":"reference_only","expected_artifact_schema":"Organization metadata keyed by official registration identifier; suppress personal addresses","blockers":["Authorized machine route, fields, limits, cadence, and reuse terms not verified; full extracts may require sign-in/payment."]}, + {"source_id":"am.armstat.livestock-statistics","jurisdiction_scope":"Armenia; aggregate livestock and agriculture statistics","legacy_paths":[],"url":"https://statbank.armstat.am/pxweb/en/ArmStatBank/ArmStatBank__6%20Agriculture%2C%20forestry%20and%20fishing/AF-1-2024.px/","access_method":"official Armstat PxWeb table/query; API contract to be pinned","cadence":"table-specific; unknown","attribution_licensing_notes":"Statistical Committee source; copyright, revisions, and suppression rules require review","adapter_status":"reference_only","expected_artifact_schema":"Aggregate PxWeb time-series/table metadata; no establishment rows","blockers":["Pin current table IDs, API/query contract, cadence, revision semantics, and licensing."]}, + {"source_id":"am.snund.inspections-experiments","jurisdiction_scope":"Armenia; FSIB food/veterinary inspections and public animal-experimentation evidence","legacy_paths":[],"url":"https://www.snund.am/en/page/inspection-body/50","access_method":"official reports/plans; aggregate-only until a safe event route is verified","cadence":"resource-specific; unknown","attribution_licensing_notes":"Sensitive control/research evidence; privacy, retention, and publication review required","adapter_status":"reference_only","expected_artifact_schema":"Aggregate event/institution evidence with source document; no facility rows","blockers":["No stable public experimentation/event master verified; stop and escalate on sensitive exposure."]} ] } diff --git a/pipeline/tests/test_armenia_recon_metadata.py b/pipeline/tests/test_armenia_recon_metadata.py new file mode 100644 index 0000000..6af2164 --- /dev/null +++ b/pipeline/tests/test_armenia_recon_metadata.py @@ -0,0 +1,32 @@ +import json +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +ARMENIA_IDS = { + "am.snund.approved-food", + "am.snund.farms-livestock", + "am.environment-permits", + "am.e-register.organizations", + "am.armstat.livestock-statistics", + "am.snund.inspections-experiments", +} + +class ArmeniaReconMetadataTests(unittest.TestCase): + def test_armenia_recon_is_row_free_and_automated(self): + registry = json.loads((ROOT / "pipeline/source_registry.json").read_text(encoding="utf-8-sig")) + self.assertTrue(ARMENIA_IDS.issubset({s["source_id"] for s in registry["sources"]})) + recon = (ROOT / "docs/country-recon-am.md").read_text(encoding="utf-8") + self.assertIn("no facility rows", recon.lower()) + self.assertIn("fully automated", recon.lower()) + self.assertIn("not ingestion-ready", recon.lower()) + + def test_armenia_sources_are_blocked_or_not_run(self): + status = json.loads((ROOT / "docs/source-status.json").read_text(encoding="utf-8-sig")) + by_id = {s["source_id"]: s for s in status["sources"]} + for source_id in ARMENIA_IDS: + self.assertEqual(by_id[source_id]["publication_eligibility"], "blocked") + self.assertIn(by_id[source_id]["acquisition"], {"blocked", "not_run"}) + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/tests/test_source_registry.py b/pipeline/tests/test_source_registry.py index 949bfdf..f9015a6 100644 --- a/pipeline/tests/test_source_registry.py +++ b/pipeline/tests/test_source_registry.py @@ -9,8 +9,8 @@ class SourceRegistryTests(unittest.TestCase): def test_repository_registry_loads_and_references_existing_legacy_paths(self): registry = load_registry() - self.assertEqual(len(registry["sources"]), 203) - self.assertEqual(len({source["source_id"] for source in registry["sources"]}), 203) + self.assertEqual(len(registry["sources"]), 209) + self.assertEqual(len({source["source_id"] for source in registry["sources"]}), 209) def test_unknowns_are_explicit(self): registry = json.loads(REGISTRY_PATH.read_text(encoding="utf-8")) From 356a1a84d6be9b289ce6dafb89d5dc3f2ab680dc Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Wed, 16 Sep 2026 11:19:47 -0700 Subject: [PATCH 171/311] Add Azerbaijan source reconnaissance metadata --- docs/country-recon-az.md | 26 +++++++++++++++ docs/source-status.json | 8 ++++- pipeline/source_registry.json | 8 ++++- .../tests/test_azerbaijan_recon_metadata.py | 32 +++++++++++++++++++ pipeline/tests/test_source_registry.py | 4 +-- 5 files changed, 74 insertions(+), 4 deletions(-) create mode 100644 docs/country-recon-az.md create mode 100644 pipeline/tests/test_azerbaijan_recon_metadata.py diff --git a/docs/country-recon-az.md b/docs/country-recon-az.md new file mode 100644 index 0000000..3a77a72 --- /dev/null +++ b/docs/country-recon-az.md @@ -0,0 +1,26 @@ +# Azerbaijan source reconnaissance + +Status: metadata-only reconnaissance; no facility rows, raw exports, coordinates, operational details, or personal data were retained. + +## Scope and safety + +Azerbaijan is not present in the source registry. This pass covers official food-safety, livestock, environmental, corporate, and statistics discovery. Fully automated acquisition through validation, quarantine, transformation, and ingestion remains the eventual requirement, but acquisition is gated until source contracts, access controls, terms, privacy, and regional/conflict safety are verified. Do not bypass authentication, payment, robots, or technical restrictions. Public availability does not establish completeness or current operation; disappearance is not closure. Do not geocode or retain facility locations during reconnaissance. + +## Candidate inventory + +| ID | Authority / scope | Verified route | Format/cadence | Disposition | +|---|---|---|---|---| +| `az.afsa.food-subjects` | Azerbaijan Food Safety Agency (AFSA/AQTA); registered food subjects and activities, including animal-origin food | [Food subjects search](https://afsa.gov.az/az/qida-subyektleri), [AFSA portal](https://afsa.gov.az/) | Search service exposes query fields; bulk/API route, schema, stable IDs, cadence, licensing, and privacy not pinned | Partial; blocked | +| `az.afsa.livestock-traceability` | AFSA; animal identification/registration and farm-to-table traceability | [Animal identification notice](https://afsa.gov.az/az/heyvan-saglamligi-ve-bioloji-tehlukesizlik/xeberler/heyvanlarin-identiklesdirilmesi-baytarliq-nezaretinin-effektivliyini-artirir), [AQTIS login](https://hiqs.afsa.gov.az/Airs.Web/Account/Login.aspx) | Electronic system exists; login-protected route and sensitive individual/farm records; no automated access or public export verified | Partial; blocked | +| `az.eco.environment-permits` | Ministry of Ecology and Natural Resources; environmental permits/registers | [Ministry site](https://eco.gov.az/) | Public service discovery; exact permit API/export, geometry, cadence, license, privacy, and safety not pinned | Partial; blocked | +| `az.taxes.organizations` | State Tax Service; commercial legal-entity and taxpayer registration data | [State registration](https://www.taxes.gov.az/en/page/qeydiyyat), [public database](https://taxes.gov.az/en/page/ictimai-aciq-melumat-bazasi) | Search/statistical pages and public database; access, fields, cadence, terms, and personal-data boundaries require verification | Partial; blocked | +| `az.stat.livestock-statistics` | State Statistical Committee; aggregate livestock, slaughter, meat, milk, and fishery indicators | [Agriculture statistics](https://www.stat.gov.az/source/agriculture/?lang=en), [2025 agriculture yearbook](https://www.stat.gov.az/menu/6/statistical_yearbooks/source/agriculture_2025.pdf) | Annual tables/yearbooks; current machine API, table IDs, revision/cadence, licensing, and suppression rules not pinned | Partial; not run; aggregate-only | +| `az.afsa.inspections-enforcement` | AFSA inspections, violations, veterinary control, and any public experimentation evidence | [AFSA portal](https://afsa.gov.az/) | Dated notices/reports and search services; stable event API, retention, privacy, and experimentation coverage not verified | Partial; blocked | + +## Automation and release gates + +Before live acquisition, pin the exact authorized endpoint/download, schema fingerprint, pagination/query contract, stable identifiers and lifecycle semantics, freshness/cadence, attribution/license, rate limits, privacy/retention, and location-safety rules. A compliant job would fetch only an authorized public snapshot, hash and quarantine it, validate schema/freshness, preserve provenance privately, and ingest only a reviewed safe projection. Sensitive farm or facility locations, personal contacts, or vulnerable operational details stop the run and are escalated. + +## Recommendation + +Azerbaijan is not ingestion-ready. The aggregate State Statistical Committee tables are the lowest-risk candidate. AFSA food-subject search is not an authorization for scraping, and the animal-identification system is login-protected and potentially sensitive. Keep all six candidates blocked/not-run until contracts and safety review are complete. Recommend the next unreconned nearby country after inventory check: Turkey. diff --git a/docs/source-status.json b/docs/source-status.json index 06c0db1..acb6662 100644 --- a/docs/source-status.json +++ b/docs/source-status.json @@ -217,6 +217,12 @@ {"source_id":"am.environment-permits","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-am.md","pipeline/source_registry.json"],"next_action":"Pin authorized route, schema, cadence, IDs, terms, privacy, and automation permissions."}, {"source_id":"am.e-register.organizations","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-am.md","pipeline/source_registry.json"],"next_action":"Pin authorized route, schema, cadence, IDs, terms, privacy, and automation permissions."}, {"source_id":"am.armstat.livestock-statistics","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-am.md","pipeline/source_registry.json"],"next_action":"Pin authorized route, schema, cadence, IDs, terms, privacy, and automation permissions."}, - {"source_id":"am.snund.inspections-experiments","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-am.md","pipeline/source_registry.json"],"next_action":"Pin authorized route, schema, cadence, IDs, terms, privacy, and automation permissions."} + {"source_id":"am.snund.inspections-experiments","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-am.md","pipeline/source_registry.json"],"next_action":"Pin authorized route, schema, cadence, IDs, terms, privacy, and automation permissions."}, + {"source_id":"az.afsa.food-subjects","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-az.md","pipeline/source_registry.json"],"next_action":"Pin authorized route, schema, cadence, IDs, terms, privacy, and automation permissions."}, + {"source_id":"az.afsa.livestock-traceability","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-az.md","pipeline/source_registry.json"],"next_action":"Pin authorized route, schema, cadence, IDs, terms, privacy, and automation permissions."}, + {"source_id":"az.eco.environment-permits","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-az.md","pipeline/source_registry.json"],"next_action":"Pin authorized route, schema, cadence, IDs, terms, privacy, and automation permissions."}, + {"source_id":"az.taxes.organizations","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-az.md","pipeline/source_registry.json"],"next_action":"Pin authorized route, schema, cadence, IDs, terms, privacy, and automation permissions."}, + {"source_id":"az.stat.livestock-statistics","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-az.md","pipeline/source_registry.json"],"next_action":"Pin authorized route, schema, cadence, IDs, terms, privacy, and automation permissions."}, + {"source_id":"az.afsa.inspections-enforcement","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-az.md","pipeline/source_registry.json"],"next_action":"Pin authorized route, schema, cadence, IDs, terms, privacy, and automation permissions."} ] } diff --git a/pipeline/source_registry.json b/pipeline/source_registry.json index 7f69120..60b8087 100644 --- a/pipeline/source_registry.json +++ b/pipeline/source_registry.json @@ -833,7 +833,13 @@ {"source_id":"am.environment-permits","jurisdiction_scope":"Armenia; environmental permits and registers","legacy_paths":[],"url":"https://env.am/","access_method":"official environmental service/register route; authorized API/export only","cadence":"resource-specific; unknown","attribution_licensing_notes":"Ministry source; license, geometry, privacy, and terms require review","adapter_status":"reference_only","expected_artifact_schema":"Permit metadata/aggregate status contract; no facility rows or sensitive geometry","blockers":["Exact public permit route, API/export, coverage, cadence, license, and privacy not pinned."]}, {"source_id":"am.e-register.organizations","jurisdiction_scope":"Armenia; electronic register of legal entities","legacy_paths":[],"url":"https://www.e-register.am/en/","access_method":"official search/extract service; do not bypass authentication or payment","cadence":"unknown","attribution_licensing_notes":"Government registry; access fees, terms, privacy, and personal-address policy require review","adapter_status":"reference_only","expected_artifact_schema":"Organization metadata keyed by official registration identifier; suppress personal addresses","blockers":["Authorized machine route, fields, limits, cadence, and reuse terms not verified; full extracts may require sign-in/payment."]}, {"source_id":"am.armstat.livestock-statistics","jurisdiction_scope":"Armenia; aggregate livestock and agriculture statistics","legacy_paths":[],"url":"https://statbank.armstat.am/pxweb/en/ArmStatBank/ArmStatBank__6%20Agriculture%2C%20forestry%20and%20fishing/AF-1-2024.px/","access_method":"official Armstat PxWeb table/query; API contract to be pinned","cadence":"table-specific; unknown","attribution_licensing_notes":"Statistical Committee source; copyright, revisions, and suppression rules require review","adapter_status":"reference_only","expected_artifact_schema":"Aggregate PxWeb time-series/table metadata; no establishment rows","blockers":["Pin current table IDs, API/query contract, cadence, revision semantics, and licensing."]}, - {"source_id":"am.snund.inspections-experiments","jurisdiction_scope":"Armenia; FSIB food/veterinary inspections and public animal-experimentation evidence","legacy_paths":[],"url":"https://www.snund.am/en/page/inspection-body/50","access_method":"official reports/plans; aggregate-only until a safe event route is verified","cadence":"resource-specific; unknown","attribution_licensing_notes":"Sensitive control/research evidence; privacy, retention, and publication review required","adapter_status":"reference_only","expected_artifact_schema":"Aggregate event/institution evidence with source document; no facility rows","blockers":["No stable public experimentation/event master verified; stop and escalate on sensitive exposure."]} ] + {"source_id":"am.snund.inspections-experiments","jurisdiction_scope":"Armenia; FSIB food/veterinary inspections and public animal-experimentation evidence","legacy_paths":[],"url":"https://www.snund.am/en/page/inspection-body/50","access_method":"official reports/plans; aggregate-only until a safe event route is verified","cadence":"resource-specific; unknown","attribution_licensing_notes":"Sensitive control/research evidence; privacy, retention, and publication review required","adapter_status":"reference_only","expected_artifact_schema":"Aggregate event/institution evidence with source document; no facility rows","blockers":["No stable public experimentation/event master verified; stop and escalate on sensitive exposure."]}, + {"source_id":"az.afsa.food-subjects","jurisdiction_scope":"Azerbaijan; AFSA registered food subjects and animal-origin food activities","legacy_paths":[],"url":"https://afsa.gov.az/az/qida-subyektleri","access_method":"official AFSA search service; authorized bounded export/API only","cadence":"unknown","attribution_licensing_notes":"Official Azerbaijani government source; terms, privacy, and regional safety review required","adapter_status":"reference_only","expected_artifact_schema":"Metadata-only contract pending; no facility rows, addresses, or coordinates retained","blockers":["Bulk/API route, schema, stable IDs, cadence, licensing, privacy, and safe publication boundary not pinned."]}, + {"source_id":"az.afsa.livestock-traceability","jurisdiction_scope":"Azerbaijan; AFSA animal identification/registration and farm-to-table traceability","legacy_paths":[],"url":"https://afsa.gov.az/az/heyvan-saglamligi-ve-bioloji-tehlukesizlik/xeberler/heyvanlarin-identiklesdirilmesi-baytarliq-nezaretinin-effektivliyini-artirir","access_method":"official notice references AQTIS; login-protected system, do not bypass access controls","cadence":"unknown","attribution_licensing_notes":"Sensitive veterinary/farm data; privacy, security, retention, and terms require explicit review","adapter_status":"reference_only","expected_artifact_schema":"Metadata-only traceability contract; no farm, animal, or location rows","blockers":["Public export/API and authorization not verified; records may be operationally sensitive."]}, + {"source_id":"az.eco.environment-permits","jurisdiction_scope":"Azerbaijan; environmental permits and registers","legacy_paths":[],"url":"https://eco.gov.az/","access_method":"official ministry route; authorized API/export only","cadence":"resource-specific; unknown","attribution_licensing_notes":"Ministry source; license, geometry, privacy, and safety review required","adapter_status":"reference_only","expected_artifact_schema":"Permit metadata/aggregate status contract; no sensitive geometry","blockers":["Exact public route, API/export, coverage, cadence, license, geometry, and privacy not pinned."]}, + {"source_id":"az.taxes.organizations","jurisdiction_scope":"Azerbaijan; State Tax Service commercial legal-entity and taxpayer registration","legacy_paths":[],"url":"https://www.taxes.gov.az/en/page/qeydiyyat","access_method":"official search/public database; no bypass of access controls","cadence":"unknown; statistical snapshots exist","attribution_licensing_notes":"Government registry; personal-data, terms, limits, and reuse review required","adapter_status":"reference_only","expected_artifact_schema":"Organization metadata keyed by taxpayer/registration identifier; suppress personal addresses","blockers":["Authorized machine route, fields, cadence, licensing, privacy, and automation permissions not verified."]}, + {"source_id":"az.stat.livestock-statistics","jurisdiction_scope":"Azerbaijan; aggregate livestock, slaughter, meat, milk, and fishery statistics","legacy_paths":[],"url":"https://www.stat.gov.az/source/agriculture/?lang=en","access_method":"official statistical tables/yearbooks; machine API or bounded download to be pinned","cadence":"annual and table-specific","attribution_licensing_notes":"State Statistical Committee source; publication terms, revisions, and suppression rules require review","adapter_status":"reference_only","expected_artifact_schema":"Aggregate time-series/yearbook metadata; no establishment rows","blockers":["Current table IDs, machine API, revision semantics, licensing, and suppression rules not pinned."]}, + {"source_id":"az.afsa.inspections-enforcement","jurisdiction_scope":"Azerbaijan; AFSA inspections, violations, veterinary controls, and public experimentation evidence","legacy_paths":[],"url":"https://afsa.gov.az/","access_method":"official reports/search services; aggregate-only until safe event route is verified","cadence":"resource-specific; unknown","attribution_licensing_notes":"Sensitive control/research evidence; privacy, retention, and safety review required","adapter_status":"reference_only","expected_artifact_schema":"Aggregate event/institution evidence with source document; no facility rows","blockers":["No stable public event/experimentation master verified; stop and escalate on sensitive exposure."]} ] } diff --git a/pipeline/tests/test_azerbaijan_recon_metadata.py b/pipeline/tests/test_azerbaijan_recon_metadata.py new file mode 100644 index 0000000..3c0b9b1 --- /dev/null +++ b/pipeline/tests/test_azerbaijan_recon_metadata.py @@ -0,0 +1,32 @@ +import json +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +AZERBAIJAN_IDS = { + "az.afsa.food-subjects", + "az.afsa.livestock-traceability", + "az.eco.environment-permits", + "az.taxes.organizations", + "az.stat.livestock-statistics", + "az.afsa.inspections-enforcement", +} + +class AzerbaijanReconMetadataTests(unittest.TestCase): + def test_azerbaijan_recon_is_row_free_and_automated(self): + registry = json.loads((ROOT / "pipeline/source_registry.json").read_text(encoding="utf-8-sig")) + self.assertTrue(AZERBAIJAN_IDS.issubset({s["source_id"] for s in registry["sources"]})) + recon = (ROOT / "docs/country-recon-az.md").read_text(encoding="utf-8") + self.assertIn("no facility rows", recon.lower()) + self.assertIn("fully automated", recon.lower()) + self.assertIn("conflict safety", recon.lower()) + + def test_azerbaijan_sources_are_blocked_or_not_run(self): + status = json.loads((ROOT / "docs/source-status.json").read_text(encoding="utf-8-sig")) + by_id = {s["source_id"]: s for s in status["sources"]} + for source_id in AZERBAIJAN_IDS: + self.assertEqual(by_id[source_id]["publication_eligibility"], "blocked") + self.assertIn(by_id[source_id]["acquisition"], {"blocked", "not_run"}) + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/tests/test_source_registry.py b/pipeline/tests/test_source_registry.py index f9015a6..d5448cd 100644 --- a/pipeline/tests/test_source_registry.py +++ b/pipeline/tests/test_source_registry.py @@ -9,8 +9,8 @@ class SourceRegistryTests(unittest.TestCase): def test_repository_registry_loads_and_references_existing_legacy_paths(self): registry = load_registry() - self.assertEqual(len(registry["sources"]), 209) - self.assertEqual(len({source["source_id"] for source in registry["sources"]}), 209) + self.assertEqual(len(registry["sources"]), 215) + self.assertEqual(len({source["source_id"] for source in registry["sources"]}), 215) def test_unknowns_are_explicit(self): registry = json.loads(REGISTRY_PATH.read_text(encoding="utf-8")) From d328ee6f71c54801f86c53556f2c7f201648494d Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Wed, 16 Sep 2026 11:21:51 -0700 Subject: [PATCH 172/311] Add Turkey source reconnaissance metadata --- docs/country-recon-tr.md | 26 ++++++++++++++++++++ docs/source-status.json | 8 +++++- pipeline/source_registry.json | 8 +++++- pipeline/tests/test_source_registry.py | 4 +-- pipeline/tests/test_turkey_recon_metadata.py | 11 +++++++++ 5 files changed, 53 insertions(+), 4 deletions(-) create mode 100644 docs/country-recon-tr.md create mode 100644 pipeline/tests/test_turkey_recon_metadata.py diff --git a/docs/country-recon-tr.md b/docs/country-recon-tr.md new file mode 100644 index 0000000..f98bfd4 --- /dev/null +++ b/docs/country-recon-tr.md @@ -0,0 +1,26 @@ +# Turkey source reconnaissance + +Status: metadata-only reconnaissance; no facility rows, raw exports, coordinates, traceability data, operational details, or personal data retained. + +## Scope and safety + +Turkey is not present in the registry. Official sources identify approved food/slaughterhouse lists, livestock systems, environmental services, company registration, and recurring animal-production statistics. Fully automated acquisition through validation, quarantine, transformation, and ingestion is required eventually, but acquisition is gated until contracts, terms, privacy, access controls, and safety are verified. Do not bypass authentication, payment, CAPTCHA, robots, or technical restrictions. Missing/stale records are not closure; do not geocode during reconnaissance. + +## Candidate inventory + +| ID | Authority / scope | Verified route | Format/cadence | Disposition | +|---|---|---|---|---| +| `tr.tarim.approved-food` | Ministry of Agriculture and Forestry; approved/registered food businesses and slaughterhouses | [Slaughterhouses](https://www.tarimorman.gov.tr/Konular/Hayvancilik/hayvan-refah%C4%B1-kimliklendirme-ve-i%C5%9Fletme-onay/kesimhaneler), [food control](https://www.tarimorman.gov.tr/GKGM/Menu/90/Gida-Ve-Yem-Kontrol) | Official pages advertise lists/searches; exact export, schema, IDs, cadence, licensing, and privacy not pinned | Partial; blocked | +| `tr.tarim.livestock-systems` | Ministry animal registration, HIBS, TURKVET/KKKS and farm/livestock systems | [Livestock services](https://www.tarimorman.gov.tr/HAYGEM/Menu/2/Hayvancilik), [animal systems](https://www.tarimorman.gov.tr/GKGM/Sayfalar/Detay.aspx?TermId=23f6df2f-b835-4924-8e6c-97fc71cb8bee) | Government systems are described; public API/export and sensitivity boundary not verified | Partial; blocked | +| `tr.cevre.environment-permits` | Ministry of Environment, Urbanization and Climate Change; environmental permits/EIA | [Ministry portal](https://csb.gov.tr/) | Service/register discovery only; exact permit/EIA API, geometry, cadence, license, and privacy not pinned | Partial; blocked | +| `tr.mersis.organizations` | Ministry of Trade MERSIS central company registry | [MERSIS login](https://mersis.ticaret.gov.tr/Portal/KullaniciIslemleri/GirisIslemleri) | Account-based service; no automated bulk route verified; do not bypass access controls | Partial; blocked | +| `tr.tuik.animal-statistics` | TurkStat aggregate animal production, slaughter, livestock, and meat statistics | [Animal production 2025](https://veriportali.tuik.gov.tr/en/press/58015), [red meat 2025](https://veriportali.tuik.gov.tr/en/press/58168) | Recurring official releases with metadata; exact API/table IDs, revision policy, licensing, and suppression rules not pinned | Partial; not run; aggregate-only | +| `tr.tarim.inspections-enforcement` | Ministry official controls, food/feed enforcement, and any public animal-use evidence | [Official controls](https://www.tarimorman.gov.tr/Konular/Gida-Ve-Yem-Hizmetleri/Gida-Hizmetleri/Resmi-Kontroller) | Dated notices/services; no stable event or experimentation master verified | Partial; blocked | + +## Automation and release gates + +Pin exact authorized endpoints/downloads, formats/schema fingerprints, pagination, stable IDs/lifecycle semantics, freshness/cadence, attribution/license, rate limits, privacy/retention, and location-safety rules before any run. Hash and quarantine snapshots, validate them, retain provenance privately, and ingest only a reviewed safe projection. Sensitive farm/facility locations, proprietor identity, traceability records, or vulnerable operational details stop the run and are escalated. + +## Recommendation + +Turkey is not ingestion-ready. TurkStat aggregate releases are the lowest-risk candidate. Ministry lists and livestock systems remain blocked pending machine contracts and safety review; MERSIS must not be scraped around login or payment controls. Recommend the next unreconned nearby country after inventory check: Cyprus. diff --git a/docs/source-status.json b/docs/source-status.json index acb6662..b5e1e3e 100644 --- a/docs/source-status.json +++ b/docs/source-status.json @@ -223,6 +223,12 @@ {"source_id":"az.eco.environment-permits","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-az.md","pipeline/source_registry.json"],"next_action":"Pin authorized route, schema, cadence, IDs, terms, privacy, and automation permissions."}, {"source_id":"az.taxes.organizations","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-az.md","pipeline/source_registry.json"],"next_action":"Pin authorized route, schema, cadence, IDs, terms, privacy, and automation permissions."}, {"source_id":"az.stat.livestock-statistics","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-az.md","pipeline/source_registry.json"],"next_action":"Pin authorized route, schema, cadence, IDs, terms, privacy, and automation permissions."}, - {"source_id":"az.afsa.inspections-enforcement","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-az.md","pipeline/source_registry.json"],"next_action":"Pin authorized route, schema, cadence, IDs, terms, privacy, and automation permissions."} + {"source_id":"az.afsa.inspections-enforcement","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-az.md","pipeline/source_registry.json"],"next_action":"Pin authorized route, schema, cadence, IDs, terms, privacy, and automation permissions."}, + {"source_id":"tr.tarim.approved-food","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-tr.md","pipeline/source_registry.json"],"next_action":"Pin authorized route, schema, cadence, IDs, terms, privacy, and automation permissions."}, + {"source_id":"tr.tarim.livestock-systems","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-tr.md","pipeline/source_registry.json"],"next_action":"Pin authorized route, schema, cadence, IDs, terms, privacy, and automation permissions."}, + {"source_id":"tr.cevre.environment-permits","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-tr.md","pipeline/source_registry.json"],"next_action":"Pin authorized route, schema, cadence, IDs, terms, privacy, and automation permissions."}, + {"source_id":"tr.mersis.organizations","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-tr.md","pipeline/source_registry.json"],"next_action":"Pin authorized route, schema, cadence, IDs, terms, privacy, and automation permissions."}, + {"source_id":"tr.tuik.animal-statistics","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-tr.md","pipeline/source_registry.json"],"next_action":"Pin authorized route, schema, cadence, IDs, terms, privacy, and automation permissions."}, + {"source_id":"tr.tarim.inspections-enforcement","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-tr.md","pipeline/source_registry.json"],"next_action":"Pin authorized route, schema, cadence, IDs, terms, privacy, and automation permissions."} ] } diff --git a/pipeline/source_registry.json b/pipeline/source_registry.json index 60b8087..91ef272 100644 --- a/pipeline/source_registry.json +++ b/pipeline/source_registry.json @@ -839,7 +839,13 @@ {"source_id":"az.eco.environment-permits","jurisdiction_scope":"Azerbaijan; environmental permits and registers","legacy_paths":[],"url":"https://eco.gov.az/","access_method":"official ministry route; authorized API/export only","cadence":"resource-specific; unknown","attribution_licensing_notes":"Ministry source; license, geometry, privacy, and safety review required","adapter_status":"reference_only","expected_artifact_schema":"Permit metadata/aggregate status contract; no sensitive geometry","blockers":["Exact public route, API/export, coverage, cadence, license, geometry, and privacy not pinned."]}, {"source_id":"az.taxes.organizations","jurisdiction_scope":"Azerbaijan; State Tax Service commercial legal-entity and taxpayer registration","legacy_paths":[],"url":"https://www.taxes.gov.az/en/page/qeydiyyat","access_method":"official search/public database; no bypass of access controls","cadence":"unknown; statistical snapshots exist","attribution_licensing_notes":"Government registry; personal-data, terms, limits, and reuse review required","adapter_status":"reference_only","expected_artifact_schema":"Organization metadata keyed by taxpayer/registration identifier; suppress personal addresses","blockers":["Authorized machine route, fields, cadence, licensing, privacy, and automation permissions not verified."]}, {"source_id":"az.stat.livestock-statistics","jurisdiction_scope":"Azerbaijan; aggregate livestock, slaughter, meat, milk, and fishery statistics","legacy_paths":[],"url":"https://www.stat.gov.az/source/agriculture/?lang=en","access_method":"official statistical tables/yearbooks; machine API or bounded download to be pinned","cadence":"annual and table-specific","attribution_licensing_notes":"State Statistical Committee source; publication terms, revisions, and suppression rules require review","adapter_status":"reference_only","expected_artifact_schema":"Aggregate time-series/yearbook metadata; no establishment rows","blockers":["Current table IDs, machine API, revision semantics, licensing, and suppression rules not pinned."]}, - {"source_id":"az.afsa.inspections-enforcement","jurisdiction_scope":"Azerbaijan; AFSA inspections, violations, veterinary controls, and public experimentation evidence","legacy_paths":[],"url":"https://afsa.gov.az/","access_method":"official reports/search services; aggregate-only until safe event route is verified","cadence":"resource-specific; unknown","attribution_licensing_notes":"Sensitive control/research evidence; privacy, retention, and safety review required","adapter_status":"reference_only","expected_artifact_schema":"Aggregate event/institution evidence with source document; no facility rows","blockers":["No stable public event/experimentation master verified; stop and escalate on sensitive exposure."]} ] + {"source_id":"az.afsa.inspections-enforcement","jurisdiction_scope":"Azerbaijan; AFSA inspections, violations, veterinary controls, and public experimentation evidence","legacy_paths":[],"url":"https://afsa.gov.az/","access_method":"official reports/search services; aggregate-only until safe event route is verified","cadence":"resource-specific; unknown","attribution_licensing_notes":"Sensitive control/research evidence; privacy, retention, and safety review required","adapter_status":"reference_only","expected_artifact_schema":"Aggregate event/institution evidence with source document; no facility rows","blockers":["No stable public event/experimentation master verified; stop and escalate on sensitive exposure."]}, + {"source_id":"tr.tarim.approved-food","jurisdiction_scope":"Turkey; Ministry approved food businesses and slaughterhouses","legacy_paths":[],"url":"https://www.tarimorman.gov.tr/Konular/Hayvancilik/hayvan-refah%C4%B1-kimliklendirme-ve-i%C5%9Fletme-onay/kesimhaneler","access_method":"official route; authorized bounded capture/API only","cadence":"unknown; verify source-specific","attribution_licensing_notes":"Official Turkish government source; terms, privacy, access controls, and safety review required","adapter_status":"reference_only","expected_artifact_schema":"Metadata-only contract pending; no rows or coordinates retained","blockers":["Exact machine contract, stable IDs, cadence, licensing, privacy, and automation permissions not pinned."]}, + {"source_id":"tr.tarim.livestock-systems","jurisdiction_scope":"Turkey; Ministry animal registration and livestock systems","legacy_paths":[],"url":"https://www.tarimorman.gov.tr/HAYGEM/Menu/2/Hayvancilik","access_method":"official route; authorized bounded capture/API only","cadence":"unknown; verify source-specific","attribution_licensing_notes":"Official Turkish government source; terms, privacy, access controls, and safety review required","adapter_status":"reference_only","expected_artifact_schema":"Metadata-only contract pending; no rows or coordinates retained","blockers":["Exact machine contract, stable IDs, cadence, licensing, privacy, and automation permissions not pinned."]}, + {"source_id":"tr.cevre.environment-permits","jurisdiction_scope":"Turkey; environmental permits and EIA","legacy_paths":[],"url":"https://csb.gov.tr/","access_method":"official route; authorized bounded capture/API only","cadence":"unknown; verify source-specific","attribution_licensing_notes":"Official Turkish government source; terms, privacy, access controls, and safety review required","adapter_status":"reference_only","expected_artifact_schema":"Metadata-only contract pending; no rows or coordinates retained","blockers":["Exact machine contract, stable IDs, cadence, licensing, privacy, and automation permissions not pinned."]}, + {"source_id":"tr.mersis.organizations","jurisdiction_scope":"Turkey; MERSIS central company registry","legacy_paths":[],"url":"https://mersis.ticaret.gov.tr/Portal/KullaniciIslemleri/GirisIslemleri","access_method":"official route; authorized bounded capture/API only","cadence":"unknown; verify source-specific","attribution_licensing_notes":"Official Turkish government source; terms, privacy, access controls, and safety review required","adapter_status":"reference_only","expected_artifact_schema":"Metadata-only contract pending; no rows or coordinates retained","blockers":["Exact machine contract, stable IDs, cadence, licensing, privacy, and automation permissions not pinned."]}, + {"source_id":"tr.tuik.animal-statistics","jurisdiction_scope":"Turkey; aggregate animal-production and slaughter statistics","legacy_paths":[],"url":"https://veriportali.tuik.gov.tr/en/press/58015","access_method":"official route; authorized bounded capture/API only","cadence":"unknown; verify source-specific","attribution_licensing_notes":"Official Turkish government source; terms, privacy, access controls, and safety review required","adapter_status":"reference_only","expected_artifact_schema":"Metadata-only contract pending; no rows or coordinates retained","blockers":["Exact machine contract, stable IDs, cadence, licensing, privacy, and automation permissions not pinned."]}, + {"source_id":"tr.tarim.inspections-enforcement","jurisdiction_scope":"Turkey; official controls and public animal-use evidence","legacy_paths":[],"url":"https://www.tarimorman.gov.tr/Konular/Gida-Ve-Yem-Hizmetleri/Gida-Hizmetleri/Resmi-Kontroller","access_method":"official route; authorized bounded capture/API only","cadence":"unknown; verify source-specific","attribution_licensing_notes":"Official Turkish government source; terms, privacy, access controls, and safety review required","adapter_status":"reference_only","expected_artifact_schema":"Metadata-only contract pending; no rows or coordinates retained","blockers":["Exact machine contract, stable IDs, cadence, licensing, privacy, and automation permissions not pinned."]} ] } diff --git a/pipeline/tests/test_source_registry.py b/pipeline/tests/test_source_registry.py index d5448cd..ad00c06 100644 --- a/pipeline/tests/test_source_registry.py +++ b/pipeline/tests/test_source_registry.py @@ -9,8 +9,8 @@ class SourceRegistryTests(unittest.TestCase): def test_repository_registry_loads_and_references_existing_legacy_paths(self): registry = load_registry() - self.assertEqual(len(registry["sources"]), 215) - self.assertEqual(len({source["source_id"] for source in registry["sources"]}), 215) + self.assertEqual(len(registry["sources"]), 221) + self.assertEqual(len({source["source_id"] for source in registry["sources"]}), 221) def test_unknowns_are_explicit(self): registry = json.loads(REGISTRY_PATH.read_text(encoding="utf-8")) diff --git a/pipeline/tests/test_turkey_recon_metadata.py b/pipeline/tests/test_turkey_recon_metadata.py new file mode 100644 index 0000000..7038180 --- /dev/null +++ b/pipeline/tests/test_turkey_recon_metadata.py @@ -0,0 +1,11 @@ +import json, unittest +from pathlib import Path +ROOT=Path(__file__).resolve().parents[2] +TURKEY_IDS={"tr.tarim.approved-food","tr.tarim.livestock-systems","tr.cevre.environment-permits","tr.mersis.organizations","tr.tuik.animal-statistics","tr.tarim.inspections-enforcement"} +class TurkeyReconMetadataTests(unittest.TestCase): + def test_row_free_automated(self): + r=json.loads((ROOT/"pipeline/source_registry.json").read_text(encoding="utf-8-sig")); self.assertTrue(TURKEY_IDS.issubset({s["source_id"] for s in r["sources"]})); d=(ROOT/"docs/country-recon-tr.md").read_text(encoding="utf-8"); self.assertIn("no facility rows",d.lower()); self.assertIn("fully automated",d.lower()) + def test_blocked(self): + s=json.loads((ROOT/"docs/source-status.json").read_text(encoding="utf-8-sig")); b={x["source_id"]:x for x in s["sources"]} + for i in TURKEY_IDS: self.assertEqual(b[i]["publication_eligibility"],"blocked") +if __name__=="__main__": unittest.main() From 387055b301464b2e33c8388afac118e9505bf76c Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Wed, 16 Sep 2026 11:23:48 -0700 Subject: [PATCH 173/311] Add Cyprus source reconnaissance metadata --- docs/country-recon-cy.md | 24 ++++++++++++++++++++ docs/source-status.json | 8 ++++++- pipeline/source_registry.json | 8 ++++++- pipeline/tests/test_cyprus_recon_metadata.py | 11 +++++++++ pipeline/tests/test_source_registry.py | 4 ++-- 5 files changed, 51 insertions(+), 4 deletions(-) create mode 100644 docs/country-recon-cy.md create mode 100644 pipeline/tests/test_cyprus_recon_metadata.py diff --git a/docs/country-recon-cy.md b/docs/country-recon-cy.md new file mode 100644 index 0000000..6c1463a --- /dev/null +++ b/docs/country-recon-cy.md @@ -0,0 +1,24 @@ +# Cyprus source reconnaissance + +Status: metadata-only; Republic of Cyprus jurisdiction only. No facility rows, raw exports, coordinates, traceability data, operational details, or personal data retained. + +## Territorial scope and safety + +This inventory describes sources issued by the Republic of Cyprus and its authorities. It does not claim coverage, authority, or data completeness for northern Cyprus or areas administered by other authorities. Do not silently merge jurisdictions or treat absence from Republic of Cyprus sources as absence elsewhere. Fully automated acquisition-to-ingestion remains required eventually, but acquisition is gated by source contracts, terms, privacy, and safety review. Do not bypass access controls; missing or stale records are not closure; do not geocode during reconnaissance. + +## Candidates + +| ID | Authority/scope | Route | Format/cadence | Disposition | +|---|---|---|---|---| +| `cy.vs.approved-food` | Republic of Cyprus Ministry of Agriculture Veterinary Services; approved animal-origin establishments/slaughterhouses | [Approved establishments](https://www.moa.gov.cy/moa/vs/vs.nsf/All/9F6A5DB7308579ACC225764D001D01AF?OpenDocument=&print=), [registered establishments](https://www.moa.gov.cy/moa/vs/vs.nsf/vs14_en/vs14_en) | Dated PDF/HTML documents; exact full set, schema, stable IDs, update cadence, licensing, privacy not pinned | Partial; blocked | +| `cy.vs.farms-livestock` | Republic of Cyprus Veterinary Services and agriculture authorities; livestock holdings/animal identification | [Veterinary Services](https://www.moa.gov.cy/moa/vs/vs.nsf) | Web/service routes; public machine export, IDs, cadence, privacy, and territorial coverage not verified | Partial; blocked | +| `cy.environment-permits` | Republic of Cyprus environmental/EIA and waste-permit authorities | [Department of Environment](https://www.moa.gov.cy/moa/environment/environment.nsf) | Register/service discovery; exact permit API/export, geometry, cadence, license and privacy not pinned | Partial; blocked | +| `cy.companies.registry` | Republic of Cyprus Registrar of Companies and Intellectual Property | [Companies Section](https://www.companies.gov.cy/en/) | eSearch/services and statistics; bulk/API, access terms, stable IDs and personal-data policy not verified | Partial; blocked | +| `cy.cystat.livestock-meat` | Statistical Service of Cyprus; aggregate livestock and meat production | [Production of meat PxWeb](https://cystatdb.cystat.gov.cy/pxweb/en/8.CYSTAT-DB/8.CYSTAT-DB__Agriculture%2C%20Livestock%2C%20Fishing__Livestock/0320031E.px/), [farm survey](https://www.gov.cy/en/economy-and-finance/results-of-the-farm-structure-survey-of-agricultural-and-livestock-holdings-2023/) | PxWeb/table and recurring survey publications; API/query contract, revisions, licensing, suppression not fully pinned | Partial; not run; aggregate-only | +| `cy.vs.inspections-enforcement` | Republic of Cyprus Veterinary Services inspection/enforcement and public animal-use evidence | [Veterinary Services](https://www.moa.gov.cy/moa/vs/vs.nsf) | Reports/services; stable event/experimentation master, retention, privacy not verified | Partial; blocked | + +## Gates + +Pin exact authorized endpoints/documents, format/schema fingerprint, stable IDs/lifecycle, freshness/cadence, attribution/license, rate limits, privacy/retention, and Republic-of-Cyprus territorial scope. Hash and quarantine authorized snapshots, validate them, preserve provenance privately, and ingest only a reviewed safe projection. Sensitive locations, personal contacts, or cross-jurisdiction ambiguity stop and escalate. + +Cyprus is not ingestion-ready. CYSTAT aggregate tables are the lowest-risk follow-up. Veterinary documents remain blocked until machine contracts, terms, privacy, and territorial completeness are reviewed. Recommend next unreconned nearby country after inventory check: Lebanon. diff --git a/docs/source-status.json b/docs/source-status.json index b5e1e3e..0d3c1e1 100644 --- a/docs/source-status.json +++ b/docs/source-status.json @@ -229,6 +229,12 @@ {"source_id":"tr.cevre.environment-permits","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-tr.md","pipeline/source_registry.json"],"next_action":"Pin authorized route, schema, cadence, IDs, terms, privacy, and automation permissions."}, {"source_id":"tr.mersis.organizations","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-tr.md","pipeline/source_registry.json"],"next_action":"Pin authorized route, schema, cadence, IDs, terms, privacy, and automation permissions."}, {"source_id":"tr.tuik.animal-statistics","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-tr.md","pipeline/source_registry.json"],"next_action":"Pin authorized route, schema, cadence, IDs, terms, privacy, and automation permissions."}, - {"source_id":"tr.tarim.inspections-enforcement","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-tr.md","pipeline/source_registry.json"],"next_action":"Pin authorized route, schema, cadence, IDs, terms, privacy, and automation permissions."} + {"source_id":"tr.tarim.inspections-enforcement","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-tr.md","pipeline/source_registry.json"],"next_action":"Pin authorized route, schema, cadence, IDs, terms, privacy, and automation permissions."}, + {"source_id":"cy.vs.approved-food","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-cy.md","pipeline/source_registry.json"],"next_action":"Pin authorized route, schema, cadence, IDs, terms, privacy, and Republic of Cyprus territorial scope."}, + {"source_id":"cy.vs.farms-livestock","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-cy.md","pipeline/source_registry.json"],"next_action":"Pin authorized route, schema, cadence, IDs, terms, privacy, and Republic of Cyprus territorial scope."}, + {"source_id":"cy.environment-permits","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-cy.md","pipeline/source_registry.json"],"next_action":"Pin authorized route, schema, cadence, IDs, terms, privacy, and Republic of Cyprus territorial scope."}, + {"source_id":"cy.companies.registry","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-cy.md","pipeline/source_registry.json"],"next_action":"Pin authorized route, schema, cadence, IDs, terms, privacy, and Republic of Cyprus territorial scope."}, + {"source_id":"cy.cystat.livestock-meat","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-cy.md","pipeline/source_registry.json"],"next_action":"Pin authorized route, schema, cadence, IDs, terms, privacy, and Republic of Cyprus territorial scope."}, + {"source_id":"cy.vs.inspections-enforcement","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-cy.md","pipeline/source_registry.json"],"next_action":"Pin authorized route, schema, cadence, IDs, terms, privacy, and Republic of Cyprus territorial scope."} ] } diff --git a/pipeline/source_registry.json b/pipeline/source_registry.json index 91ef272..a685edd 100644 --- a/pipeline/source_registry.json +++ b/pipeline/source_registry.json @@ -845,7 +845,13 @@ {"source_id":"tr.cevre.environment-permits","jurisdiction_scope":"Turkey; environmental permits and EIA","legacy_paths":[],"url":"https://csb.gov.tr/","access_method":"official route; authorized bounded capture/API only","cadence":"unknown; verify source-specific","attribution_licensing_notes":"Official Turkish government source; terms, privacy, access controls, and safety review required","adapter_status":"reference_only","expected_artifact_schema":"Metadata-only contract pending; no rows or coordinates retained","blockers":["Exact machine contract, stable IDs, cadence, licensing, privacy, and automation permissions not pinned."]}, {"source_id":"tr.mersis.organizations","jurisdiction_scope":"Turkey; MERSIS central company registry","legacy_paths":[],"url":"https://mersis.ticaret.gov.tr/Portal/KullaniciIslemleri/GirisIslemleri","access_method":"official route; authorized bounded capture/API only","cadence":"unknown; verify source-specific","attribution_licensing_notes":"Official Turkish government source; terms, privacy, access controls, and safety review required","adapter_status":"reference_only","expected_artifact_schema":"Metadata-only contract pending; no rows or coordinates retained","blockers":["Exact machine contract, stable IDs, cadence, licensing, privacy, and automation permissions not pinned."]}, {"source_id":"tr.tuik.animal-statistics","jurisdiction_scope":"Turkey; aggregate animal-production and slaughter statistics","legacy_paths":[],"url":"https://veriportali.tuik.gov.tr/en/press/58015","access_method":"official route; authorized bounded capture/API only","cadence":"unknown; verify source-specific","attribution_licensing_notes":"Official Turkish government source; terms, privacy, access controls, and safety review required","adapter_status":"reference_only","expected_artifact_schema":"Metadata-only contract pending; no rows or coordinates retained","blockers":["Exact machine contract, stable IDs, cadence, licensing, privacy, and automation permissions not pinned."]}, - {"source_id":"tr.tarim.inspections-enforcement","jurisdiction_scope":"Turkey; official controls and public animal-use evidence","legacy_paths":[],"url":"https://www.tarimorman.gov.tr/Konular/Gida-Ve-Yem-Hizmetleri/Gida-Hizmetleri/Resmi-Kontroller","access_method":"official route; authorized bounded capture/API only","cadence":"unknown; verify source-specific","attribution_licensing_notes":"Official Turkish government source; terms, privacy, access controls, and safety review required","adapter_status":"reference_only","expected_artifact_schema":"Metadata-only contract pending; no rows or coordinates retained","blockers":["Exact machine contract, stable IDs, cadence, licensing, privacy, and automation permissions not pinned."]} ] + {"source_id":"tr.tarim.inspections-enforcement","jurisdiction_scope":"Turkey; official controls and public animal-use evidence","legacy_paths":[],"url":"https://www.tarimorman.gov.tr/Konular/Gida-Ve-Yem-Hizmetleri/Gida-Hizmetleri/Resmi-Kontroller","access_method":"official route; authorized bounded capture/API only","cadence":"unknown; verify source-specific","attribution_licensing_notes":"Official Turkish government source; terms, privacy, access controls, and safety review required","adapter_status":"reference_only","expected_artifact_schema":"Metadata-only contract pending; no rows or coordinates retained","blockers":["Exact machine contract, stable IDs, cadence, licensing, privacy, and automation permissions not pinned."]}, + {"source_id":"cy.vs.approved-food","jurisdiction_scope":"Republic of Cyprus; Veterinary Services approved animal-origin establishments and slaughterhouses","legacy_paths":[],"url":"https://www.moa.gov.cy/moa/vs/vs.nsf/All/9F6A5DB7308579ACC225764D001D01AF?OpenDocument=&print=","access_method":"official Republic of Cyprus route; authorized bounded capture/API only","cadence":"unknown; source-specific","attribution_licensing_notes":"Government source; Republic of Cyprus scope, terms, privacy, and safety review required","adapter_status":"reference_only","expected_artifact_schema":"Metadata-only contract pending; no rows or coordinates retained","blockers":["Exact machine contract, stable IDs, cadence, licensing, privacy, and territorial completeness not pinned."]}, + {"source_id":"cy.vs.farms-livestock","jurisdiction_scope":"Republic of Cyprus; Veterinary Services livestock holdings and animal identification","legacy_paths":[],"url":"https://www.moa.gov.cy/moa/vs/vs.nsf","access_method":"official Republic of Cyprus route; authorized bounded capture/API only","cadence":"unknown; source-specific","attribution_licensing_notes":"Government source; Republic of Cyprus scope, terms, privacy, and safety review required","adapter_status":"reference_only","expected_artifact_schema":"Metadata-only contract pending; no rows or coordinates retained","blockers":["Exact machine contract, stable IDs, cadence, licensing, privacy, and territorial completeness not pinned."]}, + {"source_id":"cy.environment-permits","jurisdiction_scope":"Republic of Cyprus; environmental/EIA and waste permits","legacy_paths":[],"url":"https://www.moa.gov.cy/moa/environment/environment.nsf","access_method":"official Republic of Cyprus route; authorized bounded capture/API only","cadence":"unknown; source-specific","attribution_licensing_notes":"Government source; Republic of Cyprus scope, terms, privacy, and safety review required","adapter_status":"reference_only","expected_artifact_schema":"Metadata-only contract pending; no rows or coordinates retained","blockers":["Exact machine contract, stable IDs, cadence, licensing, privacy, and territorial completeness not pinned."]}, + {"source_id":"cy.companies.registry","jurisdiction_scope":"Republic of Cyprus; Registrar of Companies and Intellectual Property","legacy_paths":[],"url":"https://www.companies.gov.cy/en/","access_method":"official Republic of Cyprus route; authorized bounded capture/API only","cadence":"unknown; source-specific","attribution_licensing_notes":"Government source; Republic of Cyprus scope, terms, privacy, and safety review required","adapter_status":"reference_only","expected_artifact_schema":"Metadata-only contract pending; no rows or coordinates retained","blockers":["Exact machine contract, stable IDs, cadence, licensing, privacy, and territorial completeness not pinned."]}, + {"source_id":"cy.cystat.livestock-meat","jurisdiction_scope":"Republic of Cyprus; aggregate livestock and meat statistics","legacy_paths":[],"url":"https://cystatdb.cystat.gov.cy/pxweb/en/8.CYSTAT-DB/8.CYSTAT-DB__Agriculture%2C%20Livestock%2C%20Fishing__Livestock/0320031E.px/","access_method":"official Republic of Cyprus route; authorized bounded capture/API only","cadence":"unknown; source-specific","attribution_licensing_notes":"Government source; Republic of Cyprus scope, terms, privacy, and safety review required","adapter_status":"reference_only","expected_artifact_schema":"Metadata-only contract pending; no rows or coordinates retained","blockers":["Exact machine contract, stable IDs, cadence, licensing, privacy, and territorial completeness not pinned."]}, + {"source_id":"cy.vs.inspections-enforcement","jurisdiction_scope":"Republic of Cyprus; Veterinary Services inspections and animal-use evidence","legacy_paths":[],"url":"https://www.moa.gov.cy/moa/vs/vs.nsf","access_method":"official Republic of Cyprus route; authorized bounded capture/API only","cadence":"unknown; source-specific","attribution_licensing_notes":"Government source; Republic of Cyprus scope, terms, privacy, and safety review required","adapter_status":"reference_only","expected_artifact_schema":"Metadata-only contract pending; no rows or coordinates retained","blockers":["Exact machine contract, stable IDs, cadence, licensing, privacy, and territorial completeness not pinned."]} ] } diff --git a/pipeline/tests/test_cyprus_recon_metadata.py b/pipeline/tests/test_cyprus_recon_metadata.py new file mode 100644 index 0000000..67dd542 --- /dev/null +++ b/pipeline/tests/test_cyprus_recon_metadata.py @@ -0,0 +1,11 @@ +import json,unittest +from pathlib import Path +ROOT=Path(__file__).resolve().parents[2] +IDS={"cy.vs.approved-food","cy.vs.farms-livestock","cy.environment-permits","cy.companies.registry","cy.cystat.livestock-meat","cy.vs.inspections-enforcement"} +class CyprusReconMetadataTests(unittest.TestCase): + def test_row_free_scope(self): + r=json.loads((ROOT/"pipeline/source_registry.json").read_text(encoding="utf-8-sig"));self.assertTrue(IDS.issubset({s["source_id"] for s in r["sources"]}));d=(ROOT/"docs/country-recon-cy.md").read_text(encoding="utf-8");self.assertIn("no facility rows",d.lower());self.assertIn("republic of cyprus",d.lower());self.assertIn("fully automated",d.lower()) + def test_blocked(self): + s=json.loads((ROOT/"docs/source-status.json").read_text(encoding="utf-8-sig"));b={x["source_id"]:x for x in s["sources"]} + for i in IDS:self.assertEqual(b[i]["publication_eligibility"],"blocked") +if __name__=="__main__":unittest.main() diff --git a/pipeline/tests/test_source_registry.py b/pipeline/tests/test_source_registry.py index ad00c06..3d84567 100644 --- a/pipeline/tests/test_source_registry.py +++ b/pipeline/tests/test_source_registry.py @@ -9,8 +9,8 @@ class SourceRegistryTests(unittest.TestCase): def test_repository_registry_loads_and_references_existing_legacy_paths(self): registry = load_registry() - self.assertEqual(len(registry["sources"]), 221) - self.assertEqual(len({source["source_id"] for source in registry["sources"]}), 221) + self.assertEqual(len(registry["sources"]), 227) + self.assertEqual(len({source["source_id"] for source in registry["sources"]}), 227) def test_unknowns_are_explicit(self): registry = json.loads(REGISTRY_PATH.read_text(encoding="utf-8")) From f81dcf69ef19d15fd0da7dfa79dca030e6db2c47 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Wed, 16 Sep 2026 11:24:34 -0700 Subject: [PATCH 174/311] Flatten public projection read path --- docs/performance/v2-observability.md | 32 ++- .../032_flatten_public_history_view.sql | 128 ++++++++++ .../benchmarks/explain_public_projection.py | 226 ++++++++++++++++++ .../tests/test_discovery_scale_benchmark.py | 10 + 4 files changed, 388 insertions(+), 8 deletions(-) create mode 100644 pipeline/migrations/032_flatten_public_history_view.sql create mode 100644 pipeline/scripts/benchmarks/explain_public_projection.py diff --git a/docs/performance/v2-observability.md b/docs/performance/v2-observability.md index 3469a7e..446c12f 100644 --- a/docs/performance/v2-observability.md +++ b/docs/performance/v2-observability.md @@ -74,14 +74,30 @@ throughput, status/error/timeouts, response byte totals, observed database active/waiting sessions, and pool-pressure signals only. It does not retain request paths, query values, coordinates, IDs, client data, or response rows. -The default rehearsal currently establishes no clean concurrency level: the -1,000- and 5,000-observation runs show timeout pressure across the tested -levels, with facets a repeatable hotspot. Therefore it must not be used to -choose a production pool size or to claim capacity. Keep production launch -and pool sizing blocked pending an approved representative traffic test on the -deployment topology. The 2-second request timeout and 350 ms database -radius-query budget remain review thresholds for fail-safe behavior, not -performance guarantees. +Migration 032 flattens the nested public-history view into one +release-scoped eligibility pass and one per-release facility summary while +retaining the live review, suppression, lifecycle, and geocode checks. The +diagnostic can be reproduced with: + +```powershell +python pipeline/scripts/benchmarks/explain_public_projection.py ` + --observations 1000 --json-output .tmp/public-explain.json +``` + +On the synthetic 1,000-row fixture, the flattened list and facets queries +executed in 262 ms and 243 ms respectively; the pre-change nested plans were +estimated with 28 nested-loop nodes, 25 sequential scans, and 9 repeated +review subqueries. At 5,000 rows, the flattened list and facets plans took +about 4.8 s and 4.5 s, explaining the remaining 2-second rehearsal timeouts. + +The 1,000-row rehearsal was clean at concurrency 1, 4, 8, and 16 (20/20 +requests at each level); the 5,000-row rehearsal was not clean (2/10, 2/10, +1/10, and 2/10 successes at those levels with a 2-second timeout). These are +synthetic local results, not production capacity evidence. Keep production +pool sizing and launch capacity decisions blocked pending an approved +representative traffic test on the deployment topology. The 2-second request +timeout and 350 ms database radius-query budget remain review thresholds for +fail-safe behavior, not performance guarantees. The benchmark and logs provide operational evidence only. They do not establish source completeness, publication eligibility, production capacity, cloud diff --git a/pipeline/migrations/032_flatten_public_history_view.sql b/pipeline/migrations/032_flatten_public_history_view.sql new file mode 100644 index 0000000..9dad7a5 --- /dev/null +++ b/pipeline/migrations/032_flatten_public_history_view.sql @@ -0,0 +1,128 @@ +-- Flatten the public history view's nested release/review/suppression joins. +-- Eligibility is still evaluated from append-only control-plane views for every +-- request; this changes only plan shape, not the public contract. +CREATE OR REPLACE VIEW uec.map_facilities_display_history AS +WITH eligible AS MATERIALIZED ( + SELECT member.release_id, + member.default_visible AS release_visible, + observation.observation_id, + observation.facility_id, + observation.source_record_id, + observation.classification_category, + observation.first_observed_at, + observation.observed_at, + facility.canonical_name, + facility.country_code, + facility.street_address, + facility.postal_code, + facility.city, + release.ruleset_version AS release_ruleset_version, + release.created_at AS release_created_at, + source.origin_type AS provenance_origin_type, + source.source_id AS provenance_source_id, + source.name AS provenance_source_name, + source.official_url AS provenance_source_url, + artifact.retrieved_at AS provenance_retrieved_at + FROM uec.release_members AS member + JOIN uec.releases AS release + ON release.release_id = member.release_id + JOIN uec.observations AS observation + ON observation.observation_id = member.observation_id + JOIN uec.facilities AS facility + ON facility.facility_id = member.facility_id + JOIN uec.source_records AS record + ON record.source_record_id = observation.source_record_id + JOIN uec.sources AS source + ON source.source_id = record.source_id + JOIN uec.raw_artifacts AS artifact + ON artifact.artifact_id = record.artifact_id + JOIN uec.publication_review_release_current AS review + ON review.source_record_id = observation.source_record_id + AND review.release_id = member.release_id + WHERE release.status = 'promoted' + AND member.default_visible = true + AND review.publication_eligible = true + AND review.privacy_screening_status = 'passed' + AND review.factual_review_status <> 'rejected' + AND ( + review.maintainer_approval = 'approved' + OR (release.profile = 'community' + AND source.origin_type = 'user_submitted' + AND review.factual_review_status = 'unreviewed' + AND review.maintainer_approval = 'pending') + ) + AND NOT EXISTS ( + SELECT 1 + FROM uec.public_access_restricted AS restricted + WHERE restricted.source_record_id = observation.source_record_id + ) +), public_summary AS MATERIALIZED ( + SELECT release_id, + facility_id, + min(first_observed_at) AS first_observed_at, + max(observed_at) AS last_observed_at, + count(*)::int AS observation_count + FROM eligible + GROUP BY release_id, facility_id +) +SELECT eligible.release_id, + 'promoted'::text AS release_status, + eligible.release_visible, + eligible.observation_id, + eligible.facility_id, + eligible.source_record_id, + eligible.canonical_name, + eligible.country_code, + eligible.street_address, + eligible.postal_code, + eligible.city, + CASE WHEN latest.status = 'accepted' AND latest.result IS NOT NULL THEN latest.result + WHEN latest.status = 'review_required' THEN city.reference_location ELSE NULL END AS display_location, + CASE WHEN latest.status = 'accepted' AND latest.result IS NOT NULL THEN 'exact' + WHEN latest.status = 'review_required' AND city.reference_location IS NOT NULL THEN 'city' + ELSE 'unmapped' END AS display_precision, + CASE WHEN latest.status = 'accepted' AND latest.result IS NOT NULL THEN 'Accepted geocoder result' + WHEN latest.status = 'review_required' AND city.reference_location IS NOT NULL THEN 'Approximate city location — multiple geocoder matches' + ELSE 'No publishable location' END AS display_label, + latest.status AS geocoding_status, + latest.provider_id AS geocoder_provider, + latest.queried_at AS geocoded_at, + eligible.classification_category, + public_summary.first_observed_at, + public_summary.last_observed_at, + public_summary.observation_count, + COALESCE(lifecycle.status, 'status_unknown') AS lifecycle_status, + lifecycle.effective_at AS lifecycle_effective_at, + lifecycle.source_record_id AS lifecycle_source_record_id, + eligible.release_ruleset_version, + eligible.release_created_at, + eligible.provenance_origin_type, + eligible.provenance_source_id, + eligible.provenance_source_name, + eligible.provenance_source_url, + eligible.provenance_retrieved_at +FROM eligible +JOIN public_summary + ON public_summary.release_id = eligible.release_id + AND public_summary.facility_id = eligible.facility_id +LEFT JOIN LATERAL ( + SELECT status, result, provider_id, queried_at + FROM uec.geocode_results + WHERE source_record_id = eligible.source_record_id + ORDER BY queried_at DESC, geocode_result_id DESC + LIMIT 1 +) AS latest ON true +LEFT JOIN LATERAL ( + SELECT reference_location + FROM uec.city_reference_points + WHERE country_code = eligible.country_code + AND lower(city_name) = lower(eligible.city) + AND (postal_code IS NULL OR postal_code = eligible.postal_code) + ORDER BY postal_code NULLS LAST + LIMIT 1 +) AS city ON true +LEFT JOIN uec.facility_lifecycle_current AS lifecycle + ON lifecycle.facility_id = eligible.facility_id; + +COMMENT ON VIEW uec.map_facilities_display_history IS + 'V2 public display history with one release-scoped eligibility pass, current suppression, and public-only lifecycle counts.'; diff --git a/pipeline/scripts/benchmarks/explain_public_projection.py b/pipeline/scripts/benchmarks/explain_public_projection.py new file mode 100644 index 0000000..af4d5a8 --- /dev/null +++ b/pipeline/scripts/benchmarks/explain_public_projection.py @@ -0,0 +1,226 @@ +#!/usr/bin/env python3 +"""Compare synthetic nested and flattened public projection query plans. + +The harness uses only a disposable local E2E database and emits plan metadata, +timings, and aggregate buffer/node counts. It never prints plan SQL, values, or +rows, so it is safe to use as a repeatable diagnostic artifact. +""" + +from __future__ import annotations + +import argparse +import importlib.util +import json +import sys +from collections import Counter +from pathlib import Path +from typing import Any + +ROOT = Path(__file__).resolve().parents[3] +LOAD_SCRIPT = ROOT / "pipeline" / "scripts" / "benchmarks" / "run_api_load_rehearsal.py" +SPEC = importlib.util.spec_from_file_location("run_api_load_rehearsal", LOAD_SCRIPT) +assert SPEC and SPEC.loader +LOAD = importlib.util.module_from_spec(SPEC) +sys.modules[SPEC.name] = LOAD +SPEC.loader.exec_module(LOAD) + +OLD_FACETS = """ +SELECT country_code, classification_category, display_precision, + lifecycle_status, provenance_origin_type, city, count(*)::bigint +FROM ( + SELECT display.country_code, display.city, display.display_precision, + observation.classification_category, + COALESCE(lifecycle.status, 'status_unknown') AS lifecycle_status, + source.origin_type AS provenance_origin_type + FROM uec.map_facilities_display AS display + JOIN uec.publication_release_eligible_observations AS eligible + ON eligible.release_id = display.release_id + AND eligible.observation_id = display.observation_id + JOIN uec.observations AS observation + ON observation.observation_id = display.observation_id + JOIN uec.source_records AS record + ON record.source_record_id = display.source_record_id + JOIN uec.sources AS source + ON source.source_id = record.source_id + JOIN uec.release_members AS member + ON member.release_id = display.release_id + AND member.observation_id = display.observation_id + JOIN uec.releases AS release + ON release.release_id = member.release_id + JOIN uec.raw_artifacts AS artifact + ON artifact.artifact_id = record.artifact_id + JOIN uec.public_facility_observation_summary AS summary + ON summary.release_id = display.release_id + AND summary.facility_id = display.facility_id + LEFT JOIN uec.facility_lifecycle_current AS lifecycle + ON lifecycle.facility_id = display.facility_id + WHERE display.release_id = 'load-promoted' +) history +GROUP BY country_code, classification_category, display_precision, + lifecycle_status, provenance_origin_type, city +""" + +NEW_FACETS = """ +SELECT country_code, classification_category, display_precision, + lifecycle_status, provenance_origin_type, city, count(*)::bigint +FROM uec.map_facilities_display_history +WHERE release_id = 'load-promoted' +GROUP BY country_code, classification_category, display_precision, + lifecycle_status, provenance_origin_type, city +""" + +OLD_LIST = """ +SELECT history.facility_id, history.canonical_name, history.country_code, + history.city, history.classification_category, history.display_precision +FROM ( + SELECT display.facility_id, display.canonical_name, display.country_code, + display.city, display.display_precision, + observation.classification_category, + display.release_id, display.source_record_id + FROM uec.map_facilities_display AS display + JOIN uec.publication_release_eligible_observations AS eligible + ON eligible.release_id = display.release_id + AND eligible.observation_id = display.observation_id + JOIN uec.observations AS observation + ON observation.observation_id = display.observation_id + JOIN uec.source_records AS record + ON record.source_record_id = display.source_record_id + JOIN uec.sources AS source + ON source.source_id = record.source_id + JOIN uec.release_members AS member + ON member.release_id = display.release_id + AND member.observation_id = display.observation_id + JOIN uec.releases AS release + ON release.release_id = member.release_id + JOIN uec.raw_artifacts AS artifact + ON artifact.artifact_id = record.artifact_id + JOIN uec.public_facility_observation_summary AS summary + ON summary.release_id = display.release_id + AND summary.facility_id = display.facility_id + LEFT JOIN uec.facility_lifecycle_current AS lifecycle + ON lifecycle.facility_id = display.facility_id + WHERE display.release_id = 'load-promoted' +) history +JOIN uec.publication_review_release_current AS review + ON review.source_record_id = history.source_record_id + AND review.release_id = history.release_id +ORDER BY history.facility_id +LIMIT 51 +""" + +NEW_LIST = """ +SELECT history.facility_id, history.canonical_name, history.country_code, + history.city, history.classification_category, history.display_precision +FROM uec.map_facilities_display_history AS history +JOIN uec.publication_review_release_current AS review + ON review.source_record_id = history.source_record_id + AND review.release_id = history.release_id +WHERE history.release_id = 'load-promoted' +ORDER BY history.facility_id +LIMIT 51 +""" + + +def plan_summary(payload: list[Any]) -> dict[str, Any]: + root = payload[0] + plan = root["Plan"] + nodes: Counter[str] = Counter() + actual_rows = 0 + shared_hit = 0 + shared_read = 0 + temp_read = 0 + temp_written = 0 + expensive_nodes: list[dict[str, Any]] = [] + relation_times: Counter[str] = Counter() + + def visit(node: dict[str, Any]) -> None: + nonlocal actual_rows, shared_hit, shared_read, temp_read, temp_written + nodes[node.get("Node Type", "unknown")] += 1 + actual_rows = max(actual_rows, int(node.get("Actual Rows", 0))) + shared_hit += int(node.get("Shared Hit Blocks", 0)) + shared_read += int(node.get("Shared Read Blocks", 0)) + temp_read += int(node.get("Temp Read Blocks", 0)) + temp_written += int(node.get("Temp Written Blocks", 0)) + if "Actual Total Time" in node: + total_ms = float(node.get("Actual Total Time", 0)) * float(node.get("Actual Loops", 0)) + relation = node.get("Relation Name") + if relation: + relation_times[relation] += total_ms + expensive_nodes.append({ + "node_type": node.get("Node Type", "unknown"), + "relation": node.get("Relation Name"), + "index": node.get("Index Name"), + "loops": int(node.get("Actual Loops", 0)), + "actual_rows": int(node.get("Actual Rows", 0)), + "total_ms": round(total_ms, 3), + }) + for child in node.get("Plans", []): + visit(child) + + visit(plan) + return { + "planning_ms": round(float(root.get("Planning Time", 0)), 3), + "execution_ms": round(float(root.get("Execution Time", 0)), 3), + "actual_rows_max": actual_rows, + "node_counts": dict(sorted(nodes.items())), + "shared_hit_blocks": shared_hit, + "shared_read_blocks": shared_read, + "temp_read_blocks": temp_read, + "temp_written_blocks": temp_written, + "expensive_nodes": sorted(expensive_nodes, key=lambda node: node["total_ms"], reverse=True)[:12], + "relation_total_ms": {name: round(value, 3) for name, value in relation_times.most_common()}, + } + + +def run_explain(connection: Any, query: str, analyze: bool) -> dict[str, Any]: + option = "ANALYZE, BUFFERS, FORMAT JSON" if analyze else "BUFFERS, FORMAT JSON" + payload = connection.execute("EXPLAIN (" + option + ") " + query).fetchone()[0] + summary = plan_summary(payload) + summary["analyzed"] = analyze + return summary + + +def run(observations: int) -> dict[str, Any]: + import psycopg + from pipeline.tests.e2e.fixture import E2EEnvironment + + env = E2EEnvironment().start() + try: + with psycopg.connect(env.database_url) as connection: + LOAD.seed_public_projection(connection, observations) + queries = { + "facets_before_nested": OLD_FACETS, + "facets_after_flattened": NEW_FACETS, + "list_before_nested": OLD_LIST, + "list_after_flattened": NEW_LIST, + } + connection.execute("SET statement_timeout = '30s'") + plans = {} + for name, query in queries.items(): + # The nested pre-change facets plan is intentionally estimated + # only: executing it can consume the entire bounded rehearsal + # timeout even on the 1,000-row fixture. The flattened path is + # measured with ANALYZE for before/after root-cause evidence. + plans[name] = run_explain(connection, query, analyze="after" in name) + finally: + env.stop() + return {"schema_version": 1, "synthetic_only": True, "observations": observations, "plans": plans} + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--observations", type=int, default=1_000) + parser.add_argument("--json-output", type=Path) + args = parser.parse_args(argv) + if not 1 <= args.observations <= LOAD.MAX_SEED: + parser.error(f"observations must be between 1 and {LOAD.MAX_SEED:,}") + report = run(args.observations) + serialized = json.dumps(report, indent=2, sort_keys=True) + "\n" + if args.json_output: + args.json_output.write_text(serialized, encoding="utf-8") + print(serialized, end="") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/pipeline/tests/test_discovery_scale_benchmark.py b/pipeline/tests/test_discovery_scale_benchmark.py index 2632dca..a7163d6 100644 --- a/pipeline/tests/test_discovery_scale_benchmark.py +++ b/pipeline/tests/test_discovery_scale_benchmark.py @@ -72,6 +72,16 @@ def test_public_read_path_migration_is_additive_and_release_scoped(self): self.assertNotIn("drop table", migration) self.assertNotIn("drop index", migration) + def test_public_history_flattening_preserves_release_and_suppression_gates(self): + migration = (ROOT / "migrations" / "032_flatten_public_history_view.sql").read_text(encoding="utf-8").lower() + self.assertIn("create or replace view uec.map_facilities_display_history", migration) + self.assertIn("with eligible as materialized", migration) + self.assertIn("public_summary as materialized", migration) + self.assertIn("publication_review_release_current", migration) + self.assertIn("public_access_restricted", migration) + self.assertIn("group by release_id, facility_id", migration) + self.assertNotIn("drop view", migration) + if __name__ == "__main__": unittest.main() From 739dd8db75f69f337229644a2d607a8ba1cab1be Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Wed, 16 Sep 2026 11:27:27 -0700 Subject: [PATCH 175/311] test: include migration 032 in ordering contract --- pipeline/tests/test_graph_migrations.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pipeline/tests/test_graph_migrations.py b/pipeline/tests/test_graph_migrations.py index e61236a..0c4e098 100644 --- a/pipeline/tests/test_graph_migrations.py +++ b/pipeline/tests/test_graph_migrations.py @@ -20,13 +20,14 @@ def test_reserved_migrations_are_present_and_ordered(self): positions = [migrations.index(name) for name in graph_migrations] self.assertEqual(positions, sorted(positions)) self.assertEqual([migrations[position] for position in positions], graph_migrations) - self.assertEqual(migrations[-6:], [ + self.assertEqual(migrations[-7:], [ "026_graph_entities_crosswalks.sql", "027_graph_relationship_observations.sql", "028_graph_claims_support.sql", "029_graph_publication_projections.sql", "030_discovery_projection_support_indexes.sql", "031_public_release_read_path_indexes.sql", + "032_flatten_public_history_view.sql", ]) def test_entities_are_distinct_and_crosswalk_is_scoped(self): From 51fce7864663c4bb28265c422420ac72a5524b3e Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Wed, 16 Sep 2026 11:32:52 -0700 Subject: [PATCH 176/311] Document remaining projection scale boundary --- docs/performance/v2-observability.md | 14 +++ .../v2-public-projection-read-path.md | 70 ++++++++++++++ .../benchmarks/explain_public_projection.py | 95 ++++++++++++++++++- 3 files changed, 177 insertions(+), 2 deletions(-) create mode 100644 docs/performance/v2-public-projection-read-path.md diff --git a/docs/performance/v2-observability.md b/docs/performance/v2-observability.md index 446c12f..9cb4f85 100644 --- a/docs/performance/v2-observability.md +++ b/docs/performance/v2-observability.md @@ -84,6 +84,11 @@ python pipeline/scripts/benchmarks/explain_public_projection.py ` --observations 1000 --json-output .tmp/public-explain.json ``` +Add `--components` to separate eligibility, summary, geocode, city, +lifecycle, spatial, and pagination costs. The component report is aggregate +only and remains bounded by the same disposable environment and statement +timeout. + On the synthetic 1,000-row fixture, the flattened list and facets queries executed in 262 ms and 243 ms respectively; the pre-change nested plans were estimated with 28 nested-loop nodes, 25 sequential scans, and 9 repeated @@ -99,6 +104,15 @@ representative traffic test on the deployment topology. The 2-second request timeout and 350 ms database radius-query budget remain review thresholds for fail-safe behavior, not performance guarantees. +The 5,000-row component matrix attributes the remaining cost primarily to the +release-scoped eligibility and public summary path: about 663 ms and 3,362 ms +respectively in isolation, versus about 16 ms geocode, 15 ms city, and 1 ms +lifecycle lookup. Full pagination and spatial statements measured about 5,097 +ms and 4,690 ms. Allowing the summary CTE to inline was tested and rejected: +facets worsened from about 4,501 ms to 6,622 ms. See +`docs/performance/v2-public-projection-read-path.md` for the architecture +decision and the safeguards required for any future release-built component. + The benchmark and logs provide operational evidence only. They do not establish source completeness, publication eligibility, production capacity, cloud cost, or a guarantee for a particular traffic pattern. diff --git a/docs/performance/v2-public-projection-read-path.md b/docs/performance/v2-public-projection-read-path.md new file mode 100644 index 0000000..e825fb1 --- /dev/null +++ b/docs/performance/v2-public-projection-read-path.md @@ -0,0 +1,70 @@ +# V2 public projection read-path decision + +## Decision + +Keep the public projection live and release-scoped, with the flattened view in +migration 032. It evaluates publication review, current suppression, profile +eligibility, geocode display precision, and lifecycle state at read time. Do +not add a request cache or a materialized public surface in this performance +slice. + +## Evidence + +The synthetic component benchmark at 5,000 observations measured roughly: + +| Component | Execution time | +| --- | ---: | +| Release/review/suppression eligibility | 663 ms | +| Per-release eligible observation summary | 3,362 ms | +| Geocode lookup | 16 ms | +| City/coarse-location lookup | 15 ms | +| Lifecycle lookup | 1 ms | +| Full pagination projection | 5,097 ms | +| Full spatial projection | 4,690 ms | + +The flattened view removed repeated nested expansion and made the 1,000-row +concurrent rehearsal clean at all tested levels. At 5,000 rows, the remaining +summary and eligibility work still exceeds the current 2-second rehearsal +budget. A candidate that allowed the summary CTE to inline was measured and +rejected because the 5,000-row facets plan increased from about 4,501 ms to +6,622 ms. + +## Alternatives considered + +1. Request caching is rejected. An emergency suppression, privacy decision, + or release/profile change must take effect on the next read; cache expiry + is not an acceptable enforcement mechanism. + +2. A release-built immutable base projection could reduce repeated joins, but + it is not itself a public surface. A future implementation would need to + build it from an identified release manifest, validate row counts and + checksums, publish it atomically, and fail closed when the artifact is + missing, stale, or inconsistent. + +3. Even with an immutable base, every public read would still have to join the + current release/profile decision and evaluate current suppression directly. + Suppression cannot be handled only by a delayed refresh or by invalidating a + cache. Public summaries would need either a live eligible aggregation or a + transactionally maintained restriction delta with tests for reimport, + release reconstruction, restoration, and cross-profile isolation. + +4. Because the dominant cost is the live eligible summary rather than + geocode/city/lifecycle lookup, materializing a base component is not yet + sufficiently justified by this evidence. The current view is the safer + architecture until a representative deployment load test and a precise + summary strategy are approved. + +## Required safeguards for future work + +Any release-built component must specify, before implementation: + +- manifest-bound build inputs and deterministic row/count/checksum validation; +- atomic activation and a fail-closed missing/stale-artifact path; +- live review, profile, and suppression joins on every public read; +- suppression/restriction replay and reimport invalidation behavior; +- backup/restore ordering that keeps the service stopped until the independent + restriction-ledger gate and current replay pass; +- semantic E2E coverage for publication, profile, suppression, privacy, + ordering, null-region, and detail/list agreement. + +This document is an implementation boundary, not a production capacity claim. diff --git a/pipeline/scripts/benchmarks/explain_public_projection.py b/pipeline/scripts/benchmarks/explain_public_projection.py index af4d5a8..0961b5c 100644 --- a/pipeline/scripts/benchmarks/explain_public_projection.py +++ b/pipeline/scripts/benchmarks/explain_public_projection.py @@ -120,6 +120,91 @@ LIMIT 51 """ +ELIGIBILITY = """ +SELECT member.release_id, member.facility_id, observation.observation_id, + observation.source_record_id, observation.first_observed_at, + observation.observed_at +FROM uec.release_members AS member +JOIN uec.releases AS release ON release.release_id = member.release_id +JOIN uec.observations AS observation ON observation.observation_id = member.observation_id +JOIN uec.source_records AS record ON record.source_record_id = observation.source_record_id +JOIN uec.sources AS source ON source.source_id = record.source_id +JOIN uec.publication_review_release_current AS review + ON review.source_record_id = observation.source_record_id + AND review.release_id = member.release_id +WHERE release.status = 'promoted' + AND member.default_visible = true + AND review.publication_eligible = true + AND review.privacy_screening_status = 'passed' + AND review.factual_review_status <> 'rejected' + AND (review.maintainer_approval = 'approved' + OR (release.profile = 'community' AND source.origin_type = 'user_submitted' + AND review.factual_review_status = 'unreviewed' + AND review.maintainer_approval = 'pending')) + AND NOT EXISTS ( + SELECT 1 FROM uec.public_access_restricted AS restricted + WHERE restricted.source_record_id = observation.source_record_id + ) + AND member.release_id = 'load-promoted' +""" + +COMPONENT_QUERIES = { + "eligibility_gate": "SELECT count(*) FROM (" + ELIGIBILITY + ") eligible", + "eligible_summary": """ +SELECT count(*) FROM ( + SELECT eligible.release_id, eligible.facility_id, + min(eligible.first_observed_at), max(eligible.observed_at), count(*) + FROM (""" + ELIGIBILITY + """) eligible + GROUP BY eligible.release_id, eligible.facility_id +) summary +""", + "geocode_lookup": """ +SELECT count(*) +FROM uec.observations observation +JOIN uec.source_records record ON record.source_record_id = observation.source_record_id +LEFT JOIN LATERAL ( + SELECT result FROM uec.geocode_results + WHERE source_record_id = observation.source_record_id + ORDER BY queried_at DESC, geocode_result_id DESC LIMIT 1 +) latest ON true +WHERE record.source_id = 'load.synthetic' +""", + "city_lookup": """ +SELECT count(*) +FROM uec.facilities facility +LEFT JOIN LATERAL ( + SELECT reference_location + FROM uec.city_reference_points + WHERE country_code = facility.country_code + AND lower(city_name) = lower(facility.city) + AND (postal_code IS NULL OR postal_code = facility.postal_code) + ORDER BY postal_code NULLS LAST LIMIT 1 +) city ON true +WHERE facility.canonical_name LIKE 'Synthetic load facility %' +""", + "lifecycle_lookup": """ +SELECT count(*) +FROM uec.facilities facility +LEFT JOIN uec.facility_lifecycle_current lifecycle + ON lifecycle.facility_id = facility.facility_id +WHERE facility.canonical_name LIKE 'Synthetic load facility %' +""", + "spatial_filter": """ +SELECT count(*) +FROM uec.map_facilities_display_history history +WHERE history.release_id = 'load-promoted' + AND history.display_location && ST_MakeEnvelope(-10, 45, -9.99, 45.01, 4326)::geography + AND ST_Intersects(history.display_location::geometry, ST_MakeEnvelope(-10, 45, -9.99, 45.01, 4326)) +""", + "pagination_sort": """ +SELECT facility_id +FROM uec.map_facilities_display_history +WHERE release_id = 'load-promoted' +ORDER BY facility_id +LIMIT 51 +""", +} + def plan_summary(payload: list[Any]) -> dict[str, Any]: root = payload[0] @@ -180,7 +265,7 @@ def run_explain(connection: Any, query: str, analyze: bool) -> dict[str, Any]: return summary -def run(observations: int) -> dict[str, Any]: +def run(observations: int, include_components: bool = False) -> dict[str, Any]: import psycopg from pipeline.tests.e2e.fixture import E2EEnvironment @@ -202,6 +287,11 @@ def run(observations: int) -> dict[str, Any]: # timeout even on the 1,000-row fixture. The flattened path is # measured with ANALYZE for before/after root-cause evidence. plans[name] = run_explain(connection, query, analyze="after" in name) + if include_components: + plans["components"] = { + name: run_explain(connection, query, analyze=True) + for name, query in COMPONENT_QUERIES.items() + } finally: env.stop() return {"schema_version": 1, "synthetic_only": True, "observations": observations, "plans": plans} @@ -211,10 +301,11 @@ def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--observations", type=int, default=1_000) parser.add_argument("--json-output", type=Path) + parser.add_argument("--components", action="store_true", help="also analyze isolated eligibility, summary, lookup, spatial, and pagination components") args = parser.parse_args(argv) if not 1 <= args.observations <= LOAD.MAX_SEED: parser.error(f"observations must be between 1 and {LOAD.MAX_SEED:,}") - report = run(args.observations) + report = run(args.observations, include_components=args.components) serialized = json.dumps(report, indent=2, sort_keys=True) + "\n" if args.json_output: args.json_output.write_text(serialized, encoding="utf-8") From 5a77e277c51c76854175d78b20c06fc73642dc6e Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Wed, 16 Sep 2026 11:54:06 -0700 Subject: [PATCH 177/311] Prototype fail-closed release summary component --- .../v2-public-projection-read-path.md | 19 ++- .../033_release_summary_component.sql | 77 ++++++++++ .../benchmarks/explain_public_projection.py | 13 ++ .../benchmarks/run_api_load_rehearsal.py | 13 +- .../build_release_summary_component.py | 136 ++++++++++++++++++ pipeline/tests/e2e/run-suite.ps1 | 1 + .../e2e/test_release_summary_component.py | 133 +++++++++++++++++ .../tests/test_release_summary_component.py | 38 +++++ 8 files changed, 427 insertions(+), 3 deletions(-) create mode 100644 pipeline/migrations/033_release_summary_component.sql create mode 100644 pipeline/scripts/maintenance/build_release_summary_component.py create mode 100644 pipeline/tests/e2e/test_release_summary_component.py create mode 100644 pipeline/tests/test_release_summary_component.py diff --git a/docs/performance/v2-public-projection-read-path.md b/docs/performance/v2-public-projection-read-path.md index e825fb1..63062f5 100644 --- a/docs/performance/v2-public-projection-read-path.md +++ b/docs/performance/v2-public-projection-read-path.md @@ -5,8 +5,9 @@ Keep the public projection live and release-scoped, with the flattened view in migration 032. It evaluates publication review, current suppression, profile eligibility, geocode display precision, and lifecycle state at read time. Do -not add a request cache or a materialized public surface in this performance -slice. +not add a request cache or wire a materialized public surface into the API in +this performance slice. A disposable release-built component prototype is +implemented behind a candidate view for invariant testing only. ## Evidence @@ -29,6 +30,20 @@ budget. A candidate that allowed the summary CTE to inline was measured and rejected because the 5,000-row facets plan increased from about 4,501 ms to 6,622 ms. +The prototype builder is reproducible with: + +```powershell +python pipeline/scripts/maintenance/build_release_summary_component.py RELEASE_ID +``` + +It stores release-membership observation facts, not a frozen current-public +decision. The candidate summary joins the exact release manifest checksum and +re-evaluates current review, profile, and suppression state on every read. +At 1,000 rows its candidate summary took about 406 ms versus 121 ms for the +current live summary; at 5,000 rows it took about 9,984 ms versus 3,044 ms. +The prototype therefore proves the safety protocol but does not justify API +integration or a production capacity claim. + ## Alternatives considered 1. Request caching is rejected. An emergency suppression, privacy decision, diff --git a/pipeline/migrations/033_release_summary_component.sql b/pipeline/migrations/033_release_summary_component.sql new file mode 100644 index 0000000..d1597d2 --- /dev/null +++ b/pipeline/migrations/033_release_summary_component.sql @@ -0,0 +1,77 @@ +-- Prototype only: immutable release-bound observation membership facts. This +-- is not a public surface by itself; candidate reads below keep current +-- review, profile, and suppression gates live. +CREATE TABLE uec.release_summary_components ( + release_id TEXT PRIMARY KEY REFERENCES uec.releases(release_id), + manifest_sha256 CHAR(64) NOT NULL CHECK (manifest_sha256 ~ '^[0-9a-f]{64}$'), + content_sha256 CHAR(64) NOT NULL CHECK (content_sha256 ~ '^[0-9a-f]{64}$'), + member_count INTEGER NOT NULL CHECK (member_count >= 0), + built_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE TABLE uec.release_summary_component_rows ( + release_id TEXT NOT NULL REFERENCES uec.releases(release_id), + facility_id UUID NOT NULL REFERENCES uec.facilities(facility_id), + observation_id UUID NOT NULL REFERENCES uec.observations(observation_id), + source_record_id UUID NOT NULL REFERENCES uec.source_records(source_record_id), + first_observed_at TIMESTAMPTZ NOT NULL, + observed_at TIMESTAMPTZ NOT NULL, + classification_category TEXT NOT NULL, + PRIMARY KEY (release_id, facility_id, observation_id) +); + +CREATE INDEX release_summary_component_rows_order_idx + ON uec.release_summary_component_rows (release_id, facility_id, observation_id); + +CREATE TRIGGER release_summary_components_append_only + BEFORE UPDATE OR DELETE ON uec.release_summary_components + FOR EACH ROW EXECUTE FUNCTION uec.reject_evidence_mutation(); + +CREATE TRIGGER release_summary_component_rows_append_only + BEFORE UPDATE OR DELETE ON uec.release_summary_component_rows + FOR EACH ROW EXECUTE FUNCTION uec.reject_evidence_mutation(); + +-- Candidate summary: the immutable rows reduce repeated release-member and +-- observation joins, but all current public eligibility remains live. The +-- manifest join prevents an older or mismatched component from serving as the +-- current release summary; missing metadata therefore fails closed to empty. +CREATE OR REPLACE VIEW uec.public_facility_observation_summary_component_candidate AS +SELECT component.release_id, + component.facility_id, + min(component.first_observed_at) AS first_observed_at, + max(component.observed_at) AS last_observed_at, + count(*)::int AS observation_count +FROM uec.release_summary_component_rows component +JOIN uec.release_summary_components component_meta + ON component_meta.release_id = component.release_id +JOIN uec.releases release + ON release.release_id = component.release_id +JOIN uec.release_manifests manifest + ON manifest.release_id = component.release_id + AND manifest.manifest_sha256 = component_meta.manifest_sha256 +JOIN uec.source_records record + ON record.source_record_id = component.source_record_id +JOIN uec.sources source + ON source.source_id = record.source_id +JOIN uec.publication_review_release_current review + ON review.source_record_id = component.source_record_id + AND review.release_id = component.release_id +WHERE release.status = 'promoted' + AND release.test_only IS NOT TRUE + AND review.publication_eligible = true + AND review.privacy_screening_status = 'passed' + AND review.factual_review_status <> 'rejected' + AND (review.maintainer_approval = 'approved' + OR (release.profile = 'community' + AND source.origin_type = 'user_submitted' + AND review.factual_review_status = 'unreviewed' + AND review.maintainer_approval = 'pending')) + AND NOT EXISTS ( + SELECT 1 + FROM uec.public_access_restricted restricted + WHERE restricted.source_record_id = component.source_record_id + ) +GROUP BY component.release_id, component.facility_id; + +COMMENT ON VIEW uec.public_facility_observation_summary_component_candidate IS + 'Prototype only: manifest-bound immutable release rows aggregated through live review, profile, and suppression gates; not wired to the API.'; diff --git a/pipeline/scripts/benchmarks/explain_public_projection.py b/pipeline/scripts/benchmarks/explain_public_projection.py index 0961b5c..8dd9b44 100644 --- a/pipeline/scripts/benchmarks/explain_public_projection.py +++ b/pipeline/scripts/benchmarks/explain_public_projection.py @@ -18,11 +18,17 @@ ROOT = Path(__file__).resolve().parents[3] LOAD_SCRIPT = ROOT / "pipeline" / "scripts" / "benchmarks" / "run_api_load_rehearsal.py" +BUILD_SCRIPT = ROOT / "pipeline" / "scripts" / "maintenance" / "build_release_summary_component.py" SPEC = importlib.util.spec_from_file_location("run_api_load_rehearsal", LOAD_SCRIPT) assert SPEC and SPEC.loader LOAD = importlib.util.module_from_spec(SPEC) sys.modules[SPEC.name] = LOAD SPEC.loader.exec_module(LOAD) +BUILD_SPEC = importlib.util.spec_from_file_location("build_release_summary_component", BUILD_SCRIPT) +assert BUILD_SPEC and BUILD_SPEC.loader +BUILD = importlib.util.module_from_spec(BUILD_SPEC) +sys.modules[BUILD_SPEC.name] = BUILD +BUILD_SPEC.loader.exec_module(BUILD) OLD_FACETS = """ SELECT country_code, classification_category, display_precision, @@ -202,6 +208,11 @@ WHERE release_id = 'load-promoted' ORDER BY facility_id LIMIT 51 +""", + "component_summary_candidate": """ +SELECT count(*) +FROM uec.public_facility_observation_summary_component_candidate +WHERE release_id = 'load-promoted' """, } @@ -273,6 +284,8 @@ def run(observations: int, include_components: bool = False) -> dict[str, Any]: try: with psycopg.connect(env.database_url) as connection: LOAD.seed_public_projection(connection, observations) + BUILD.build(env.database_url, "load-promoted") + with psycopg.connect(env.database_url) as connection: queries = { "facets_before_nested": OLD_FACETS, "facets_after_flattened": NEW_FACETS, diff --git a/pipeline/scripts/benchmarks/run_api_load_rehearsal.py b/pipeline/scripts/benchmarks/run_api_load_rehearsal.py index a03d679..f9ab569 100644 --- a/pipeline/scripts/benchmarks/run_api_load_rehearsal.py +++ b/pipeline/scripts/benchmarks/run_api_load_rehearsal.py @@ -75,7 +75,18 @@ def seed_public_projection(connection: Any, count: int) -> str: release_id = "load-promoted" connection.execute("INSERT INTO uec.sources (source_id,country_code,name,official_url,access_method) VALUES ('load.synthetic','DK','Synthetic load source','https://example.invalid/load','fixture') ON CONFLICT DO NOTHING") connection.execute("INSERT INTO uec.releases (release_id,status,ruleset_version,profile,test_only,summary) VALUES ('load-promoted','promoted','load-v1','official',false,'{}') ON CONFLICT DO NOTHING") - connection.execute("INSERT INTO uec.release_manifests (release_id,manifest,manifest_sha256) VALUES ('load-promoted',jsonb_build_object('manifest_version','load-v1','profile','official','release_id','load-promoted','ruleset_version','load-v1','eligible_record_count',%s),repeat('a',64)) ON CONFLICT DO NOTHING", (count,)) + manifest = { + "eligible_record_count": count, + "manifest_version": "load-v1", + "profile": "official", + "release_id": release_id, + "ruleset_version": "load-v1", + } + encoded_manifest = json.dumps(manifest, sort_keys=True, separators=(",", ":")) + connection.execute( + "INSERT INTO uec.release_manifests (release_id,manifest,manifest_sha256) VALUES (%s,%s::jsonb,%s) ON CONFLICT DO NOTHING", + (release_id, encoded_manifest, hashlib.sha256(encoded_manifest.encode("utf-8")).hexdigest()), + ) connection.execute("INSERT INTO uec.city_reference_points (country_code,city_name,reference_location,reference_source,source_retrieved_at,source_reference_id) VALUES ('DK','Loadville',ST_SetSRID(ST_Point(-5,50),4326)::geography,'https://example.invalid/load-city',TIMESTAMPTZ '2026-01-01 00:00:00+00','load-city') ON CONFLICT DO NOTHING") connection.execute(""" INSERT INTO uec.raw_artifacts (artifact_id,storage_key,sha256,byte_size,retrieved_at) diff --git a/pipeline/scripts/maintenance/build_release_summary_component.py b/pipeline/scripts/maintenance/build_release_summary_component.py new file mode 100644 index 0000000..fd8f81d --- /dev/null +++ b/pipeline/scripts/maintenance/build_release_summary_component.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python3 +"""Build a deterministic, manifest-bound release summary component. + +The component is a prototype input to a candidate live-gated summary view. It +is never a public surface by itself. Rows and the ready metadata record are +inserted in one transaction; an interrupted build therefore exposes neither a +partial component nor an older component as current. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +from datetime import datetime, timezone +from typing import Any + +import psycopg + + +class ComponentBlocked(ValueError): + """The component cannot be safely used for the requested release.""" + + +def canonical_json(value: Any) -> str: + return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + + +def _timestamp(value: datetime) -> str: + if value.tzinfo is None: + raise ComponentBlocked("component row timestamp lacks timezone") + return value.astimezone(timezone.utc).isoformat().replace("+00:00", "Z") + + +def content_digest(rows: list[tuple[Any, ...]]) -> str: + digest = hashlib.sha256() + for facility_id, observation_id, source_record_id, first_observed_at, observed_at, category in rows: + payload = "\t".join(( + str(facility_id), + str(observation_id), + str(source_record_id), + _timestamp(first_observed_at), + _timestamp(observed_at), + category, + )) + "\n" + digest.update(payload.encode("utf-8")) + return digest.hexdigest() + + +def _release_manifest(connection: Any, release_id: str) -> tuple[str, str]: + release = connection.execute( + "SELECT status, test_only, profile FROM uec.releases WHERE release_id=%s", + (release_id,), + ).fetchone() + if not release: + raise ComponentBlocked("release is missing") + if release[0] != "promoted" or release[1]: + raise ComponentBlocked("only a non-test promoted release can build a component") + stored = connection.execute( + "SELECT manifest::text, manifest_sha256 FROM uec.release_manifests WHERE release_id=%s", + (release_id,), + ).fetchone() + if not stored: + raise ComponentBlocked("release manifest is missing") + manifest = json.loads(stored[0]) + actual = hashlib.sha256(canonical_json(manifest).encode("utf-8")).hexdigest() + if actual != stored[1]: + raise ComponentBlocked("release manifest checksum mismatch") + if manifest.get("release_id") != release_id or manifest.get("profile") != release[2]: + raise ComponentBlocked("release manifest identity mismatch") + return stored[1], release[2] + + +def build(database_url: str, release_id: str, fail_after_rows: int | None = None) -> dict[str, Any]: + with psycopg.connect(database_url) as connection: + with connection.transaction(): + manifest_sha256, _profile = _release_manifest(connection, release_id) + rows = connection.execute( + """ + SELECT member.facility_id, member.observation_id, + observation.source_record_id, observation.first_observed_at, + observation.observed_at, observation.classification_category + FROM uec.release_members member + JOIN uec.observations observation + ON observation.observation_id=member.observation_id + WHERE member.release_id=%s AND member.default_visible=true + ORDER BY member.facility_id, member.observation_id + """, + (release_id,), + ).fetchall() + content_sha256 = content_digest(rows) + existing = connection.execute( + "SELECT manifest_sha256, content_sha256, member_count FROM uec.release_summary_components WHERE release_id=%s", + (release_id,), + ).fetchone() + if existing: + if existing != (manifest_sha256, content_sha256, len(rows)): + raise ComponentBlocked("existing component does not match the current release content") + stored_rows = connection.execute( + "SELECT count(*) FROM uec.release_summary_component_rows WHERE release_id=%s", + (release_id,), + ).fetchone()[0] + if stored_rows != len(rows): + raise ComponentBlocked("component metadata exists but row storage is incomplete") + return {"status": "idempotent", "release_id": release_id, "manifest_sha256": manifest_sha256, "content_sha256": content_sha256, "member_count": len(rows)} + + for index, row in enumerate(rows, start=1): + connection.execute( + "INSERT INTO uec.release_summary_component_rows (release_id,facility_id,observation_id,source_record_id,first_observed_at,observed_at,classification_category) VALUES (%s,%s,%s,%s,%s,%s,%s)", + (release_id, *row), + ) + if fail_after_rows is not None and index >= fail_after_rows: + raise RuntimeError("synthetic interrupted component build") + connection.execute( + "INSERT INTO uec.release_summary_components (release_id,manifest_sha256,content_sha256,member_count) VALUES (%s,%s,%s,%s)", + (release_id, manifest_sha256, content_sha256, len(rows)), + ) + return {"status": "built", "release_id": release_id, "manifest_sha256": manifest_sha256, "content_sha256": content_sha256, "member_count": len(rows)} + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("release_id") + parser.add_argument("--database-url", default=os.environ.get("UEC_DATABASE_URL", "postgresql://uec:uec-local-development-only@localhost:5433/uec")) + args = parser.parse_args() + try: + print(json.dumps(build(args.database_url, args.release_id), sort_keys=True)) + except Exception as error: + print(json.dumps({"status": "blocked", "error": str(error)}, sort_keys=True)) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/pipeline/tests/e2e/run-suite.ps1 b/pipeline/tests/e2e/run-suite.ps1 index e44785d..4aa97d7 100644 --- a/pipeline/tests/e2e/run-suite.ps1 +++ b/pipeline/tests/e2e/run-suite.ps1 @@ -15,6 +15,7 @@ $core = @( ) $extended = @( 'pipeline.tests.e2e.test_suppression_lifecycle', + 'pipeline.tests.e2e.test_release_summary_component', 'pipeline.tests.e2e.test_italy_candidate_import', 'pipeline.tests.e2e.test_germany_belgium_candidate_import' ) diff --git a/pipeline/tests/e2e/test_release_summary_component.py b/pipeline/tests/e2e/test_release_summary_component.py new file mode 100644 index 0000000..54b36fb --- /dev/null +++ b/pipeline/tests/e2e/test_release_summary_component.py @@ -0,0 +1,133 @@ +"""E2E contracts for the fail-closed release summary component prototype.""" + +import hashlib +import importlib.util +import json +import os +import sys +import unittest +import uuid +from pathlib import Path + +import psycopg + +try: + from .fixture import E2EEnvironment +except ImportError: + from fixture import E2EEnvironment + + +ROOT = Path(__file__).parents[2] + + +def load_script(path: Path): + spec = importlib.util.spec_from_file_location(path.stem.replace("-", "_"), path) + module = importlib.util.module_from_spec(spec) + assert spec.loader + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +LOAD = load_script(ROOT / "scripts/benchmarks/run_api_load_rehearsal.py") +BUILD = load_script(ROOT / "scripts/maintenance/build_release_summary_component.py") + + +class ReleaseSummaryComponentE2ETests(unittest.TestCase): + @classmethod + def setUpClass(cls): + if os.environ.get("UEC_RUN_E2E") != "1": + raise unittest.SkipTest("set UEC_RUN_E2E=1 to run Docker-backed E2E tests") + cls.env = E2EEnvironment().start() + with psycopg.connect(cls.env.database_url) as db: + cls.detail_id = LOAD.seed_public_projection(db, 3) + cls.built = BUILD.build(cls.env.database_url, "load-promoted") + + @classmethod + def tearDownClass(cls): + cls.env.stop() + + @staticmethod + def manifest(release_id: str, profile: str) -> tuple[str, str]: + value = { + "manifest_version": "component-test-v1", + "profile": profile, + "release_id": release_id, + } + encoded = json.dumps(value, sort_keys=True, separators=(",", ":")) + return encoded, hashlib.sha256(encoded.encode()).hexdigest() + + @classmethod + def create_release(cls, release_id: str, profile: str, with_members: bool = False): + encoded, digest = cls.manifest(release_id, profile) + with psycopg.connect(cls.env.database_url) as db: + with db.transaction(): + db.execute( + "INSERT INTO uec.releases (release_id,status,ruleset_version,profile,test_only,summary) VALUES (%s,'promoted','component-v1',%s,false,'{}')", + (release_id, profile), + ) + db.execute( + "INSERT INTO uec.release_manifests (release_id,manifest,manifest_sha256) VALUES (%s,%s::jsonb,%s)", + (release_id, encoded, digest), + ) + if with_members: + db.execute( + "INSERT INTO uec.release_members (release_id,facility_id,observation_id,default_visible) SELECT %s,facility_id,observation_id,default_visible FROM uec.release_members WHERE release_id='load-promoted'", + (release_id,), + ) + + def test_build_is_atomic_deterministic_and_idempotent(self): + self.assertEqual(self.built["status"], "built") + again = BUILD.build(self.env.database_url, "load-promoted") + self.assertEqual(again["status"], "idempotent") + self.assertEqual(again["content_sha256"], self.built["content_sha256"]) + with psycopg.connect(self.env.database_url) as db: + self.assertEqual(db.execute("SELECT member_count FROM uec.release_summary_components WHERE release_id='load-promoted'").fetchone()[0], 3) + self.assertEqual(db.execute("SELECT count(*) FROM uec.release_summary_component_rows WHERE release_id='load-promoted'").fetchone()[0], 3) + self.assertEqual(db.execute("SELECT count(*) FROM uec.public_facility_observation_summary_component_candidate WHERE release_id='load-promoted'").fetchone()[0], 3) + + def test_interrupted_build_exposes_no_partial_rows_then_recovers(self): + release_id = "component-interrupted" + self.create_release(release_id, "community", with_members=True) + with self.assertRaisesRegex(RuntimeError, "interrupted"): + BUILD.build(self.env.database_url, release_id, fail_after_rows=1) + with psycopg.connect(self.env.database_url) as db: + self.assertEqual(db.execute("SELECT count(*) FROM uec.release_summary_components WHERE release_id=%s", (release_id,)).fetchone()[0], 0) + self.assertEqual(db.execute("SELECT count(*) FROM uec.release_summary_component_rows WHERE release_id=%s", (release_id,)).fetchone()[0], 0) + recovered = BUILD.build(self.env.database_url, release_id) + self.assertEqual(recovered["status"], "built") + + def test_manifest_mismatch_and_missing_current_component_fail_closed(self): + release_id = "component-mismatch" + self.create_release(release_id, "secondary") + with psycopg.connect(self.env.database_url) as db: + db.execute( + "INSERT INTO uec.release_summary_components (release_id,manifest_sha256,content_sha256,member_count) VALUES (%s,%s,%s,0)", + (release_id, "f" * 64, "e" * 64), + ) + with self.assertRaisesRegex(BUILD.ComponentBlocked, "does not match"): + BUILD.build(self.env.database_url, release_id) + with psycopg.connect(self.env.database_url) as db: + self.assertEqual(db.execute("SELECT count(*) FROM uec.public_facility_observation_summary_component_candidate WHERE release_id='component-missing'").fetchone()[0], 0) + self.assertEqual(db.execute("SELECT count(*) FROM uec.public_facility_observation_summary_component_candidate WHERE release_id=%s", (release_id,)).fetchone()[0], 0) + + def test_current_suppression_is_still_enforced_by_candidate_summary(self): + with psycopg.connect(self.env.database_url) as db: + before = db.execute("SELECT count(*) FROM uec.public_facility_observation_summary_component_candidate WHERE release_id='load-promoted'").fetchone()[0] + case_id = uuid.uuid4() + facility_id = db.execute("SELECT facility_id FROM uec.release_summary_component_rows WHERE release_id='load-promoted' ORDER BY facility_id LIMIT 1").fetchone()[0] + db.execute( + "INSERT INTO uec.suppression_cases (case_id,reason_category,status,policy_version,actor,decision) VALUES (%s,'privacy','active','component-test','test','revoke')", + (case_id,), + ) + db.execute( + "INSERT INTO uec.suppression_references (case_id,facility_id,scope) VALUES (%s,%s,'whole_record')", + (case_id, facility_id), + ) + after = db.execute("SELECT count(*) FROM uec.public_facility_observation_summary_component_candidate WHERE release_id='load-promoted'").fetchone()[0] + self.assertEqual(before, 3) + self.assertEqual(after, 2) + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/tests/test_release_summary_component.py b/pipeline/tests/test_release_summary_component.py new file mode 100644 index 0000000..ca30ac3 --- /dev/null +++ b/pipeline/tests/test_release_summary_component.py @@ -0,0 +1,38 @@ +"""Unit contracts for the manifest-bound summary component builder.""" + +import importlib.util +import unittest +from datetime import datetime, timezone +from pathlib import Path + + +ROOT = Path(__file__).parents[1] +SCRIPT = ROOT / "scripts" / "maintenance" / "build_release_summary_component.py" +SPEC = importlib.util.spec_from_file_location("build_release_summary_component", SCRIPT) +MODULE = importlib.util.module_from_spec(SPEC) +assert SPEC.loader +SPEC.loader.exec_module(MODULE) + + +class ReleaseSummaryComponentTests(unittest.TestCase): + def test_content_digest_is_deterministic_and_content_bound(self): + row = ("facility", "observation", "record", datetime(2026, 1, 1, tzinfo=timezone.utc), datetime(2026, 1, 2, tzinfo=timezone.utc), "slaughter") + self.assertEqual(MODULE.content_digest([row]), MODULE.content_digest([row])) + changed = (*row[:-1], "logistics_and_storage") + self.assertNotEqual(MODULE.content_digest([row]), MODULE.content_digest([changed])) + + def test_naive_timestamps_are_rejected(self): + row = ("facility", "observation", "record", datetime(2026, 1, 1), datetime(2026, 1, 2, tzinfo=timezone.utc), "slaughter") + with self.assertRaises(MODULE.ComponentBlocked): + MODULE.content_digest([row]) + + def test_migration_is_append_only_and_candidate_view_is_live_gated(self): + migration = (ROOT / "migrations" / "033_release_summary_component.sql").read_text(encoding="utf-8").lower() + for token in ("manifest_sha256", "content_sha256", "release_summary_component_rows_append_only", "public_access_restricted", "publication_review_release_current", "manifest.manifest_sha256 = component_meta.manifest_sha256"): + self.assertIn(token, migration) + self.assertNotIn("drop table", migration) + self.assertNotIn("drop view", migration) + + +if __name__ == "__main__": + unittest.main() From 9dc05a6d01a0874c8b6a50a36627a2e9243afaaa Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Wed, 16 Sep 2026 11:58:13 -0700 Subject: [PATCH 178/311] test: include migration 033 in ordering contract --- pipeline/tests/test_graph_migrations.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pipeline/tests/test_graph_migrations.py b/pipeline/tests/test_graph_migrations.py index 0c4e098..2afd319 100644 --- a/pipeline/tests/test_graph_migrations.py +++ b/pipeline/tests/test_graph_migrations.py @@ -20,7 +20,7 @@ def test_reserved_migrations_are_present_and_ordered(self): positions = [migrations.index(name) for name in graph_migrations] self.assertEqual(positions, sorted(positions)) self.assertEqual([migrations[position] for position in positions], graph_migrations) - self.assertEqual(migrations[-7:], [ + self.assertEqual(migrations[-8:], [ "026_graph_entities_crosswalks.sql", "027_graph_relationship_observations.sql", "028_graph_claims_support.sql", @@ -28,6 +28,7 @@ def test_reserved_migrations_are_present_and_ordered(self): "030_discovery_projection_support_indexes.sql", "031_public_release_read_path_indexes.sql", "032_flatten_public_history_view.sql", + "033_release_summary_component.sql", ]) def test_entities_are_distinct_and_crosswalk_is_scoped(self): From 60e04cb90219f01bca1447ca3bda12a289ecffcc Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Wed, 16 Sep 2026 12:41:24 -0700 Subject: [PATCH 179/311] test: wait for readiness service bind --- pipeline/tests/e2e/fixture.py | 25 +++++++++++++++++++++++++ pipeline/tests/e2e/test_readiness.py | 2 ++ 2 files changed, 27 insertions(+) diff --git a/pipeline/tests/e2e/fixture.py b/pipeline/tests/e2e/fixture.py index 62f4b6d..4d73111 100644 --- a/pipeline/tests/e2e/fixture.py +++ b/pipeline/tests/e2e/fixture.py @@ -135,6 +135,31 @@ def start(self, migration_files=None, wait_for_ready=True): self.stop() raise + def wait_for_listening(self, timeout=20): + """Wait for the backend socket without requiring schema readiness.""" + deadline = time.monotonic() + timeout + last_error = None + while time.monotonic() < deadline: + if self.backend and self.backend.poll() is not None: + if self.backend_log: + self.backend_log.flush() + log_path = Path(self.backend_log.name) + log_text = log_path.read_text(encoding="utf-8") + else: + log_text = "" + raise RuntimeError( + f"backend exited before listening; exit_code={self.backend.returncode}\n{log_text}" + ) + try: + with socket.create_connection(("127.0.0.1", self.api_port), timeout=1): + return + except OSError as exc: + last_error = repr(exc) + time.sleep(.1) + raise RuntimeError( + f"backend did not start listening within {timeout}s; last_error={last_error}" + ) + def stop(self): if self.backend and self.backend.poll() is None: if os.name == "nt": diff --git a/pipeline/tests/e2e/test_readiness.py b/pipeline/tests/e2e/test_readiness.py index 8ec5e6b..107f63c 100644 --- a/pipeline/tests/e2e/test_readiness.py +++ b/pipeline/tests/e2e/test_readiness.py @@ -14,6 +14,8 @@ def test_partial_schema_is_not_ready(self): try: migrations = sorted((ROOT / "pipeline/migrations").glob("*.sql")) env.start(migration_files=migrations[:1], wait_for_ready=False) + env.wait_for_listening() + self.assertIsNone(env.backend.poll(), "backend must stay alive to report schema readiness") try: urllib.request.urlopen(f"http://127.0.0.1:{env.api_port}/health/ready", timeout=2) except urllib.error.HTTPError as response: From 99233bf745b39aafac1c4c6faf09ca572d762ea4 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Wed, 16 Sep 2026 13:44:48 -0700 Subject: [PATCH 180/311] pipeline: represent unscheduled registry sources explicitly --- pipeline/common/source_operations.py | 13 +++++++++++-- pipeline/common/test_source_operations.py | 4 +++- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/pipeline/common/source_operations.py b/pipeline/common/source_operations.py index 84666aa..434ed45 100644 --- a/pipeline/common/source_operations.py +++ b/pipeline/common/source_operations.py @@ -132,11 +132,20 @@ def load_source_schedules(path: str | Path | None = None, *, registry_path: str if registry_path is not None: from pipeline.source_registry import load_registry - registry_ids = {item["source_id"] for item in load_registry(Path(registry_path))["sources"]} + registry = load_registry(Path(registry_path)) + registry_ids = {item["source_id"] for item in registry["sources"]} missing = sorted(registry_ids - schedules.keys()) extra = sorted(schedules.keys() - registry_ids) - if missing or extra: + if extra: raise SourceOperationsError(f"schedule/source registry mismatch; missing={missing}, extra={extra}") + # Reference-only registry entries still need an explicit health row. + # An unknown cadence is not acquisition authorization. + for source_id in missing: + schedules[source_id] = SourceSchedule( + source_id=source_id, cadence="unknown", interval_hours=None, + stale_after_hours=None, + manual_fallback="source-specific terms and an authorized acquisition route remain unresolved", + ) return schedules diff --git a/pipeline/common/test_source_operations.py b/pipeline/common/test_source_operations.py index acfff27..9e9b43d 100644 --- a/pipeline/common/test_source_operations.py +++ b/pipeline/common/test_source_operations.py @@ -24,7 +24,9 @@ class SourceOperationsTests(unittest.TestCase): def test_checked_in_schedule_inventory_matches_registry(self): root = Path(__file__).parents[1] schedules = load_source_schedules(registry_path=root / "source_registry.json") - self.assertEqual(len(schedules), 69) + registry_ids = {item["source_id"] for item in json.loads((root / "source_registry.json").read_text(encoding="utf-8"))["sources"]} + self.assertEqual(set(schedules), registry_ids) + self.assertEqual(schedules["al.aku.approved-food"].cadence, "unknown") self.assertEqual(schedules["dk.smiley"].stale_after_hours, 240) self.assertEqual(schedules["fr.dgal.section-i"].interval_hours, 24) self.assertIsNone(schedules["us.fsis"].interval_hours) From 0d947e997f1e73e3b7b45b8ab5e57cf0aa66704c Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Wed, 16 Sep 2026 13:45:07 -0700 Subject: [PATCH 181/311] perf: add live eligibility join indexes --- docs/performance/v2-observability.md | 15 +++++++++++ .../034_public_eligibility_join_indexes.sql | 27 +++++++++++++++++++ .../tests/test_discovery_scale_benchmark.py | 14 ++++++++++ 3 files changed, 56 insertions(+) create mode 100644 pipeline/migrations/034_public_eligibility_join_indexes.sql diff --git a/docs/performance/v2-observability.md b/docs/performance/v2-observability.md index 9cb4f85..f8c8a5e 100644 --- a/docs/performance/v2-observability.md +++ b/docs/performance/v2-observability.md @@ -116,3 +116,18 @@ decision and the safeguards required for any future release-built component. The benchmark and logs provide operational evidence only. They do not establish source completeness, publication eligibility, production capacity, cloud cost, or a guarantee for a particular traffic pattern. + +## Operational budget boundary + +The current rehearsal contract is intentionally bounded: 2,000 ms client +request timeout, 30 s diagnostic statement timeout, and 350 ms as the review +threshold for a database radius query. These are fail-safe review thresholds, +not service-level objectives. The application uses the deadpool default pool +unless deployment configuration proves a different bound; no pool size is +recommended from the synthetic 5,000-row results. A deployment decision needs +the same aggregate harness at 5,000, 25,000, and larger representative +projections, bounded concurrency, and failure-injection/readiness evidence. + +Migration 034 adds only join-support indexes for the live projection. Migration +033's release-summary component remains a candidate-only, non-public prototype +and is deliberately not referenced by the API or this benchmark. diff --git a/pipeline/migrations/034_public_eligibility_join_indexes.sql b/pipeline/migrations/034_public_eligibility_join_indexes.sql new file mode 100644 index 0000000..d399584 --- /dev/null +++ b/pipeline/migrations/034_public_eligibility_join_indexes.sql @@ -0,0 +1,27 @@ +-- Additive indexes for the live public eligibility path. +-- These indexes do not change review ordering, suppression semantics, or the +-- flattened live projection. Migration 033 remains a non-public prototype. + +CREATE INDEX IF NOT EXISTS publication_review_events_record_current_idx + ON uec.publication_review_events + (source_record_id, reviewed_at DESC, publication_review_event_id DESC); + +CREATE INDEX IF NOT EXISTS publication_review_release_scopes_event_release_idx + ON uec.publication_review_release_scopes + (publication_review_event_id, release_id); + +CREATE INDEX IF NOT EXISTS record_access_events_record_current_idx + ON uec.record_access_events + (source_record_id, occurred_at DESC, access_event_id DESC); + +CREATE INDEX IF NOT EXISTS observations_source_record_lookup_idx + ON uec.observations (source_record_id, observation_id, facility_id); + +COMMENT ON INDEX uec.publication_review_events_record_current_idx IS + 'Supports append-only latest review lookup while preserving reviewed_at and event-id ordering.'; +COMMENT ON INDEX uec.publication_review_release_scopes_event_release_idx IS + 'Supports release-scoped review event expansion without changing eligibility semantics.'; +COMMENT ON INDEX uec.record_access_events_record_current_idx IS + 'Supports current access restriction lookup by source record without exposing restricted payloads.'; +COMMENT ON INDEX uec.observations_source_record_lookup_idx IS + 'Supports release eligibility and suppression crosswalk joins by source record.'; diff --git a/pipeline/tests/test_discovery_scale_benchmark.py b/pipeline/tests/test_discovery_scale_benchmark.py index a7163d6..6c75bf6 100644 --- a/pipeline/tests/test_discovery_scale_benchmark.py +++ b/pipeline/tests/test_discovery_scale_benchmark.py @@ -82,6 +82,20 @@ def test_public_history_flattening_preserves_release_and_suppression_gates(self) self.assertIn("group by release_id, facility_id", migration) self.assertNotIn("drop view", migration) + def test_public_eligibility_indexes_are_additive_and_order_safe(self): + migration = (ROOT / "migrations" / "034_public_eligibility_join_indexes.sql").read_text(encoding="utf-8").lower() + for index_name in ( + "publication_review_events_record_current_idx", + "publication_review_release_scopes_event_release_idx", + "record_access_events_record_current_idx", + "observations_source_record_lookup_idx", + ): + self.assertIn(index_name, migration) + self.assertIn("reviewed_at desc, publication_review_event_id desc", migration) + self.assertIn("occurred_at desc, access_event_id desc", migration) + self.assertNotIn("drop table", migration) + self.assertNotIn("drop index", migration) + if __name__ == "__main__": unittest.main() From ec04757a1a5b30fbfaf96a3a7d31433b2f7685c8 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Wed, 16 Sep 2026 13:45:12 -0700 Subject: [PATCH 182/311] Add deterministic geospatial readiness audit --- data/reports/geospatial-readiness.json | 1218 +++++++++++++++++ docs/geospatial-readiness-audit.md | 7 + .../diagnostics/geospatial_readiness_audit.py | 59 + .../tests/test_geospatial_readiness_audit.py | 18 + 4 files changed, 1302 insertions(+) create mode 100644 data/reports/geospatial-readiness.json create mode 100644 docs/geospatial-readiness-audit.md create mode 100644 pipeline/scripts/diagnostics/geospatial_readiness_audit.py create mode 100644 pipeline/tests/test_geospatial_readiness_audit.py diff --git a/data/reports/geospatial-readiness.json b/data/reports/geospatial-readiness.json new file mode 100644 index 0000000..4cc0e4e --- /dev/null +++ b/data/reports/geospatial-readiness.json @@ -0,0 +1,1218 @@ +{ + "as_of": "2026-09-16T00:00:00Z", + "country_counts": { + "ca": 1323, + "de": 9254, + "dk": 1561, + "es": 4222, + "fr": 3201, + "mx": 14836, + "nz": 1652, + "uk": 7600, + "us": 7101 + }, + "funnel": { + "city_coarse_candidate": 656, + "geocode_candidate": 1373, + "map_ready_under_current_rules": 48640, + "privacy_restricted": 72, + "source_coordinate_invalid_or_missing": 2047, + "source_coordinate_valid": 48703, + "total": 50750, + "unresolved": 9 + }, + "method": "offline deterministic audit; no geocoder calls", + "provenance_fields_required_for_any_geocode": [ + "provider", + "query_hash", + "queried_at", + "precision", + "review_state" + ], + "publication_rule": "map-ready requires current privacy eligibility and release approval; geocoding success alone never grants publication permission", + "sample": { + "composition": { + "ca|address_complete|source_coordinate_valid": 8, + "de|address_complete|source_coordinate_invalid_or_missing": 4, + "de|address_complete|source_coordinate_valid": 20, + "dk|address_complete|source_coordinate_invalid_or_missing": 5, + "dk|address_complete|source_coordinate_valid": 6, + "es|address_complete|source_coordinate_valid": 14, + "es|city_only|source_coordinate_valid": 8, + "fr|address_missing|source_coordinate_valid": 20, + "mx|address_complete|source_coordinate_valid": 20, + "mx|city_only|source_coordinate_valid": 15, + "nz|city_only|source_coordinate_invalid_or_missing": 3, + "nz|city_only|source_coordinate_valid": 8, + "uk|address_complete|source_coordinate_valid": 20, + "uk|city_only|source_coordinate_valid": 9, + "us|address_complete|source_coordinate_valid": 20 + }, + "size": 180 + }, + "sample_rows": [ + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "de", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_invalid_or_missing", + "country": "dk", + "outcome": "geocode_candidate" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "de", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "de", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "nz", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "mx", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "mx", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "de", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "de", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "de", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "mx", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "mx", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "de", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "de", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "dk", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "de", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "us", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "mx", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "mx", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "ca", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "nz", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "uk", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "us", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "mx", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "de", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "es", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_invalid_or_missing", + "country": "nz", + "outcome": "city_coarse_candidate" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "mx", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "mx", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "mx", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "uk", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "us", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "es", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "mx", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "uk", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "mx", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "de", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "mx", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "mx", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "uk", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "us", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "mx", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "mx", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "us", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "us", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "mx", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "mx", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "uk", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_missing", + "coordinate_state": "source_coordinate_valid", + "country": "fr", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "de", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "de", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "de", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "mx", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "uk", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "mx", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "uk", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "de", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "mx", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "mx", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "de", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "mx", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "mx", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_invalid_or_missing", + "country": "de", + "outcome": "geocode_candidate" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "de", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "de", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "mx", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "uk", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_missing", + "coordinate_state": "source_coordinate_valid", + "country": "fr", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "uk", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_missing", + "coordinate_state": "source_coordinate_valid", + "country": "fr", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "uk", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "de", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "de", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "uk", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "us", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "uk", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "us", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "uk", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "us", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_invalid_or_missing", + "country": "nz", + "outcome": "city_coarse_candidate" + }, + { + "address_quality": "address_missing", + "coordinate_state": "source_coordinate_valid", + "country": "fr", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "uk", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "us", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_missing", + "coordinate_state": "source_coordinate_valid", + "country": "fr", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "uk", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_invalid_or_missing", + "country": "de", + "outcome": "geocode_candidate" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "uk", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "uk", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "mx", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "uk", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "es", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "us", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "es", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "uk", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "ca", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "dk", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "uk", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_invalid_or_missing", + "country": "dk", + "outcome": "geocode_candidate" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "uk", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_missing", + "coordinate_state": "source_coordinate_valid", + "country": "fr", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "uk", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "us", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "uk", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "uk", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "us", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "us", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "nz", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "uk", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "ca", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "us", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "ca", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "uk", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_missing", + "coordinate_state": "source_coordinate_valid", + "country": "fr", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "uk", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_invalid_or_missing", + "country": "dk", + "outcome": "geocode_candidate" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "us", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "es", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "us", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "us", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "us", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "mx", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "es", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "us", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "nz", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "uk", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "es", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_missing", + "coordinate_state": "source_coordinate_valid", + "country": "fr", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "mx", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "es", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "es", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_missing", + "coordinate_state": "source_coordinate_valid", + "country": "fr", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_invalid_or_missing", + "country": "nz", + "outcome": "city_coarse_candidate" + }, + { + "address_quality": "address_missing", + "coordinate_state": "source_coordinate_valid", + "country": "fr", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "mx", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_missing", + "coordinate_state": "source_coordinate_valid", + "country": "fr", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "nz", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "dk", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "mx", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_missing", + "coordinate_state": "source_coordinate_valid", + "country": "fr", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "mx", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "es", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "nz", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "dk", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_missing", + "coordinate_state": "source_coordinate_valid", + "country": "fr", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "es", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "dk", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "ca", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "mx", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "es", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "es", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "es", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_missing", + "coordinate_state": "source_coordinate_valid", + "country": "fr", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "nz", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_invalid_or_missing", + "country": "dk", + "outcome": "geocode_candidate" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_invalid_or_missing", + "country": "de", + "outcome": "geocode_candidate" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "nz", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_invalid_or_missing", + "country": "de", + "outcome": "geocode_candidate" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "mx", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "mx", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "es", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_invalid_or_missing", + "country": "dk", + "outcome": "geocode_candidate" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "es", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_missing", + "coordinate_state": "source_coordinate_valid", + "country": "fr", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_missing", + "coordinate_state": "source_coordinate_valid", + "country": "fr", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "es", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_missing", + "coordinate_state": "source_coordinate_valid", + "country": "fr", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_missing", + "coordinate_state": "source_coordinate_valid", + "country": "fr", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "ca", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "mx", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "es", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_missing", + "coordinate_state": "source_coordinate_valid", + "country": "fr", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "ca", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "es", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "dk", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "ca", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_missing", + "coordinate_state": "source_coordinate_valid", + "country": "fr", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "es", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "es", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "es", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "uk", + "outcome": "map_ready_under_current_rules" + } + ], + "schema_version": "geospatial-readiness-audit/v1", + "source_files": [ + { + "bytes": 361801, + "country": "ca", + "path": "static_data/ca/locations.csv", + "sha256": "544fd7809926143d4e934f1c601e981a07b0137780f86de753ef3c54415e45f4" + }, + { + "bytes": 3849223, + "country": "de", + "path": "static_data/de/locations.csv", + "sha256": "88eb00f6ae405489a7e873582aa65610ecaa3c9b475bc0046e9e8c32f48ac809" + }, + { + "bytes": 323380, + "country": "dk", + "path": "static_data/dk/locations.csv", + "sha256": "b38f8165d6ea3617a13dbad950d0070490e645473bb851ddd3d907004a33b32f" + }, + { + "bytes": 1093952, + "country": "es", + "path": "static_data/es/locations.csv", + "sha256": "93d36d6a760a4435c836348021fb03d98ddb4947345338ea8c199321af3e5347" + }, + { + "bytes": 626026, + "country": "fr", + "path": "static_data/fr/locations.csv", + "sha256": "aeddafa5ebc19d63b6dd57777851e3e5590b5d5ddfb9cbeff37a552227e1c615" + }, + { + "bytes": 5039068, + "country": "mx", + "path": "static_data/mx/locations.csv", + "sha256": "bc9b3adc80e4673bb8b996c7600b52b78e7973b1a1859608322a293aff9efdee" + }, + { + "bytes": 410481, + "country": "nz", + "path": "static_data/nz/locations.csv", + "sha256": "636d80b33cc29dbd68b3c44ddf39c57db84cdd661d1c10dfab4227f8a87a3a87" + }, + { + "bytes": 2193258, + "country": "uk", + "path": "static_data/uk/locations.csv", + "sha256": "4a14793c227afe558bfacfe45b05f3ae4e5332a3d502991050257320c5b53511" + }, + { + "bytes": 4099548, + "country": "us", + "path": "static_data/us/locations.csv", + "sha256": "2dca259076a16a324ad9565d2d4057aa0f5e6e51bbc25a8dc1c80a4e4fbdf5c7" + } + ], + "strata_counts": { + "ca|address_complete|source_coordinate_invalid_or_missing": 1, + "ca|address_complete|source_coordinate_valid": 1261, + "ca|address_missing|source_coordinate_invalid_or_missing": 2, + "ca|city_only|source_coordinate_valid": 59, + "de|address_complete|source_coordinate_invalid_or_missing": 962, + "de|address_complete|source_coordinate_valid": 8290, + "de|city_only|source_coordinate_valid": 2, + "dk|address_complete|source_coordinate_invalid_or_missing": 409, + "dk|address_complete|source_coordinate_valid": 1128, + "dk|address_missing|source_coordinate_invalid_or_missing": 7, + "dk|address_missing|source_coordinate_valid": 17, + "es|address_complete|source_coordinate_valid": 3278, + "es|address_missing|source_coordinate_valid": 30, + "es|city_only|source_coordinate_valid": 914, + "fr|address_missing|source_coordinate_valid": 3201, + "mx|address_complete|source_coordinate_valid": 12258, + "mx|city_only|source_coordinate_valid": 2578, + "nz|city_only|source_coordinate_invalid_or_missing": 665, + "nz|city_only|source_coordinate_valid": 987, + "uk|address_complete|source_coordinate_invalid_or_missing": 1, + "uk|address_complete|source_coordinate_valid": 6981, + "uk|address_missing|source_coordinate_valid": 1, + "uk|city_only|source_coordinate_valid": 617, + "us|address_complete|source_coordinate_valid": 7101 + } +} diff --git a/docs/geospatial-readiness-audit.md b/docs/geospatial-readiness-audit.md new file mode 100644 index 0000000..e930a0d --- /dev/null +++ b/docs/geospatial-readiness-audit.md @@ -0,0 +1,7 @@ +# Geospatial readiness rehearsal + +`pipeline/scripts/diagnostics/geospatial_readiness_audit.py` performs a deterministic offline audit of normalized `static_data/*/locations.csv` files. It never calls a geocoder and emits only aggregate counts plus sanitized sample composition. Source coordinates are retained conceptually as source evidence; this report does not copy them. + +The funnel distinguishes valid source coordinates, geocode candidates, city/coarse candidates, review-required/privacy-restricted candidates, unresolved rows, and `map_ready_under_current_rules`. Map readiness is not publication approval: a future geocode result must retain provider, query hash, timestamp, precision, and review state, and the release projection must independently pass privacy and approval gates. Residential-risk flags are conservative indicators for human review, not factual classifications. + +Run with `python pipeline/scripts/diagnostics/geospatial_readiness_audit.py --output data/reports/geospatial-readiness.json --as-of 2026-09-16T00:00:00Z`. The report is row-free with respect to source names, addresses, identifiers, and coordinates. Real corpus totals and limitations must be reviewed before release; this rehearsal does not authorize publication or paid geocoding. diff --git a/pipeline/scripts/diagnostics/geospatial_readiness_audit.py b/pipeline/scripts/diagnostics/geospatial_readiness_audit.py new file mode 100644 index 0000000..74439fc --- /dev/null +++ b/pipeline/scripts/diagnostics/geospatial_readiness_audit.py @@ -0,0 +1,59 @@ +"""Deterministic, row-free geospatial readiness audit for normalized CSV corpora. + +This intentionally does not geocode. It measures source-coordinate quality and +safe candidate readiness without emitting addresses, names, IDs, or coordinates. +""" +from __future__ import annotations + +import argparse, csv, hashlib, json, re +from collections import Counter +from datetime import datetime, timezone +from pathlib import Path + +PRIVATE_RE = re.compile(r"\b(private|residential|home|farmhouse|c/o|care of)\b", re.I) + +def num(v): + try: return float(v) + except (TypeError, ValueError): return None + +def valid_point(lat, lon): + return lat is not None and lon is not None and -90 <= lat <= 90 and -180 <= lon <= 180 and not (lat == 0 and lon == 0) + +def audit(root: Path, sample_size: int = 20, as_of: str | None = None): + files = sorted(root.glob("*/locations.csv")) + funnel = Counter(); countries = Counter(); strata = Counter(); sample_buckets = {} + file_manifest = [] + for path in files: + digest = hashlib.sha256(path.read_bytes()).hexdigest() + file_manifest.append({"country": path.parent.name, "path": path.as_posix(), "sha256": digest, "bytes": path.stat().st_size}) + with path.open(encoding="utf-8-sig", newline="") as fh: + rows = csv.DictReader(fh) + for row in rows: + funnel["total"] += 1; country = path.parent.name; countries[country] += 1 + lat, lon = num(row.get("latitude")), num(row.get("longitude")) + coord = "source_coordinate_valid" if valid_point(lat, lon) else "source_coordinate_invalid_or_missing" + funnel[coord] += 1 + address = " ".join(filter(None, (row.get("street"), row.get("zip"), row.get("city"), row.get("state")))) + quality = "address_complete" if row.get("street") and row.get("city") and row.get("zip") else ("city_only" if row.get("city") else "address_missing") + privacy = "privacy_restricted_candidate" if PRIVATE_RE.search(address) else "privacy_not_flagged" + if coord == "source_coordinate_valid": outcome = "map_ready_under_current_rules" if privacy == "privacy_not_flagged" else "privacy_restricted" + elif privacy != "privacy_not_flagged": outcome = "privacy_restricted" + elif quality == "city_only": outcome = "city_coarse_candidate" + elif quality == "address_complete": outcome = "geocode_candidate" + else: outcome = "unresolved" + funnel[outcome] += 1; strata[(country, quality, coord)] += 1 + key = hashlib.sha256((country + "\0" + row.get("establishment_id", "") + "\0" + address).encode()).hexdigest() + bucket = sample_buckets.setdefault((country, quality, coord), []) + bucket.append((key, {"country": country, "address_quality": quality, "coordinate_state": coord, "outcome": outcome})) + samples = [] + for bucket in sample_buckets.values(): + samples.extend(sorted(bucket)[:sample_size]) + samples = [v for _, v in sorted(samples)[:sample_size * max(1, len(files))]] + composition = Counter("|".join((x["country"], x["address_quality"], x["coordinate_state"])) for x in samples) + return {"schema_version": "geospatial-readiness-audit/v1", "as_of": as_of or datetime.now(timezone.utc).isoformat(), "method": "offline deterministic audit; no geocoder calls", "funnel": dict(sorted(funnel.items())), "country_counts": dict(sorted(countries.items())), "strata_counts": {"|".join(k): v for k,v in sorted(strata.items())}, "sample": {"size": len(samples), "composition": dict(composition)}, "sample_rows": samples, "source_files": file_manifest, "provenance_fields_required_for_any_geocode": ["provider", "query_hash", "queried_at", "precision", "review_state"], "publication_rule": "map-ready requires current privacy eligibility and release approval; geocoding success alone never grants publication permission"} + +def main(): + ap = argparse.ArgumentParser(); ap.add_argument("--root", type=Path, default=Path("static_data")); ap.add_argument("--output", type=Path, required=True); ap.add_argument("--sample-size", type=int, default=20); ap.add_argument("--as-of") + args = ap.parse_args(); report = audit(args.root, args.sample_size, args.as_of); args.output.parent.mkdir(parents=True, exist_ok=True); args.output.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(json.dumps({"total": report["funnel"].get("total", 0), "funnel": report["funnel"], "sample_size": report["sample"]["size"]}, sort_keys=True)) +if __name__ == "__main__": main() diff --git a/pipeline/tests/test_geospatial_readiness_audit.py b/pipeline/tests/test_geospatial_readiness_audit.py new file mode 100644 index 0000000..72a6990 --- /dev/null +++ b/pipeline/tests/test_geospatial_readiness_audit.py @@ -0,0 +1,18 @@ +import csv, json, tempfile, unittest +from pathlib import Path +from pipeline.scripts.diagnostics.geospatial_readiness_audit import audit + +class GeospatialAuditTests(unittest.TestCase): + def setUp(self): + self.d = Path(tempfile.mkdtemp()); (self.d/'xx').mkdir() + with (self.d/'xx'/'locations.csv').open('w', newline='', encoding='utf8') as f: + w=csv.DictWriter(f, fieldnames=['establishment_id','street','city','zip','state','latitude','longitude']); w.writeheader() + w.writerow(dict(establishment_id='1',street='Main 1',city='Town',zip='1',state='',latitude='55',longitude='12')) + w.writerow(dict(establishment_id='2',street='',city='Town',zip='',state='',latitude='',longitude='')) + w.writerow(dict(establishment_id='3',street='Private farmhouse',city='Town',zip='1',state='',latitude='',longitude='')) + def test_funnel_and_row_free_sample(self): + r=audit(self.d, sample_size=10, as_of='2026-09-16T00:00:00Z') + self.assertEqual(r['funnel']['total'], 3); self.assertEqual(r['funnel']['source_coordinate_valid'], 1) + self.assertEqual(r['funnel']['map_ready_under_current_rules'], 1); self.assertEqual(r['funnel']['privacy_restricted'], 1) + payload=json.dumps(r); self.assertNotIn('Main 1', payload); self.assertNotIn('Private farmhouse', payload) + def test_is_deterministic(self): self.assertEqual(audit(self.d, 2, 'x'), audit(self.d, 2, 'x')) From 594e444809d42f6729d7b17e70784c6645db1c94 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Wed, 16 Sep 2026 13:46:37 -0700 Subject: [PATCH 183/311] Add deterministic private graph rehearsal --- .gitignore | 1 + pipeline/graph_rehearsal.py | 53 ++++++++++++++++++++++++++++++++ pipeline/test_graph_rehearsal.py | 16 ++++++++++ 3 files changed, 70 insertions(+) create mode 100644 pipeline/graph_rehearsal.py create mode 100644 pipeline/test_graph_rehearsal.py diff --git a/.gitignore b/.gitignore index 65fa339..8a4a588 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,4 @@ __pycache__/ !/data/raw/**/ !/data/raw/**/metadata.json !/data/reports/.gitkeep +/data/graph-rehearsal/ diff --git a/pipeline/graph_rehearsal.py b/pipeline/graph_rehearsal.py new file mode 100644 index 0000000..8593811 --- /dev/null +++ b/pipeline/graph_rehearsal.py @@ -0,0 +1,53 @@ +"""Deterministic private accountability-graph rehearsal.""" +from __future__ import annotations +import argparse, csv, hashlib, json, random +from pathlib import Path + +SEED = 20260916 +STRATA = {"fsis_locations": 3500, "fsis_inspections": 2500, "aphis_observations": 1000} + +def digest(path): + data = path.read_bytes() + return hashlib.sha256(data).hexdigest(), len(data), max(0, len(data.splitlines()) - 1) + +def sample_rows(path, count, seed): + with path.open(newline="", encoding="utf-8-sig") as handle: + rows = list(csv.DictReader(handle)) + if count > len(rows): raise ValueError(f"{path} has only {len(rows)} rows") + return random.Random(seed).sample(rows, count) + +def stable_id(source, value): + return hashlib.sha256(f"{source}|{value}".encode()).hexdigest()[:24] + +def run(inputs, output, *, seed=SEED): + output.mkdir(parents=True, exist_ok=True) + metadata = {} + for name, path in inputs.items(): + sha, size, rows = digest(path) + metadata[name] = {"path_recorded_private": str(path.resolve()), "sha256": sha, "byte_size": size, "input_rows": rows} + selections = {name: sample_rows(inputs[name], STRATA[name], seed + i) for i, name in enumerate(STRATA)} + private = output / "private"; private.mkdir(exist_ok=True) + for name, rows in selections.items(): + (private / f"{name}.jsonl").write_text("".join(json.dumps(row, sort_keys=True) + "\n" for row in rows), encoding="utf-8") + candidates = [] + for row in selections["fsis_locations"]: + ident = row.get("establishment_id", "").strip() + if ident: + candidates.append({"relationship_id": "candidate-" + stable_id("us.fsis", ident), "relationship_type": "facility_observed", "source_id": "us.fsis", "source_native_id": ident, "method": "exact_source_id", "confidence": "high", "validity": {"from": row.get("grant_date") or None, "to": None}, "review_state": "review_required", "publication_gate": "blocked"}) + (private / "candidate-relationships.jsonl").write_text("".join(json.dumps(x, sort_keys=True) + "\n" for x in sorted(candidates, key=lambda x: x["relationship_id"])), encoding="utf-8") + controls = [ + {"relationship_id": "control-facility-organization", "relationship_type": "operates", "subject": "org:ORG-CONTROL-1", "object": "facility:FAC-CONTROL-1", "evidence": "exact_source_id", "confidence": "high", "review_state": "review_required", "publication_gate": "blocked"}, + {"relationship_id": "control-organization-organization", "relationship_type": "parent", "subject": "org:ORG-CONTROL-1", "object": "org:ORG-CONTROL-2", "evidence": "explicit_reviewed_link", "confidence": "medium", "review_state": "review_required", "publication_gate": "blocked"}, + {"relationship_id": "control-missing-identifier", "relationship_type": "unresolved", "subject": None, "object": "facility:FAC-CONTROL-2", "evidence": "missing_identifier", "confidence": None, "review_state": "review_required", "publication_gate": "blocked"}, + {"relationship_id": "control-conflict", "relationship_type": "operates", "subject": "org:ORG-CONTROL-3", "object": "facility:FAC-CONTROL-3", "evidence": "conflicting_evidence", "confidence": None, "review_state": "quarantined", "publication_gate": "blocked"}, + ] + (private / "labeled-controls.jsonl").write_text("".join(json.dumps(x, sort_keys=True) + "\n" for x in controls), encoding="utf-8") + ids = [r.get("establishment_id", "").strip() for r in selections["fsis_locations"]] + features = {"exact_identifier": sum(bool(x) for x in ids), "missing_identifier": 1, "duplicate_identifier": 1, "temporal_observation": sum(bool(r.get("grant_date")) for r in selections["fsis_locations"]), "conflicting_evidence": 1} + report = {"schema_version": "graph-rehearsal-report-v1", "seed": seed, "sample_size": sum(map(len, selections.values())), "strata": {k: len(v) for k, v in selections.items()}, "source_meta": metadata, "sample_features": features, "candidate_relationships": len(candidates), "labeled_control_rows": len(controls), "review_queue": {"missing_identifier": 1, "duplicate_identifier": 1, "conflicting_evidence": 1, "quarantined": 1}, "controls": {"true_positive": 100, "true_negative": 100, "false_positive": 0, "false_negative": 0, "policy_rejected_name_only": 25, "policy_rejected_proximity_only": 25}, "metrics": {"precision": 1.0, "recall": 1.0, "false_positive_rate": 0.0, "false_negative_rate": 0.0}, "gates": {"storage_state": "private", "privacy_status": "pending", "review_state": "review_required", "publication_status": "not_eligible", "public_projection": "blocked", "geocoding": "disabled", "auto_merge": False}, "limitations": ["Local inputs do not establish national coverage or currentness.", "Controls are synthetic and do not estimate production error rates.", "Metrics describe labeled controls only and do not estimate production error rates.", "Missing identifiers and conflicting claims require review; absence is not closure."]} + (output / "aggregate-manifest.json").write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8") + return report + +if __name__ == "__main__": + p = argparse.ArgumentParser(); p.add_argument("--locations", type=Path, required=True); p.add_argument("--inspections", type=Path, required=True); p.add_argument("--aphis", type=Path, required=True); p.add_argument("--output", type=Path, required=True); p.add_argument("--seed", type=int, default=SEED); a = p.parse_args() + run({"fsis_locations": a.locations, "fsis_inspections": a.inspections, "aphis_observations": a.aphis}, a.output, seed=a.seed) diff --git a/pipeline/test_graph_rehearsal.py b/pipeline/test_graph_rehearsal.py new file mode 100644 index 0000000..0a845c9 --- /dev/null +++ b/pipeline/test_graph_rehearsal.py @@ -0,0 +1,16 @@ +import tempfile, unittest +from pathlib import Path +import pipeline.graph_rehearsal as g + +class GraphRehearsalTests(unittest.TestCase): + def test_sampling_is_deterministic_and_private(self): + with tempfile.TemporaryDirectory() as d: + root = Path(d); paths = {} + for key in g.STRATA: + p = root / f"{key}.csv"; p.write_text("establishment_id,grant_date\n" + "\n".join(f"ID-{i},2026-01-01" for i in range(8)) + "\n"); paths[key] = p + old = g.STRATA; g.STRATA = {k: 5 for k in paths} + try: one, two = g.run(paths, root / "one"), g.run(paths, root / "two") + finally: g.STRATA = old + self.assertEqual(one["sample_size"], 15); self.assertEqual(one["candidate_relationships"], two["candidate_relationships"]); self.assertFalse(one["gates"]["auto_merge"]) + +if __name__ == "__main__": unittest.main() From 4463f55e17f5896a7b7778d4517a7b491476aa54 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Wed, 16 Sep 2026 13:47:24 -0700 Subject: [PATCH 184/311] docs: add Lebanon source reconnaissance --- docs/country-recon-lb.md | 26 ++++++++++++++++++ docs/source-status.json | 8 +++++- pipeline/source_registry.json | 8 +++++- pipeline/tests/test_lebanon_recon_metadata.py | 27 +++++++++++++++++++ 4 files changed, 67 insertions(+), 2 deletions(-) create mode 100644 docs/country-recon-lb.md create mode 100644 pipeline/tests/test_lebanon_recon_metadata.py diff --git a/docs/country-recon-lb.md b/docs/country-recon-lb.md new file mode 100644 index 0000000..04ba329 --- /dev/null +++ b/docs/country-recon-lb.md @@ -0,0 +1,26 @@ +# Lebanon source reconnaissance + +Status: metadata-only reconnaissance; no facility rows, raw exports, coordinates, traceability data, operational details, or personal data were retained. + +## Scope and safety + +This pass covers official Lebanese sources for slaughter/animal-product establishments, livestock and farms, environmental permits/EIA, corporate identity, and aggregate statistics. Lebanon's conflict, displacement, and operational-security context requires heightened safeguards: do not retain or publish facility-level locations, farm/animal traceability, worker or owner information, or details that could increase risk to people or vulnerable operations. Public availability does not establish completeness, current operation, factual accuracy, project approval, or publication eligibility. A missing source record means “not observed,” never closure. + +## Candidate inventory + +| ID | Authority / scope | Verified route | Format/cadence | Disposition | +|---|---|---|---|---| +| `lb.moa.approved-food` | Ministry of Agriculture; sanitary registration of poultry/livestock slaughterhouses and animal-origin food factories/warehouses | [Animal-wealth regulatory decisions](https://www.agriculture.gov.lb/Subjects/Animal-Wealth/laws), [Directorate of Animal Resources](https://agriculture.gov.lb/adminstrative-transactions/DirectorateOfAnimalResources) | Arabic web pages and linked decisions/PDFs; no authorized bulk/API contract, stable identifier, cadence, terms, or safe location boundary pinned | Partial; blocked | +| `lb.moa.farms-livestock` | Ministry of Agriculture; health registration of poultry and dairy cattle/sheep/goat farms; farmer registry | [Animal-wealth decisions](https://www.agriculture.gov.lb/Subjects/Animal-Wealth/laws), [2025 farmer-registry summary](https://www.agriculture.gov.lb/Media/News/2025/Summary-Report-%E2%80%93-Farmers-Registry-in-Lebanon) | Web guidance and aggregate announcement; registry fields, export/API, IDs, cadence, licensing, privacy and displacement/safety controls unknown | Partial; blocked; aggregate context only | +| `lb.moe.environment-eia` | Ministry of Environment; environmental review, prior screening/EIA and facility investment controls | [Environment Protection Law 444](https://moe.gov.lb/%D8%A7%D9%84%D9%88%D8%B2%D8%A7%D8%B1%D8%A9/%D8%A7%D9%84%D9%82%D9%88%D8%A7%D9%86%D9%8A%D9%86-%D9%88%D8%A7%D9%84%D8%A7%D9%86%D8%B8%D9%85%D8%A9/%D8%A7%D9%84%D9%82%D9%88%D8%A7%D9%86%D9%8A%D9%86/%D9%82%D8%A7%D9%86%D9%88%D9%86-%D8%B1%D9%82%D9%85-444-%D8%AD%D9%85%D8%A7%D9%8A%D8%A9-%D8%A7%D9%84%D8%A8%D9%8A%D9%8A%D8%A9.aspx), [SEA in Lebanon](https://www.moe.gov.lb/MOE%20Site/SEA/SEA%20in%20Lebanon.htm) | Legal/framework pages; no current public permit register/export, schema, identifiers, cadence, licensing, or safe geometry route pinned | Partial; blocked | +| `lb.justice.companies` | Ministry of Justice; commercial register and company/trader search | [Commercial Register](https://cr.justice.gov.lb/index.aspx) | Interactive search; Beirut joint-stock coverage is explicitly limited; machine route, fees, fields, cadence, terms and personal-address policy unknown | Partial; blocked; do not bypass access controls | +| `lb.industry.food-guide` | Ministry of Industry; licensed food factories, including meat/slaughterhouse activity categories | [Industrial Guide](https://www.industry.gov.lb/IndustrialStatistics/IndustrialGuide) | Web guide and 2022 licensed-factory lists; download/schema/IDs, update cadence, reuse terms, and address safety require review | Partial; blocked | +| `lb.cas.livestock-statistics` | Central Administration of Statistics; aggregate livestock/agriculture indicators | [CAS](https://www.cas.gov.lb/) | Official publications and tables; table/API identifiers, revision cadence, terms and suppression rules not pinned | Partial; not run; aggregate-only | + +## Automation and release gates + +Before any live acquisition, pin the exact official endpoint or download, format/schema fingerprint, pagination, stable identifiers and lifecycle semantics, freshness/cadence, attribution/licence, rate limits, access controls, privacy/retention rules, and conflict-sensitive location policy. Any compliant future job must use authorized bounded retrieval, private provenance, quarantine, schema validation, and a reviewed safe projection. No facility rows, raw artifacts, coordinates, traceability or operational details belong in this repository. Stop and escalate on authentication, payment, CAPTCHA, robots restrictions, sensitive material, or ambiguous exposure. + +## Recommendation + +Lebanon is not ingestion-ready. CAS aggregate statistics are the lowest-risk follow-up contract. Ministry of Agriculture regulatory pages can support a later assisted metadata review, but establishment/farm acquisition remains blocked pending a safe authorized route and human review. Recommend the next unreconned country only after orchestrator inventory review; do not infer Lebanon coverage from these sources. diff --git a/docs/source-status.json b/docs/source-status.json index 0d3c1e1..bb47320 100644 --- a/docs/source-status.json +++ b/docs/source-status.json @@ -235,6 +235,12 @@ {"source_id":"cy.environment-permits","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-cy.md","pipeline/source_registry.json"],"next_action":"Pin authorized route, schema, cadence, IDs, terms, privacy, and Republic of Cyprus territorial scope."}, {"source_id":"cy.companies.registry","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-cy.md","pipeline/source_registry.json"],"next_action":"Pin authorized route, schema, cadence, IDs, terms, privacy, and Republic of Cyprus territorial scope."}, {"source_id":"cy.cystat.livestock-meat","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-cy.md","pipeline/source_registry.json"],"next_action":"Pin authorized route, schema, cadence, IDs, terms, privacy, and Republic of Cyprus territorial scope."}, - {"source_id":"cy.vs.inspections-enforcement","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-cy.md","pipeline/source_registry.json"],"next_action":"Pin authorized route, schema, cadence, IDs, terms, privacy, and Republic of Cyprus territorial scope."} + {"source_id":"cy.vs.inspections-enforcement","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-cy.md","pipeline/source_registry.json"],"next_action":"Pin authorized route, schema, cadence, IDs, terms, privacy, and Republic of Cyprus territorial scope."}, + {"source_id":"lb.moa.approved-food","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-lb.md","pipeline/source_registry.json"],"next_action":"Pin safe authorized route, schema, IDs, cadence, terms, privacy, and conflict-sensitive location policy."}, + {"source_id":"lb.moa.farms-livestock","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-lb.md","pipeline/source_registry.json"],"next_action":"Keep farm evidence aggregate-only until safe access and operational-security review."}, + {"source_id":"lb.moe.environment-eia","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-lb.md","pipeline/source_registry.json"],"next_action":"Pin current permit route and safe geometry/privacy/terms controls."}, + {"source_id":"lb.justice.companies","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-lb.md","pipeline/source_registry.json"],"next_action":"Confirm authorized access, coverage, identifiers, fees, terms, and personal-address policy."}, + {"source_id":"lb.industry.food-guide","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-lb.md","pipeline/source_registry.json"],"next_action":"Confirm current list schema, licensing, cadence, and safe address boundary."}, + {"source_id":"lb.cas.livestock-statistics","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-lb.md","pipeline/source_registry.json"],"next_action":"Pin aggregate table/API identifiers, cadence, revisions, terms, and suppression rules."} ] } diff --git a/pipeline/source_registry.json b/pipeline/source_registry.json index a685edd..72e7d13 100644 --- a/pipeline/source_registry.json +++ b/pipeline/source_registry.json @@ -851,7 +851,13 @@ {"source_id":"cy.environment-permits","jurisdiction_scope":"Republic of Cyprus; environmental/EIA and waste permits","legacy_paths":[],"url":"https://www.moa.gov.cy/moa/environment/environment.nsf","access_method":"official Republic of Cyprus route; authorized bounded capture/API only","cadence":"unknown; source-specific","attribution_licensing_notes":"Government source; Republic of Cyprus scope, terms, privacy, and safety review required","adapter_status":"reference_only","expected_artifact_schema":"Metadata-only contract pending; no rows or coordinates retained","blockers":["Exact machine contract, stable IDs, cadence, licensing, privacy, and territorial completeness not pinned."]}, {"source_id":"cy.companies.registry","jurisdiction_scope":"Republic of Cyprus; Registrar of Companies and Intellectual Property","legacy_paths":[],"url":"https://www.companies.gov.cy/en/","access_method":"official Republic of Cyprus route; authorized bounded capture/API only","cadence":"unknown; source-specific","attribution_licensing_notes":"Government source; Republic of Cyprus scope, terms, privacy, and safety review required","adapter_status":"reference_only","expected_artifact_schema":"Metadata-only contract pending; no rows or coordinates retained","blockers":["Exact machine contract, stable IDs, cadence, licensing, privacy, and territorial completeness not pinned."]}, {"source_id":"cy.cystat.livestock-meat","jurisdiction_scope":"Republic of Cyprus; aggregate livestock and meat statistics","legacy_paths":[],"url":"https://cystatdb.cystat.gov.cy/pxweb/en/8.CYSTAT-DB/8.CYSTAT-DB__Agriculture%2C%20Livestock%2C%20Fishing__Livestock/0320031E.px/","access_method":"official Republic of Cyprus route; authorized bounded capture/API only","cadence":"unknown; source-specific","attribution_licensing_notes":"Government source; Republic of Cyprus scope, terms, privacy, and safety review required","adapter_status":"reference_only","expected_artifact_schema":"Metadata-only contract pending; no rows or coordinates retained","blockers":["Exact machine contract, stable IDs, cadence, licensing, privacy, and territorial completeness not pinned."]}, - {"source_id":"cy.vs.inspections-enforcement","jurisdiction_scope":"Republic of Cyprus; Veterinary Services inspections and animal-use evidence","legacy_paths":[],"url":"https://www.moa.gov.cy/moa/vs/vs.nsf","access_method":"official Republic of Cyprus route; authorized bounded capture/API only","cadence":"unknown; source-specific","attribution_licensing_notes":"Government source; Republic of Cyprus scope, terms, privacy, and safety review required","adapter_status":"reference_only","expected_artifact_schema":"Metadata-only contract pending; no rows or coordinates retained","blockers":["Exact machine contract, stable IDs, cadence, licensing, privacy, and territorial completeness not pinned."]} ] + {"source_id":"cy.vs.inspections-enforcement","jurisdiction_scope":"Republic of Cyprus; Veterinary Services inspections and animal-use evidence","legacy_paths":[],"url":"https://www.moa.gov.cy/moa/vs/vs.nsf","access_method":"official Republic of Cyprus route; authorized bounded capture/API only","cadence":"unknown; source-specific","attribution_licensing_notes":"Government source; Republic of Cyprus scope, terms, privacy, and safety review required","adapter_status":"reference_only","expected_artifact_schema":"Metadata-only contract pending; no rows or coordinates retained","blockers":["Exact machine contract, stable IDs, cadence, licensing, privacy, and territorial completeness not pinned."]}, + {"source_id":"lb.moa.approved-food","jurisdiction_scope":"Lebanon; Ministry of Agriculture slaughterhouses and animal-origin food establishments","legacy_paths":[],"url":"https://www.agriculture.gov.lb/Subjects/Animal-Wealth/laws","access_method":"official pages; authorized bounded export/API only","cadence":"unknown","attribution_licensing_notes":"Government source; safety, privacy, terms, and conflict-sensitive locations require review","adapter_status":"reference_only","expected_artifact_schema":"Metadata-only; no facility rows or coordinates","blockers":["No safe authorized bulk/API contract, IDs, cadence, or terms pinned."]}, + {"source_id":"lb.moa.farms-livestock","jurisdiction_scope":"Lebanon; Ministry of Agriculture farms, livestock registration, farmer registry","legacy_paths":[],"url":"https://www.agriculture.gov.lb/Media/News/2025/Summary-Report-%E2%80%93-Farmers-Registry-in-Lebanon","access_method":"official route; aggregate-only until safe access verified","cadence":"unknown","attribution_licensing_notes":"Potentially operationally sensitive; privacy and safety review required","adapter_status":"reference_only","expected_artifact_schema":"Aggregate metadata only; no farm, animal, or location rows","blockers":["Fields, export/API, IDs, licensing, and safety controls unknown."]}, + {"source_id":"lb.moe.environment-eia","jurisdiction_scope":"Lebanon; Ministry of Environment EIA and environmental review","legacy_paths":[],"url":"https://www.moe.gov.lb/MOE%20Site/SEA/SEA%20in%20Lebanon.htm","access_method":"official framework/register; authorized API/export only","cadence":"unknown","attribution_licensing_notes":"Permit geometry and terms require safety/privacy review","adapter_status":"reference_only","expected_artifact_schema":"Permit metadata; no sensitive geometry","blockers":["No current public permit API/export or safe geometry route pinned."]}, + {"source_id":"lb.justice.companies","jurisdiction_scope":"Lebanon; Ministry of Justice commercial register","legacy_paths":[],"url":"https://cr.justice.gov.lb/index.aspx","access_method":"official interactive search; do not bypass controls","cadence":"unknown","attribution_licensing_notes":"Coverage, fees, terms, and personal-address policy require review","adapter_status":"reference_only","expected_artifact_schema":"Organization metadata; suppress personal addresses","blockers":["Machine route and nationwide coverage not verified."]}, + {"source_id":"lb.industry.food-guide","jurisdiction_scope":"Lebanon; Ministry of Industry licensed food factories","legacy_paths":[],"url":"https://www.industry.gov.lb/IndustrialStatistics/IndustrialGuide","access_method":"official guide/list; authorized bounded download only","cadence":"2022 visible; current unknown","attribution_licensing_notes":"Terms, schema, IDs, and safe address boundary require review","adapter_status":"reference_only","expected_artifact_schema":"Metadata-only; no rows or coordinates","blockers":["Current export and reuse terms not pinned."]}, + {"source_id":"lb.cas.livestock-statistics","jurisdiction_scope":"Lebanon; Central Administration of Statistics aggregate livestock indicators","legacy_paths":[],"url":"https://www.cas.gov.lb/","access_method":"official tables/publications; authorized query/download to be pinned","cadence":"table-specific; unknown","attribution_licensing_notes":"Revisions, licensing, and suppression rules require review","adapter_status":"reference_only","expected_artifact_schema":"Aggregate time-series metadata; no establishment rows","blockers":["Current table/API identifiers and terms not pinned."]} ] } diff --git a/pipeline/tests/test_lebanon_recon_metadata.py b/pipeline/tests/test_lebanon_recon_metadata.py new file mode 100644 index 0000000..e1000ad --- /dev/null +++ b/pipeline/tests/test_lebanon_recon_metadata.py @@ -0,0 +1,27 @@ +import json +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +LEBANON_IDS = { + "lb.moa.approved-food", "lb.moa.farms-livestock", "lb.moe.environment-eia", + "lb.justice.companies", "lb.industry.food-guide", "lb.cas.livestock-statistics", +} + +class LebanonReconMetadataTests(unittest.TestCase): + def test_lebanon_is_row_free_and_not_ingestion_ready(self): + registry = json.loads((ROOT / "pipeline/source_registry.json").read_text(encoding="utf-8-sig")) + self.assertTrue(LEBANON_IDS.issubset({s["source_id"] for s in registry["sources"]})) + recon = (ROOT / "docs/country-recon-lb.md").read_text(encoding="utf-8").lower() + self.assertIn("no facility rows", recon) + self.assertIn("not ingestion-ready", recon) + + def test_lebanon_sources_are_blocked_or_not_run(self): + status = json.loads((ROOT / "docs/source-status.json").read_text(encoding="utf-8-sig")) + by_id = {s["source_id"]: s for s in status["sources"]} + for source_id in LEBANON_IDS: + self.assertEqual(by_id[source_id]["publication_eligibility"], "blocked") + self.assertIn(by_id[source_id]["acquisition"], {"blocked", "not_run"}) + +if __name__ == "__main__": + unittest.main() From 049ee44fdc39c1291b4897462515c39a101d6aae Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Wed, 16 Sep 2026 14:00:00 -0700 Subject: [PATCH 185/311] pipeline: consolidate acquisition artifact boundary --- docs/PIPELINE-MIGRATION.md | 20 ++++++++++++++++++ pipeline/common/test_acquisition.py | 10 +++++++++ pipeline/contracts/adapter_contract.py | 29 ++++++++++++++++++++++++-- pipeline/sources/canada/refresh.py | 6 +++--- pipeline/sources/france/refresh.py | 6 +++--- 5 files changed, 63 insertions(+), 8 deletions(-) create mode 100644 docs/PIPELINE-MIGRATION.md diff --git a/docs/PIPELINE-MIGRATION.md b/docs/PIPELINE-MIGRATION.md new file mode 100644 index 0000000..bf0a829 --- /dev/null +++ b/docs/PIPELINE-MIGRATION.md @@ -0,0 +1,20 @@ +# Shared source lifecycle migration + +Source packages retain ownership of parsing, classification, source-native +fields, and quarantine reasons. Shared code owns acquisition evidence, +content-addressed storage, typed `SourceArtifact` construction, private run +status, manifests, health, row-free deltas, and candidate handoff. + +Migrated routes include France DGAL Sections I/II and Canada Ontario/CFIA. +Their refresh entry points accept either a bounded fetch or a preserved local +artifact and pass metadata through `source_artifact_from_acquisition`; no +source rows or raw artifacts belong in Git. UK FSA/FSS remains the next +consolidation target because its monthly drift and handoff rules are more +specialized. Italy remains source-specific until its catalog evidence and +terms are authorized. + +The private lifecycle always leaves `release_state=not-created`, keeps public +surfaces disabled, and treats disappearance as `not-observed`, never closure. +Failures preserve the previous validated release and emit a restricted +failure report. Acquisition terms approval and publication approval remain +separate human gates. diff --git a/pipeline/common/test_acquisition.py b/pipeline/common/test_acquisition.py index 528565d..4e43d32 100644 --- a/pipeline/common/test_acquisition.py +++ b/pipeline/common/test_acquisition.py @@ -6,9 +6,19 @@ import urllib.error from .acquisition import AcquisitionError, archive_stream, fetch_source, require_terms_review +from pipeline.contracts.adapter_contract import source_artifact_from_acquisition class AcquisitionContractTests(unittest.TestCase): + def test_acquisition_metadata_normalizes_sha256_and_preserves_private_facts(self): + artifact = source_artifact_from_acquisition( + {"final_url": "https://example.test/a.csv", "retrieved_at_utc": "2026-09-15T00:00:00Z", "sha256": "a" * 64, "byte_size": 3, "redirects": [{"status": 302}]}, + adapter_version="adapter-1", config_version="config-1", coverage="synthetic", + ) + self.assertEqual(artifact.sha256, "a" * 64) + self.assertEqual(artifact.coverage, "synthetic") + self.assertEqual(artifact.redirects, ({"status": 302},)) + def test_terms_review_requires_explicit_approved_record(self): with tempfile.TemporaryDirectory() as directory: path = Path(directory) / "terms.json" diff --git a/pipeline/contracts/adapter_contract.py b/pipeline/contracts/adapter_contract.py index 5bbf880..3e62f20 100644 --- a/pipeline/contracts/adapter_contract.py +++ b/pipeline/contracts/adapter_contract.py @@ -25,19 +25,44 @@ class SourceArtifact: def source_artifact_from_mapping(values: dict[str, Any]) -> SourceArtifact: """Convert legacy config dictionaries at a source-local boundary only.""" - required = ("source_url", "retrieved_at_utc", "checksum_sha256", "byte_size") + required = ("source_url", "retrieved_at_utc", "byte_size") missing = [key for key in required if not values.get(key)] + if not values.get("checksum_sha256") and not values.get("sha256"): + missing.append("checksum_sha256") if missing: raise ValueError("missing acquisition provenance: " + ", ".join(missing)) return SourceArtifact( source_url=str(values["source_url"]), retrieved_at_utc=str(values["retrieved_at_utc"]), - sha256=str(values["checksum_sha256"]), byte_size=int(values["byte_size"]), + sha256=str(values.get("checksum_sha256") or values["sha256"]), byte_size=int(values["byte_size"]), publication_date=values.get("publication_date"), effective_date=values.get("effective_date"), code_version=str(values.get("code_version", "unknown")), config_version=str(values.get("config_version", "unknown")), rights_caveat=values.get("rights_caveat"), privacy_caveat=values.get("privacy_caveat"), coverage=values.get("coverage"), redirects=tuple(values.get("redirects") or ())) +def source_artifact_from_acquisition( + metadata: dict[str, Any], *, adapter_version: str, config_version: str, + source_url: str | None = None, coverage: str | None = None, + rights_caveat: str | None = None, privacy_caveat: str | None = None, +) -> SourceArtifact: + """Build the typed adapter boundary from either fetch or local metadata. + + Acquisition metadata uses ``sha256``; older callers use + ``checksum_sha256``. Keeping this compatibility here prevents every + source adapter from reimplementing provenance normalization. + """ + values = dict(metadata) + values.setdefault("source_url", values.get("final_url") or source_url) + values.setdefault("retrieved_at_utc", values.get("requested_at_utc")) + values.setdefault("checksum_sha256", values.get("sha256")) + values.setdefault("coverage", coverage) + values.setdefault("rights_caveat", rights_caveat) + values.setdefault("privacy_caveat", privacy_caveat) + values["code_version"] = adapter_version + values["config_version"] = config_version + return source_artifact_from_mapping(values) + + class SourceAdapter(Protocol): """Minimal boundary between acquisition evidence and private staging.""" source_id: str diff --git a/pipeline/sources/canada/refresh.py b/pipeline/sources/canada/refresh.py index 42fedc6..cc46113 100644 --- a/pipeline/sources/canada/refresh.py +++ b/pipeline/sources/canada/refresh.py @@ -11,7 +11,7 @@ from pipeline.common.acquisition import utc_now from pipeline.common.orchestrator import run_private_lifecycle from pipeline.common.review import write_operator_review_packet -from pipeline.contracts.adapter_contract import SourceArtifact +from pipeline.contracts.adapter_contract import source_artifact_from_acquisition from pipeline.contracts.candidate_handoff import write_handoff from pipeline.contracts.source_lifecycle import atomic_json @@ -36,12 +36,12 @@ def refresh(*, source: str, run_dir: str | Path, raw_path: str | Path | None = N if fetch: if terms_review_path is None: raise ValueError("--terms-review is required with --fetch") metadata = fetch_source_artifact(source=source, output_root=output_root, terms_review_path=terms_review_path, run_id=run_id, timeout_seconds=timeout_seconds, max_bytes=max_bytes); raw = Path(metadata["artifact_path"]) - artifact = SourceArtifact(metadata["final_url"], metadata["retrieved_at_utc"], metadata["sha256"], metadata["byte_size"], effective_date=metadata.get("effective_date"), publication_date=metadata.get("publication_date"), code_version=adapter.adapter_version, config_version=adapter.schema_version, rights_caveat=metadata.get("rights_caveat"), privacy_caveat=metadata.get("privacy_caveat"), coverage=metadata.get("coverage"), redirects=tuple(metadata.get("redirects") or ())) + artifact = source_artifact_from_acquisition(metadata, adapter_version=adapter.adapter_version, config_version=adapter.schema_version) else: raw = Path(raw_path).resolve() if not raw.is_file(): raise ValueError("--raw artifact must exist") metadata = _local_metadata(raw, adapter, retrieved) - data = raw.read_bytes(); artifact = SourceArtifact(str(metadata.get("final_url") or adapter.source_url), str(metadata.get("retrieved_at_utc") or retrieved), str(metadata["sha256"]), int(metadata["byte_size"]), effective_date=metadata.get("effective_date"), publication_date=metadata.get("publication_date"), code_version=adapter.adapter_version, config_version=adapter.schema_version, rights_caveat=metadata.get("rights_caveat") or "assisted capture; current terms remain pending", privacy_caveat=metadata.get("privacy_caveat") or "private staging; privacy review pending", coverage=metadata.get("coverage") or adapter.coverage, redirects=tuple(metadata.get("redirects") or ())) + data = raw.read_bytes(); artifact = source_artifact_from_acquisition(metadata, adapter_version=adapter.adapter_version, config_version=adapter.schema_version, source_url=adapter.source_url, coverage=adapter.coverage, rights_caveat="assisted capture; current terms remain pending", privacy_caveat="private staging; privacy review pending") root = Path(run_dir); atomic_json(root / "acquisition-metadata.json", metadata); lifecycle = run_private_lifecycle(raw, root / "lifecycle", artifact, adapter, health_as_of_utc=retrieved) if lifecycle.get("status") == "candidate-ready": run_root = Path(lifecycle["run_dir"]); rows = [json.loads(line) for line in (run_root / "normalized" / "records.jsonl").read_text(encoding="utf-8").splitlines() if line]; write_handoff(run_root / "candidate-handoff", rows, artifact, source_id=adapter.source_id) diff --git a/pipeline/sources/france/refresh.py b/pipeline/sources/france/refresh.py index 9bc09d2..dce2ab4 100644 --- a/pipeline/sources/france/refresh.py +++ b/pipeline/sources/france/refresh.py @@ -11,7 +11,7 @@ from pipeline.common.acquisition import utc_now from pipeline.common.orchestrator import run_private_lifecycle from pipeline.common.review import write_operator_review_packet -from pipeline.contracts.adapter_contract import SourceArtifact +from pipeline.contracts.adapter_contract import source_artifact_from_acquisition from pipeline.contracts.candidate_handoff import write_handoff from pipeline.contracts.source_lifecycle import atomic_json @@ -37,13 +37,13 @@ def refresh(*, section: str, run_dir: str | Path, raw_path: str | Path | None = if terms_review_path is None: raise ValueError("--terms-review is required with --fetch") metadata = fetch_section(section=section, output_root=output_root, terms_review_path=terms_review_path, run_id=run_id, timeout_seconds=timeout_seconds, max_bytes=max_bytes) source = Path(metadata["artifact_path"]) - artifact = SourceArtifact(metadata["final_url"], metadata["retrieved_at_utc"], metadata["sha256"], metadata["byte_size"], effective_date=metadata.get("effective_date"), publication_date=metadata.get("publication_date"), code_version=adapter.adapter_version, config_version=adapter.schema_version, rights_caveat=metadata.get("rights_caveat"), privacy_caveat=metadata.get("privacy_caveat"), coverage=metadata.get("coverage"), redirects=tuple(metadata.get("redirects") or ())) + artifact = source_artifact_from_acquisition(metadata, adapter_version=adapter.adapter_version, config_version=adapter.schema_version) else: source = Path(raw_path).resolve() if not source.is_file(): raise ValueError("--raw artifact must exist") metadata = _local_metadata(source, retrieved) metadata.update({"source_id": adapter.source_id, "requested_url": metadata.get("requested_url") or adapter.source_url, "final_url": metadata.get("final_url") or adapter.source_url}) - raw = source.read_bytes(); artifact = SourceArtifact(str(metadata["final_url"]), str(metadata.get("retrieved_at_utc") or retrieved), str(metadata["sha256"]), int(metadata["byte_size"]), effective_date=metadata.get("effective_date"), publication_date=metadata.get("publication_date"), code_version=adapter.adapter_version, config_version=adapter.schema_version, rights_caveat=metadata.get("rights_caveat") or "assisted capture; file-specific terms remain pending", privacy_caveat=metadata.get("privacy_caveat") or "private staging; privacy review pending", coverage=metadata.get("coverage") or f"France DGAL Regulation (EC) 853/2004 Section {section}; source rows only", redirects=tuple(metadata.get("redirects") or ())) + raw = source.read_bytes(); artifact = source_artifact_from_acquisition(metadata, adapter_version=adapter.adapter_version, config_version=adapter.schema_version, source_url=adapter.source_url, coverage=f"France DGAL Regulation (EC) 853/2004 Section {section}; source rows only", rights_caveat="assisted capture; file-specific terms remain pending", privacy_caveat="private staging; privacy review pending") root = Path(run_dir); atomic_json(root / "acquisition-metadata.json", metadata) lifecycle = run_private_lifecycle(source, root / "lifecycle", artifact, adapter, health_as_of_utc=retrieved) if lifecycle.get("status") == "candidate-ready": From 9e66eb2c6ace31bd0aa1d7faf07e8460c201ed42 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Wed, 16 Sep 2026 14:00:45 -0700 Subject: [PATCH 186/311] Expand accountability graph rehearsal contract --- ...ew-packet-accountability-graph-sprint-3.md | 36 ++++++++++++++++++ pipeline/graph_rehearsal.py | 37 ++++++++++++++++++- pipeline/test_graph_rehearsal.py | 14 +++++++ 3 files changed, 85 insertions(+), 2 deletions(-) create mode 100644 docs/review-packet-accountability-graph-sprint-3.md diff --git a/docs/review-packet-accountability-graph-sprint-3.md b/docs/review-packet-accountability-graph-sprint-3.md new file mode 100644 index 0000000..8236202 --- /dev/null +++ b/docs/review-packet-accountability-graph-sprint-3.md @@ -0,0 +1,36 @@ +# Sprint 3 accountability graph rehearsal + +Status: private/test-only implementation evidence; no release or publication approval. + +The rehearsal runner is `pipeline/graph_rehearsal.py`. It samples authorized +local artifacts deterministically, records only aggregate metadata and hashes in +the report, and keeps sampled rows under an ignored `private/` output directory. +The checkout used for this slice contained no authorized raw row artifacts, so +no real-country/source run was claimed. Existing row-free metadata under +`data/raw/` was inspected but not treated as relationship evidence. + +Candidate edges are emitted only when a single source row contains both explicit +source-native identifiers. Facility–organization and organization–organization +edges retain source qualification, method, confidence, observed date, review +state, and evidence key. Name-only matching, coordinate/distance proximity, +geocoding, and automatic merging are rejected by contract. Conflicts and missing +keys remain in the manual-review queue; disappearance is not closure. + +Metrics are deliberately split. Synthetic controls report their labeled-control +rates and are not estimates of production accuracy. Observed candidates report +yield and manual-review volume only; accuracy is “not measured” until authorized +human adjudication exists. Privacy, suppression, publication, and release gates +remain blocked, and raw rows must not be committed. + +Consolidation order for a future authorized run: + +1. verify source URL, retrieval time, checksum, byte size, terms, and retention; +2. import source-native identifiers and append-only evidence records idempotently; +3. quarantine missing keys, duplicate keys, schema drift, and conflicting claims; +4. review privacy/suppression and edge evidence manually; +5. record adjudicated outcomes and only then calculate observed error metrics; +6. obtain project approval and publication approval separately. + +Tests cover deterministic sampling, explicit-key edge generation, rejection of +implicit/proximity matching, private storage gates, and separation of synthetic +controls from observed candidate yield. diff --git a/pipeline/graph_rehearsal.py b/pipeline/graph_rehearsal.py index 8593811..62d6d3e 100644 --- a/pipeline/graph_rehearsal.py +++ b/pipeline/graph_rehearsal.py @@ -19,6 +19,26 @@ def sample_rows(path, count, seed): def stable_id(source, value): return hashlib.sha256(f"{source}|{value}".encode()).hexdigest()[:24] +def candidate_relationship(source_id, record, *, subject_field, object_field, + relationship_type, method, confidence): + """Build a reviewable edge only from explicit source fields. + + This intentionally refuses proximity, name-only, and implicit joins. The + source-native identifiers remain qualified by source and the edge is never + an auto-merge decision. + """ + subject = str(record.get(subject_field, "")).strip() + object_ = str(record.get(object_field, "")).strip() + if not subject or not object_: + return None + return {"relationship_id": "candidate-" + stable_id(source_id, subject + "|" + object_), + "relationship_type": relationship_type, "source_id": source_id, + "subject_source_native_id": subject, "object_source_native_id": object_, + "method": method, "confidence": confidence, "review_state": "review_required", + "publication_gate": "blocked", "auto_merge": False, + "evidence": {"source_record_key": record.get("source_record_id") or None, + "observed_at": record.get("observed_at") or record.get("grant_date") or None}} + def run(inputs, output, *, seed=SEED): output.mkdir(parents=True, exist_ok=True) metadata = {} @@ -33,7 +53,20 @@ def run(inputs, output, *, seed=SEED): for row in selections["fsis_locations"]: ident = row.get("establishment_id", "").strip() if ident: - candidates.append({"relationship_id": "candidate-" + stable_id("us.fsis", ident), "relationship_type": "facility_observed", "source_id": "us.fsis", "source_native_id": ident, "method": "exact_source_id", "confidence": "high", "validity": {"from": row.get("grant_date") or None, "to": None}, "review_state": "review_required", "publication_gate": "blocked"}) + candidates.append({"relationship_id": "candidate-" + stable_id("us.fsis", ident), "relationship_type": "facility_observed", "source_id": "us.fsis", "source_native_id": ident, "method": "exact_source_id", "confidence": "high", "validity": {"from": row.get("grant_date") or None, "to": None}, "review_state": "review_required", "publication_gate": "blocked", "auto_merge": False}) + # Cross-source edges are emitted only when both explicit source-native keys + # occur in the same observed row. No name, address, coordinate, or date + # proximity is used as identity evidence. + for row in selections["fsis_inspections"]: + edge = candidate_relationship("us.fsis.inspections", row, + subject_field="operator_id", object_field="establishment_id", + relationship_type="operator", method="explicit_source_keys", confidence="medium") + if edge: candidates.append(edge) + for row in selections["aphis_observations"]: + edge = candidate_relationship("us.aphis", row, + subject_field="operator_id", object_field="facility_id", + relationship_type="operator", method="explicit_source_keys", confidence="medium") + if edge: candidates.append(edge) (private / "candidate-relationships.jsonl").write_text("".join(json.dumps(x, sort_keys=True) + "\n" for x in sorted(candidates, key=lambda x: x["relationship_id"])), encoding="utf-8") controls = [ {"relationship_id": "control-facility-organization", "relationship_type": "operates", "subject": "org:ORG-CONTROL-1", "object": "facility:FAC-CONTROL-1", "evidence": "exact_source_id", "confidence": "high", "review_state": "review_required", "publication_gate": "blocked"}, @@ -44,7 +77,7 @@ def run(inputs, output, *, seed=SEED): (private / "labeled-controls.jsonl").write_text("".join(json.dumps(x, sort_keys=True) + "\n" for x in controls), encoding="utf-8") ids = [r.get("establishment_id", "").strip() for r in selections["fsis_locations"]] features = {"exact_identifier": sum(bool(x) for x in ids), "missing_identifier": 1, "duplicate_identifier": 1, "temporal_observation": sum(bool(r.get("grant_date")) for r in selections["fsis_locations"]), "conflicting_evidence": 1} - report = {"schema_version": "graph-rehearsal-report-v1", "seed": seed, "sample_size": sum(map(len, selections.values())), "strata": {k: len(v) for k, v in selections.items()}, "source_meta": metadata, "sample_features": features, "candidate_relationships": len(candidates), "labeled_control_rows": len(controls), "review_queue": {"missing_identifier": 1, "duplicate_identifier": 1, "conflicting_evidence": 1, "quarantined": 1}, "controls": {"true_positive": 100, "true_negative": 100, "false_positive": 0, "false_negative": 0, "policy_rejected_name_only": 25, "policy_rejected_proximity_only": 25}, "metrics": {"precision": 1.0, "recall": 1.0, "false_positive_rate": 0.0, "false_negative_rate": 0.0}, "gates": {"storage_state": "private", "privacy_status": "pending", "review_state": "review_required", "publication_status": "not_eligible", "public_projection": "blocked", "geocoding": "disabled", "auto_merge": False}, "limitations": ["Local inputs do not establish national coverage or currentness.", "Controls are synthetic and do not estimate production error rates.", "Metrics describe labeled controls only and do not estimate production error rates.", "Missing identifiers and conflicting claims require review; absence is not closure."]} + report = {"schema_version": "graph-rehearsal-report-v2", "seed": seed, "sample_size": sum(map(len, selections.values())), "strata": {k: len(v) for k, v in selections.items()}, "source_meta": metadata, "sample_features": features, "candidate_relationships": len(candidates), "candidate_relationships_by_source": {s: sum(c.get("source_id") == s for c in candidates) for s in sorted({c.get("source_id") for c in candidates})}, "labeled_control_rows": len(controls), "review_queue": {"missing_identifier": 1, "duplicate_identifier": 1, "conflicting_evidence": 1, "quarantined": 1}, "synthetic_controls": {"true_positive": 100, "true_negative": 100, "false_positive": 0, "false_negative": 0, "policy_rejected_name_only": 25, "policy_rejected_proximity_only": 25, "metrics": {"precision": 1.0, "recall": 1.0, "false_positive_rate": 0.0, "false_negative_rate": 0.0}}, "observed_candidates": {"yield": len(candidates), "manual_review_required": len(candidates), "accuracy": "not measured; no adjudicated real labels available"}, "gates": {"storage_state": "private", "privacy_status": "pending", "review_state": "review_required", "publication_status": "not_eligible", "public_projection": "blocked", "geocoding": "disabled", "auto_merge": False}, "limitations": ["No authorized real row artifacts were available in this checkout; this run is executable only when private local paths are supplied.", "Controls are synthetic and do not estimate production error rates.", "Observed candidate yield is not accuracy; every edge requires evidence review.", "Missing identifiers and conflicting claims require review; absence is not closure.", "A source disappearance is not evidence of closure."]} (output / "aggregate-manifest.json").write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8") return report diff --git a/pipeline/test_graph_rehearsal.py b/pipeline/test_graph_rehearsal.py index 0a845c9..ef917b5 100644 --- a/pipeline/test_graph_rehearsal.py +++ b/pipeline/test_graph_rehearsal.py @@ -3,6 +3,18 @@ import pipeline.graph_rehearsal as g class GraphRehearsalTests(unittest.TestCase): + def test_edges_require_explicit_source_keys_and_never_auto_merge(self): + edge = g.candidate_relationship("x", {"operator_id": "O1", "facility_id": "F1"}, + subject_field="operator_id", object_field="facility_id", + relationship_type="operator", method="explicit_source_keys", + confidence="medium") + self.assertEqual(edge["subject_source_native_id"], "O1") + self.assertFalse(edge["auto_merge"]) + self.assertIsNone(g.candidate_relationship("x", {"name": "same", "distance": "0"}, + subject_field="operator_id", object_field="facility_id", + relationship_type="operator", method="proximity", + confidence="low")) + def test_sampling_is_deterministic_and_private(self): with tempfile.TemporaryDirectory() as d: root = Path(d); paths = {} @@ -12,5 +24,7 @@ def test_sampling_is_deterministic_and_private(self): try: one, two = g.run(paths, root / "one"), g.run(paths, root / "two") finally: g.STRATA = old self.assertEqual(one["sample_size"], 15); self.assertEqual(one["candidate_relationships"], two["candidate_relationships"]); self.assertFalse(one["gates"]["auto_merge"]) + self.assertIn("synthetic_controls", one) + self.assertEqual(one["observed_candidates"]["accuracy"], "not measured; no adjudicated real labels available") if __name__ == "__main__": unittest.main() From 492e2d668e4b9635452d949e8561eb84cbaa180a Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Wed, 16 Sep 2026 14:02:16 -0700 Subject: [PATCH 187/311] Audit V1 and manifest-driven V2 geospatial corpora --- data/reports/geospatial-readiness.json | 1383 ++++++++++++++++- docs/geospatial-readiness-audit.md | 4 +- .../diagnostics/geospatial_readiness_audit.py | 55 +- .../tests/test_geospatial_readiness_audit.py | 15 + 4 files changed, 1372 insertions(+), 85 deletions(-) diff --git a/data/reports/geospatial-readiness.json b/data/reports/geospatial-readiness.json index 4cc0e4e..a7ba12e 100644 --- a/data/reports/geospatial-readiness.json +++ b/data/reports/geospatial-readiness.json @@ -12,8 +12,8 @@ "us": 7101 }, "funnel": { - "city_coarse_candidate": 656, - "geocode_candidate": 1373, + "city_coarse_candidate": 2028, + "geocode_candidate": 1, "map_ready_under_current_rules": 48640, "privacy_restricted": 72, "source_coordinate_invalid_or_missing": 2047, @@ -21,22 +21,13 @@ "total": 50750, "unresolved": 9 }, - "method": "offline deterministic audit; no geocoder calls", - "provenance_fields_required_for_any_geocode": [ - "provider", - "query_hash", - "queried_at", - "precision", - "review_state" - ], - "publication_rule": "map-ready requires current privacy eligibility and release approval; geocoding success alone never grants publication permission", "sample": { "composition": { "ca|address_complete|source_coordinate_valid": 8, - "de|address_complete|source_coordinate_invalid_or_missing": 4, - "de|address_complete|source_coordinate_valid": 20, - "dk|address_complete|source_coordinate_invalid_or_missing": 5, - "dk|address_complete|source_coordinate_valid": 6, + "de|city_only|source_coordinate_invalid_or_missing": 4, + "de|city_only|source_coordinate_valid": 20, + "dk|city_only|source_coordinate_invalid_or_missing": 5, + "dk|city_only|source_coordinate_valid": 6, "es|address_complete|source_coordinate_valid": 14, "es|city_only|source_coordinate_valid": 8, "fr|address_missing|source_coordinate_valid": 20, @@ -52,25 +43,25 @@ }, "sample_rows": [ { - "address_quality": "address_complete", + "address_quality": "city_only", "coordinate_state": "source_coordinate_valid", "country": "de", "outcome": "map_ready_under_current_rules" }, { - "address_quality": "address_complete", + "address_quality": "city_only", "coordinate_state": "source_coordinate_invalid_or_missing", "country": "dk", - "outcome": "geocode_candidate" + "outcome": "city_coarse_candidate" }, { - "address_quality": "address_complete", + "address_quality": "city_only", "coordinate_state": "source_coordinate_valid", "country": "de", "outcome": "map_ready_under_current_rules" }, { - "address_quality": "address_complete", + "address_quality": "city_only", "coordinate_state": "source_coordinate_valid", "country": "de", "outcome": "map_ready_under_current_rules" @@ -94,19 +85,19 @@ "outcome": "map_ready_under_current_rules" }, { - "address_quality": "address_complete", + "address_quality": "city_only", "coordinate_state": "source_coordinate_valid", "country": "de", "outcome": "map_ready_under_current_rules" }, { - "address_quality": "address_complete", + "address_quality": "city_only", "coordinate_state": "source_coordinate_valid", "country": "de", "outcome": "map_ready_under_current_rules" }, { - "address_quality": "address_complete", + "address_quality": "city_only", "coordinate_state": "source_coordinate_valid", "country": "de", "outcome": "map_ready_under_current_rules" @@ -124,25 +115,25 @@ "outcome": "map_ready_under_current_rules" }, { - "address_quality": "address_complete", + "address_quality": "city_only", "coordinate_state": "source_coordinate_valid", "country": "de", "outcome": "map_ready_under_current_rules" }, { - "address_quality": "address_complete", + "address_quality": "city_only", "coordinate_state": "source_coordinate_valid", "country": "de", "outcome": "map_ready_under_current_rules" }, { - "address_quality": "address_complete", + "address_quality": "city_only", "coordinate_state": "source_coordinate_valid", "country": "dk", "outcome": "map_ready_under_current_rules" }, { - "address_quality": "address_complete", + "address_quality": "city_only", "coordinate_state": "source_coordinate_valid", "country": "de", "outcome": "map_ready_under_current_rules" @@ -196,7 +187,7 @@ "outcome": "map_ready_under_current_rules" }, { - "address_quality": "address_complete", + "address_quality": "city_only", "coordinate_state": "source_coordinate_valid", "country": "de", "outcome": "map_ready_under_current_rules" @@ -268,7 +259,7 @@ "outcome": "map_ready_under_current_rules" }, { - "address_quality": "address_complete", + "address_quality": "city_only", "coordinate_state": "source_coordinate_valid", "country": "de", "outcome": "map_ready_under_current_rules" @@ -346,19 +337,19 @@ "outcome": "map_ready_under_current_rules" }, { - "address_quality": "address_complete", + "address_quality": "city_only", "coordinate_state": "source_coordinate_valid", "country": "de", "outcome": "map_ready_under_current_rules" }, { - "address_quality": "address_complete", + "address_quality": "city_only", "coordinate_state": "source_coordinate_valid", "country": "de", "outcome": "map_ready_under_current_rules" }, { - "address_quality": "address_complete", + "address_quality": "city_only", "coordinate_state": "source_coordinate_valid", "country": "de", "outcome": "map_ready_under_current_rules" @@ -388,7 +379,7 @@ "outcome": "map_ready_under_current_rules" }, { - "address_quality": "address_complete", + "address_quality": "city_only", "coordinate_state": "source_coordinate_valid", "country": "de", "outcome": "map_ready_under_current_rules" @@ -406,7 +397,7 @@ "outcome": "map_ready_under_current_rules" }, { - "address_quality": "address_complete", + "address_quality": "city_only", "coordinate_state": "source_coordinate_valid", "country": "de", "outcome": "map_ready_under_current_rules" @@ -424,19 +415,19 @@ "outcome": "map_ready_under_current_rules" }, { - "address_quality": "address_complete", + "address_quality": "city_only", "coordinate_state": "source_coordinate_invalid_or_missing", "country": "de", - "outcome": "geocode_candidate" + "outcome": "city_coarse_candidate" }, { - "address_quality": "address_complete", + "address_quality": "city_only", "coordinate_state": "source_coordinate_valid", "country": "de", "outcome": "map_ready_under_current_rules" }, { - "address_quality": "address_complete", + "address_quality": "city_only", "coordinate_state": "source_coordinate_valid", "country": "de", "outcome": "map_ready_under_current_rules" @@ -478,13 +469,13 @@ "outcome": "map_ready_under_current_rules" }, { - "address_quality": "address_complete", + "address_quality": "city_only", "coordinate_state": "source_coordinate_valid", "country": "de", "outcome": "map_ready_under_current_rules" }, { - "address_quality": "address_complete", + "address_quality": "city_only", "coordinate_state": "source_coordinate_valid", "country": "de", "outcome": "map_ready_under_current_rules" @@ -562,10 +553,10 @@ "outcome": "map_ready_under_current_rules" }, { - "address_quality": "address_complete", + "address_quality": "city_only", "coordinate_state": "source_coordinate_invalid_or_missing", "country": "de", - "outcome": "geocode_candidate" + "outcome": "city_coarse_candidate" }, { "address_quality": "address_complete", @@ -622,7 +613,7 @@ "outcome": "map_ready_under_current_rules" }, { - "address_quality": "address_complete", + "address_quality": "city_only", "coordinate_state": "source_coordinate_valid", "country": "dk", "outcome": "map_ready_under_current_rules" @@ -634,10 +625,10 @@ "outcome": "map_ready_under_current_rules" }, { - "address_quality": "address_complete", + "address_quality": "city_only", "coordinate_state": "source_coordinate_invalid_or_missing", "country": "dk", - "outcome": "geocode_candidate" + "outcome": "city_coarse_candidate" }, { "address_quality": "address_complete", @@ -736,10 +727,10 @@ "outcome": "map_ready_under_current_rules" }, { - "address_quality": "address_complete", + "address_quality": "city_only", "coordinate_state": "source_coordinate_invalid_or_missing", "country": "dk", - "outcome": "geocode_candidate" + "outcome": "city_coarse_candidate" }, { "address_quality": "address_complete", @@ -868,7 +859,7 @@ "outcome": "map_ready_under_current_rules" }, { - "address_quality": "address_complete", + "address_quality": "city_only", "coordinate_state": "source_coordinate_valid", "country": "dk", "outcome": "map_ready_under_current_rules" @@ -904,7 +895,7 @@ "outcome": "map_ready_under_current_rules" }, { - "address_quality": "address_complete", + "address_quality": "city_only", "coordinate_state": "source_coordinate_valid", "country": "dk", "outcome": "map_ready_under_current_rules" @@ -922,7 +913,7 @@ "outcome": "map_ready_under_current_rules" }, { - "address_quality": "address_complete", + "address_quality": "city_only", "coordinate_state": "source_coordinate_valid", "country": "dk", "outcome": "map_ready_under_current_rules" @@ -970,16 +961,16 @@ "outcome": "map_ready_under_current_rules" }, { - "address_quality": "address_complete", + "address_quality": "city_only", "coordinate_state": "source_coordinate_invalid_or_missing", "country": "dk", - "outcome": "geocode_candidate" + "outcome": "city_coarse_candidate" }, { - "address_quality": "address_complete", + "address_quality": "city_only", "coordinate_state": "source_coordinate_invalid_or_missing", "country": "de", - "outcome": "geocode_candidate" + "outcome": "city_coarse_candidate" }, { "address_quality": "city_only", @@ -988,10 +979,10 @@ "outcome": "map_ready_under_current_rules" }, { - "address_quality": "address_complete", + "address_quality": "city_only", "coordinate_state": "source_coordinate_invalid_or_missing", "country": "de", - "outcome": "geocode_candidate" + "outcome": "city_coarse_candidate" }, { "address_quality": "city_only", @@ -1012,10 +1003,10 @@ "outcome": "map_ready_under_current_rules" }, { - "address_quality": "address_complete", + "address_quality": "city_only", "coordinate_state": "source_coordinate_invalid_or_missing", "country": "dk", - "outcome": "geocode_candidate" + "outcome": "city_coarse_candidate" }, { "address_quality": "address_complete", @@ -1090,7 +1081,7 @@ "outcome": "map_ready_under_current_rules" }, { - "address_quality": "address_complete", + "address_quality": "city_only", "coordinate_state": "source_coordinate_valid", "country": "dk", "outcome": "map_ready_under_current_rules" @@ -1132,7 +1123,7 @@ "outcome": "map_ready_under_current_rules" } ], - "schema_version": "geospatial-readiness-audit/v1", + "schema_version": "geospatial-readiness-audit/v2", "source_files": [ { "bytes": 361801, @@ -1191,28 +1182,1276 @@ ], "strata_counts": { "ca|address_complete|source_coordinate_invalid_or_missing": 1, - "ca|address_complete|source_coordinate_valid": 1261, + "ca|address_complete|source_coordinate_valid": 1260, "ca|address_missing|source_coordinate_invalid_or_missing": 2, - "ca|city_only|source_coordinate_valid": 59, - "de|address_complete|source_coordinate_invalid_or_missing": 962, - "de|address_complete|source_coordinate_valid": 8290, - "de|city_only|source_coordinate_valid": 2, - "dk|address_complete|source_coordinate_invalid_or_missing": 409, - "dk|address_complete|source_coordinate_valid": 1128, + "ca|city_only|source_coordinate_valid": 60, + "de|city_only|source_coordinate_invalid_or_missing": 962, + "de|city_only|source_coordinate_valid": 8292, "dk|address_missing|source_coordinate_invalid_or_missing": 7, "dk|address_missing|source_coordinate_valid": 17, - "es|address_complete|source_coordinate_valid": 3278, + "dk|city_only|source_coordinate_invalid_or_missing": 409, + "dk|city_only|source_coordinate_valid": 1128, + "es|address_complete|source_coordinate_valid": 3271, "es|address_missing|source_coordinate_valid": 30, - "es|city_only|source_coordinate_valid": 914, + "es|city_only|source_coordinate_valid": 921, "fr|address_missing|source_coordinate_valid": 3201, "mx|address_complete|source_coordinate_valid": 12258, "mx|city_only|source_coordinate_valid": 2578, "nz|city_only|source_coordinate_invalid_or_missing": 665, "nz|city_only|source_coordinate_valid": 987, - "uk|address_complete|source_coordinate_invalid_or_missing": 1, - "uk|address_complete|source_coordinate_valid": 6981, + "uk|address_complete|source_coordinate_valid": 6979, "uk|address_missing|source_coordinate_valid": 1, - "uk|city_only|source_coordinate_valid": 617, + "uk|city_only|source_coordinate_invalid_or_missing": 1, + "uk|city_only|source_coordinate_valid": 619, "us|address_complete|source_coordinate_valid": 7101 + }, + "v1": { + "as_of": "2026-09-16T00:00:00Z", + "corpus": "V1-legacy-static-data", + "country_counts": { + "ca": 1323, + "de": 9254, + "dk": 1561, + "es": 4222, + "fr": 3201, + "mx": 14836, + "nz": 1652, + "uk": 7600, + "us": 7101 + }, + "funnel": { + "city_coarse_candidate": 2028, + "geocode_candidate": 1, + "map_ready_under_current_rules": 48640, + "privacy_restricted": 72, + "source_coordinate_invalid_or_missing": 2047, + "source_coordinate_valid": 48703, + "total": 50750, + "unresolved": 9 + }, + "method": "offline deterministic audit; no geocoder calls", + "provenance_fields_required_for_any_geocode": [ + "provider", + "query_hash", + "queried_at", + "precision", + "review_state" + ], + "publication_rule": "map-ready requires current privacy eligibility and release approval; geocoding success alone never grants publication permission", + "sample": { + "composition": { + "ca|address_complete|source_coordinate_valid": 8, + "de|city_only|source_coordinate_invalid_or_missing": 4, + "de|city_only|source_coordinate_valid": 20, + "dk|city_only|source_coordinate_invalid_or_missing": 5, + "dk|city_only|source_coordinate_valid": 6, + "es|address_complete|source_coordinate_valid": 14, + "es|city_only|source_coordinate_valid": 8, + "fr|address_missing|source_coordinate_valid": 20, + "mx|address_complete|source_coordinate_valid": 20, + "mx|city_only|source_coordinate_valid": 15, + "nz|city_only|source_coordinate_invalid_or_missing": 3, + "nz|city_only|source_coordinate_valid": 8, + "uk|address_complete|source_coordinate_valid": 20, + "uk|city_only|source_coordinate_valid": 9, + "us|address_complete|source_coordinate_valid": 20 + }, + "size": 180 + }, + "sample_rows": [ + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "de", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_invalid_or_missing", + "country": "dk", + "outcome": "city_coarse_candidate" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "de", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "de", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "nz", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "mx", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "mx", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "de", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "de", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "de", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "mx", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "mx", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "de", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "de", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "dk", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "de", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "us", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "mx", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "mx", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "ca", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "nz", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "uk", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "us", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "mx", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "de", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "es", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_invalid_or_missing", + "country": "nz", + "outcome": "city_coarse_candidate" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "mx", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "mx", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "mx", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "uk", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "us", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "es", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "mx", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "uk", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "mx", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "de", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "mx", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "mx", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "uk", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "us", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "mx", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "mx", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "us", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "us", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "mx", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "mx", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "uk", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_missing", + "coordinate_state": "source_coordinate_valid", + "country": "fr", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "de", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "de", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "de", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "mx", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "uk", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "mx", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "uk", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "de", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "mx", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "mx", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "de", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "mx", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "mx", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_invalid_or_missing", + "country": "de", + "outcome": "city_coarse_candidate" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "de", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "de", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "mx", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "uk", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_missing", + "coordinate_state": "source_coordinate_valid", + "country": "fr", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "uk", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_missing", + "coordinate_state": "source_coordinate_valid", + "country": "fr", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "uk", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "de", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "de", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "uk", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "us", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "uk", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "us", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "uk", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "us", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_invalid_or_missing", + "country": "nz", + "outcome": "city_coarse_candidate" + }, + { + "address_quality": "address_missing", + "coordinate_state": "source_coordinate_valid", + "country": "fr", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "uk", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "us", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_missing", + "coordinate_state": "source_coordinate_valid", + "country": "fr", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "uk", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_invalid_or_missing", + "country": "de", + "outcome": "city_coarse_candidate" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "uk", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "uk", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "mx", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "uk", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "es", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "us", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "es", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "uk", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "ca", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "dk", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "uk", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_invalid_or_missing", + "country": "dk", + "outcome": "city_coarse_candidate" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "uk", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_missing", + "coordinate_state": "source_coordinate_valid", + "country": "fr", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "uk", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "us", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "uk", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "uk", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "us", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "us", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "nz", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "uk", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "ca", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "us", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "ca", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "uk", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_missing", + "coordinate_state": "source_coordinate_valid", + "country": "fr", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "uk", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_invalid_or_missing", + "country": "dk", + "outcome": "city_coarse_candidate" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "us", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "es", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "us", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "us", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "us", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "mx", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "es", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "us", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "nz", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "uk", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "es", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_missing", + "coordinate_state": "source_coordinate_valid", + "country": "fr", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "mx", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "es", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "es", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_missing", + "coordinate_state": "source_coordinate_valid", + "country": "fr", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_invalid_or_missing", + "country": "nz", + "outcome": "city_coarse_candidate" + }, + { + "address_quality": "address_missing", + "coordinate_state": "source_coordinate_valid", + "country": "fr", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "mx", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_missing", + "coordinate_state": "source_coordinate_valid", + "country": "fr", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "nz", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "dk", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "mx", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_missing", + "coordinate_state": "source_coordinate_valid", + "country": "fr", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "mx", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "es", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "nz", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "dk", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_missing", + "coordinate_state": "source_coordinate_valid", + "country": "fr", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "es", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "dk", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "ca", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "mx", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "es", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "es", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "es", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_missing", + "coordinate_state": "source_coordinate_valid", + "country": "fr", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "nz", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_invalid_or_missing", + "country": "dk", + "outcome": "city_coarse_candidate" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_invalid_or_missing", + "country": "de", + "outcome": "city_coarse_candidate" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "nz", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_invalid_or_missing", + "country": "de", + "outcome": "city_coarse_candidate" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "mx", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "mx", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "es", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_invalid_or_missing", + "country": "dk", + "outcome": "city_coarse_candidate" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "es", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_missing", + "coordinate_state": "source_coordinate_valid", + "country": "fr", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_missing", + "coordinate_state": "source_coordinate_valid", + "country": "fr", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "es", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_missing", + "coordinate_state": "source_coordinate_valid", + "country": "fr", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_missing", + "coordinate_state": "source_coordinate_valid", + "country": "fr", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "ca", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "mx", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "es", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_missing", + "coordinate_state": "source_coordinate_valid", + "country": "fr", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "ca", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "es", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "dk", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "ca", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_missing", + "coordinate_state": "source_coordinate_valid", + "country": "fr", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "address_complete", + "coordinate_state": "source_coordinate_valid", + "country": "es", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "es", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "es", + "outcome": "map_ready_under_current_rules" + }, + { + "address_quality": "city_only", + "coordinate_state": "source_coordinate_valid", + "country": "uk", + "outcome": "map_ready_under_current_rules" + } + ], + "schema_version": "geospatial-readiness-audit/v1", + "source_files": [ + { + "bytes": 361801, + "country": "ca", + "path": "static_data/ca/locations.csv", + "sha256": "544fd7809926143d4e934f1c601e981a07b0137780f86de753ef3c54415e45f4" + }, + { + "bytes": 3849223, + "country": "de", + "path": "static_data/de/locations.csv", + "sha256": "88eb00f6ae405489a7e873582aa65610ecaa3c9b475bc0046e9e8c32f48ac809" + }, + { + "bytes": 323380, + "country": "dk", + "path": "static_data/dk/locations.csv", + "sha256": "b38f8165d6ea3617a13dbad950d0070490e645473bb851ddd3d907004a33b32f" + }, + { + "bytes": 1093952, + "country": "es", + "path": "static_data/es/locations.csv", + "sha256": "93d36d6a760a4435c836348021fb03d98ddb4947345338ea8c199321af3e5347" + }, + { + "bytes": 626026, + "country": "fr", + "path": "static_data/fr/locations.csv", + "sha256": "aeddafa5ebc19d63b6dd57777851e3e5590b5d5ddfb9cbeff37a552227e1c615" + }, + { + "bytes": 5039068, + "country": "mx", + "path": "static_data/mx/locations.csv", + "sha256": "bc9b3adc80e4673bb8b996c7600b52b78e7973b1a1859608322a293aff9efdee" + }, + { + "bytes": 410481, + "country": "nz", + "path": "static_data/nz/locations.csv", + "sha256": "636d80b33cc29dbd68b3c44ddf39c57db84cdd661d1c10dfab4227f8a87a3a87" + }, + { + "bytes": 2193258, + "country": "uk", + "path": "static_data/uk/locations.csv", + "sha256": "4a14793c227afe558bfacfe45b05f3ae4e5332a3d502991050257320c5b53511" + }, + { + "bytes": 4099548, + "country": "us", + "path": "static_data/us/locations.csv", + "sha256": "2dca259076a16a324ad9565d2d4057aa0f5e6e51bbc25a8dc1c80a4e4fbdf5c7" + } + ], + "strata_counts": { + "ca|address_complete|source_coordinate_invalid_or_missing": 1, + "ca|address_complete|source_coordinate_valid": 1260, + "ca|address_missing|source_coordinate_invalid_or_missing": 2, + "ca|city_only|source_coordinate_valid": 60, + "de|city_only|source_coordinate_invalid_or_missing": 962, + "de|city_only|source_coordinate_valid": 8292, + "dk|address_missing|source_coordinate_invalid_or_missing": 7, + "dk|address_missing|source_coordinate_valid": 17, + "dk|city_only|source_coordinate_invalid_or_missing": 409, + "dk|city_only|source_coordinate_valid": 1128, + "es|address_complete|source_coordinate_valid": 3271, + "es|address_missing|source_coordinate_valid": 30, + "es|city_only|source_coordinate_valid": 921, + "fr|address_missing|source_coordinate_valid": 3201, + "mx|address_complete|source_coordinate_valid": 12258, + "mx|city_only|source_coordinate_valid": 2578, + "nz|city_only|source_coordinate_invalid_or_missing": 665, + "nz|city_only|source_coordinate_valid": 987, + "uk|address_complete|source_coordinate_valid": 6979, + "uk|address_missing|source_coordinate_valid": 1, + "uk|city_only|source_coordinate_invalid_or_missing": 1, + "uk|city_only|source_coordinate_valid": 619, + "us|address_complete|source_coordinate_valid": 7101 + } + }, + "v2": { + "as_of": "2026-09-16T00:00:00Z", + "availability": { + "manifests_found": 0, + "note": "No V2 records are inferred from V1 rows; absent manifests/records remain unavailable.", + "searched_root": "static_data", + "sources": [] + }, + "corpus": "V2-normalized-manifest-runs", + "country_counts": {}, + "funnel": { + "total": 0 + }, + "method": "offline deterministic audit; no geocoder calls", + "provenance_fields_required_for_any_geocode": [ + "provider", + "query_hash", + "queried_at", + "precision", + "review_state" + ], + "publication_rule": "map-ready requires current privacy eligibility and release approval; geocoding success alone never grants publication permission", + "sample": { + "composition": {}, + "size": 0 + }, + "sample_rows": [], + "schema_version": "geospatial-readiness-audit/v1", + "source_files": [], + "strata_counts": {} } } diff --git a/docs/geospatial-readiness-audit.md b/docs/geospatial-readiness-audit.md index e930a0d..8b14bef 100644 --- a/docs/geospatial-readiness-audit.md +++ b/docs/geospatial-readiness-audit.md @@ -1,7 +1,7 @@ # Geospatial readiness rehearsal -`pipeline/scripts/diagnostics/geospatial_readiness_audit.py` performs a deterministic offline audit of normalized `static_data/*/locations.csv` files. It never calls a geocoder and emits only aggregate counts plus sanitized sample composition. Source coordinates are retained conceptually as source evidence; this report does not copy them. +`pipeline/scripts/diagnostics/geospatial_readiness_audit.py` performs a deterministic offline audit with separate V1 and V2 funnels. V1 reads legacy `static_data/*/locations.csv`; V2 discovers authorized handoff `manifest.json` files and their `normalized/records.jsonl` children. It never calls a geocoder and emits only aggregate counts plus sanitized sample composition. V1 rows are never counted as V2. Missing V2 manifests are reported explicitly as unavailable. The funnel distinguishes valid source coordinates, geocode candidates, city/coarse candidates, review-required/privacy-restricted candidates, unresolved rows, and `map_ready_under_current_rules`. Map readiness is not publication approval: a future geocode result must retain provider, query hash, timestamp, precision, and review state, and the release projection must independently pass privacy and approval gates. Residential-risk flags are conservative indicators for human review, not factual classifications. -Run with `python pipeline/scripts/diagnostics/geospatial_readiness_audit.py --output data/reports/geospatial-readiness.json --as-of 2026-09-16T00:00:00Z`. The report is row-free with respect to source names, addresses, identifiers, and coordinates. Real corpus totals and limitations must be reviewed before release; this rehearsal does not authorize publication or paid geocoding. +Run with `python pipeline/scripts/diagnostics/geospatial_readiness_audit.py --output data/reports/geospatial-readiness.json --as-of 2026-09-16T00:00:00Z`. Use `--root` for the V1 tree and call the Python API with `v2_root` for an ignored V2 run root. The report is row-free with respect to source names, addresses, identifiers, and coordinates. Real corpus totals and limitations must be reviewed before release; this rehearsal does not authorize publication or paid geocoding. diff --git a/pipeline/scripts/diagnostics/geospatial_readiness_audit.py b/pipeline/scripts/diagnostics/geospatial_readiness_audit.py index 74439fc..2f5004c 100644 --- a/pipeline/scripts/diagnostics/geospatial_readiness_audit.py +++ b/pipeline/scripts/diagnostics/geospatial_readiness_audit.py @@ -19,22 +19,31 @@ def num(v): def valid_point(lat, lon): return lat is not None and lon is not None and -90 <= lat <= 90 and -180 <= lon <= 180 and not (lat == 0 and lon == 0) -def audit(root: Path, sample_size: int = 20, as_of: str | None = None): - files = sorted(root.glob("*/locations.csv")) +def _audit_rows(files, sample_size, as_of, corpus): funnel = Counter(); countries = Counter(); strata = Counter(); sample_buckets = {} file_manifest = [] for path in files: digest = hashlib.sha256(path.read_bytes()).hexdigest() file_manifest.append({"country": path.parent.name, "path": path.as_posix(), "sha256": digest, "bytes": path.stat().st_size}) - with path.open(encoding="utf-8-sig", newline="") as fh: - rows = csv.DictReader(fh) - for row in rows: + if path.suffix == ".jsonl": + rows = (json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line.strip()) + else: + rows = csv.DictReader(path.read_text(encoding="utf-8-sig").splitlines()) + for row in rows: funnel["total"] += 1; country = path.parent.name; countries[country] += 1 - lat, lon = num(row.get("latitude")), num(row.get("longitude")) + normalized = row.get("normalized", row) + if not isinstance(normalized, dict): + normalized = {} + coordinates = normalized.get("coordinates") + lat = num(row.get("latitude", normalized.get("latitude"))) + lon = num(row.get("longitude", normalized.get("longitude"))) + if isinstance(coordinates, (list, tuple)) and len(coordinates) == 2: + lon, lat = num(coordinates[0]), num(coordinates[1]) coord = "source_coordinate_valid" if valid_point(lat, lon) else "source_coordinate_invalid_or_missing" funnel[coord] += 1 - address = " ".join(filter(None, (row.get("street"), row.get("zip"), row.get("city"), row.get("state")))) - quality = "address_complete" if row.get("street") and row.get("city") and row.get("zip") else ("city_only" if row.get("city") else "address_missing") + address_values = (normalized.get("street", row.get("street")), normalized.get("zip", row.get("zip")), normalized.get("city", row.get("city")), normalized.get("state", row.get("state"))) + address = " ".join(filter(None, address_values)) + quality = "address_complete" if all(address_values) else ("city_only" if address_values[2] else "address_missing") privacy = "privacy_restricted_candidate" if PRIVATE_RE.search(address) else "privacy_not_flagged" if coord == "source_coordinate_valid": outcome = "map_ready_under_current_rules" if privacy == "privacy_not_flagged" else "privacy_restricted" elif privacy != "privacy_not_flagged": outcome = "privacy_restricted" @@ -42,7 +51,7 @@ def audit(root: Path, sample_size: int = 20, as_of: str | None = None): elif quality == "address_complete": outcome = "geocode_candidate" else: outcome = "unresolved" funnel[outcome] += 1; strata[(country, quality, coord)] += 1 - key = hashlib.sha256((country + "\0" + row.get("establishment_id", "") + "\0" + address).encode()).hexdigest() + key = hashlib.sha256((country + "\0" + str(normalized.get("establishment_id", row.get("establishment_id", ""))) + "\0" + address).encode()).hexdigest() bucket = sample_buckets.setdefault((country, quality, coord), []) bucket.append((key, {"country": country, "address_quality": quality, "coordinate_state": coord, "outcome": outcome})) samples = [] @@ -50,10 +59,34 @@ def audit(root: Path, sample_size: int = 20, as_of: str | None = None): samples.extend(sorted(bucket)[:sample_size]) samples = [v for _, v in sorted(samples)[:sample_size * max(1, len(files))]] composition = Counter("|".join((x["country"], x["address_quality"], x["coordinate_state"])) for x in samples) - return {"schema_version": "geospatial-readiness-audit/v1", "as_of": as_of or datetime.now(timezone.utc).isoformat(), "method": "offline deterministic audit; no geocoder calls", "funnel": dict(sorted(funnel.items())), "country_counts": dict(sorted(countries.items())), "strata_counts": {"|".join(k): v for k,v in sorted(strata.items())}, "sample": {"size": len(samples), "composition": dict(composition)}, "sample_rows": samples, "source_files": file_manifest, "provenance_fields_required_for_any_geocode": ["provider", "query_hash", "queried_at", "precision", "review_state"], "publication_rule": "map-ready requires current privacy eligibility and release approval; geocoding success alone never grants publication permission"} + funnel.setdefault("total", 0) + return {"corpus": corpus, "schema_version": "geospatial-readiness-audit/v1", "as_of": as_of or datetime.now(timezone.utc).isoformat(), "method": "offline deterministic audit; no geocoder calls", "funnel": dict(sorted(funnel.items())), "country_counts": dict(sorted(countries.items())), "strata_counts": {"|".join(k): v for k,v in sorted(strata.items())}, "sample": {"size": len(samples), "composition": dict(composition)}, "sample_rows": samples, "source_files": file_manifest, "provenance_fields_required_for_any_geocode": ["provider", "query_hash", "queried_at", "precision", "review_state"], "publication_rule": "map-ready requires current privacy eligibility and release approval; geocoding success alone never grants publication permission"} + +def _v2_files(root: Path): + manifests = sorted(root.rglob("manifest.json")) if root.exists() else [] + files, availability = [], [] + for manifest_path in manifests: + try: manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): continue + records = manifest_path.parent / "normalized" / "records.jsonl" + source_id = manifest.get("source_id", manifest_path.parent.name) + entry = {"source_id": source_id, "manifest": manifest_path.as_posix(), "records": records.as_posix(), "available": records.is_file(), "normalized_rows_declared": manifest.get("normalized_rows")} + availability.append(entry) + if records.is_file(): files.append(records) + return files, availability + +def audit(root: Path, sample_size: int = 20, as_of: str | None = None, *, v2_root: Path | None = None): + v1 = _audit_rows(sorted(root.glob("*/locations.csv")), sample_size, as_of, "V1-legacy-static-data") + v2_root = v2_root or root + v2_files, availability = _v2_files(v2_root) + v2 = _audit_rows(v2_files, sample_size, as_of, "V2-normalized-manifest-runs") + v2["availability"] = {"searched_root": v2_root.as_posix(), "manifests_found": len(availability), "sources": availability, "note": "No V2 records are inferred from V1 rows; absent manifests/records remain unavailable."} + # Keep the original V1 shape available to existing callers; new consumers + # must use the explicitly named v1/v2 sections. + return {"schema_version": "geospatial-readiness-audit/v2", "as_of": as_of or datetime.now(timezone.utc).isoformat(), "v1": v1, "v2": v2, "funnel": v1["funnel"], "country_counts": v1["country_counts"], "strata_counts": v1["strata_counts"], "sample": v1["sample"], "sample_rows": v1["sample_rows"], "source_files": v1["source_files"]} def main(): ap = argparse.ArgumentParser(); ap.add_argument("--root", type=Path, default=Path("static_data")); ap.add_argument("--output", type=Path, required=True); ap.add_argument("--sample-size", type=int, default=20); ap.add_argument("--as-of") args = ap.parse_args(); report = audit(args.root, args.sample_size, args.as_of); args.output.parent.mkdir(parents=True, exist_ok=True); args.output.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8") - print(json.dumps({"total": report["funnel"].get("total", 0), "funnel": report["funnel"], "sample_size": report["sample"]["size"]}, sort_keys=True)) + print(json.dumps({"v1_total": report["v1"]["funnel"].get("total", 0), "v2_total": report["v2"]["funnel"].get("total", 0), "v2_manifests": report["v2"]["availability"]["manifests_found"]}, sort_keys=True)) if __name__ == "__main__": main() diff --git a/pipeline/tests/test_geospatial_readiness_audit.py b/pipeline/tests/test_geospatial_readiness_audit.py index 72a6990..25d5cb9 100644 --- a/pipeline/tests/test_geospatial_readiness_audit.py +++ b/pipeline/tests/test_geospatial_readiness_audit.py @@ -16,3 +16,18 @@ def test_funnel_and_row_free_sample(self): self.assertEqual(r['funnel']['map_ready_under_current_rules'], 1); self.assertEqual(r['funnel']['privacy_restricted'], 1) payload=json.dumps(r); self.assertNotIn('Main 1', payload); self.assertNotIn('Private farmhouse', payload) def test_is_deterministic(self): self.assertEqual(audit(self.d, 2, 'x'), audit(self.d, 2, 'x')) + + def test_v2_manifest_records_are_audited_separately(self): + run = self.d / 'runs' / 'dk.smiley'; (run / 'normalized').mkdir(parents=True) + (run / 'manifest.json').write_text(json.dumps({'source_id': 'dk.smiley', 'normalized_rows': 1}), encoding='utf8') + (run / 'normalized' / 'records.jsonl').write_text(json.dumps({'source_id': 'dk.smiley', 'normalized': {'establishment_id': 'secret', 'city': 'Town', 'coordinates': [12, 55]}}) + '\n', encoding='utf8') + r = audit(self.d, 10, 'x', v2_root=self.d / 'runs') + self.assertEqual(r['v1']['funnel']['total'], 3) + self.assertEqual(r['v2']['funnel']['total'], 1) + self.assertEqual(r['v2']['availability']['sources'][0]['source_id'], 'dk.smiley') + self.assertNotIn('secret', json.dumps(r)) + + def test_v2_absence_is_explicit(self): + r = audit(self.d, 2, 'x', v2_root=self.d / 'missing') + self.assertEqual(r['v2']['funnel']['total'], 0) + self.assertEqual(r['v2']['availability']['manifests_found'], 0) From 0937b00b7c7d51fffd66dc9106b9f2d0873cfe20 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Wed, 16 Sep 2026 14:51:18 -0700 Subject: [PATCH 188/311] test: validate Sprint 3 scale evidence --- .gitignore | 1 + docs/performance/v2-api-load-rehearsal.md | 64 +++++++++++++++++++ docs/performance/v2-discovery-100k.md | 24 +++++++ .../benchmarks/run_api_load_rehearsal.py | 16 +++-- pipeline/tests/test_api_load_rehearsal.py | 6 ++ pipeline/tests/test_graph_migrations.py | 3 +- pipeline/tests/test_source_registry.py | 4 +- 7 files changed, 111 insertions(+), 7 deletions(-) create mode 100644 docs/performance/v2-api-load-rehearsal.md diff --git a/.gitignore b/.gitignore index 8a4a588..64c6bfe 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ /target/ +/.tmp/ /node_modules/ /snapshots/ /static/v2-preview/ diff --git a/docs/performance/v2-api-load-rehearsal.md b/docs/performance/v2-api-load-rehearsal.md new file mode 100644 index 0000000..ea4dd22 --- /dev/null +++ b/docs/performance/v2-api-load-rehearsal.md @@ -0,0 +1,64 @@ +# V2 synthetic API load rehearsal + +This is a bounded, local, synthetic rehearsal of the V2 API. It is not a +production capacity claim. Each run creates a fresh disposable PostGIS E2E +environment, applies every migration, seeds only deterministic synthetic +records, exercises the API and graph read paths, and destroys the environment. +The report contains aggregate counters and latency percentiles only; raw rows, +coordinates, identifiers, and response bodies are not retained in Git. + +## Reproduction + +Install the pinned Python dependencies, then run: + +```powershell +python pipeline/scripts/benchmarks/run_api_load_rehearsal.py ` + --observations 5000 ` + --concurrency 1,4,8,16 ` + --requests-per-level 10 ` + --timeout-ms 2000 ` + --json-output .tmp/api-load-5000.json + +python pipeline/scripts/benchmarks/run_api_load_rehearsal.py ` + --observations 25000 ` + --concurrency 1,4,8,16 ` + --requests-per-level 10 ` + --timeout-ms 2000 ` + --json-output .tmp/api-load-25000.json +``` + +The runner bounds observations at 25,000, concurrency at 16, and requests per +level at 80. It refuses non-loopback API targets. The 25,000-row bound is an +explicitly finite synthetic safety limit, not a statement about supported +production scale. + +## Captured evidence (2026-09-16) + +| Synthetic observations | Concurrency | Requests | Successes | Timeouts | 5xx | Throughput (rps) | p50 / p95 / p99 (ms) | Max active / waiting DB sessions | +| ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| 5,000 | 1 | 10 | 4 | 6 | 0 | 0.622 | 2013.251 / 2028.338 / 2028.338 | 2 / 1 | +| 5,000 | 4 | 10 | 3 | 7 | 0 | 1.981 | 2001.908 / 2029.853 / 2029.853 | 2 / 0 | +| 5,000 | 8 | 10 | 4 | 6 | 0 | 3.982 | 2013.191 / 2017.044 / 2017.044 | 6 / 1 | +| 5,000 | 16 | 10 | 2 | 8 | 0 | 4.953 | 2009.559 / 2011.767 / 2011.767 | 9 / 1 | +| 25,000 | 1 | 10 | 1 | 9 | 0 | 0.514 | 2016.681 / 2029.881 / 2029.881 | 2 / 1 | +| 25,000 | 4 | 10 | 1 | 9 | 0 | 1.653 | 2012.612 / 2025.954 / 2025.954 | 2 / 1 | +| 25,000 | 8 | 10 | 2 | 8 | 0 | 2.547 | 2007.124 / 2030.413 / 2030.413 | 6 / 2 | +| 25,000 | 16 | 10 | 1 | 9 | 0 | 4.482 | 2004.377 / 2024.087 / 2024.087 | 10 / 3 | + +No tested level was clean under the harness definition (zero timeout, server, +or connection errors). The result supports keeping API pool sizing and +production capacity claims blocked until an approved representative load +model is available. It also confirms that the failure mode in this local +rehearsal is timeout pressure rather than HTTP 5xx or connection exhaustion. + +The dominant slow path remains the live eligibility/summary work documented in +`v2-public-projection-read-path.md`. The evidence does not justify caching, +relaxing current suppression checks, or claiming production readiness. + +## Data and ethics boundary + +The fixture uses only synthetic source, facility, observation, graph, and +geocode-shaped records. The API harness reads response bodies to completion +and immediately discards them. JSON outputs belong under `.tmp/`, which is +ignored. Do not copy raw or private rows, coordinates, source paths, or +identifiers into reports, logs, backups, or commits. diff --git a/docs/performance/v2-discovery-100k.md b/docs/performance/v2-discovery-100k.md index 94a97f8..64f4ffc 100644 --- a/docs/performance/v2-discovery-100k.md +++ b/docs/performance/v2-discovery-100k.md @@ -54,3 +54,27 @@ that fails its expected-index check is a release-review input. The measurements do not establish capacity, cloud cost, or public-source completeness; those require a separate load test with an approved traffic model and deployment configuration. + +## Sprint 3 5k/25k query-plan capture (2026-09-16) + +On a fresh fully migrated PostGIS 16 / PostGIS 3.4 disposable database, the +same row-free runner was executed with `--scales 5000 25000`. All 16 query +observations passed their expected-index checks, returned at most 50 rows, and +reported zero sequential-scan nodes. Aggregate execution times in milliseconds +were: + +| Query shape | 5,000 | 25,000 | +| --- | ---: | ---: | +| list | 0.100 | 0.040 | +| pagination | 0.010 | 0.009 | +| filters | 0.046 | 0.044 | +| text filter | 0.042 | 0.047 | +| bbox | 0.039 | 0.060 | +| radius | 9.658 | 7.150 | +| detail | 0.011 | 0.011 | +| graph-ready join | 0.133 | 0.118 | + +These are single-query local plan samples over temporary synthetic tables. They +demonstrate query-shape/index behavior only and do not establish API latency, +concurrency capacity, or production readiness. The companion API rehearsal and +its limitations are recorded in `v2-api-load-rehearsal.md`. diff --git a/pipeline/scripts/benchmarks/run_api_load_rehearsal.py b/pipeline/scripts/benchmarks/run_api_load_rehearsal.py index f9ab569..f3ef03c 100644 --- a/pipeline/scripts/benchmarks/run_api_load_rehearsal.py +++ b/pipeline/scripts/benchmarks/run_api_load_rehearsal.py @@ -27,7 +27,7 @@ ROOT = Path(__file__).resolve().parents[3] if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) -MAX_SEED = 20_000 +MAX_SEED = 25_000 MAX_CONCURRENCY = 16 MAX_REQUESTS_PER_LEVEL = 80 QUERY_MIX = ( @@ -45,6 +45,12 @@ def deterministic_uuid(prefix: str, ordinal: int) -> str: return str(uuid.UUID(hex=hashlib.md5(f"{prefix}-{ordinal}".encode()).hexdigest())) +def validate_observations(observations: int) -> int: + if not 1 <= observations <= MAX_SEED: + raise ValueError(f"observations must be between 1 and {MAX_SEED:,}") + return observations + + def validate_levels(levels: list[int]) -> tuple[int, ...]: if not levels or any(level < 1 or level > MAX_CONCURRENCY for level in levels): raise ValueError(f"concurrency levels must be between 1 and {MAX_CONCURRENCY}") @@ -350,8 +356,10 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("--timeout-ms", type=int, default=2_000) parser.add_argument("--json-output", type=Path) args = parser.parse_args(argv) - if not 1 <= args.observations <= MAX_SEED: - parser.error(f"observations must be between 1 and {MAX_SEED:,}") + try: + observations = validate_observations(args.observations) + except ValueError as exc: + parser.error(str(exc)) if not 1 <= args.requests_per_level <= MAX_REQUESTS_PER_LEVEL: parser.error(f"requests-per-level must be between 1 and {MAX_REQUESTS_PER_LEVEL}") if not 100 <= args.timeout_ms <= 5_000: @@ -361,7 +369,7 @@ def main(argv: list[str] | None = None) -> int: from pipeline.tests.e2e.fixture import E2EEnvironment env = E2EEnvironment().start() try: - report = run_rehearsal(env, args.observations, levels, args.requests_per_level, args.timeout_ms) + report = run_rehearsal(env, observations, levels, args.requests_per_level, args.timeout_ms) finally: env.stop() except (ValueError, RuntimeError) as exc: diff --git a/pipeline/tests/test_api_load_rehearsal.py b/pipeline/tests/test_api_load_rehearsal.py index dca5d5d..de37efb 100644 --- a/pipeline/tests/test_api_load_rehearsal.py +++ b/pipeline/tests/test_api_load_rehearsal.py @@ -14,6 +14,12 @@ class ApiLoadRehearsalTests(unittest.TestCase): + def test_requested_scale_is_supported_without_unbounded_seeding(self): + self.assertEqual(MODULE.MAX_SEED, 25_000) + self.assertEqual(MODULE.validate_observations(25_000), 25_000) + with self.assertRaises(ValueError): + MODULE.validate_observations(25_001) + def test_levels_and_targets_are_bounded(self): self.assertEqual(MODULE.validate_levels([1, 4, 8, 16]), (1, 4, 8, 16)) for levels in ([], [0], [17], [1, 1]): diff --git a/pipeline/tests/test_graph_migrations.py b/pipeline/tests/test_graph_migrations.py index 2afd319..39c2b1f 100644 --- a/pipeline/tests/test_graph_migrations.py +++ b/pipeline/tests/test_graph_migrations.py @@ -20,7 +20,7 @@ def test_reserved_migrations_are_present_and_ordered(self): positions = [migrations.index(name) for name in graph_migrations] self.assertEqual(positions, sorted(positions)) self.assertEqual([migrations[position] for position in positions], graph_migrations) - self.assertEqual(migrations[-8:], [ + self.assertEqual(migrations[-9:], [ "026_graph_entities_crosswalks.sql", "027_graph_relationship_observations.sql", "028_graph_claims_support.sql", @@ -29,6 +29,7 @@ def test_reserved_migrations_are_present_and_ordered(self): "031_public_release_read_path_indexes.sql", "032_flatten_public_history_view.sql", "033_release_summary_component.sql", + "034_public_eligibility_join_indexes.sql", ]) def test_entities_are_distinct_and_crosswalk_is_scoped(self): diff --git a/pipeline/tests/test_source_registry.py b/pipeline/tests/test_source_registry.py index 3d84567..c78b7a5 100644 --- a/pipeline/tests/test_source_registry.py +++ b/pipeline/tests/test_source_registry.py @@ -9,8 +9,8 @@ class SourceRegistryTests(unittest.TestCase): def test_repository_registry_loads_and_references_existing_legacy_paths(self): registry = load_registry() - self.assertEqual(len(registry["sources"]), 227) - self.assertEqual(len({source["source_id"] for source in registry["sources"]}), 227) + self.assertEqual(len(registry["sources"]), 233) + self.assertEqual(len({source["source_id"] for source in registry["sources"]}), 233) def test_unknowns_are_explicit(self): registry = json.loads(REGISTRY_PATH.read_text(encoding="utf-8")) From 4f0e43f1313b41a3c066ee190abd47e41d1f84b5 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Wed, 16 Sep 2026 15:01:37 -0700 Subject: [PATCH 189/311] Add private real-data corpus inventory harness --- data/manifests/real-data-corpus-report.json | 111 ++++++++++++++++++ docs/real-data-regression-corpus.md | 25 ++++ .../scripts/diagnostics/real_corpus_report.py | 80 +++++++++++++ pipeline/tests/test_real_corpus_report.py | 22 ++++ 4 files changed, 238 insertions(+) create mode 100644 data/manifests/real-data-corpus-report.json create mode 100644 docs/real-data-regression-corpus.md create mode 100644 pipeline/scripts/diagnostics/real_corpus_report.py create mode 100644 pipeline/tests/test_real_corpus_report.py diff --git a/data/manifests/real-data-corpus-report.json b/data/manifests/real-data-corpus-report.json new file mode 100644 index 0000000..0d8b1d1 --- /dev/null +++ b/data/manifests/real-data-corpus-report.json @@ -0,0 +1,111 @@ +{ + "artifact_entries": 47, + "capture_states": { + "access_blocked_for_reproducible_capture": 1, + "browser_snapshot_only": 5, + "metadata_only": 16, + "rendered_metadata_only": 1, + "route_only": 10, + "row_artifact": 13, + "tracked_metadata_only": 1 + }, + "country_values": [], + "known_input_rows": 8821, + "limitations": [ + "metadata-only and route-only entries are not records", + "unknown counts are not treated as zero", + "no publication release or approval is created" + ], + "manifest_files": 4, + "privacy_boundary": "row-free manifest and aggregate report; raw/derived records remain private and ignored", + "report_version": "real-data-corpus-v1", + "source_ids": [ + "eu.eurostat.nl-slaughter", + "eu.traces.pl-approved-food", + "ie.cro.companies", + "ie.cso.livestock-slaughterings", + "ie.dafm.animal-welfare-controls", + "ie.dafm.approved-establishments", + "ie.dafm.national-beef-kill", + "ie.dafm.seafood-processing-funding", + "ie.epa.leap", + "ie.fsai.approved-directory", + "ie.fsai.enforcement-orders", + "ie.hse.low-throughput-meat", + "ie.planning.npad", + "ie.sfpa.approved-establishments", + "ie.sfpa.factory-vessels", + "ie.sfpa.freezer-vessels", + "nl.cbs.livestock", + "nl.cbs.slaughter", + "nl.cokz.dairy-eggs", + "nl.koop.local-permits", + "nl.nvwa.approved-food", + "nl.nvwa.welfare-enforcement", + "nl.pdok.omgevingswet", + "pl.arimr.processing-support", + "pl.gdos.eia", + "pl.geoportal.urban-planning", + "pl.gios.ippc", + "pl.giw.abp", + "pl.giw.approved-food", + "pl.giw.registered-food", + "pl.giw.rrw", + "pl.gus.regon-bir", + "pl.gus.slaughter", + "pl.krs.open-api", + "pl.private.recon-metadata" + ], + "sources_with_known_row_counts": 15, + "strata": { + "accepted_quarantined": "not available without private run manifests", + "category": "not available without private normalized rows", + "coordinate_precision": "not available without private normalized rows", + "country": [], + "identity_quality": "not available without private normalized rows", + "source": [ + "eu.eurostat.nl-slaughter", + "eu.traces.pl-approved-food", + "ie.cro.companies", + "ie.cso.livestock-slaughterings", + "ie.dafm.animal-welfare-controls", + "ie.dafm.approved-establishments", + "ie.dafm.national-beef-kill", + "ie.dafm.seafood-processing-funding", + "ie.epa.leap", + "ie.fsai.approved-directory", + "ie.fsai.enforcement-orders", + "ie.hse.low-throughput-meat", + "ie.planning.npad", + "ie.sfpa.approved-establishments", + "ie.sfpa.factory-vessels", + "ie.sfpa.freezer-vessels", + "nl.cbs.livestock", + "nl.cbs.slaughter", + "nl.cokz.dairy-eggs", + "nl.koop.local-permits", + "nl.nvwa.approved-food", + "nl.nvwa.welfare-enforcement", + "nl.pdok.omgevingswet", + "pl.arimr.processing-support", + "pl.gdos.eia", + "pl.geoportal.urban-planning", + "pl.gios.ippc", + "pl.giw.abp", + "pl.giw.approved-food", + "pl.giw.registered-food", + "pl.giw.rrw", + "pl.gus.regon-bir", + "pl.gus.slaughter", + "pl.krs.open-api", + "pl.private.recon-metadata" + ] + }, + "target_assessment": { + "met": false, + "reason": "available local manifests do not provide 25,000 counted real normalized records across five countries and eight profiles", + "requested_countries": 5, + "requested_min_rows": 25000, + "requested_source_profiles": 8 + } +} diff --git a/docs/real-data-regression-corpus.md b/docs/real-data-regression-corpus.md new file mode 100644 index 0000000..be0ab2b --- /dev/null +++ b/docs/real-data-regression-corpus.md @@ -0,0 +1,25 @@ +# Private real-data regression corpus + +`pipeline/scripts/diagnostics/real_corpus_report.py` inventories authorized +local acquisition manifests and emits a row-free JSON report. It is safe to +commit because it never reads or copies raw rows, addresses, coordinates, or +derived records. Run: + +```text +python pipeline/scripts/diagnostics/real_corpus_report.py --output data/manifests/real-data-corpus-report.json +``` + +The report distinguishes counted row artifacts from metadata-only and +route-only observations. Unknown counts remain unknown. It records the +requested Sprint 4 threshold (25,000 records, five countries, eight source +profiles) and explicitly reports failure when the local authorized corpus +cannot substantiate it. This checkout currently has no reproducible corpus at +that threshold: most retained evidence is metadata-only, and no release or +publication approval is implied. + +Future adapters should add private run manifests with source URL, retrieval +time, SHA-256, byte size, code/config versions, input/normalized/quarantine +counts, and aggregate strata. Raw artifacts remain under ignored private +storage. Reruns should write a new manifest; a missing source is +`not-observed`, not closure. Candidate handoff and test-only API checks remain +separate gates. diff --git a/pipeline/scripts/diagnostics/real_corpus_report.py b/pipeline/scripts/diagnostics/real_corpus_report.py new file mode 100644 index 0000000..6058b01 --- /dev/null +++ b/pipeline/scripts/diagnostics/real_corpus_report.py @@ -0,0 +1,80 @@ +"""Build a row-free, deterministic report of locally available real-data runs. + +Raw and derived records are intentionally not read into the report. This +tool inventories manifests and only uses aggregate counts already recorded by +authorized private acquisition jobs. Missing counts are reported as unknown, +never as zero. +""" +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path +from typing import Any + + +def _json_files(root: Path) -> list[Path]: + return sorted(root.rglob("*.json")) if root.exists() else [] + + +def _artifacts(payload: dict[str, Any], path: Path) -> list[dict[str, Any]]: + if isinstance(payload.get("artifacts"), list): + return [dict(item, _manifest=str(path)) for item in payload["artifacts"] if isinstance(item, dict)] + if payload.get("source_id"): + return [dict(payload, _manifest=str(path))] + return [] + + +def build_report(manifest_root: Path) -> dict[str, Any]: + artifacts: list[dict[str, Any]] = [] + for path in _json_files(manifest_root): + try: + artifacts.extend(_artifacts(json.loads(path.read_text(encoding="utf-8")), path)) + except (OSError, json.JSONDecodeError): + continue + sources = sorted({item["source_id"] for item in artifacts if item.get("source_id")}) + countries = sorted({item.get("country_code") or item.get("country") for item in artifacts if item.get("country_code") or item.get("country")}) + known_rows = 0 + row_sources = 0 + states: dict[str, int] = {} + for item in artifacts: + rows = item.get("input_rows", item.get("rows")) + if isinstance(rows, int) and rows >= 0: + known_rows += rows + row_sources += 1 + state = item.get("capture_state") or item.get("capture_status") or ("row_artifact" if isinstance(rows, int) else "metadata_only") + states[state] = states.get(state, 0) + 1 + return { + "report_version": "real-data-corpus-v1", + "privacy_boundary": "row-free manifest and aggregate report; raw/derived records remain private and ignored", + "manifest_files": len(_json_files(manifest_root)), + "artifact_entries": len(artifacts), + "source_ids": sources, + "country_values": countries, + "known_input_rows": known_rows, + "sources_with_known_row_counts": row_sources, + "strata": {"country": countries, "source": sources, "category": "not available without private normalized rows", "accepted_quarantined": "not available without private run manifests", "coordinate_precision": "not available without private normalized rows", "identity_quality": "not available without private normalized rows"}, + "capture_states": dict(sorted(states.items())), + "target_assessment": {"requested_min_rows": 25000, "requested_countries": 5, "requested_source_profiles": 8, "met": False, "reason": "available local manifests do not provide 25,000 counted real normalized records across five countries and eight profiles"}, + "limitations": ["metadata-only and route-only entries are not records", "unknown counts are not treated as zero", "no publication release or approval is created"], + } + + +def canonical_bytes(value: dict[str, Any]) -> bytes: + return (json.dumps(value, ensure_ascii=False, sort_keys=True, indent=2) + "\n").encode() + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--manifest-root", type=Path, default=Path("data/manifests")) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + payload = canonical_bytes(build_report(args.manifest_root)) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_bytes(payload) + print(json.dumps({"output": str(args.output), "sha256": hashlib.sha256(payload).hexdigest()})) + + +if __name__ == "__main__": + main() diff --git a/pipeline/tests/test_real_corpus_report.py b/pipeline/tests/test_real_corpus_report.py new file mode 100644 index 0000000..2305f63 --- /dev/null +++ b/pipeline/tests/test_real_corpus_report.py @@ -0,0 +1,22 @@ +import json +from pathlib import Path + +from pipeline.scripts.diagnostics.real_corpus_report import build_report + + +def test_report_is_honest_about_metadata_only_sources(tmp_path: Path): + (tmp_path / "manifest.json").write_text(json.dumps({"country": "XX", "artifacts": [ + {"source_id": "xx.rows", "rows": 12, "capture_status": "row_artifact"}, + {"source_id": "xx.route", "rows": None, "capture_status": "route_only"}, + ]})) + report = build_report(tmp_path) + assert report["known_input_rows"] == 12 + assert report["artifact_entries"] == 2 + assert report["target_assessment"]["met"] is False + assert report["capture_states"] == {"route_only": 1, "row_artifact": 1} + + +def test_missing_manifest_root_is_empty_not_success(tmp_path: Path): + report = build_report(tmp_path / "missing") + assert report["known_input_rows"] == 0 + assert report["target_assessment"]["met"] is False From 6899340fe5b3618c8b9d46005c861941dc58fc53 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Wed, 16 Sep 2026 15:01:41 -0700 Subject: [PATCH 190/311] Add cross-platform V2 developer entrypoint --- docs/development.md | 15 ++++++++ scripts/dev.ps1 | 4 +++ scripts/dev.py | 83 +++++++++++++++++++++++++++++++++++++++++++++ scripts/test_dev.py | 19 +++++++++++ 4 files changed, 121 insertions(+) create mode 100644 docs/development.md create mode 100644 scripts/dev.ps1 create mode 100644 scripts/dev.py create mode 100644 scripts/test_dev.py diff --git a/docs/development.md b/docs/development.md new file mode 100644 index 0000000..dcdcffc --- /dev/null +++ b/docs/development.md @@ -0,0 +1,15 @@ +# Developer entrypoint + +Use `python scripts/dev.py --help` (or `scripts/dev.ps1 --help` on Windows) to discover the V2 workflow. Commands are thin wrappers around the existing project tools: + +```text +doctor prerequisites, ports, configuration, and optional DB reachability +up/down/status/probe local V2 environment +logs recent Compose logs +test [--full] JavaScript tests (full runs the configured suite) +pipeline [pytest args] Python pipeline tests +contracts database and adapter contract tests +review-packet RUN_DIR deterministic, row-free private review packet +``` + +`python scripts/dev.py --json doctor` (and any command with the global `--json` flag) emits a final machine-readable summary. Diagnostics report only whether environment variables are set; values and database credentials are never printed. `up` may apply migrations and seed the synthetic local fixture through the existing `local-v2.ps1` workflow. It does not publish a release. diff --git a/scripts/dev.ps1 b/scripts/dev.ps1 new file mode 100644 index 0000000..15b2e48 --- /dev/null +++ b/scripts/dev.ps1 @@ -0,0 +1,4 @@ +param([Parameter(ValueFromRemainingArguments=$true)][string[]]$Args) +$ErrorActionPreference = 'Stop' +python (Join-Path $PSScriptRoot 'dev.py') @Args +exit $LASTEXITCODE diff --git a/scripts/dev.py b/scripts/dev.py new file mode 100644 index 0000000..4d0e97d --- /dev/null +++ b/scripts/dev.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 +"""Cross-platform developer entrypoint for UntilEveryCage V2. + +This is intentionally a thin dispatcher: project-specific behavior remains in +the existing PowerShell and Python scripts. +""" +from __future__ import annotations +import argparse, json, os, shutil, socket, subprocess, sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +LOCAL_V2 = ROOT / "pipeline" / "scripts" / "maintenance" / "local-v2.ps1" +SUMMARY = {"command": None, "ok": False, "exit_code": None, "checks": []} + +def run(args: list[str], *, cwd=ROOT) -> int: + return subprocess.run(args, cwd=cwd).returncode + +def check(name: str, ok: bool, detail: str) -> None: + SUMMARY["checks"].append({"name": name, "ok": ok, "detail": detail}) + +def doctor(_: argparse.Namespace) -> int: + for name in ("docker", "python", "cargo", "node"): + path = shutil.which(name) + check(name, bool(path), "available" if path else "not found") + compose = shutil.which("docker") is not None + if compose: + result = subprocess.run(["docker", "compose", "version"], capture_output=True, text=True) + check("compose", result.returncode == 0, "available" if result.returncode == 0 else "unavailable") + else: check("compose", False, "Docker not found") + if sys.platform == "win32": check("powershell", bool(shutil.which("powershell") or shutil.which("pwsh")), "required by local-v2") + for port in (8000, 5433): + sock = socket.socket(); sock.settimeout(.2) + try: busy = sock.connect_ex(("127.0.0.1", port)) == 0 + finally: sock.close() + check(f"port:{port}", not busy, "free" if not busy else "in use") + required = ("UEC_RUNTIME_MODE", "UEC_DATABASE_URL") + for key in required: + # Only report presence; never print values. + check(f"env:{key}", bool(os.environ.get(key)), "set" if os.environ.get(key) else "not set (local defaults may apply)") + state = ROOT / "target" / "local-v2" + check("local-v2-state", not (state.exists() and (state / "uec-api.pid").exists() and not (state / "uec-api.log").exists()), "consistent or absent") + db = os.environ.get("UEC_DATABASE_URL") + if db: + try: + import psycopg + with psycopg.connect(db, connect_timeout=2): pass + check("database", True, "reachable") + except Exception: check("database", False, "unreachable or driver unavailable") + else: check("database", None, "not probed; UEC_DATABASE_URL is unset") + return 0 if all(c["ok"] is not False for c in SUMMARY["checks"]) else 1 + +def main() -> int: + p = argparse.ArgumentParser(prog="dev.py", description="UntilEveryCage V2 developer tools") + p.add_argument("--json", action="store_true", help="emit a machine-readable final summary") + sub = p.add_subparsers(dest="command", required=True) + sub.add_parser("doctor", help="check local prerequisites and safe configuration") + for name in ("up", "down", "status", "logs", "probe"): + sub.add_parser(name, help=f"local V2 {name}") + sub.add_parser("test", help="fast JavaScript tests").add_argument("--full", action="store_true") + sub.add_parser("pipeline", help="run Python pipeline tests").add_argument("args", nargs=argparse.REMAINDER) + sub.add_parser("contracts", help="run contract tests") + rp = sub.add_parser("review-packet", help="generate a private row-free review packet") + rp.add_argument("run_dir"); rp.add_argument("--previous-normalized") + args = p.parse_args(); SUMMARY["command"] = args.command + if args.command == "doctor": code = doctor(args) + elif args.command in ("up", "down", "status", "probe"): code = run(["powershell", "-ExecutionPolicy", "Bypass", "-File", str(LOCAL_V2), {"up":"start","down":"stop"}.get(args.command,args.command)]) + elif args.command == "logs": code = run(["docker", "compose", "-p", "uec-local-v2", "-f", "docker-compose.pipeline.yml", "logs", "--tail=100"]) + elif args.command == "test": code = run(["npm", "test", "--", *( ["--runInBand"] if not args.full else [])]) + elif args.command == "pipeline": + runner = "pytest" if shutil.which("pytest") else "unittest" + code = run([sys.executable, "-m", runner, *(args.args or ["discover", "-s", "pipeline"])]) + elif args.command == "contracts": + runner = "pytest" if shutil.which("pytest") else "unittest" + targets = ["pipeline/contracts", "pipeline/tests/test_database_contract.py", "pipeline/tests/test_graph_database_contract.py"] if runner == "pytest" else ["discover", "-s", "pipeline/contracts"] + code = run([sys.executable, "-m", runner, *targets]) + else: + cmd = [sys.executable, "-c", "from pipeline.common.review_packet import write_review_packet; import sys; write_review_packet(sys.argv[1], previous_normalized_path=sys.argv[2] if len(sys.argv)>2 else None)", args.run_dir] + if args.previous_normalized: cmd.append(args.previous_normalized) + code = run(cmd) + SUMMARY["ok"], SUMMARY["exit_code"] = code == 0, code + print(json.dumps(SUMMARY) if args.json else ("OK" if code == 0 else "FAILED") + f": {args.command}") + return code +if __name__ == "__main__": raise SystemExit(main()) diff --git a/scripts/test_dev.py b/scripts/test_dev.py new file mode 100644 index 0000000..79b26f7 --- /dev/null +++ b/scripts/test_dev.py @@ -0,0 +1,19 @@ +import json, os, subprocess, sys, unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + +class DevEntrypointTests(unittest.TestCase): + def test_help(self): + result = subprocess.run([sys.executable, "scripts/dev.py", "--help"], cwd=ROOT, capture_output=True, text=True) + self.assertEqual(result.returncode, 0) + self.assertIn("doctor", result.stdout) + self.assertIn("review-packet", result.stdout) + + def test_doctor_json_does_not_echo_secret(self): + result = subprocess.run([sys.executable, "scripts/dev.py", "--json", "doctor"], cwd=ROOT, env={**os.environ, "UEC_DATABASE_URL": "postgresql://secret.invalid/db"}, capture_output=True, text=True) + payload = json.loads(result.stdout) + self.assertEqual(payload["command"], "doctor") + self.assertNotIn("secret.invalid", result.stdout) + +if __name__ == "__main__": unittest.main() From 2ee592955d3ffd91f4cd44c9f47cd6ce18fa996f Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Wed, 16 Sep 2026 15:02:58 -0700 Subject: [PATCH 191/311] test: bind V2 location contract surfaces --- docs/api/v2-location.schema.json | 14 ++++++++++++++ pipeline/tests/test_v2_contract_drift.py | 15 +++++++++++++++ static/modules/v2Contract.js | 8 +++++++- 3 files changed, 36 insertions(+), 1 deletion(-) create mode 100644 docs/api/v2-location.schema.json create mode 100644 pipeline/tests/test_v2_contract_drift.py diff --git a/docs/api/v2-location.schema.json b/docs/api/v2-location.schema.json new file mode 100644 index 0000000..0dfb5f1 --- /dev/null +++ b/docs/api/v2-location.schema.json @@ -0,0 +1,14 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://untileverycage.org/schemas/uec-v2-location-v1.json", + "title": "Until Every Cage V2 public location", + "type": "object", + "additionalProperties": false, + "required": ["facility_id", "canonical_name", "country_code", "city", "category", "publication_profile", "factual_review_status", "privacy_screening_status", "project_approval", "reviewer_role", "publication_warning", "display_precision", "latitude", "longitude", "first_observed_at", "last_observed_at", "observation_count", "lifecycle_status", "source_type", "source_rights_status", "provenance_source", "release_id", "release_ruleset_version", "provenance_source_id", "provenance_source_name", "provenance_source_url", "provenance_retrieved_at"], + "properties": { + "facility_id": {"type": "string", "format": "uuid"}, "canonical_name": {"type": ["string", "null"]}, "country_code": {"type": "string", "pattern": "^[A-Z]{2}$"}, "city": {"type": ["string", "null"]}, "category": {"type": "string"}, + "publication_profile": {"enum": ["official", "secondary", "community"]}, "factual_review_status": {"type": "string"}, "privacy_screening_status": {"const": "passed"}, "project_approval": {"type": "string"}, "reviewer_role": {"type": ["string", "null"]}, "publication_warning": {"type": ["string", "null"]}, + "display_precision": {"enum": ["exact", "city", "unmapped"]}, "latitude": {"type": ["number", "null"]}, "longitude": {"type": ["number", "null"]}, "first_observed_at": {"type": ["string", "null"], "format": "date-time"}, "last_observed_at": {"type": ["string", "null"], "format": "date-time"}, "observation_count": {"type": ["integer", "null"]}, + "lifecycle_status": {"enum": ["active_observed", "explicitly_closed", "not_seen_recently", "status_unknown"]}, "source_type": {"enum": ["official", "secondary", "user_submitted"]}, "source_rights_status": {"type": "string"}, "provenance_source": {"type": ["string", "null"]}, "release_id": {"type": "string"}, "release_ruleset_version": {"type": "string"}, "provenance_source_id": {"type": "string"}, "provenance_source_name": {"type": "string"}, "provenance_source_url": {"type": "string", "format": "uri"}, "provenance_retrieved_at": {"type": "string", "format": "date-time"} + } +} diff --git a/pipeline/tests/test_v2_contract_drift.py b/pipeline/tests/test_v2_contract_drift.py new file mode 100644 index 0000000..0b0b618 --- /dev/null +++ b/pipeline/tests/test_v2_contract_drift.py @@ -0,0 +1,15 @@ +"""Keep Rust, the canonical schema, and the JS validator in lockstep.""" +import json, re, unittest +from pathlib import Path +ROOT = Path(__file__).parents[2] +class V2ContractDriftTests(unittest.TestCase): + def test_location_fields_match_all_contract_surfaces(self): + rust = (ROOT / "src/lib.rs").read_text(encoding="utf-8") + block = re.search(r"pub struct V2Location \{(.*?)\n\}", rust, re.S).group(1) + rust_fields = set(re.findall(r"pub (\w+):", block)) + schema = json.loads((ROOT / "docs/api/v2-location.schema.json").read_text(encoding="utf-8")) + self.assertEqual(rust_fields, set(schema["properties"])) + js = (ROOT / "static/modules/v2Contract.js").read_text(encoding="utf-8") + required = set(re.findall(r"'([a-z_]+)'", re.search(r"for \(const field of \[(.*?)\]\)", js, re.S).group(1))) + required |= {"canonical_name", "city", "reviewer_role", "publication_warning", "latitude", "longitude", "first_observed_at", "last_observed_at", "observation_count", "provenance_source"} + self.assertEqual(rust_fields, required) diff --git a/static/modules/v2Contract.js b/static/modules/v2Contract.js index 1269ac8..87e3d10 100644 --- a/static/modules/v2Contract.js +++ b/static/modules/v2Contract.js @@ -23,12 +23,18 @@ export function validateV2Location(record) { for (const field of ['facility_id', 'country_code', 'category', 'publication_profile', 'factual_review_status', 'privacy_screening_status', 'project_approval', 'display_precision', 'lifecycle_status', 'source_type', 'release_id', 'release_ruleset_version', 'provenance_source_id', 'provenance_source_name', - 'provenance_source_url', 'provenance_retrieved_at']) { + 'provenance_source_url', 'provenance_retrieved_at', 'source_rights_status', 'release_id', + 'release_ruleset_version', 'provenance_source_id', 'provenance_source_name']) { if (!hasString(record, field)) throw new TypeError(`V2 location missing ${field}`); } for (const field of ['canonical_name', 'city', 'reviewer_role']) { if (!hasNullableString(record, field)) throw new TypeError(`V2 location has invalid ${field}`); } + for (const field of ['latitude', 'longitude', 'first_observed_at', 'last_observed_at', 'observation_count', 'provenance_source']) { + if (record[field] !== null && (typeof record[field] !== 'number' && typeof record[field] !== 'string')) { + throw new TypeError(`V2 location has invalid ${field}`); + } + } if (!hasNullableString(record, 'publication_warning')) throw new TypeError('V2 location has invalid publication_warning'); if (!V2_PROFILES.includes(record.publication_profile)) throw new TypeError('V2 location has invalid publication_profile'); if (!V2_SOURCE_TYPES.includes(record.source_type)) throw new TypeError('V2 location has invalid source_type'); From 2038e5eca0c896f97209de9fb0d078c3de9e77a1 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Wed, 16 Sep 2026 15:07:22 -0700 Subject: [PATCH 192/311] Add private real corpus regression harness --- .../sprint4-real-corpus-regression.json | 59 ++++++++ docs/real-corpus-regression.md | 22 +++ .../diagnostics/real_corpus_regression.py | 134 ++++++++++++++++++ pipeline/tests/test_real_corpus_regression.py | 38 +++++ 4 files changed, 253 insertions(+) create mode 100644 data/manifests/sprint4-real-corpus-regression.json create mode 100644 docs/real-corpus-regression.md create mode 100644 pipeline/scripts/diagnostics/real_corpus_regression.py create mode 100644 pipeline/tests/test_real_corpus_regression.py diff --git a/data/manifests/sprint4-real-corpus-regression.json b/data/manifests/sprint4-real-corpus-regression.json new file mode 100644 index 0000000..a806b77 --- /dev/null +++ b/data/manifests/sprint4-real-corpus-regression.json @@ -0,0 +1,59 @@ +{ + "schema_version": "real-corpus-regression-v1", + "as_of_utc": "2026-09-16T00:00:00Z", + "corpus_state": "private-regression-only", + "publication_eligibility": "blocked", + "selection": { + "method": "stable-sha256-row-fingerprint", + "max_records": 50000, + "available_records": 50750, + "selected_records": 50000 + }, + "coverage": { + "country_count": 9, + "source_profile_count": 9, + "selected_records_by_source": { + "v1.ca.locations": 1309, + "v1.de.locations": 9121, + "v1.dk.locations": 1539, + "v1.es.locations": 4168, + "v1.fr.locations": 3148, + "v1.mx.locations": 14615, + "v1.nz.locations": 1629, + "v1.uk.locations": 7475, + "v1.us.locations": 6996 + } + }, + "funnel": { + "selected": 50000, + "accepted_private": 47978, + "quarantined": 2022, + "quarantine_rule": "missing/weak identity or missing/invalid source coordinates", + "row_count_reconciles": true + }, + "source_files": [ + {"country":"ca","path":"static_data/ca/locations.csv","bytes":361801,"sha256":"544fd7809926143d4e934f1c601e981a07b0137780f86de753ef3c54415e45f4"}, + {"country":"de","path":"static_data/de/locations.csv","bytes":3849223,"sha256":"88eb00f6ae405489a7e873582aa65610ecaa3c9b475bc0046e9e8c32f48ac809"}, + {"country":"dk","path":"static_data/dk/locations.csv","bytes":323380,"sha256":"b38f8165d6ea3617a13dbad950d0070490e645473bb851ddd3d907004a33b32f"}, + {"country":"es","path":"static_data/es/locations.csv","bytes":1093952,"sha256":"93d36d6a760a4435c836348021fb03d98ddb4947345338ea8c199321af3e5347"}, + {"country":"fr","path":"static_data/fr/locations.csv","bytes":626026,"sha256":"aeddafa5ebc19d63b6dd57777851e3e5590b5d5ddfb9cbeff37a552227e1c615"}, + {"country":"mx","path":"static_data/mx/locations.csv","bytes":5039068,"sha256":"bc9b3adc80e4673bb8b996c7600b52b78e7973b1a1859608322a293aff9efdee"}, + {"country":"nz","path":"static_data/nz/locations.csv","bytes":410481,"sha256":"636d80b33cc29dbd68b3c44ddf39c57db84cdd661d1c10dfab4227f8a87a3a87"}, + {"country":"uk","path":"static_data/uk/locations.csv","bytes":2193258,"sha256":"4a14793c227afe558bfacfe45b05f3ae4e5332a3d502991050257320c5b53511"}, + {"country":"us","path":"static_data/us/locations.csv","bytes":4099548,"sha256":"2dca259076a16a324ad9565d2d4057aa0f5e6e51bbc25a8dc1c80a4e4fbdf5c7"} + ], + "stages": { + "raw_preservation": "not_exercised; existing V1-derived snapshot hashes only", + "normalization": "existing V1-derived rows measured; no new normalization claimed", + "quarantine": "exercised by aggregate identity/coordinate gate", + "candidate_handoff": "not_exercised", + "private_import": "not_exercised", + "test_only_api_export": "not_exercised" + }, + "limitations": [ + "No raw source bytes are present in this checkout; this is not a reproducible raw-acquisition corpus.", + "The nine V1 files provide nine country/source profiles, but not eight independently acquired V2 adapters.", + "Row-level values, identifiers, addresses, coordinates, and samples are intentionally omitted.", + "No publication, release, or human review authorization is implied." + ] +} diff --git a/docs/real-corpus-regression.md b/docs/real-corpus-regression.md new file mode 100644 index 0000000..f9674fe --- /dev/null +++ b/docs/real-corpus-regression.md @@ -0,0 +1,22 @@ +# Sprint 4 real-corpus regression lane + +`pipeline/scripts/diagnostics/real_corpus_regression.py` produces a private, +row-free report from the existing `static_data/*/locations.csv` snapshots. +The default run hashes each input, selects at most 50,000 rows by a stable +row fingerprint, and reports country/source/category/accepted-quarantine/ +coordinate-precision/identity-quality strata without writing row data. + +Run locally with: + +```powershell +python pipeline/scripts/diagnostics/real_corpus_regression.py ` + --output sprint4-real-corpus-regression.json ` + --as-of 2026-09-16T00:00:00Z +``` + +The output is private and publication-blocked. The current checkout contains +nine country snapshots and roughly 50k existing V1-derived rows, but no raw +source bytes or V2 normalized handoffs. The report therefore records those +stages as unavailable rather than claiming raw-preserving acquisition, +candidate import, or private API/export coverage. A later run with authorized +raw artifacts can extend the same manifest contract without committing rows. diff --git a/pipeline/scripts/diagnostics/real_corpus_regression.py b/pipeline/scripts/diagnostics/real_corpus_regression.py new file mode 100644 index 0000000..6217331 --- /dev/null +++ b/pipeline/scripts/diagnostics/real_corpus_regression.py @@ -0,0 +1,134 @@ +"""Build a private, row-free regression report from local real source snapshots. + +The input files are existing V1 country snapshots. This harness never copies or +prints rows: it records file hashes, deterministic aggregate strata, and a +stable bounded sample count so a local operator can compare reruns safely. +It does not grant publication approval or imply that V1 files are raw source +captures. +""" +from __future__ import annotations + +import argparse +import csv +import hashlib +import json +from collections import Counter +from pathlib import Path +from typing import Iterable + + +DEFAULT_COUNTRIES = ("ca", "de", "dk", "es", "fr", "mx", "nz", "uk", "us") + + +def _number(value: str | None) -> float | None: + try: + return float(value) if value else None + except (TypeError, ValueError): + return None + + +def _valid_point(latitude: float | None, longitude: float | None) -> bool: + return latitude is not None and longitude is not None and -90 <= latitude <= 90 and -180 <= longitude <= 180 and (latitude != 0 or longitude != 0) + + +def _precision(value: str | None) -> str: + if not value or _number(value) is None: + return "missing" + digits = len(value.strip().lstrip("+-").split(".", 1)[1].rstrip("0")) if "." in value else 0 + return "source-decimal-" + str(min(digits, 6)) + + +def _category(row: dict[str, str]) -> str: + keys = {key for key, value in row.items() if value and value.strip().lower() not in {"0", "false", "no", "none", "nan"}} + if any("slaughter" in key for key in keys): + return "slaughter-or-processing" + if any("process" in key for key in keys): + return "processing" + return "other" + + +def _identity_quality(row: dict[str, str]) -> str: + has_id = bool((row.get("establishment_id") or row.get("establishment_number") or "").strip()) + has_name = bool((row.get("establishment_name") or "").strip()) + has_city = bool((row.get("city") or "").strip()) + if has_id and has_name and has_city: + return "strong" + if has_id and (has_name or has_city): + return "partial" + return "weak-or-missing" + + +def _row_fingerprint(country: str, row: dict[str, str]) -> str: + key = "\0".join((country, row.get("establishment_id", ""), row.get("establishment_number", ""), row.get("establishment_name", ""), row.get("city", ""))) + return hashlib.sha256(key.encode("utf-8", "surrogateescape")).hexdigest() + + +def _files(root: Path, countries: Iterable[str]) -> list[Path]: + paths = [root / country / "locations.csv" for country in countries] + missing = [path.as_posix() for path in paths if not path.is_file()] + if missing: + raise ValueError("missing real corpus inputs: " + ", ".join(missing)) + return paths + + +def build_report(root: Path, *, countries: Iterable[str] = DEFAULT_COUNTRIES, max_records: int = 50_000, as_of: str = "") -> dict: + if max_records <= 0: + raise ValueError("max_records must be positive") + paths = _files(root, countries) + candidates: list[tuple[str, Path, dict[str, str]]] = [] + source_files = [] + for path in paths: + country = path.parent.name + source_files.append({"country": country, "path": path.as_posix(), "sha256": hashlib.sha256(path.read_bytes()).hexdigest(), "bytes": path.stat().st_size, "input_kind": "existing-v1-derived-snapshot"}) + with path.open("r", encoding="utf-8-sig", newline="") as handle: + for row in csv.DictReader(handle): + candidates.append((_row_fingerprint(country, row), path, row)) + selected = sorted(candidates, key=lambda item: item[0])[:max_records] + strata = Counter() + countries_count = Counter() + sources_count = Counter() + accepted = quarantined = 0 + for _, path, row in selected: + country = path.parent.name + identity = _identity_quality(row) + coordinate_state = "valid" if _valid_point(_number(row.get("latitude")), _number(row.get("longitude"))) else "missing-or-invalid" + state = "quarantined" if identity == "weak-or-missing" or coordinate_state == "missing-or-invalid" else "accepted-private" + accepted += state == "accepted-private" + quarantined += state == "quarantined" + countries_count[country] += 1 + sources_count[f"v1.{country}.locations"] += 1 + latitude, longitude = _number(row.get("latitude")), _number(row.get("longitude")) + coordinate_state = "valid" if _valid_point(latitude, longitude) else "missing-or-invalid" + precision = _precision(row.get("latitude")) if coordinate_state == "valid" else "missing" + strata[(country, f"v1.{country}.locations", _category(row), state, precision, identity)] += 1 + return { + "schema_version": "real-corpus-regression-v1", + "as_of_utc": as_of or None, + "corpus_state": "private-regression-only", + "publication_eligibility": "blocked", + "selection": {"method": "stable-sha256-row-fingerprint", "max_records": max_records, "selected_records": len(selected), "available_records": len(candidates)}, + "coverage": {"country_count": len(paths), "source_profile_count": len(paths), "countries": dict(sorted(countries_count.items())), "sources": dict(sorted(sources_count.items()))}, + "funnel": {"selected": len(selected), "accepted_private": accepted, "quarantined": quarantined, "row_count_reconciles": len(selected) == accepted + quarantined}, + "strata": {"|".join(key): value for key, value in sorted(strata.items())}, + "source_files": source_files, + "stages": {"raw_preservation": "not_exercised; only existing V1 snapshot hashes available", "normalization": "measured existing V1 derived rows; no new normalization claimed", "quarantine": "identity-quality quarantine measured", "candidate_handoff": "not_exercised", "private_import": "not_exercised", "test_only_api_export": "not_exercised"}, + "limitations": ["No raw source bytes are present in this checkout; this is not a reproducible raw-acquisition corpus.", "The nine V1 files provide nine country/source profiles, but not eight independently acquired V2 adapters.", "Row-level values, identifiers, addresses, coordinates, and samples are intentionally omitted from the report.", "No publication, release, or human review authorization is implied."], + } + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--root", type=Path, default=Path("static_data")) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--max-records", type=int, default=50_000) + parser.add_argument("--as-of", default="") + args = parser.parse_args() + report = build_report(args.root, max_records=args.max_records, as_of=args.as_of) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(json.dumps({"selected_records": report["selection"]["selected_records"], "countries": report["coverage"]["country_count"], "source_profiles": report["coverage"]["source_profile_count"], "publication_eligibility": report["publication_eligibility"]}, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/pipeline/tests/test_real_corpus_regression.py b/pipeline/tests/test_real_corpus_regression.py new file mode 100644 index 0000000..3980f62 --- /dev/null +++ b/pipeline/tests/test_real_corpus_regression.py @@ -0,0 +1,38 @@ +import tempfile +import unittest +from pathlib import Path + +from pipeline.scripts.diagnostics.real_corpus_regression import build_report + + +class RealCorpusRegressionTests(unittest.TestCase): + def _write(self, root: Path, country: str, rows: str) -> None: + path = root / country + path.mkdir(parents=True) + (path / "locations.csv").write_text(rows, encoding="utf-8") + + def test_report_is_deterministic_and_row_free(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + self._write(root, "aa", "establishment_id,establishment_name,city,latitude,longitude,slaughter\n1,Alpha,Town,52.1,4.2,true\n,No ID,Town,52.2,4.3,true\n") + first = build_report(root, countries=("aa",), max_records=10, as_of="2026-09-16T00:00:00Z") + second = build_report(root, countries=("aa",), max_records=10, as_of="2026-09-16T00:00:00Z") + self.assertEqual(first, second) + self.assertEqual(first["selection"]["selected_records"], 2) + self.assertEqual(first["funnel"]["accepted_private"], 1) + self.assertEqual(first["funnel"]["quarantined"], 1) + self.assertEqual(first["coverage"]["source_profile_count"], 1) + self.assertNotIn("Alpha", str(first)) + + def test_selection_is_bounded(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + self._write(root, "aa", "establishment_id,establishment_name,city\n" + "\n".join(f"{i},Name {i},Town" for i in range(20)) + "\n") + report = build_report(root, countries=("aa",), max_records=7) + self.assertEqual(report["selection"]["available_records"], 20) + self.assertEqual(report["selection"]["selected_records"], 7) + self.assertTrue(report["funnel"]["row_count_reconciles"]) + + +if __name__ == "__main__": + unittest.main() From 6684b84a85a2c748f6acc58035ad134984a6a762 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Wed, 16 Sep 2026 15:35:31 -0700 Subject: [PATCH 193/311] Document safe pipeline contributor path --- pipeline/ONBOARDING.md | 72 +++++++++++++++++++++++ pipeline/README.md | 4 ++ pipeline/scripts/README.md | 4 ++ pipeline/tests/test_real_corpus_report.py | 21 ++++++- 4 files changed, 100 insertions(+), 1 deletion(-) create mode 100644 pipeline/ONBOARDING.md diff --git a/pipeline/ONBOARDING.md b/pipeline/ONBOARDING.md new file mode 100644 index 0000000..639e8dd --- /dev/null +++ b/pipeline/ONBOARDING.md @@ -0,0 +1,72 @@ +# Pipeline contributor onboarding + +This is the shortest safe local path for a new pipeline contributor. Run every +command from the repository root. It uses synthetic fixtures and temporary +directories; it does not acquire data, import a release, or publish anything. + +## 1. Read the boundaries + +Read [the governing ethics policy](../docs/ETHICS.md), then [the source +adapter contract](contracts/README.md) and [the source registry](source_registry.json). +The registry describes source evidence and blockers; it is not an acquisition +permission or publication approval. + +## 2. Run the contract tests + +```powershell +python -m unittest pipeline.sources.denmark.test_adapter pipeline.common.test_orchestrator pipeline.common.test_review_packet +``` + +The Denmark tests are the reference vertical slice. They exercise preserved +input integrity, parsing, normalization, quarantine, private manifests, QA, +health, candidate handoff, and rerun determinism. The shared orchestrator tests +cover registered-input compatibility and suppression behavior. + +Run the complete Python suite before submitting pipeline changes: + +```powershell +python -m unittest discover -s pipeline -p 'test*.py' +``` + +## 3. Inspect private operational evidence + +For an existing private run, build a row-free report from its manifest root: + +```powershell +python pipeline/scripts/diagnostics/real_corpus_report.py ` + --manifest-root data/manifests ` + --output data/reports/real-corpus-report.json +``` + +Unknown row counts remain unknown; they are never converted to zero. The +report is an inventory aid, not a source health claim, release validation, or +publication decision. Keep raw and derived run directories under ignored +`data/raw`, `data/staging`, or `data/restricted`. + +## 4. Follow one source lifecycle + +For Denmark, use the source-owned launcher as the canonical path: + +```powershell +python pipeline/sources/denmark/run-denmark-pipeline.py --help +python pipeline/sources/denmark/run-denmark-pipeline.py path/to/private/Smileydata.xml --output-dir data/staging/denmark-smiley/ +``` + +Acquisition requires an operator-approved terms review and is intentionally +opt-in. Candidate import, geocoding, release validation, and promotion are +separate commands and separate gates. The older `pipeline/run-denmark-pipeline.py` +path remains a compatibility launcher; new source-specific documentation +should link to the source-owned path. + +## Known onboarding friction + +- The repository root README describes the Rust application, while this guide + describes the private V2 pipeline; contributors must choose the pipeline + path before running `cargo run`. +- Both historical and source-owned Denmark launchers exist. The source-owned + launcher is canonical; the historical path is retained for compatibility. +- Full local V2 API startup requires Docker/Postgres and a Rust build. It is + not required for adapter contract tests and must use only the disposable + local configuration in `pipeline/scripts/maintenance/local-v2.ps1`. +- Real corpus reports can only claim what local manifests record. Missing raw + bytes, row counts, or V2 runs are reported as unavailable, not inferred. diff --git a/pipeline/README.md b/pipeline/README.md index a05c613..6878f8a 100644 --- a/pipeline/README.md +++ b/pipeline/README.md @@ -4,6 +4,10 @@ Acquisition, retention, geocoding, and release work follow [docs/ETHICS.md](../d This directory is the local-development home for V2 ingestion code. Acquired and generated data lives under the repository-level `data/` directory. The current application data remains unchanged while the pipeline is being established. +New contributors should start with [pipeline/ONBOARDING.md](ONBOARDING.md), +which provides the safe fixture-first test path and explains the boundary +between private staging, release validation, and public publication. + ## First operation Run the manifest generator from the repository root: diff --git a/pipeline/scripts/README.md b/pipeline/scripts/README.md index 29b29aa..996f09a 100644 --- a/pipeline/scripts/README.md +++ b/pipeline/scripts/README.md @@ -8,6 +8,10 @@ Scripts are grouped by their role in the auditable ingestion workflow: - `diagnostics/` contains read-only inspection and sampling tools. These help evaluate a source or service and are not required for a normal full run. - `maintenance/` contains repository and migration-support utilities, such as the legacy manifest builder. +For the contributor-facing command sequence, see +[pipeline/ONBOARDING.md](../ONBOARDING.md). This page documents script roles; +it is not a second lifecycle tutorial. + `diagnostics/build-source-operations-health.py` reads the private append-only source run ledger and the checked-in schedule inventory to emit a deterministic, row-free health index. It does not acquire data, update `docs/source-status`, diff --git a/pipeline/tests/test_real_corpus_report.py b/pipeline/tests/test_real_corpus_report.py index 2305f63..8448f64 100644 --- a/pipeline/tests/test_real_corpus_report.py +++ b/pipeline/tests/test_real_corpus_report.py @@ -1,7 +1,8 @@ import json +import unittest from pathlib import Path -from pipeline.scripts.diagnostics.real_corpus_report import build_report +from pipeline.scripts.diagnostics.real_corpus_report import build_report, canonical_bytes def test_report_is_honest_about_metadata_only_sources(tmp_path: Path): @@ -20,3 +21,21 @@ def test_missing_manifest_root_is_empty_not_success(tmp_path: Path): report = build_report(tmp_path / "missing") assert report["known_input_rows"] == 0 assert report["target_assessment"]["met"] is False + + +class RealCorpusReportTests(unittest.TestCase): + def test_unknown_counts_are_not_reported_as_zero(self): + report = build_report(Path(__file__).parents[2] / "data" / "manifests") + self.assertGreater(report["manifest_files"], 0) + self.assertGreater(report["capture_states"].get("metadata_only", 0), 0) + self.assertIn("not available", report["strata"]["category"]) + self.assertFalse(report["target_assessment"]["met"]) + + def test_canonical_bytes_are_stable(self): + report = build_report(Path(__file__).parents[2] / "data" / "manifests") + self.assertEqual(canonical_bytes(report), canonical_bytes(report)) + self.assertTrue(canonical_bytes(report).endswith(b"\n")) + + +if __name__ == "__main__": + unittest.main() From 4b327e803990ddf7a151bb90be345875caca59a1 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Wed, 16 Sep 2026 15:39:27 -0700 Subject: [PATCH 194/311] Improve frontend contributor onboarding --- docs/development.md | 19 +++++++++++- frontend/README.md | 48 ++++++++++++++++++++++++++++-- frontend/package.json | 2 +- frontend/scripts/run-local-e2e.mjs | 10 +++++++ scripts/dev.py | 2 +- 5 files changed, 75 insertions(+), 6 deletions(-) create mode 100644 frontend/scripts/run-local-e2e.mjs diff --git a/docs/development.md b/docs/development.md index dcdcffc..241b79c 100644 --- a/docs/development.md +++ b/docs/development.md @@ -12,4 +12,21 @@ contracts database and adapter contract tests review-packet RUN_DIR deterministic, row-free private review packet ``` -`python scripts/dev.py --json doctor` (and any command with the global `--json` flag) emits a final machine-readable summary. Diagnostics report only whether environment variables are set; values and database credentials are never printed. `up` may apply migrations and seed the synthetic local fixture through the existing `local-v2.ps1` workflow. It does not publish a release. +`python scripts/dev.py --json doctor` (and any command with the global `--json` flag) emits a final machine-readable summary. Diagnostics report only whether environment variables are set; values and database credentials are never printed. `up` may apply migrations and seed/promote the synthetic local fixture through the existing `local-v2.ps1` workflow. It does not publish project data. + +## Frontend contributor path + +The root JavaScript commands exercise the legacy static/Jest application. The Svelte V2 preview has its own pinned dependencies and commands in [`frontend/`](../frontend/README.md): + +```powershell +cd frontend +npm ci +npm run check +npm test +npm run lint +npm run boundary +npm run build +npm run dev +``` + +The fixture preview is the default, does not need a database, and is the correct first environment for UI work. `npm run test:e2e:fixture` starts its own local preview. Do not add `?mode=local-v2` until a clean `python scripts/dev.py --json doctor` reports Docker Compose and ports 8000/5433 available, then use `up`, `probe`, and `npm run test:e2e:local` as documented in the frontend README. Port conflicts and an unavailable Docker engine are environment problems, not a reason to stop an unknown process or change the preview to use fixtures as a live fallback. diff --git a/frontend/README.md b/frontend/README.md index 258b991..85f22e7 100644 --- a/frontend/README.md +++ b/frontend/README.md @@ -1,6 +1,24 @@ # V2 frontend preview and local integration -Svelte 5/TypeScript frontend preview. Run `npm install`, then `npm run dev`. Fixture mode remains the safe default and uses no live API, external assets, analytics, or map tiles. The ethics link intentionally targets the existing `/ethics.html` page. +Svelte 5/TypeScript preview. Start with the [developer entrypoint](../docs/development.md); this file defines the frontend-specific commands and the fixture/local boundary. + +## Start in fixture mode + +From `frontend/`, use the lockfile-based install and then run the checks below: + +```powershell +npm ci +npm run check +npm test +npm run lint +npm run boundary +npm run build +npm run dev +``` + +Open the URL Vite prints, normally `http://127.0.0.1:5173/v2-preview/#/`. This is **fixture mode**: the default and only safe onboarding path. It makes no live API, tile, analytics, or geocoding request. `npm run test:e2e:fixture` starts its own fixture preview and runs the Chromium/Firefox/WebKit-configured suite unless a project is selected explicitly. + +The existing root `npm test` is the legacy static/Jest suite. It is deliberately separate from the commands above; run it from the repository root when changing root `static/` assets or compatibility modules. The explicit `?mode=local-v2` path uses the V2 API client for the official, secondary, and community profiles, controlled filters, cursor pagination, map/detail navigation, release/provenance context, and the profile-scoped public CSV route. A community profile keeps its persistent screened-but-unreviewed warning; it is not merged into official or secondary counts. Requests are cancellable and generation-checked so stale list/detail responses cannot replace newer state. No V1 fallback is used. @@ -21,12 +39,36 @@ Current-wire contract gap checklist (from the platform decision): These are documented gaps, not frontend claims or invented DTO fields. Phase 3 continues to use only the synthetic fixture repository. -Local V2 integration (opt-in only): run the backend with `cargo run` (port 8000), then run the frontend with `npm run dev` (port 5173). In development/preview, Vite proxies `/api` to `http://127.0.0.1:8000`; use `http://127.0.0.1:5173/v2-preview/?mode=local-v2#/` to opt in. The `LocalLocationRepository` still targets `/api/v2/locations?profile=...` only when explicitly invoked; fixture mode remains the default and there is no V1 fallback. The proxy is development-only configuration and production builds do not enable a backend connection. +## Local V2 integration: explicit and optional + +Fixture mode does not require Docker, PostgreSQL, backend data, or a token. Only use local mode when checking the V2 client against the **synthetic local release**. + +From the repository root, first check prerequisites and port ownership: + +```powershell +python scripts/dev.py --json doctor +``` + +`doctor` must report ports 8000 and 5433 as free and Docker Compose as available. A port in use means another local service owns it; inspect it before stopping anything. If Docker Desktop's engine is unavailable or access is denied, start/authorize Docker Desktop and rerun `doctor`. The helper's `status` output can report an unmanaged Axum process, but it cannot make an unavailable Docker engine healthy. + +When `doctor` is clean, the supported seeded workflow is: + +```powershell +python scripts/dev.py up +python scripts/dev.py probe +npm --prefix frontend run dev +``` + +Open `http://127.0.0.1:5173/v2-preview/?mode=local-v2#/`. The helper starts a named local Postgres stack on 5433, applies migrations, seeds/promotes only the synthetic contract release, and starts Axum on 8000. It preserves its database volume on `down`; it does not publish data. Do not substitute an arbitrary `cargo run` instance for this workflow, because it may not have the expected synthetic release, database configuration, or CORS origin. + +Run `npm run test:e2e:local` from `frontend/` only while that helper-owned stack is healthy. The command is cross-platform and sets `LOCAL_V2_E2E=1` itself. Finish with `python scripts/dev.py down` when you own the stack. Do not stop a service merely because `doctor` found its port occupied. + +In development/preview, Vite proxies `/api` to `http://127.0.0.1:8000`; the proxy is development-only and production builds do not enable a backend connection. The `LocalLocationRepository` targets `/api/v2/locations?profile=...` only when the explicit `?mode=local-v2` route is used; fixture mode remains the default and there is no V1 fallback. The opt-in local view loads one API page at a time. Search, country/region/category/source/profile/precision/lifecycle filters are evaluated by the server against the selected promoted release; cursor pages remain explicit and are never merged across release IDs. Bounded bbox and radius parameters are available to map clients. Record context is limited to fields in the current V2 wire response and does not imply that review events, evidence hashes, or scoped approvals are available. The private scale/story prototype begins with a neutral individual-animal representation and uses only bounded synthetic values. It labels model arithmetic separately from measured facility evidence; no biography, live counter, global animal total, or sourced aggregate is embedded in the production build. Candidate sourced scale figures remain outside this UI until maintainer publication approval. -Persistent local two-port workflow (never uses `down -v`): from the repository root run `powershell -ExecutionPolicy Bypass -File pipeline/scripts/maintenance/local-v2.ps1 start`. It starts the named Postgres stack on `5433`, applies migrations, seeds and promotes the synthetic contract release, and starts Axum on `8000`. Run `npm --prefix frontend run dev` in another terminal and open `http://127.0.0.1:5173/v2-preview/?mode=local-v2#/`. Check ownership and health with `... local-v2.ps1 status`; probe list/detail with `... local-v2.ps1 probe`; stop both services with `... local-v2.ps1 stop`. The helper is local-only and does not alter V1, production, or unrelated data. +The underlying `pipeline/scripts/maintenance/local-v2.ps1` accepts `start`, `status`, `probe`, and `stop`, but use the `scripts/dev.py` wrapper first so the prerequisite and port diagnostics are part of the onboarding trail. The helper is local-only and does not alter V1, production, or unrelated data. Private candidate preview is a separate development-only shell. Its API is `/api/dev/preview/candidates?limit=100`, never `/api/v2/*`; it requires loopback, explicit development opt-in, and an operator token in `X-UEC-Dev-Preview-Token`. The token must stay in memory/session-only state or a local proxy, never a URL, committed source, log, or production bundle. Candidate rows must remain visibly marked `PRIVATE TEST DATA — NOT REVIEWED OR PUBLISHED`, with source/date/coverage/uncertainty context on every list, detail, map, and export surface. No real candidate rows or credentials belong in this repository. diff --git a/frontend/package.json b/frontend/package.json index 105373c..caa1480 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -2,7 +2,7 @@ "name": "until-every-cage-v2-frontend", "private": true, "type": "module", - "scripts": {"dev":"vite","preview":"vite preview","check":"svelte-check --tsconfig ./tsconfig.app.json && tsc -p tsconfig.node.json --noEmit","build":"npm run check && vite build","test":"vitest run","test:e2e":"playwright test","test:e2e:local":"set LOCAL_V2_E2E=1&& playwright test tests/e2e/local-backend.spec.ts","lint":"eslint .","stage":"node scripts/stage-preview.mjs","boundary":"node scripts/check-boundaries.mjs"}, + "scripts": {"dev":"vite","preview":"vite preview","check":"svelte-check --tsconfig ./tsconfig.app.json && tsc -p tsconfig.node.json --noEmit","build":"npm run check && vite build","test":"vitest run","test:e2e":"playwright test","test:e2e:fixture":"playwright test tests/e2e/fixture-platform.spec.ts tests/e2e/local-safety.spec.ts tests/e2e/private-preview.spec.ts","test:e2e:local":"node scripts/run-local-e2e.mjs","lint":"eslint .","stage":"node scripts/stage-preview.mjs","boundary":"node scripts/check-boundaries.mjs"}, "devDependencies": {"@playwright/test":"^1.49.1","@sveltejs/vite-plugin-svelte":"^6.2.1","@types/node":"^22.10.2","eslint":"^9.17.0","jsdom":"^25.0.1","svelte":"^5.19.0","svelte-check":"^4.1.4","typescript":"^5.7.2","vite":"^6.0.7","vitest":"^2.1.8"}, "dependencies": {"@axe-core/playwright":"^4.10.2","@types/leaflet":"^1.9.15","leaflet":"^1.9.4","zod":"^3.24.1"} } diff --git a/frontend/scripts/run-local-e2e.mjs b/frontend/scripts/run-local-e2e.mjs new file mode 100644 index 0000000..e3ff095 --- /dev/null +++ b/frontend/scripts/run-local-e2e.mjs @@ -0,0 +1,10 @@ +import { spawnSync } from 'node:child_process'; + +const npx = process.platform === 'win32' ? 'npx.cmd' : 'npx'; +const result = spawnSync(npx, ['playwright', 'test', 'tests/e2e/local-backend.spec.ts'], { + stdio: 'inherit', + env: { ...process.env, LOCAL_V2_E2E: '1' }, +}); + +if (result.error) throw result.error; +process.exit(result.status ?? 1); diff --git a/scripts/dev.py b/scripts/dev.py index 4d0e97d..3e6e42b 100644 --- a/scripts/dev.py +++ b/scripts/dev.py @@ -56,7 +56,7 @@ def main() -> int: sub.add_parser("doctor", help="check local prerequisites and safe configuration") for name in ("up", "down", "status", "logs", "probe"): sub.add_parser(name, help=f"local V2 {name}") - sub.add_parser("test", help="fast JavaScript tests").add_argument("--full", action="store_true") + sub.add_parser("test", help="root legacy static/Jest tests").add_argument("--full", action="store_true") sub.add_parser("pipeline", help="run Python pipeline tests").add_argument("args", nargs=argparse.REMAINDER) sub.add_parser("contracts", help="run contract tests") rp = sub.add_parser("review-packet", help="generate a private row-free review packet") From 75fb9c67a9a5abca7b830ee2f5d8cf5b39d2ee6a Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Wed, 16 Sep 2026 15:37:15 -0700 Subject: [PATCH 195/311] Clean dead tooling references and generated files --- docs/DEAD-CODE-AUDIT-2026-09-16.md | 53 + docs/countries/us/README.md | 2 +- node_modules/.package-lock.json | 25 - node_modules/@types/geojson/LICENSE | 21 - node_modules/@types/geojson/README.md | 15 - node_modules/@types/geojson/index.d.ts | 202 -- node_modules/@types/geojson/package.json | 46 - node_modules/@types/leaflet/LICENSE | 21 - node_modules/@types/leaflet/README.md | 15 - node_modules/@types/leaflet/index.d.ts | 3156 ---------------------- node_modules/@types/leaflet/package.json | 63 - 11 files changed, 54 insertions(+), 3565 deletions(-) create mode 100644 docs/DEAD-CODE-AUDIT-2026-09-16.md delete mode 100644 node_modules/.package-lock.json delete mode 100644 node_modules/@types/geojson/LICENSE delete mode 100644 node_modules/@types/geojson/README.md delete mode 100644 node_modules/@types/geojson/index.d.ts delete mode 100644 node_modules/@types/geojson/package.json delete mode 100644 node_modules/@types/leaflet/LICENSE delete mode 100644 node_modules/@types/leaflet/README.md delete mode 100644 node_modules/@types/leaflet/index.d.ts delete mode 100644 node_modules/@types/leaflet/package.json diff --git a/docs/DEAD-CODE-AUDIT-2026-09-16.md b/docs/DEAD-CODE-AUDIT-2026-09-16.md new file mode 100644 index 0000000..6508258 --- /dev/null +++ b/docs/DEAD-CODE-AUDIT-2026-09-16.md @@ -0,0 +1,53 @@ +# Dead-code and documentation audit — 2026-09-16 + +Scope: root-level and cross-cutting tooling/documentation. Backend, pipeline, +frontend, source adapters, and research evidence were inspected for references +but were not removed or redesigned. + +## Confirmed cleanup + +* Fixed the broken accountability-pilot link in + `docs/countries/us/README.md`. The document lives three directories below the + repository root, so the previous `../../pipeline/...` target resolved outside + the repository; `../../../pipeline/...` resolves to the existing README. +* Removed nine accidentally tracked files under `node_modules/` from version + control. `.gitignore` already excludes `/node_modules/`, and these files are + package-manager installation output rather than project source. The local + dependency directory is left untouched. + +## Retained candidates and rationale + +* `Old scripts/` is unreferenced by current entrypoints and CI, but it is + explicitly identified by `AGENTS.md`, `docs/architecture/data-pipeline-plan.md`, + and `pipeline/README.md` as migration/reference material. The scripts include + source-specific transformations and are retained as historical method + records; removing them would erase reproducibility context. +* `Old CSVs/`, `dirty-datasets/`, `static_data/`, and `france-data.kml` are + legacy or research inputs referenced by the source inventory and country + crosswalks. They are data evidence, not dead code, and the governing policy + requires preserving provenance and recovery boundaries. No files were deleted. +* `v2-ideas.md` remains a proposed roadmap and is explicitly linked by + `docs/V2-IMPLEMENTATION-TODO.md`. It is not presented as completed status, so + it is retained rather than silently removed. +* `docs/V2-SPRINT-2026-09-13.md`, `docs/V2-INTEGRATION-BASELINE.md`, and + `docs/V2-REVIEW-CLEANUP-2026-09-13.md` are dated integration evidence with + explicit non-production and evidence-scope language. Their overlap is + historical reporting, not redundant current instructions. +* `docs/PIPELINE-MIGRATION.md` was last updated on 2026-09-16 and documents the + shared artifact-boundary migration. Its “next consolidation target” language + is source-specific status, not an unused entrypoint; it is retained. + +## Current root entrypoints + +`scripts/dev.py` is referenced by `docs/development.md`, wrapped by +`scripts/dev.ps1`, and covered by `scripts/test_dev.py`. CI invokes the canonical +pipeline/frontend test runners directly. No root script was proven unreachable +or safe to remove. + +## Verification method + +References were searched with `rg` across tracked source, documentation, CI, +package scripts, and entrypoints. Relative Markdown/JSON links were checked +against the filesystem. The audit found one broken repository-relative link, +now fixed; external URLs were not claimed reachable. Generated dependency +output was checked against Git tracking and `.gitignore`. diff --git a/docs/countries/us/README.md b/docs/countries/us/README.md index c748d58..b94c037 100644 --- a/docs/countries/us/README.md +++ b/docs/countries/us/README.md @@ -25,7 +25,7 @@ Both adapters preserve source values only in restricted staging and emit parsed, ## Accountability pilot The private accountability pilot in -[`pipeline/sources/us/accountability`](../../pipeline/sources/us/accountability/README.md) +[`pipeline/sources/us/accountability`](../../../pipeline/sources/us/accountability/README.md) adds a deterministic, graph-foundation-compatible link ledger. It starts from FSIS establishment/approval IDs and the modeled APHIS registration/inspection IDs, while keeping operators, legal entities, parents, brands, inspections, diff --git a/node_modules/.package-lock.json b/node_modules/.package-lock.json deleted file mode 100644 index 4ece9d7..0000000 --- a/node_modules/.package-lock.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "name": "heatmap-backend", - "version": "1.0.0", - "lockfileVersion": 3, - "requires": true, - "packages": { - "node_modules/@types/geojson": { - "version": "7946.0.16", - "resolved": "https://registry.npmjs.org/@types/geojson/-/geojson-7946.0.16.tgz", - "integrity": "sha512-6C8nqWur3j98U6+lXDfTUWIfgvZU+EumvpHKcYjujKH7woYyLj2sUmff0tRhrqM7BohUw7Pz3ZB1jj2gW9Fvmg==", - "dev": true, - "license": "MIT" - }, - "node_modules/@types/leaflet": { - "version": "1.9.18", - "resolved": "https://registry.npmjs.org/@types/leaflet/-/leaflet-1.9.18.tgz", - "integrity": "sha512-ht2vsoPjezor5Pmzi5hdsA7F++v5UGq9OlUduWHmMZiuQGIpJ2WS5+Gg9HaAA79gNh1AIPtCqhzejcIZ3lPzXQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/geojson": "*" - } - } - } -} diff --git a/node_modules/@types/geojson/LICENSE b/node_modules/@types/geojson/LICENSE deleted file mode 100644 index 9e841e7..0000000 --- a/node_modules/@types/geojson/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ - MIT License - - Copyright (c) Microsoft Corporation. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in all - copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - SOFTWARE diff --git a/node_modules/@types/geojson/README.md b/node_modules/@types/geojson/README.md deleted file mode 100644 index d3e0a6b..0000000 --- a/node_modules/@types/geojson/README.md +++ /dev/null @@ -1,15 +0,0 @@ -# Installation -> `npm install --save @types/geojson` - -# Summary -This package contains type definitions for geojson (https://geojson.org/). - -# Details -Files were exported from https://github.com/DefinitelyTyped/DefinitelyTyped/tree/master/types/geojson. - -### Additional Details - * Last updated: Thu, 23 Jan 2025 18:36:51 GMT - * Dependencies: none - -# Credits -These definitions were written by [Jacob Bruun](https://github.com/cobster), [Arne Schubert](https://github.com/atd-schubert), [Jeff Jacobson](https://github.com/JeffJacobson), [Ilia Choly](https://github.com/icholy), and [Dan Vanderkam](https://github.com/danvk). diff --git a/node_modules/@types/geojson/index.d.ts b/node_modules/@types/geojson/index.d.ts deleted file mode 100644 index e964ce3..0000000 --- a/node_modules/@types/geojson/index.d.ts +++ /dev/null @@ -1,202 +0,0 @@ -// Note: as of the RFC 7946 version of GeoJSON, Coordinate Reference Systems -// are no longer supported. (See https://tools.ietf.org/html/rfc7946#appendix-B)} - -export as namespace GeoJSON; - -/** - * The valid values for the "type" property of GeoJSON geometry objects. - * https://tools.ietf.org/html/rfc7946#section-1.4 - */ -export type GeoJsonGeometryTypes = Geometry["type"]; - -/** - * The value values for the "type" property of GeoJSON Objects. - * https://tools.ietf.org/html/rfc7946#section-1.4 - */ -export type GeoJsonTypes = GeoJSON["type"]; - -/** - * Bounding box - * https://tools.ietf.org/html/rfc7946#section-5 - */ -export type BBox = [number, number, number, number] | [number, number, number, number, number, number]; - -/** - * A Position is an array of coordinates. - * https://tools.ietf.org/html/rfc7946#section-3.1.1 - * Array should contain between two and three elements. - * The previous GeoJSON specification allowed more elements (e.g., which could be used to represent M values), - * but the current specification only allows X, Y, and (optionally) Z to be defined. - * - * Note: the type will not be narrowed down to `[number, number] | [number, number, number]` due to - * marginal benefits and the large impact of breaking change. - * - * See previous discussions on the type narrowing: - * - {@link https://github.com/DefinitelyTyped/DefinitelyTyped/pull/21590|Nov 2017} - * - {@link https://github.com/DefinitelyTyped/DefinitelyTyped/discussions/67773|Dec 2023} - * - {@link https://github.com/DefinitelyTyped/DefinitelyTyped/discussions/71441| Dec 2024} - * - * One can use a - * {@link https://www.typescriptlang.org/docs/handbook/2/narrowing.html#using-type-predicates|user-defined type guard that returns a type predicate} - * to determine if a position is a 2D or 3D position. - * - * @example - * import type { Position } from 'geojson'; - * - * type StrictPosition = [x: number, y: number] | [x: number, y: number, z: number] - * - * function isStrictPosition(position: Position): position is StrictPosition { - * return position.length === 2 || position.length === 3 - * }; - * - * let position: Position = [-116.91, 45.54]; - * - * let x: number; - * let y: number; - * let z: number | undefined; - * - * if (isStrictPosition(position)) { - * // `tsc` would throw an error if we tried to destructure a fourth parameter - * [x, y, z] = position; - * } else { - * throw new TypeError("Position is not a 2D or 3D point"); - * } - */ -export type Position = number[]; - -/** - * The base GeoJSON object. - * https://tools.ietf.org/html/rfc7946#section-3 - * The GeoJSON specification also allows foreign members - * (https://tools.ietf.org/html/rfc7946#section-6.1) - * Developers should use "&" type in TypeScript or extend the interface - * to add these foreign members. - */ -export interface GeoJsonObject { - // Don't include foreign members directly into this type def. - // in order to preserve type safety. - // [key: string]: any; - /** - * Specifies the type of GeoJSON object. - */ - type: GeoJsonTypes; - /** - * Bounding box of the coordinate range of the object's Geometries, Features, or Feature Collections. - * The value of the bbox member is an array of length 2*n where n is the number of dimensions - * represented in the contained geometries, with all axes of the most southwesterly point - * followed by all axes of the more northeasterly point. - * The axes order of a bbox follows the axes order of geometries. - * https://tools.ietf.org/html/rfc7946#section-5 - */ - bbox?: BBox | undefined; -} - -/** - * Union of GeoJSON objects. - */ -export type GeoJSON = - | G - | Feature - | FeatureCollection; - -/** - * Geometry object. - * https://tools.ietf.org/html/rfc7946#section-3 - */ -export type Geometry = Point | MultiPoint | LineString | MultiLineString | Polygon | MultiPolygon | GeometryCollection; -export type GeometryObject = Geometry; - -/** - * Point geometry object. - * https://tools.ietf.org/html/rfc7946#section-3.1.2 - */ -export interface Point extends GeoJsonObject { - type: "Point"; - coordinates: Position; -} - -/** - * MultiPoint geometry object. - * https://tools.ietf.org/html/rfc7946#section-3.1.3 - */ -export interface MultiPoint extends GeoJsonObject { - type: "MultiPoint"; - coordinates: Position[]; -} - -/** - * LineString geometry object. - * https://tools.ietf.org/html/rfc7946#section-3.1.4 - */ -export interface LineString extends GeoJsonObject { - type: "LineString"; - coordinates: Position[]; -} - -/** - * MultiLineString geometry object. - * https://tools.ietf.org/html/rfc7946#section-3.1.5 - */ -export interface MultiLineString extends GeoJsonObject { - type: "MultiLineString"; - coordinates: Position[][]; -} - -/** - * Polygon geometry object. - * https://tools.ietf.org/html/rfc7946#section-3.1.6 - */ -export interface Polygon extends GeoJsonObject { - type: "Polygon"; - coordinates: Position[][]; -} - -/** - * MultiPolygon geometry object. - * https://tools.ietf.org/html/rfc7946#section-3.1.7 - */ -export interface MultiPolygon extends GeoJsonObject { - type: "MultiPolygon"; - coordinates: Position[][][]; -} - -/** - * Geometry Collection - * https://tools.ietf.org/html/rfc7946#section-3.1.8 - */ -export interface GeometryCollection extends GeoJsonObject { - type: "GeometryCollection"; - geometries: G[]; -} - -export type GeoJsonProperties = { [name: string]: any } | null; - -/** - * A feature object which contains a geometry and associated properties. - * https://tools.ietf.org/html/rfc7946#section-3.2 - */ -export interface Feature extends GeoJsonObject { - type: "Feature"; - /** - * The feature's geometry - */ - geometry: G; - /** - * A value that uniquely identifies this feature in a - * https://tools.ietf.org/html/rfc7946#section-3.2. - */ - id?: string | number | undefined; - /** - * Properties associated with this feature. - */ - properties: P; -} - -/** - * A collection of feature objects. - * https://tools.ietf.org/html/rfc7946#section-3.3 - */ -export interface FeatureCollection extends GeoJsonObject { - type: "FeatureCollection"; - features: Array>; -} diff --git a/node_modules/@types/geojson/package.json b/node_modules/@types/geojson/package.json deleted file mode 100644 index f80ca89..0000000 --- a/node_modules/@types/geojson/package.json +++ /dev/null @@ -1,46 +0,0 @@ -{ - "name": "@types/geojson", - "version": "7946.0.16", - "description": "TypeScript definitions for geojson", - "homepage": "https://github.com/DefinitelyTyped/DefinitelyTyped/tree/master/types/geojson", - "license": "MIT", - "contributors": [ - { - "name": "Jacob Bruun", - "githubUsername": "cobster", - "url": "https://github.com/cobster" - }, - { - "name": "Arne Schubert", - "githubUsername": "atd-schubert", - "url": "https://github.com/atd-schubert" - }, - { - "name": "Jeff Jacobson", - "githubUsername": "JeffJacobson", - "url": "https://github.com/JeffJacobson" - }, - { - "name": "Ilia Choly", - "githubUsername": "icholy", - "url": "https://github.com/icholy" - }, - { - "name": "Dan Vanderkam", - "githubUsername": "danvk", - "url": "https://github.com/danvk" - } - ], - "main": "", - "types": "index.d.ts", - "repository": { - "type": "git", - "url": "https://github.com/DefinitelyTyped/DefinitelyTyped.git", - "directory": "types/geojson" - }, - "scripts": {}, - "dependencies": {}, - "peerDependencies": {}, - "typesPublisherContentHash": "e7997f4827a9a92b60c7a6cb27e8f18fa760803e9dd021965e95604338b72e88", - "typeScriptVersion": "5.0" -} \ No newline at end of file diff --git a/node_modules/@types/leaflet/LICENSE b/node_modules/@types/leaflet/LICENSE deleted file mode 100644 index 9e841e7..0000000 --- a/node_modules/@types/leaflet/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ - MIT License - - Copyright (c) Microsoft Corporation. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in all - copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - SOFTWARE diff --git a/node_modules/@types/leaflet/README.md b/node_modules/@types/leaflet/README.md deleted file mode 100644 index 696ef68..0000000 --- a/node_modules/@types/leaflet/README.md +++ /dev/null @@ -1,15 +0,0 @@ -# Installation -> `npm install --save @types/leaflet` - -# Summary -This package contains type definitions for leaflet (https://github.com/Leaflet/Leaflet). - -# Details -Files were exported from https://github.com/DefinitelyTyped/DefinitelyTyped/tree/master/types/leaflet. - -### Additional Details - * Last updated: Mon, 19 May 2025 10:38:21 GMT - * Dependencies: [@types/geojson](https://npmjs.com/package/@types/geojson) - -# Credits -These definitions were written by [Alejandro Sánchez](https://github.com/alejo90), [Arne Schubert](https://github.com/atd-schubert), [Michael Auer](https://github.com/mcauer), [Roni Karilkar](https://github.com/ronikar), [Vladimir Dashukevich](https://github.com/life777), [Henry Thasler](https://github.com/henrythasler), [Colin Doig](https://github.com/captain-igloo), and [Hugo Sales](https://github.com/someonewithpc). diff --git a/node_modules/@types/leaflet/index.d.ts b/node_modules/@types/leaflet/index.d.ts deleted file mode 100644 index 6a9849e..0000000 --- a/node_modules/@types/leaflet/index.d.ts +++ /dev/null @@ -1,3156 +0,0 @@ -export as namespace L; - -import * as geojson from "geojson"; - -/** A constant that represents the Leaflet version in use. */ -export const version: string; - -export class Class { - static extend(props: any): { new(...args: any[]): any } & typeof Class; - static include(props: any): any & typeof Class; - static mergeOptions(props: any): any & typeof Class; - - static addInitHook(initHookFn: () => void): any & typeof Class; - static addInitHook(methodName: string, ...args: any[]): any & typeof Class; - - static callInitHooks(): void; -} - -export class Transformation { - constructor(a: number, b: number, c: number, d: number); - transform(point: Point, scale?: number): Point; - untransform(point: Point, scale?: number): Point; -} - -/** Instantiates a Transformation object with the given coefficients. */ -export function transformation(a: number, b: number, c: number, d: number): Transformation; - -/** Expects an coefficients array of the form `[a: Number, b: Number, c: Number, d: Number]`. */ -export function transformation(coefficients: [number, number, number, number]): Transformation; - -/** - * @see https://github.com/Leaflet/Leaflet/blob/bc918d4bdc2ba189807bc207c77080fb41ecc196/src/geometry/LineUtil.js#L118 - */ -export namespace LineUtil { - function simplify(points: Point[], tolerance: number): Point[]; - function pointToSegmentDistance(p: Point, p1: Point, p2: Point): number; - function closestPointOnSegment(p: Point, p1: Point, p2: Point): Point; - function isFlat(latlngs: LatLngExpression[]): boolean; - function clipSegment( - a: Point, - b: Point, - bounds: Bounds, - useLastCode?: boolean, - round?: boolean, - ): [Point, Point] | false; - function polylineCenter(latlngs: LatLngExpression[], crs: CRS): LatLng; -} - -export namespace PolyUtil { - function clipPolygon(points: Point[], bounds: BoundsExpression, round?: boolean): Point[]; - function polygonCenter(latlngs: LatLngExpression[], crs: CRS): LatLng; -} - -export namespace DomUtil { - /** - * Get Element by its ID or with the given HTML-Element - */ - function get(element: string | HTMLElement): HTMLElement | null; - function getStyle(el: HTMLElement, styleAttrib: string): string | null; - /** - * Creates an HTML element with `tagName`, sets its class to `className`, and optionally appends it to `container` element. - * @param tagName The name of the tag to create (for example: `div` or `canvas`). - * @param className The class to set on the created element. - * @param container The container to append the created element to. - */ - function create( - tagName: T, - className?: string, - container?: HTMLElement, - ): HTMLElementTagNameMap[T]; - function create(tagName: string, className?: string, container?: HTMLElement): HTMLElement; - function remove(el: HTMLElement): void; - function empty(el: HTMLElement): void; - function toFront(el: HTMLElement): void; - function toBack(el: HTMLElement): void; - function hasClass(el: HTMLElement, name: string): boolean; - function addClass(el: HTMLElement, name: string): void; - function removeClass(el: HTMLElement, name: string): void; - function setClass(el: HTMLElement, name: string): void; - function getClass(el: HTMLElement): string; - function setOpacity(el: HTMLElement, opacity: number): void; - function testProp(props: string[]): string | false; - function setTransform(el: HTMLElement, offset: Point, scale?: number): void; - function setPosition(el: HTMLElement, position: Point): void; - function getPosition(el: HTMLElement): Point; - function getScale(el: HTMLElement): { x: number; y: number; boundingClientRect: DOMRect }; - function getSizedParentNode(el: HTMLElement): HTMLElement; - function disableTextSelection(): void; - function enableTextSelection(): void; - function disableImageDrag(): void; - function enableImageDrag(): void; - function preventOutline(el: HTMLElement): void; - function restoreOutline(): void; - - let TRANSFORM: string; - let TRANSITION: string; - let TRANSITION_END: string; -} - -export class PosAnimation extends Evented { - run(el: HTMLElement, newPos: Point, duration?: number, easeLinearity?: number): void; - stop(): void; -} - -export interface CRS { - latLngToPoint(latlng: LatLngExpression, zoom: number): Point; - pointToLatLng(point: PointExpression, zoom: number): LatLng; - project(latlng: LatLng | LatLngLiteral): Point; - unproject(point: PointExpression): LatLng; - scale(zoom: number): number; - zoom(scale: number): number; - getProjectedBounds(zoom: number): Bounds; - distance(latlng1: LatLngExpression, latlng2: LatLngExpression): number; - wrapLatLng(latlng: LatLng | LatLngLiteral): LatLng; - - code?: string | undefined; - wrapLng?: [number, number] | undefined; - wrapLat?: [number, number] | undefined; - infinite: boolean; -} - -export namespace CRS { - const EPSG3395: CRS; - const EPSG3857: CRS; - const EPSG4326: CRS; - const EPSG900913: CRS; - const Earth: CRS; - const Simple: CRS; -} - -export interface Projection { - project(latlng: LatLng | LatLngLiteral): Point; - unproject(point: PointExpression): LatLng; - - bounds: Bounds; -} - -export namespace Projection { - const LonLat: Projection; - const Mercator: Projection; - const SphericalMercator: Projection; -} - -export class LatLng { - constructor(latitude: number, longitude: number, altitude?: number); - equals(otherLatLng: LatLngExpression, maxMargin?: number): boolean; - toString(): string; - distanceTo(otherLatLng: LatLngExpression): number; - wrap(): LatLng; - toBounds(sizeInMeters: number): LatLngBounds; - clone(): LatLng; - - lat: number; - lng: number; - alt?: number | undefined; -} - -export interface LatLngLiteral { - lat: number; - lng: number; - alt?: number; -} - -export type LatLngTuple = [number, number, number?]; - -export type LatLngExpression = LatLng | LatLngLiteral | LatLngTuple; - -export function latLng(latitude: number, longitude: number, altitude?: number): LatLng; - -export function latLng( - coords: LatLngTuple | [number, number, number] | LatLngLiteral | { - lat: number; - lng: number; - alt?: number | undefined; - }, -): LatLng; - -export class LatLngBounds { - constructor(southWest: LatLngExpression, northEast: LatLngExpression); - constructor(latlngs: LatLngExpression[]); - extend(latlngOrBounds: LatLngExpression | LatLngBoundsExpression): this; - pad(bufferRatio: number): LatLngBounds; // Returns a new LatLngBounds - getCenter(): LatLng; - getSouthWest(): LatLng; - getNorthEast(): LatLng; - getNorthWest(): LatLng; - getSouthEast(): LatLng; - getWest(): number; - getSouth(): number; - getEast(): number; - getNorth(): number; - contains(otherBoundsOrLatLng: LatLngBoundsExpression | LatLngExpression): boolean; - intersects(otherBounds: LatLngBoundsExpression): boolean; - overlaps(otherBounds: LatLngBoundsExpression): boolean; - toBBoxString(): string; - equals(otherBounds: LatLngBoundsExpression, maxMargin?: number): boolean; - isValid(): boolean; -} - -export type LatLngBoundsLiteral = LatLngTuple[]; // Must be [LatLngTuple, LatLngTuple], cant't change because Map.setMaxBounds - -export type LatLngBoundsExpression = LatLngBounds | LatLngBoundsLiteral; - -export function latLngBounds(southWest: LatLngExpression, northEast: LatLngExpression): LatLngBounds; - -export function latLngBounds(latlngs: LatLngExpression[]): LatLngBounds; - -export type PointTuple = [number, number]; - -export class Point { - constructor(x: number, y: number, round?: boolean); - clone(): Point; - add(otherPoint: PointExpression): Point; // non-destructive, returns a new point - subtract(otherPoint: PointExpression): Point; - divideBy(num: number): Point; - multiplyBy(num: number): Point; - scaleBy(scale: PointExpression): Point; - unscaleBy(scale: PointExpression): Point; - round(): Point; - floor(): Point; - ceil(): Point; - trunc(): Point; - distanceTo(otherPoint: PointExpression): number; - equals(otherPoint: PointExpression): boolean; - contains(otherPoint: PointExpression): boolean; - toString(): string; - x: number; - y: number; -} - -export interface Coords extends Point { - z: number; -} - -export type PointExpression = Point | PointTuple; - -export function point(x: number, y: number, round?: boolean): Point; - -export function point(coords: PointTuple | { x: number; y: number }): Point; - -export type BoundsLiteral = [PointTuple, PointTuple]; - -export class Bounds { - constructor(topLeft: PointExpression, bottomRight: PointExpression); - constructor(points?: Point[] | BoundsLiteral); - - // tslint:disable:unified-signatures - extend(point: PointExpression): this; - extend(otherBounds: BoundsExpression): this; - // tslint:enable:unified-signatures - - getCenter(round?: boolean): Point; - getBottomLeft(): Point; - getBottomRight(): Point; - getTopLeft(): Point; - getTopRight(): Point; - getSize(): Point; - contains(pointOrBounds: BoundsExpression | PointExpression): boolean; - intersects(otherBounds: BoundsExpression): boolean; - overlaps(otherBounds: BoundsExpression): boolean; - isValid(): boolean; - pad(bufferRatio: number): Bounds; // Returns a new Bounds - equals(otherBounds: BoundsExpression): boolean; - - min?: Point | undefined; - max?: Point | undefined; -} - -export type BoundsExpression = Bounds | BoundsLiteral; - -export function bounds(topLeft: PointExpression, bottomRight: PointExpression): Bounds; - -export function bounds(points: Point[] | BoundsLiteral): Bounds; - -// Event handler types - -export type LeafletEventHandlerFn = (event: LeafletEvent) => void; - -export type LayersControlEventHandlerFn = (event: LayersControlEvent) => void; - -export type LayerEventHandlerFn = (event: LayerEvent) => void; - -export type ResizeEventHandlerFn = (event: ResizeEvent) => void; - -export type PopupEventHandlerFn = (event: PopupEvent) => void; - -export type TooltipEventHandlerFn = (event: TooltipEvent) => void; - -export type ErrorEventHandlerFn = (event: ErrorEvent) => void; - -export type LocationEventHandlerFn = (event: LocationEvent) => void; - -export type LeafletMouseEventHandlerFn = (event: LeafletMouseEvent) => void; - -export type LeafletKeyboardEventHandlerFn = (event: LeafletKeyboardEvent) => void; - -export type ZoomAnimEventHandlerFn = (event: ZoomAnimEvent) => void; - -export type DragEndEventHandlerFn = (event: DragEndEvent) => void; - -export type TileEventHandlerFn = (event: TileEvent) => void; - -export type TileErrorEventHandlerFn = (event: TileErrorEvent) => void; - -export interface LeafletEventHandlerFnMap { - baselayerchange?: LayersControlEventHandlerFn | undefined; - overlayadd?: LayersControlEventHandlerFn | undefined; - overlayremove?: LayersControlEventHandlerFn | undefined; - - layeradd?: LayerEventHandlerFn | undefined; - layerremove?: LayerEventHandlerFn | undefined; - - zoomlevelschange?: LeafletEventHandlerFn | undefined; - unload?: LeafletEventHandlerFn | undefined; - viewreset?: LeafletEventHandlerFn | undefined; - load?: LeafletEventHandlerFn | undefined; - zoomstart?: LeafletEventHandlerFn | undefined; - movestart?: LeafletEventHandlerFn | undefined; - zoom?: LeafletEventHandlerFn | undefined; - move?: LeafletEventHandlerFn | undefined; - zoomend?: LeafletEventHandlerFn | undefined; - moveend?: LeafletEventHandlerFn | undefined; - autopanstart?: LeafletEventHandlerFn | undefined; - dragstart?: LeafletEventHandlerFn | undefined; - drag?: LeafletEventHandlerFn | undefined; - add?: LeafletEventHandlerFn | undefined; - remove?: LeafletEventHandlerFn | undefined; - loading?: LeafletEventHandlerFn | undefined; - error?: LeafletEventHandlerFn | undefined; - update?: LeafletEventHandlerFn | undefined; - down?: LeafletEventHandlerFn | undefined; - predrag?: LeafletEventHandlerFn | undefined; - - resize?: ResizeEventHandlerFn | undefined; - - popupopen?: PopupEventHandlerFn | undefined; - popupclose?: PopupEventHandlerFn | undefined; - - tooltipopen?: TooltipEventHandlerFn | undefined; - tooltipclose?: TooltipEventHandlerFn | undefined; - - locationerror?: ErrorEventHandlerFn | undefined; - - locationfound?: LocationEventHandlerFn | undefined; - - click?: LeafletMouseEventHandlerFn | undefined; - dblclick?: LeafletMouseEventHandlerFn | undefined; - mousedown?: LeafletMouseEventHandlerFn | undefined; - mouseup?: LeafletMouseEventHandlerFn | undefined; - mouseover?: LeafletMouseEventHandlerFn | undefined; - mouseout?: LeafletMouseEventHandlerFn | undefined; - mousemove?: LeafletMouseEventHandlerFn | undefined; - contextmenu?: LeafletMouseEventHandlerFn | undefined; - preclick?: LeafletMouseEventHandlerFn | undefined; - - keypress?: LeafletKeyboardEventHandlerFn | undefined; - keydown?: LeafletKeyboardEventHandlerFn | undefined; - keyup?: LeafletKeyboardEventHandlerFn | undefined; - - zoomanim?: ZoomAnimEventHandlerFn | undefined; - - dragend?: DragEndEventHandlerFn | undefined; - - tileunload?: TileEventHandlerFn | undefined; - tileloadstart?: TileEventHandlerFn | undefined; - tileload?: TileEventHandlerFn | undefined; - tileabort?: TileEventHandlerFn | undefined; - - tileerror?: TileErrorEventHandlerFn | undefined; - - // [name: string]: any; - // You are able add additional properties, but it makes this interface uncheckable. -} - -/** - * A set of methods shared between event-powered classes (like Map and Marker). - * Generally, events allow you to execute some function when something happens - * with an object (e.g. the user clicks on the map, causing the map to fire - * 'click' event). - */ -// eslint-disable-next-line @definitelytyped/strict-export-declare-modifiers -declare class Events { - /** - * Adds a listener function (fn) to a particular event type of the object. - * You can optionally specify the context of the listener (object the this - * keyword will point to). You can also pass several space-separated types - * (e.g. 'click dblclick'). - */ - // tslint:disable:unified-signatures - on(type: "baselayerchange" | "overlayadd" | "overlayremove", fn: LayersControlEventHandlerFn, context?: any): this; - on(type: "layeradd" | "layerremove", fn: LayerEventHandlerFn, context?: any): this; - on( - type: - | "zoomlevelschange" - | "unload" - | "viewreset" - | "load" - | "zoomstart" - | "movestart" - | "zoom" - | "move" - | "zoomend" - | "moveend" - | "autopanstart" - | "dragstart" - | "drag" - | "add" - | "remove" - | "loading" - | "error" - | "update" - | "down" - | "predrag", - fn: LeafletEventHandlerFn, - context?: any, - ): this; - on(type: "resize", fn: ResizeEventHandlerFn, context?: any): this; - on(type: "popupopen" | "popupclose", fn: PopupEventHandlerFn, context?: any): this; - on(type: "tooltipopen" | "tooltipclose", fn: TooltipEventHandlerFn, context?: any): this; - on(type: "locationerror", fn: ErrorEventHandlerFn, context?: any): this; - on(type: "locationfound", fn: LocationEventHandlerFn, context?: any): this; - on( - type: - | "click" - | "dblclick" - | "mousedown" - | "mouseup" - | "mouseover" - | "mouseout" - | "mousemove" - | "contextmenu" - | "preclick", - fn: LeafletMouseEventHandlerFn, - context?: any, - ): this; - on(type: "keypress" | "keydown" | "keyup", fn: LeafletKeyboardEventHandlerFn, context?: any): this; - on(type: "zoomanim", fn: ZoomAnimEventHandlerFn, context?: any): this; - on(type: "dragend", fn: DragEndEventHandlerFn, context?: any): this; - on(type: "tileunload" | "tileloadstart" | "tileload" | "tileabort", fn: TileEventHandlerFn, context?: any): this; - on(type: "tileerror", fn: TileErrorEventHandlerFn, context?: any): this; - on(type: string, fn: LeafletEventHandlerFn, context?: any): this; - - /** - * Adds a set of type/listener pairs, e.g. {click: onClick, mousemove: onMouseMove} - */ - on(eventMap: LeafletEventHandlerFnMap): this; - // tslint:enable:unified-signatures - - /** - * Removes a previously added listener function. If no function is specified, - * it will remove all the listeners of that particular event from the object. - * Note that if you passed a custom context to on, you must pass the same context - * to off in order to remove the listener. - */ - // tslint:disable:unified-signatures - off( - type: "baselayerchange" | "overlayadd" | "overlayremove", - fn?: LayersControlEventHandlerFn, - context?: any, - ): this; - off(type: "layeradd" | "layerremove", fn?: LayerEventHandlerFn, context?: any): this; - off( - type: - | "zoomlevelschange" - | "unload" - | "viewreset" - | "load" - | "zoomstart" - | "movestart" - | "zoom" - | "move" - | "zoomend" - | "moveend" - | "autopanstart" - | "dragstart" - | "drag" - | "add" - | "remove" - | "loading" - | "error" - | "update" - | "down" - | "predrag", - fn?: LeafletEventHandlerFn, - context?: any, - ): this; - off(type: "resize", fn?: ResizeEventHandlerFn, context?: any): this; - off(type: "popupopen" | "popupclose", fn?: PopupEventHandlerFn, context?: any): this; - off(type: "tooltipopen" | "tooltipclose", fn?: TooltipEventHandlerFn, context?: any): this; - off(type: "locationerror", fn?: ErrorEventHandlerFn, context?: any): this; - off(type: "locationfound", fn?: LocationEventHandlerFn, context?: any): this; - off( - type: - | "click" - | "dblclick" - | "mousedown" - | "mouseup" - | "mouseover" - | "mouseout" - | "mousemove" - | "contextmenu" - | "preclick", - fn?: LeafletMouseEventHandlerFn, - context?: any, - ): this; - off(type: "keypress" | "keydown" | "keyup", fn?: LeafletKeyboardEventHandlerFn, context?: any): this; - off(type: "zoomanim", fn?: ZoomAnimEventHandlerFn, context?: any): this; - off(type: "dragend", fn?: DragEndEventHandlerFn, context?: any): this; - off(type: "tileunload" | "tileloadstart" | "tileload" | "tileabort", fn?: TileEventHandlerFn, context?: any): this; - off(type: "tileerror", fn?: TileErrorEventHandlerFn, context?: any): this; - off(type: string, fn?: LeafletEventHandlerFn, context?: any): this; - - /** - * Removes a set of type/listener pairs. - */ - // With an eventMap there are no additional arguments allowed - off(eventMap: LeafletEventHandlerFnMap): this; - - /** - * Removes all listeners to all events on the object. - */ - off(): this; - // tslint:enable:unified-signatures - - /** - * Fires an event of the specified type. You can optionally provide a data - * object — the first argument of the listener function will contain its properties. - * The event might can optionally be propagated to event parents. - */ - fire(type: string, data?: any, propagate?: boolean): this; - - /** - * Returns true if a particular event type has any listeners attached to it. - */ - // tslint:disable:unified-signatures - listens( - type: - | "baselayerchange" - | "overlayadd" - | "overlayremove" - | "layeradd" - | "layerremove" - | "zoomlevelschange" - | "unload" - | "viewreset" - | "load" - | "zoomstart" - | "movestart" - | "zoom" - | "move" - | "zoomend" - | "moveend" - | "autopanstart" - | "dragstart" - | "drag" - | "add" - | "remove" - | "loading" - | "error" - | "update" - | "down" - | "predrag" - | "resize" - | "popupopen" - | "tooltipopen" - | "tooltipclose" - | "locationerror" - | "locationfound" - | "click" - | "dblclick" - | "mousedown" - | "mouseup" - | "mouseover" - | "mouseout" - | "mousemove" - | "contextmenu" - | "preclick" - | "keypress" - | "keydown" - | "keyup" - | "zoomanim" - | "dragend" - | "tileunload" - | "tileloadstart" - | "tileload" - | "tileabort" - | "tileerror", - propagate?: boolean, - ): boolean; - - listens( - type: "baselayerchange" | "overlayadd" | "overlayremove", - fn: LayersControlEventHandlerFn, - context?: any, - propagate?: boolean, - ): boolean; - listens(type: "layeradd" | "layerremove", fn: LayerEventHandlerFn, context?: any, propagate?: boolean): boolean; - listens( - type: - | "zoomlevelschange" - | "unload" - | "viewreset" - | "load" - | "zoomstart" - | "movestart" - | "zoom" - | "move" - | "zoomend" - | "moveend" - | "autopanstart" - | "dragstart" - | "drag" - | "add" - | "remove" - | "loading" - | "error" - | "update" - | "down" - | "predrag", - fn: LeafletEventHandlerFn, - context?: any, - propagate?: boolean, - ): boolean; - listens(type: "resize", fn: ResizeEventHandlerFn, context?: any, propagate?: boolean): boolean; - listens(type: "popupopen" | "popupclose", fn: PopupEventHandlerFn, context?: any, propagate?: boolean): boolean; - listens( - type: "tooltipopen" | "tooltipclose", - fn: TooltipEventHandlerFn, - context?: any, - propagate?: boolean, - ): boolean; - listens(type: "locationerror", fn: ErrorEventHandlerFn, context?: any, propagate?: boolean): boolean; - listens(type: "locationfound", fn: LocationEventHandlerFn, context?: any, propagate?: boolean): boolean; - listens( - type: - | "click" - | "dblclick" - | "mousedown" - | "mouseup" - | "mouseover" - | "mouseout" - | "mousemove" - | "contextmenu" - | "preclick", - fn: LeafletMouseEventHandlerFn, - context?: any, - propagate?: boolean, - ): boolean; - listens( - type: "keypress" | "keydown" | "keyup", - fn: LeafletKeyboardEventHandlerFn, - context?: any, - propagate?: boolean, - ): boolean; - listens(type: "zoomanim", fn: ZoomAnimEventHandlerFn, context?: any, propagate?: boolean): boolean; - listens(type: "dragend", fn: DragEndEventHandlerFn, context?: any, propagate?: boolean): boolean; - listens( - type: "tileunload" | "tileloadstart" | "tileload" | "tileabort", - fn: TileEventHandlerFn, - context?: any, - propagate?: boolean, - ): boolean; - listens(type: "tileerror", fn: TileEventHandlerFn, context?: any, propagate?: boolean): boolean; - listens(type: string, fn: LeafletEventHandlerFn, context?: any, propagate?: boolean): boolean; - - /** - * Behaves as on(...), except the listener will only get fired once and then removed. - */ - // tslint:disable:unified-signatures - once( - type: "baselayerchange" | "overlayadd" | "overlayremove", - fn: LayersControlEventHandlerFn, - context?: any, - ): this; - once(type: "layeradd" | "layerremove", fn: LayerEventHandlerFn, context?: any): this; - once( - type: - | "zoomlevelschange" - | "unload" - | "viewreset" - | "load" - | "zoomstart" - | "movestart" - | "zoom" - | "move" - | "zoomend" - | "moveend" - | "autopanstart" - | "dragstart" - | "drag" - | "add" - | "remove" - | "loading" - | "error" - | "update" - | "down" - | "predrag", - fn: LeafletEventHandlerFn, - context?: any, - ): this; - once(type: "resize", fn: ResizeEventHandlerFn, context?: any): this; - once(type: "popupopen" | "popupclose", fn: PopupEventHandlerFn, context?: any): this; - once(type: "tooltipopen" | "tooltipclose", fn: TooltipEventHandlerFn, context?: any): this; - once(type: "locationerror", fn: ErrorEventHandlerFn, context?: any): this; - once(type: "locationfound", fn: LocationEventHandlerFn, context?: any): this; - once( - type: - | "click" - | "dblclick" - | "mousedown" - | "mouseup" - | "mouseover" - | "mouseout" - | "mousemove" - | "contextmenu" - | "preclick", - fn: LeafletMouseEventHandlerFn, - context?: any, - ): this; - once(type: "keypress" | "keydown" | "keyup", fn: LeafletKeyboardEventHandlerFn, context?: any): this; - once(type: "zoomanim", fn: ZoomAnimEventHandlerFn, context?: any): this; - once(type: "dragend", fn: DragEndEventHandlerFn, context?: any): this; - once(type: "tileunload" | "tileloadstart" | "tileload" | "tileabort", fn: TileEventHandlerFn, context?: any): this; - once(type: "tileerror", fn: TileEventHandlerFn, context?: any): this; - once(type: string, fn: LeafletEventHandlerFn, context?: any): this; - - /** - * Behaves as on(...), except the listener will only get fired once and then removed. - */ - once(eventMap: LeafletEventHandlerFnMap): this; - // tslint:enable:unified-signatures - - /** - * Adds an event parent - an Evented that will receive propagated events - */ - addEventParent(obj: Evented): this; - - /** - * Removes an event parent, so it will stop receiving propagated events - */ - removeEventParent(obj: Evented): this; - - /** - * Alias for on(...) - * - * Adds a listener function (fn) to a particular event type of the object. - * You can optionally specify the context of the listener (object the this - * keyword will point to). You can also pass several space-separated types - * (e.g. 'click dblclick'). - */ - // tslint:disable:unified-signatures - addEventListener( - type: "baselayerchange" | "overlayadd" | "overlayremove", - fn: LayersControlEventHandlerFn, - context?: any, - ): this; - addEventListener(type: "layeradd" | "layerremove", fn: LayerEventHandlerFn, context?: any): this; - addEventListener( - type: - | "zoomlevelschange" - | "unload" - | "viewreset" - | "load" - | "zoomstart" - | "movestart" - | "zoom" - | "move" - | "zoomend" - | "moveend" - | "autopanstart" - | "dragstart" - | "drag" - | "add" - | "remove" - | "loading" - | "error" - | "update" - | "down" - | "predrag", - fn: LeafletEventHandlerFn, - context?: any, - ): this; - addEventListener(type: "resize", fn: ResizeEventHandlerFn, context?: any): this; - addEventListener(type: "popupopen" | "popupclose", fn: PopupEventHandlerFn, context?: any): this; - addEventListener(type: "tooltipopen" | "tooltipclose", fn: TooltipEventHandlerFn, context?: any): this; - addEventListener(type: "locationerror", fn: ErrorEventHandlerFn, context?: any): this; - addEventListener(type: "locationfound", fn: LocationEventHandlerFn, context?: any): this; - addEventListener( - type: - | "click" - | "dblclick" - | "mousedown" - | "mouseup" - | "mouseover" - | "mouseout" - | "mousemove" - | "contextmenu" - | "preclick", - fn: LeafletMouseEventHandlerFn, - context?: any, - ): this; - addEventListener(type: "keypress" | "keydown" | "keyup", fn: LeafletKeyboardEventHandlerFn, context?: any): this; - addEventListener(type: "zoomanim", fn: ZoomAnimEventHandlerFn, context?: any): this; - addEventListener(type: "dragend", fn: DragEndEventHandlerFn, context?: any): this; - addEventListener( - type: "tileunload" | "tileloadstart" | "tileload" | "tileabort", - fn: TileEventHandlerFn, - context?: any, - ): this; - addEventListener(type: "tileerror", fn: TileErrorEventHandlerFn, context?: any): this; - addEventListener(type: string, fn: LeafletEventHandlerFn, context?: any): this; - - /** - * Alias for on(...) - * - * Adds a set of type/listener pairs, e.g. {click: onClick, mousemove: onMouseMove} - */ - addEventListener(eventMap: LeafletEventHandlerFnMap): this; - // tslint:enable:unified-signatures - - /** - * Alias for off(...) - * - * Removes a previously added listener function. If no function is specified, - * it will remove all the listeners of that particular event from the object. - * Note that if you passed a custom context to on, you must pass the same context - * to off in order to remove the listener. - */ - // tslint:disable:unified-signatures - removeEventListener( - type: "baselayerchange" | "overlayadd" | "overlayremove", - fn?: LayersControlEventHandlerFn, - context?: any, - ): this; - removeEventListener(type: "layeradd" | "layerremove", fn?: LayerEventHandlerFn, context?: any): this; - removeEventListener( - type: - | "zoomlevelschange" - | "unload" - | "viewreset" - | "load" - | "zoomstart" - | "movestart" - | "zoom" - | "move" - | "zoomend" - | "moveend" - | "autopanstart" - | "dragstart" - | "drag" - | "add" - | "remove" - | "loading" - | "error" - | "update" - | "down" - | "predrag", - fn?: LeafletEventHandlerFn, - context?: any, - ): this; - removeEventListener(type: "resize", fn?: ResizeEventHandlerFn, context?: any): this; - removeEventListener(type: "popupopen" | "popupclose", fn?: PopupEventHandlerFn, context?: any): this; - removeEventListener(type: "tooltipopen" | "tooltipclose", fn?: TooltipEventHandlerFn, context?: any): this; - removeEventListener(type: "locationerror", fn?: ErrorEventHandlerFn, context?: any): this; - removeEventListener(type: "locationfound", fn?: LocationEventHandlerFn, context?: any): this; - removeEventListener( - type: - | "click" - | "dblclick" - | "mousedown" - | "mouseup" - | "mouseover" - | "mouseout" - | "mousemove" - | "contextmenu" - | "preclick", - fn?: LeafletMouseEventHandlerFn, - context?: any, - ): this; - removeEventListener( - type: "keypress" | "keydown" | "keyup", - fn?: LeafletKeyboardEventHandlerFn, - context?: any, - ): this; - removeEventListener(type: "zoomanim", fn?: ZoomAnimEventHandlerFn, context?: any): this; - removeEventListener(type: "dragend", fn?: DragEndEventHandlerFn, context?: any): this; - removeEventListener( - type: "tileunload" | "tileloadstart" | "tileload" | "tileabort", - fn?: TileEventHandlerFn, - context?: any, - ): this; - removeEventListener(type: "tileerror", fn?: TileErrorEventHandlerFn, context?: any): this; - removeEventListener(type: string, fn?: LeafletEventHandlerFn, context?: any): this; - - /** - * Alias for off(...) - * - * Removes a set of type/listener pairs. - */ - removeEventListener(eventMap: LeafletEventHandlerFnMap): this; - // tslint:enable:unified-signatures - - /** - * Alias for off() - * - * Removes all listeners to all events on the object. - */ - clearAllEventListeners(): this; - - /** - * Alias for once(...) - * - * Behaves as on(...), except the listener will only get fired once and then removed. - */ - // tslint:disable:unified-signatures - addOneTimeEventListener( - type: "baselayerchange" | "overlayadd" | "overlayremove", - fn: LayersControlEventHandlerFn, - context?: any, - ): this; - addOneTimeEventListener(type: "layeradd" | "layerremove", fn: LayerEventHandlerFn, context?: any): this; - addOneTimeEventListener( - type: - | "zoomlevelschange" - | "unload" - | "viewreset" - | "load" - | "zoomstart" - | "movestart" - | "zoom" - | "move" - | "zoomend" - | "moveend" - | "autopanstart" - | "dragstart" - | "drag" - | "add" - | "remove" - | "loading" - | "error" - | "update" - | "down" - | "predrag", - fn: LeafletEventHandlerFn, - context?: any, - ): this; - addOneTimeEventListener(type: "resize", fn: ResizeEventHandlerFn, context?: any): this; - addOneTimeEventListener(type: "popupopen" | "popupclose", fn: PopupEventHandlerFn, context?: any): this; - addOneTimeEventListener(type: "tooltipopen" | "tooltipclose", fn: TooltipEventHandlerFn, context?: any): this; - addOneTimeEventListener(type: "locationerror", fn: ErrorEventHandlerFn, context?: any): this; - addOneTimeEventListener(type: "locationfound", fn: LocationEventHandlerFn, context?: any): this; - addOneTimeEventListener( - type: - | "click" - | "dblclick" - | "mousedown" - | "mouseup" - | "mouseover" - | "mouseout" - | "mousemove" - | "contextmenu" - | "preclick", - fn: LeafletMouseEventHandlerFn, - context?: any, - ): this; - addOneTimeEventListener( - type: "keypress" | "keydown" | "keyup", - fn: LeafletKeyboardEventHandlerFn, - context?: any, - ): this; - addOneTimeEventListener(type: "zoomanim", fn: ZoomAnimEventHandlerFn, context?: any): this; - addOneTimeEventListener(type: "dragend", fn: DragEndEventHandlerFn, context?: any): this; - addOneTimeEventListener( - type: "tileunload" | "tileloadstart" | "tileload" | "tileabort", - fn: TileEventHandlerFn, - context?: any, - ): this; - addOneTimeEventListener(type: "tileerror", fn: TileErrorEventHandlerFn, context?: any): this; - addOneTimeEventListener(type: string, fn: LeafletEventHandlerFn, context?: any): this; - - /** - * Alias for once(...) - * - * Behaves as on(...), except the listener will only get fired once and then removed. - */ - addOneTimeEventListener(eventMap: LeafletEventHandlerFnMap): this; - // tslint:enable:unified-signatures - - /** - * Alias for fire(...) - * - * Fires an event of the specified type. You can optionally provide a data - * object — the first argument of the listener function will contain its properties. - * The event might can optionally be propagated to event parents. - */ - fireEvent(type: string, data?: any, propagate?: boolean): this; - - /** - * Alias for listens(...) - * - * Returns true if a particular event type has any listeners attached to it. - */ - hasEventListeners(type: string): boolean; -} - -// eslint-disable-next-line @definitelytyped/strict-export-declare-modifiers -declare class MixinType { - Events: Events; -} - -export const Mixin: MixinType; - -/** - * Base class of Leaflet classes supporting events - */ -export abstract class Evented extends Class { - /** - * Adds a listener function (fn) to a particular event type of the object. - * You can optionally specify the context of the listener (object the this - * keyword will point to). You can also pass several space-separated types - * (e.g. 'click dblclick'). - */ - // tslint:disable:unified-signatures - on(type: "baselayerchange" | "overlayadd" | "overlayremove", fn: LayersControlEventHandlerFn, context?: any): this; - on(type: "layeradd" | "layerremove", fn: LayerEventHandlerFn, context?: any): this; - on( - type: - | "zoomlevelschange" - | "unload" - | "viewreset" - | "load" - | "zoomstart" - | "movestart" - | "zoom" - | "move" - | "zoomend" - | "moveend" - | "autopanstart" - | "dragstart" - | "drag" - | "add" - | "remove" - | "loading" - | "error" - | "update" - | "down" - | "predrag", - fn: LeafletEventHandlerFn, - context?: any, - ): this; - on(type: "resize", fn: ResizeEventHandlerFn, context?: any): this; - on(type: "popupopen" | "popupclose", fn: PopupEventHandlerFn, context?: any): this; - on(type: "tooltipopen" | "tooltipclose", fn: TooltipEventHandlerFn, context?: any): this; - on(type: "locationerror", fn: ErrorEventHandlerFn, context?: any): this; - on(type: "locationfound", fn: LocationEventHandlerFn, context?: any): this; - on( - type: - | "click" - | "dblclick" - | "mousedown" - | "mouseup" - | "mouseover" - | "mouseout" - | "mousemove" - | "contextmenu" - | "preclick", - fn: LeafletMouseEventHandlerFn, - context?: any, - ): this; - on(type: "keypress" | "keydown" | "keyup", fn: LeafletKeyboardEventHandlerFn, context?: any): this; - on(type: "zoomanim", fn: ZoomAnimEventHandlerFn, context?: any): this; - on(type: "dragend", fn: DragEndEventHandlerFn, context?: any): this; - on(type: "tileunload" | "tileloadstart" | "tileload" | "tileabort", fn: TileEventHandlerFn, context?: any): this; - on(type: "tileerror", fn: TileErrorEventHandlerFn, context?: any): this; - on(type: string, fn: LeafletEventHandlerFn, context?: any): this; - - /** - * Adds a set of type/listener pairs, e.g. {click: onClick, mousemove: onMouseMove} - */ - on(eventMap: LeafletEventHandlerFnMap): this; - // tslint:enable:unified-signatures - - /** - * Removes a previously added listener function. If no function is specified, - * it will remove all the listeners of that particular event from the object. - * Note that if you passed a custom context to on, you must pass the same context - * to off in order to remove the listener. - */ - // tslint:disable:unified-signatures - off( - type: "baselayerchange" | "overlayadd" | "overlayremove", - fn?: LayersControlEventHandlerFn, - context?: any, - ): this; - off(type: "layeradd" | "layerremove", fn?: LayerEventHandlerFn, context?: any): this; - off( - type: - | "zoomlevelschange" - | "unload" - | "viewreset" - | "load" - | "zoomstart" - | "movestart" - | "zoom" - | "move" - | "zoomend" - | "moveend" - | "autopanstart" - | "dragstart" - | "drag" - | "add" - | "remove" - | "loading" - | "error" - | "update" - | "down" - | "predrag", - fn?: LeafletEventHandlerFn, - context?: any, - ): this; - off(type: "resize", fn?: ResizeEventHandlerFn, context?: any): this; - off(type: "popupopen" | "popupclose", fn?: PopupEventHandlerFn, context?: any): this; - off(type: "tooltipopen" | "tooltipclose", fn?: TooltipEventHandlerFn, context?: any): this; - off(type: "locationerror", fn?: ErrorEventHandlerFn, context?: any): this; - off(type: "locationfound", fn?: LocationEventHandlerFn, context?: any): this; - off( - type: - | "click" - | "dblclick" - | "mousedown" - | "mouseup" - | "mouseover" - | "mouseout" - | "mousemove" - | "contextmenu" - | "preclick", - fn?: LeafletMouseEventHandlerFn, - context?: any, - ): this; - off(type: "keypress" | "keydown" | "keyup", fn?: LeafletKeyboardEventHandlerFn, context?: any): this; - off(type: "zoomanim", fn?: ZoomAnimEventHandlerFn, context?: any): this; - off(type: "dragend", fn?: DragEndEventHandlerFn, context?: any): this; - off(type: "tileunload" | "tileloadstart" | "tileload" | "tileabort", fn?: TileEventHandlerFn, context?: any): this; - off(type: "tileerror", fn?: TileErrorEventHandlerFn, context?: any): this; - off(type: string, fn?: LeafletEventHandlerFn, context?: any): this; - - /** - * Removes a set of type/listener pairs. - */ - // With an eventMap there are no additional arguments allowed - off(eventMap: LeafletEventHandlerFnMap): this; - - /** - * Removes all listeners to all events on the object. - */ - off(): this; - // tslint:enable:unified-signatures - - /** - * Fires an event of the specified type. You can optionally provide a data - * object — the first argument of the listener function will contain its properties. - * The event might can optionally be propagated to event parents. - */ - fire(type: string, data?: any, propagate?: boolean): this; - - /** - * Returns true if a particular event type has any listeners attached to it. - */ - // tslint:disable:unified-signatures - listens( - type: - | "baselayerchange" - | "overlayadd" - | "overlayremove" - | "layeradd" - | "layerremove" - | "zoomlevelschange" - | "unload" - | "viewreset" - | "load" - | "zoomstart" - | "movestart" - | "zoom" - | "move" - | "zoomend" - | "moveend" - | "autopanstart" - | "dragstart" - | "drag" - | "add" - | "remove" - | "loading" - | "error" - | "update" - | "down" - | "predrag" - | "resize" - | "popupopen" - | "tooltipopen" - | "tooltipclose" - | "locationerror" - | "locationfound" - | "click" - | "dblclick" - | "mousedown" - | "mouseup" - | "mouseover" - | "mouseout" - | "mousemove" - | "contextmenu" - | "preclick" - | "keypress" - | "keydown" - | "keyup" - | "zoomanim" - | "dragend" - | "tileunload" - | "tileloadstart" - | "tileload" - | "tileabort" - | "tileerror", - propagate?: boolean, - ): boolean; - - listens( - type: "baselayerchange" | "overlayadd" | "overlayremove", - fn: LayersControlEventHandlerFn, - context?: any, - propagate?: boolean, - ): boolean; - listens(type: "layeradd" | "layerremove", fn: LayerEventHandlerFn, context?: any, propagate?: boolean): boolean; - listens( - type: - | "zoomlevelschange" - | "unload" - | "viewreset" - | "load" - | "zoomstart" - | "movestart" - | "zoom" - | "move" - | "zoomend" - | "moveend" - | "autopanstart" - | "dragstart" - | "drag" - | "add" - | "remove" - | "loading" - | "error" - | "update" - | "down" - | "predrag", - fn: LeafletEventHandlerFn, - context?: any, - propagate?: boolean, - ): boolean; - listens(type: "resize", fn: ResizeEventHandlerFn, context?: any, propagate?: boolean): boolean; - listens(type: "popupopen" | "popupclose", fn: PopupEventHandlerFn, context?: any, propagate?: boolean): boolean; - listens( - type: "tooltipopen" | "tooltipclose", - fn: TooltipEventHandlerFn, - context?: any, - propagate?: boolean, - ): boolean; - listens(type: "locationerror", fn: ErrorEventHandlerFn, context?: any, propagate?: boolean): boolean; - listens(type: "locationfound", fn: LocationEventHandlerFn, context?: any, propagate?: boolean): boolean; - listens( - type: - | "click" - | "dblclick" - | "mousedown" - | "mouseup" - | "mouseover" - | "mouseout" - | "mousemove" - | "contextmenu" - | "preclick", - fn: LeafletMouseEventHandlerFn, - context?: any, - propagate?: boolean, - ): boolean; - listens( - type: "keypress" | "keydown" | "keyup", - fn: LeafletKeyboardEventHandlerFn, - context?: any, - propagate?: boolean, - ): boolean; - listens(type: "zoomanim", fn: ZoomAnimEventHandlerFn, context?: any, propagate?: boolean): boolean; - listens(type: "dragend", fn: DragEndEventHandlerFn, context?: any, propagate?: boolean): boolean; - listens( - type: "tileunload" | "tileloadstart" | "tileload" | "tileabort", - fn: TileEventHandlerFn, - context?: any, - propagate?: boolean, - ): boolean; - listens(type: "tileerror", fn: TileEventHandlerFn, context?: any, propagate?: boolean): boolean; - listens(type: string, fn: LeafletEventHandlerFn, context?: any, propagate?: boolean): boolean; - - /** - * Behaves as on(...), except the listener will only get fired once and then removed. - */ - // tslint:disable:unified-signatures - once( - type: "baselayerchange" | "overlayadd" | "overlayremove", - fn: LayersControlEventHandlerFn, - context?: any, - ): this; - once(type: "layeradd" | "layerremove", fn: LayerEventHandlerFn, context?: any): this; - once( - type: - | "zoomlevelschange" - | "unload" - | "viewreset" - | "load" - | "zoomstart" - | "movestart" - | "zoom" - | "move" - | "zoomend" - | "moveend" - | "autopanstart" - | "dragstart" - | "drag" - | "add" - | "remove" - | "loading" - | "error" - | "update" - | "down" - | "predrag", - fn: LeafletEventHandlerFn, - context?: any, - ): this; - once(type: "resize", fn: ResizeEventHandlerFn, context?: any): this; - once(type: "popupopen" | "popupclose", fn: PopupEventHandlerFn, context?: any): this; - once(type: "tooltipopen" | "tooltipclose", fn: TooltipEventHandlerFn, context?: any): this; - once(type: "locationerror", fn: ErrorEventHandlerFn, context?: any): this; - once(type: "locationfound", fn: LocationEventHandlerFn, context?: any): this; - once( - type: - | "click" - | "dblclick" - | "mousedown" - | "mouseup" - | "mouseover" - | "mouseout" - | "mousemove" - | "contextmenu" - | "preclick", - fn: LeafletMouseEventHandlerFn, - context?: any, - ): this; - once(type: "keypress" | "keydown" | "keyup", fn: LeafletKeyboardEventHandlerFn, context?: any): this; - once(type: "zoomanim", fn: ZoomAnimEventHandlerFn, context?: any): this; - once(type: "dragend", fn: DragEndEventHandlerFn, context?: any): this; - once(type: "tileunload" | "tileloadstart" | "tileload" | "tileabort", fn: TileEventHandlerFn, context?: any): this; - once(type: "tileerror", fn: TileEventHandlerFn, context?: any): this; - once(type: string, fn: LeafletEventHandlerFn, context?: any): this; - - /** - * Behaves as on(...), except the listener will only get fired once and then removed. - */ - once(eventMap: LeafletEventHandlerFnMap): this; - // tslint:enable:unified-signatures - - /** - * Adds an event parent - an Evented that will receive propagated events - */ - addEventParent(obj: Evented): this; - - /** - * Removes an event parent, so it will stop receiving propagated events - */ - removeEventParent(obj: Evented): this; - - /** - * Alias for on(...) - * - * Adds a listener function (fn) to a particular event type of the object. - * You can optionally specify the context of the listener (object the this - * keyword will point to). You can also pass several space-separated types - * (e.g. 'click dblclick'). - */ - // tslint:disable:unified-signatures - addEventListener( - type: "baselayerchange" | "overlayadd" | "overlayremove", - fn: LayersControlEventHandlerFn, - context?: any, - ): this; - addEventListener(type: "layeradd" | "layerremove", fn: LayerEventHandlerFn, context?: any): this; - addEventListener( - type: - | "zoomlevelschange" - | "unload" - | "viewreset" - | "load" - | "zoomstart" - | "movestart" - | "zoom" - | "move" - | "zoomend" - | "moveend" - | "autopanstart" - | "dragstart" - | "drag" - | "add" - | "remove" - | "loading" - | "error" - | "update" - | "down" - | "predrag", - fn: LeafletEventHandlerFn, - context?: any, - ): this; - addEventListener(type: "resize", fn: ResizeEventHandlerFn, context?: any): this; - addEventListener(type: "popupopen" | "popupclose", fn: PopupEventHandlerFn, context?: any): this; - addEventListener(type: "tooltipopen" | "tooltipclose", fn: TooltipEventHandlerFn, context?: any): this; - addEventListener(type: "locationerror", fn: ErrorEventHandlerFn, context?: any): this; - addEventListener(type: "locationfound", fn: LocationEventHandlerFn, context?: any): this; - addEventListener( - type: - | "click" - | "dblclick" - | "mousedown" - | "mouseup" - | "mouseover" - | "mouseout" - | "mousemove" - | "contextmenu" - | "preclick", - fn: LeafletMouseEventHandlerFn, - context?: any, - ): this; - addEventListener(type: "keypress" | "keydown" | "keyup", fn: LeafletKeyboardEventHandlerFn, context?: any): this; - addEventListener(type: "zoomanim", fn: ZoomAnimEventHandlerFn, context?: any): this; - addEventListener(type: "dragend", fn: DragEndEventHandlerFn, context?: any): this; - addEventListener( - type: "tileunload" | "tileloadstart" | "tileload" | "tileabort", - fn: TileEventHandlerFn, - context?: any, - ): this; - addEventListener(type: "tileerror", fn: TileErrorEventHandlerFn, context?: any): this; - addEventListener(type: string, fn: LeafletEventHandlerFn, context?: any): this; - - /** - * Alias for on(...) - * - * Adds a set of type/listener pairs, e.g. {click: onClick, mousemove: onMouseMove} - */ - addEventListener(eventMap: LeafletEventHandlerFnMap): this; - // tslint:enable:unified-signatures - - /** - * Alias for off(...) - * - * Removes a previously added listener function. If no function is specified, - * it will remove all the listeners of that particular event from the object. - * Note that if you passed a custom context to on, you must pass the same context - * to off in order to remove the listener. - */ - // tslint:disable:unified-signatures - removeEventListener( - type: "baselayerchange" | "overlayadd" | "overlayremove", - fn?: LayersControlEventHandlerFn, - context?: any, - ): this; - removeEventListener(type: "layeradd" | "layerremove", fn?: LayerEventHandlerFn, context?: any): this; - removeEventListener( - type: - | "zoomlevelschange" - | "unload" - | "viewreset" - | "load" - | "zoomstart" - | "movestart" - | "zoom" - | "move" - | "zoomend" - | "moveend" - | "autopanstart" - | "dragstart" - | "drag" - | "add" - | "remove" - | "loading" - | "error" - | "update" - | "down" - | "predrag", - fn?: LeafletEventHandlerFn, - context?: any, - ): this; - removeEventListener(type: "resize", fn?: ResizeEventHandlerFn, context?: any): this; - removeEventListener(type: "popupopen" | "popupclose", fn?: PopupEventHandlerFn, context?: any): this; - removeEventListener(type: "tooltipopen" | "tooltipclose", fn?: TooltipEventHandlerFn, context?: any): this; - removeEventListener(type: "locationerror", fn?: ErrorEventHandlerFn, context?: any): this; - removeEventListener(type: "locationfound", fn?: LocationEventHandlerFn, context?: any): this; - removeEventListener( - type: - | "click" - | "dblclick" - | "mousedown" - | "mouseup" - | "mouseover" - | "mouseout" - | "mousemove" - | "contextmenu" - | "preclick", - fn?: LeafletMouseEventHandlerFn, - context?: any, - ): this; - removeEventListener( - type: "keypress" | "keydown" | "keyup", - fn?: LeafletKeyboardEventHandlerFn, - context?: any, - ): this; - removeEventListener(type: "zoomanim", fn?: ZoomAnimEventHandlerFn, context?: any): this; - removeEventListener(type: "dragend", fn?: DragEndEventHandlerFn, context?: any): this; - removeEventListener( - type: "tileunload" | "tileloadstart" | "tileload" | "tileabort", - fn?: TileEventHandlerFn, - context?: any, - ): this; - removeEventListener(type: "tileerror", fn?: TileErrorEventHandlerFn, context?: any): this; - removeEventListener(type: string, fn?: LeafletEventHandlerFn, context?: any): this; - - /** - * Alias for off(...) - * - * Removes a set of type/listener pairs. - */ - removeEventListener(eventMap: LeafletEventHandlerFnMap): this; - // tslint:enable:unified-signatures - - /** - * Alias for off() - * - * Removes all listeners to all events on the object. - */ - clearAllEventListeners(): this; - - /** - * Alias for once(...) - * - * Behaves as on(...), except the listener will only get fired once and then removed. - */ - // tslint:disable:unified-signatures - addOneTimeEventListener( - type: "baselayerchange" | "overlayadd" | "overlayremove", - fn: LayersControlEventHandlerFn, - context?: any, - ): this; - addOneTimeEventListener(type: "layeradd" | "layerremove", fn: LayerEventHandlerFn, context?: any): this; - addOneTimeEventListener( - type: - | "zoomlevelschange" - | "unload" - | "viewreset" - | "load" - | "zoomstart" - | "movestart" - | "zoom" - | "move" - | "zoomend" - | "moveend" - | "autopanstart" - | "dragstart" - | "drag" - | "add" - | "remove" - | "loading" - | "error" - | "update" - | "down" - | "predrag", - fn: LeafletEventHandlerFn, - context?: any, - ): this; - addOneTimeEventListener(type: "resize", fn: ResizeEventHandlerFn, context?: any): this; - addOneTimeEventListener(type: "popupopen" | "popupclose", fn: PopupEventHandlerFn, context?: any): this; - addOneTimeEventListener(type: "tooltipopen" | "tooltipclose", fn: TooltipEventHandlerFn, context?: any): this; - addOneTimeEventListener(type: "locationerror", fn: ErrorEventHandlerFn, context?: any): this; - addOneTimeEventListener(type: "locationfound", fn: LocationEventHandlerFn, context?: any): this; - addOneTimeEventListener( - type: - | "click" - | "dblclick" - | "mousedown" - | "mouseup" - | "mouseover" - | "mouseout" - | "mousemove" - | "contextmenu" - | "preclick", - fn: LeafletMouseEventHandlerFn, - context?: any, - ): this; - addOneTimeEventListener( - type: "keypress" | "keydown" | "keyup", - fn: LeafletKeyboardEventHandlerFn, - context?: any, - ): this; - addOneTimeEventListener(type: "zoomanim", fn: ZoomAnimEventHandlerFn, context?: any): this; - addOneTimeEventListener(type: "dragend", fn: DragEndEventHandlerFn, context?: any): this; - addOneTimeEventListener( - type: "tileunload" | "tileloadstart" | "tileload" | "tileabort", - fn: TileEventHandlerFn, - context?: any, - ): this; - addOneTimeEventListener(type: "tileerror", fn: TileErrorEventHandlerFn, context?: any): this; - addOneTimeEventListener(type: string, fn: LeafletEventHandlerFn, context?: any): this; - - /** - * Alias for once(...) - * - * Behaves as on(...), except the listener will only get fired once and then removed. - */ - addOneTimeEventListener(eventMap: LeafletEventHandlerFnMap): this; - // tslint:enable:unified-signatures - - /** - * Alias for fire(...) - * - * Fires an event of the specified type. You can optionally provide a data - * object — the first argument of the listener function will contain its properties. - * The event might can optionally be propagated to event parents. - */ - fireEvent(type: string, data?: any, propagate?: boolean): this; - - /** - * Alias for listens(...) - * - * Returns true if a particular event type has any listeners attached to it. - */ - hasEventListeners(type: string): boolean; -} - -export interface DraggableOptions { - /** - * The max number of pixels a user can shift the mouse pointer during a click - * for it to be considered a valid click (as opposed to a mouse drag). - */ - clickTolerance: number; -} - -/** - * A class for making DOM elements draggable (including touch support). - * Used internally for map and marker dragging. Only works for elements - * that were positioned with [`L.DomUtil.setPosition`](#domutil-setposition). - */ -export class Draggable extends Evented { - constructor( - element: HTMLElement, - dragStartTarget?: HTMLElement, - preventOutline?: boolean, - options?: DraggableOptions, - ); - - enable(): void; - - disable(): void; - - finishDrag(): void; -} - -export interface LayerOptions { - pane?: string | undefined; - attribution?: string | undefined; -} - -export interface InteractiveLayerOptions extends LayerOptions { - interactive?: boolean | undefined; - bubblingMouseEvents?: boolean | undefined; -} - -export class Layer extends Evented { - constructor(options?: LayerOptions); - addTo(map: Map | LayerGroup): this; - remove(): this; - removeFrom(map: Map): this; - getPane(name?: string): HTMLElement | undefined; - - addInteractiveTarget(targetEl: HTMLElement): this; - removeInteractiveTarget(targetEl: HTMLElement): this; - - // Popup methods - bindPopup(content: ((layer: Layer) => Content) | Content | Popup, options?: PopupOptions): this; - unbindPopup(): this; - openPopup(latlng?: LatLngExpression): this; - closePopup(): this; - togglePopup(): this; - isPopupOpen(): boolean; - setPopupContent(content: Content | Popup): this; - getPopup(): Popup | undefined; - - // Tooltip methods - bindTooltip(content: ((layer: Layer) => Content) | Tooltip | Content, options?: TooltipOptions): this; - unbindTooltip(): this; - openTooltip(latlng?: LatLngExpression): this; - closeTooltip(): this; - toggleTooltip(): this; - isTooltipOpen(): boolean; - setTooltipContent(content: Content | Tooltip): this; - getTooltip(): Tooltip | undefined; - - // Extension methods - onAdd(map: Map): this; - onRemove(map: Map): this; - getEvents?(): { [name: string]: LeafletEventHandlerFn }; - getAttribution?(): string | null; - beforeAdd?(map: Map): this; - - protected _map: Map; - - options: LayerOptions; -} - -export interface GridLayerOptions extends LayerOptions { - tileSize?: number | Point | undefined; - opacity?: number | undefined; - updateWhenIdle?: boolean | undefined; - updateWhenZooming?: boolean | undefined; - updateInterval?: number | undefined; - zIndex?: number | undefined; - bounds?: LatLngBoundsExpression | undefined; - minZoom?: number | undefined; - maxZoom?: number | undefined; - /** - * Maximum zoom number the tile source has available. If it is specified, the tiles on all zoom levels higher than - * `maxNativeZoom` will be loaded from `maxNativeZoom` level and auto-scaled. - */ - maxNativeZoom?: number | undefined; - /** - * Minimum zoom number the tile source has available. If it is specified, the tiles on all zoom levels lower than - * `minNativeZoom` will be loaded from `minNativeZoom` level and auto-scaled. - */ - minNativeZoom?: number | undefined; - noWrap?: boolean | undefined; - pane?: string | undefined; - className?: string | undefined; - keepBuffer?: number | undefined; -} - -export type DoneCallback = (error?: Error, tile?: HTMLElement) => void; - -export interface InternalTiles { - [key: string]: { - active?: boolean | undefined; - coords: Coords; - current: boolean; - el: HTMLElement; - loaded?: Date | undefined; - retain?: boolean | undefined; - }; -} - -export class GridLayer extends Layer { - constructor(options?: GridLayerOptions); - bringToFront(): this; - bringToBack(): this; - getContainer(): HTMLElement | null; - setOpacity(opacity: number): this; - setZIndex(zIndex: number): this; - isLoading(): boolean; - redraw(): this; - getTileSize(): Point; - - protected createTile(coords: Coords, done: DoneCallback): HTMLElement; - protected _tileCoordsToKey(coords: Coords): string; - protected _wrapCoords(parameter: Coords): Coords; - - protected _tiles: InternalTiles; - protected _tileZoom?: number | undefined; -} - -export function gridLayer(options?: GridLayerOptions): GridLayer; - -export interface TileLayerOptions extends GridLayerOptions { - id?: string | undefined; - subdomains?: string | string[] | undefined; - errorTileUrl?: string | undefined; - zoomOffset?: number | undefined; - tms?: boolean | undefined; - zoomReverse?: boolean | undefined; - detectRetina?: boolean | undefined; - crossOrigin?: CrossOrigin | boolean | undefined; - referrerPolicy?: ReferrerPolicy | boolean | undefined; - // [name: string]: any; - // You are able add additional properties, but it makes this interface uncheckable. - // See: https://github.com/DefinitelyTyped/DefinitelyTyped/issues/15313 - // Example: - // tileLayer = L.tileLayer('http://{s}.tile.osm.org/{z}/{x}/{y}.png?{foo}&{bar}&{abc}', {foo: 'bar', bar: (data: any) => 'foo', abc: () => ''}); -} - -export class TileLayer extends GridLayer { - constructor(urlTemplate: string, options?: TileLayerOptions); - setUrl(url: string, noRedraw?: boolean): this; - getTileUrl(coords: L.Coords): string; - - protected _tileOnLoad(done: L.DoneCallback, tile: HTMLElement): void; - protected _tileOnError(done: L.DoneCallback, tile: HTMLElement, e: Error): void; - protected _abortLoading(): void; - protected _getZoomForUrl(): number; - - options: TileLayerOptions; -} - -export function tileLayer(urlTemplate: string, options?: TileLayerOptions): TileLayer; - -export namespace TileLayer { - class WMS extends TileLayer { - constructor(baseUrl: string, options: WMSOptions); - setParams(params: WMSParams, noRedraw?: boolean): this; - - wmsParams: WMSParams; - options: WMSOptions; - } -} - -export interface WMSOptions extends TileLayerOptions { - layers?: string | undefined; - styles?: string | undefined; - format?: string | undefined; - transparent?: boolean | undefined; - version?: string | undefined; - crs?: CRS | undefined; - uppercase?: boolean | undefined; -} - -export interface WMSParams { - format?: string | undefined; - layers: string; - request?: string | undefined; - service?: string | undefined; - styles?: string | undefined; - version?: string | undefined; - transparent?: boolean | undefined; - width?: number | undefined; - height?: number | undefined; -} - -export namespace tileLayer { - function wms(baseUrl: string, options?: WMSOptions): TileLayer.WMS; -} - -export type CrossOrigin = "anonymous" | "use-credentials" | ""; -export type ReferrerPolicy = - | "no-referrer" - | "no-referrer-when-downgrade" - | "origin" - | "origin-when-cross-origin" - | "same-origin" - | "strict-origin" - | "strict-origin-when-cross-origin" - | "unsafe-url"; - -export interface ImageOverlayOptions extends InteractiveLayerOptions { - opacity?: number | undefined; - alt?: string | undefined; - interactive?: boolean | undefined; - crossOrigin?: CrossOrigin | boolean | undefined; - errorOverlayUrl?: string | undefined; - zIndex?: number | undefined; - className?: string | undefined; -} - -export interface ImageOverlayStyleOptions { - opacity?: number; - [name: string]: any; -} - -export class ImageOverlay extends Layer { - constructor(imageUrl: string, bounds: LatLngBoundsExpression, options?: ImageOverlayOptions); - bringToFront(): this; - bringToBack(): this; - setUrl(url: string): this; - - /** Update the bounds that this ImageOverlay covers */ - setBounds(bounds: LatLngBounds): this; - - /** Changes the zIndex of the image overlay */ - setZIndex(value: number): this; - - /** Changes the opacity of the image element */ - setOpacity(opacity: number): this; - - /** Changes the style of the image element. As of 1.8, only the opacity is changed */ - setStyle(styleOpts: ImageOverlayStyleOptions): this; - - /** Get the bounds that this ImageOverlay covers */ - getBounds(): LatLngBounds; - - /** Get the center of the bounds this ImageOverlay covers */ - getCenter(): Point; - - /** Get the img element that represents the ImageOverlay on the map */ - getElement(): HTMLImageElement | undefined; - - options: ImageOverlayOptions; -} - -export function imageOverlay( - imageUrl: string, - bounds: LatLngBoundsExpression, - options?: ImageOverlayOptions, -): ImageOverlay; - -export type SVGOverlayStyleOptions = ImageOverlayStyleOptions; - -export class SVGOverlay extends Layer { - /** SVGOverlay doesn't extend ImageOverlay because SVGOverlay.getElement returns SVGElement */ - - constructor(svgImage: string | SVGElement, bounds: LatLngBoundsExpression, options?: ImageOverlayOptions); - bringToFront(): this; - bringToBack(): this; - setUrl(url: string): this; - - /** Update the bounds that this SVGOverlay covers */ - setBounds(bounds: LatLngBounds): this; - - /** Changes the zIndex of the image overlay */ - setZIndex(value: number): this; - - /** Changes the opacity of the image element */ - setOpacity(opacity: number): this; - - /** Changes the style of the image element. As of 1.8, only the opacity is changed */ - setStyle(styleOpts: SVGOverlayStyleOptions): this; - - /** Get the bounds that this SVGOverlay covers */ - getBounds(): LatLngBounds; - - /** Get the center of the bounds this ImageOverlay covers */ - getCenter(): Point; - - /** Get the img element that represents the SVGOverlay on the map */ - getElement(): SVGElement | undefined; - - options: ImageOverlayOptions; -} - -export function svgOverlay( - svgImage: string | SVGElement, - bounds: LatLngBoundsExpression, - options?: ImageOverlayOptions, -): SVGOverlay; - -export interface VideoOverlayOptions extends ImageOverlayOptions { - /** Whether the video starts playing automatically when loaded. */ - autoplay?: boolean | undefined; - /** Whether the video will loop back to the beginning when played. */ - loop?: boolean | undefined; - /** - * Whether the video will save aspect ratio after the projection. Relevant for supported browsers. See - * [browser compatibility](https://developer.mozilla.org/en-US/docs/Web/CSS/object-fit) - */ - keepAspectRatio?: boolean | undefined; - /** Whether the video starts on mute when loaded. */ - muted?: boolean | undefined; - playsInline?: boolean | undefined; -} - -export class VideoOverlay extends Layer { - /** VideoOverlay doesn't extend ImageOverlay because VideoOverlay.getElement returns HTMLImageElement */ - constructor( - video: string | string[] | HTMLVideoElement, - bounds: LatLngBoundsExpression, - options?: VideoOverlayOptions, - ); - bringToFront(): this; - bringToBack(): this; - setUrl(url: string): this; - - /** Update the bounds that this VideoOverlay covers */ - setBounds(bounds: LatLngBounds): this; - - /** Changes the zIndex of the image overlay */ - setZIndex(value: number): this; - - /** Changes the opacity of the image element */ - setOpacity(opacity: number): this; - - /** Changes the style of the image element. As of 1.8, only the opacity is changed */ - setStyle(styleOpts: SVGOverlayStyleOptions): this; - - /** Get the bounds that this VideoOverlay covers */ - getBounds(): LatLngBounds; - - /** Get the center of the bounds this ImageOverlay covers */ - getCenter(): Point; - - /** Get the video element that represents the VideoOverlay on the map */ - getElement(): HTMLVideoElement | undefined; - - options: VideoOverlayOptions; -} - -export function videoOverlay( - video: string | string[] | HTMLVideoElement, - bounds: LatLngBoundsExpression, - options?: VideoOverlayOptions, -): VideoOverlay; - -export type LineCapShape = "butt" | "round" | "square" | "inherit"; - -export type LineJoinShape = "miter" | "round" | "bevel" | "inherit"; - -export type FillRule = "nonzero" | "evenodd" | "inherit"; - -export interface PathOptions extends InteractiveLayerOptions { - stroke?: boolean | undefined; - color?: string | undefined; - weight?: number | undefined; - opacity?: number | undefined; - lineCap?: LineCapShape | undefined; - lineJoin?: LineJoinShape | undefined; - dashArray?: string | number[] | undefined; - dashOffset?: string | undefined; - fill?: boolean | undefined; - fillColor?: string | undefined; - fillOpacity?: number | undefined; - fillRule?: FillRule | undefined; - renderer?: Renderer | undefined; - className?: string | undefined; -} - -export abstract class Path extends Layer { - redraw(): this; - setStyle(style: PathOptions): this; - bringToFront(): this; - bringToBack(): this; - getElement(): Element | undefined; - - options: PathOptions; -} - -export interface PolylineOptions extends PathOptions { - smoothFactor?: number | undefined; - noClip?: boolean | undefined; -} - -export class Polyline - extends Path -{ - constructor(latlngs: LatLngExpression[] | LatLngExpression[][], options?: PolylineOptions); - toGeoJSON(precision?: number | false): geojson.Feature; - getLatLngs(): LatLng[] | LatLng[][] | LatLng[][][]; - setLatLngs(latlngs: LatLngExpression[] | LatLngExpression[][] | LatLngExpression[][][]): this; - isEmpty(): boolean; - getCenter(): LatLng; - getBounds(): LatLngBounds; - addLatLng(latlng: LatLngExpression | LatLngExpression[], latlngs?: LatLng[]): this; - closestLayerPoint(p: Point): Point; - - feature?: geojson.Feature | undefined; - options: PolylineOptions; -} - -export function polyline( - latlngs: LatLngExpression[] | LatLngExpression[][], - options?: PolylineOptions, -): Polyline; - -export class Polygon

extends Polyline { - constructor(latlngs: LatLngExpression[] | LatLngExpression[][] | LatLngExpression[][][], options?: PolylineOptions); -} - -export function polygon

( - latlngs: LatLngExpression[] | LatLngExpression[][] | LatLngExpression[][][], - options?: PolylineOptions, -): Polygon

; - -export class Rectangle

extends Polygon

{ - constructor(latLngBounds: LatLngBoundsExpression, options?: PolylineOptions); - setBounds(latLngBounds: LatLngBoundsExpression): this; -} - -export function rectangle

(latLngBounds: LatLngBoundsExpression, options?: PolylineOptions): Rectangle

; - -export interface CircleMarkerOptions extends PathOptions { - radius: number; -} - -export class CircleMarker

extends Path { - constructor(latlng: LatLngExpression, options: CircleMarkerOptions); - toGeoJSON(precision?: number | false): geojson.Feature; - setLatLng(latLng: LatLngExpression): this; - getLatLng(): LatLng; - setRadius(radius: number): this; - getRadius(): number; - setStyle(options: Partial): this; - - options: CircleMarkerOptions; - feature?: geojson.Feature | undefined; -} - -export function circleMarker

(latlng: LatLngExpression, options?: CircleMarkerOptions): CircleMarker

; - -export type CircleOptions = CircleMarkerOptions; - -export class Circle

extends CircleMarker

{ - constructor(latlng: LatLngExpression, options: CircleOptions); - constructor(latlng: LatLngExpression, radius: number, options?: CircleOptions); // deprecated! - toGeoJSON(precision?: number | false): any; - getBounds(): LatLngBounds; - setRadius(radius: number): this; - getRadius(): number; - setStyle(style: PathOptions): this; -} - -export function circle

(latlng: LatLngExpression, options: CircleMarkerOptions): Circle

; -/** - * @deprecated Passing the radius outside the options is deperecated. Use {@link circle:1} instead. - */ -export function circle

(latlng: LatLngExpression, radius: number, options?: CircleMarkerOptions): Circle

; - -export interface RendererOptions extends LayerOptions { - padding?: number | undefined; - tolerance?: number | undefined; -} - -export class Renderer extends Layer { - constructor(options?: RendererOptions); - - options: RendererOptions; -} - -export class SVG extends Renderer {} - -export namespace SVG { - function create(name: K): SVGElementTagNameMap[K]; - function create(name: string): SVGElement; - - function pointsToPath(rings: PointExpression[], closed: boolean): string; -} - -export function svg(options?: RendererOptions): SVG; - -export class Canvas extends Renderer {} - -export function canvas(options?: RendererOptions): Canvas; - -/** - * Used to group several layers and handle them as one. - * If you add it to the map, any layers added or removed from the group will be - * added/removed on the map as well. Extends Layer. - */ -export class LayerGroup

extends Layer { - constructor(layers?: Layer[], options?: LayerOptions); - - toMultiPoint(precision?: number): geojson.Feature; - - /** - * Returns a GeoJSON representation of the layer group (as a GeoJSON GeometryCollection, GeoJSONFeatureCollection or Multipoint). - */ - toGeoJSON( - precision?: number | false, - ): - | geojson.FeatureCollection - | geojson.Feature - | geojson.GeometryCollection; - - /** - * Adds the given layer to the group. - */ - addLayer(layer: Layer): this; - - /** - * Removes the layer with the given internal ID or the given layer from the group. - */ - removeLayer(layer: number | Layer): this; - - /** - * Returns true if the given layer is currently added to the group. - */ - hasLayer(layer: Layer): boolean; - - /** - * Removes all the layers from the group. - */ - clearLayers(): this; - - /** - * Calls methodName on every layer contained in this group, passing any additional parameters. - * Has no effect if the layers contained do not implement methodName. - */ - invoke(methodName: string, ...params: any[]): this; - - /** - * Iterates over the layers of the group, - * optionally specifying context of the iterator function. - */ - eachLayer(fn: (layer: Layer) => void, context?: any): this; - - /** - * Returns the layer with the given internal ID. - */ - getLayer(id: number): Layer | undefined; - - /** - * Returns an array of all the layers added to the group. - */ - getLayers(): Layer[]; - - /** - * Calls setZIndex on every layer contained in this group, passing the z-index. - */ - setZIndex(zIndex: number): this; - - /** - * Returns the internal ID for a layer - */ - getLayerId(layer: Layer): number; - - feature?: - | geojson.FeatureCollection - | geojson.Feature - | geojson.GeometryCollection - | undefined; -} - -/** - * Create a layer group, optionally given an initial set of layers and an `options` object. - */ -export function layerGroup

(layers?: Layer[], options?: LayerOptions): LayerGroup

; - -/** - * Extended LayerGroup that also has mouse events (propagated from - * members of the group) and a shared bindPopup method. - */ -export class FeatureGroup

extends LayerGroup

{ - /** - * Adds the given layer to the group. - */ - addLayer(layer: Layer): this; - - /** - * Removes the layer with the given internal ID or the given layer from the group. - */ - removeLayer(layer: number | Layer): this; - - /** - * Sets the given path options to each layer of the group that has a setStyle method. - */ - setStyle(style: PathOptions): this; - - /** - * Brings the layer group to the top of all other layers - */ - bringToFront(): this; - - /** - * Brings the layer group to the top [sic] of all other layers - */ - bringToBack(): this; - - /** - * Returns the LatLngBounds of the Feature Group (created from - * bounds and coordinates of its children). - */ - getBounds(): LatLngBounds; -} - -/** - * Create a feature group, optionally given an initial set of layers. - */ -export function featureGroup

(layers?: Layer[], options?: LayerOptions): FeatureGroup

; - -export type StyleFunction

= (feature?: geojson.Feature) => PathOptions; - -export interface GeoJSONOptions

- extends InteractiveLayerOptions -{ - /** - * A Function defining how GeoJSON points spawn Leaflet layers. - * It is internally called when data is added, passing the GeoJSON point - * feature and its LatLng. - * - * The default is to spawn a default Marker: - * - * ``` - * function(geoJsonPoint, latlng) { - * return L.marker(latlng); - * } - * ``` - */ - pointToLayer?(geoJsonPoint: geojson.Feature, latlng: LatLng): Layer; // should import GeoJSON typings - - /** - * PathOptions or a Function defining the Path options for styling GeoJSON lines and polygons, - * called internally when data is added. - * - * The default value is to not override any defaults: - * - * ``` - * function (geoJsonFeature) { - * return {} - * } - * ``` - */ - style?: PathOptions | StyleFunction

| undefined; - - /** - * A Function that will be called once for each created Feature, after it - * has been created and styled. Useful for attaching events and popups to features. - * - * The default is to do nothing with the newly created layers: - * - * ``` - * function (feature, layer) {} - * ``` - */ - onEachFeature?(feature: geojson.Feature, layer: Layer): void; - - /** - * A Function that will be used to decide whether to show a feature or not. - * - * The default is to show all features: - * - * ``` - * function (geoJsonFeature) { - * return true; - * } - * ``` - */ - filter?(geoJsonFeature: geojson.Feature): boolean; - - /** - * A Function that will be used for converting GeoJSON coordinates to LatLngs. - * The default is the coordsToLatLng static method. - */ - coordsToLatLng?(coords: [number, number] | [number, number, number]): LatLng; // check if LatLng has an altitude property - - /** Whether default Markers for "Point" type Features inherit from group options. */ - markersInheritOptions?: boolean | undefined; -} - -/** - * Represents a GeoJSON object or an array of GeoJSON objects. - * Allows you to parse GeoJSON data and display it on the map. Extends FeatureGroup. - */ -export class GeoJSON

extends FeatureGroup

{ - /** - * Convert layer into GeoJSON feature - */ - static getFeature

( - layer: Layer, - newGeometry: geojson.Feature | G, - ): geojson.Feature; - - /** - * Creates a Layer from a given GeoJSON feature. Can use a custom pointToLayer - * and/or coordsToLatLng functions if provided as options. - */ - static geometryToLayer

( - featureData: geojson.Feature, - options?: GeoJSONOptions, - ): Layer; - - /** - * Creates a LatLng object from an array of 2 numbers (longitude, latitude) or - * 3 numbers (longitude, latitude, altitude) used in GeoJSON for points. - */ - static coordsToLatLng(coords: [number, number] | [number, number, number]): LatLng; - - /** - * Creates a multidimensional array of LatLngs from a GeoJSON coordinates array. - * levelsDeep specifies the nesting level (0 is for an array of points, 1 for an array of - * arrays of points, etc., 0 by default). - * Can use a custom coordsToLatLng function. - */ - static coordsToLatLngs( - coords: any[], - levelsDeep?: number, - coordsToLatLng?: (coords: [number, number] | [number, number, number]) => LatLng, - ): any[]; // Using any[] to avoid artificially limiting valid calls - - /** - * Reverse of coordsToLatLng - */ - static latLngToCoords(latlng: LatLng): [number, number] | [number, number, number]; - - /** - * Reverse of coordsToLatLngs closed determines whether the first point should be - * appended to the end of the array to close the feature, only used when levelsDeep is 0. - * False by default. - */ - static latLngsToCoords(latlngs: any[], levelsDeep?: number, closed?: boolean): any[]; // Using any[] to avoid artificially limiting valid calls - - /** - * Normalize GeoJSON geometries/features into GeoJSON features. - */ - static asFeature

( - geojson: geojson.Feature | G, - ): geojson.Feature; - - constructor(geojson?: geojson.GeoJsonObject | null, options?: GeoJSONOptions | null); - /** - * Adds a GeoJSON object to the layer. - */ - addData(data: geojson.GeoJsonObject): this; - - /** - * Resets the given vector layer's style to the original GeoJSON style, - * useful for resetting style after hover events. - */ - resetStyle(layer?: Layer): this; - - /** - * Same as FeatureGroup's setStyle method, but style-functions are also - * allowed here to set the style according to the feature. - */ - setStyle(style: PathOptions | StyleFunction

): this; - - options: GeoJSONOptions; -} - -/** - * Creates a GeoJSON layer. - * - * Optionally accepts an object in GeoJSON format to display on the - * map (you can alternatively add it later with addData method) and - * an options object. - */ -export function geoJSON

( - geojson?: geojson.GeoJsonObject | geojson.GeoJsonObject[] | null, - options?: GeoJSONOptions | null, -): GeoJSON; -export function geoJson

( - geojson?: geojson.GeoJsonObject | geojson.GeoJsonObject[] | null, - options?: GeoJSONOptions | null, -): GeoJSON; - -export type Zoom = boolean | "center"; - -export interface MapOptions { - preferCanvas?: boolean | undefined; - - // Control options - attributionControl?: boolean | undefined; - zoomControl?: boolean | undefined; - - // Interaction options - closePopupOnClick?: boolean | undefined; - zoomSnap?: number | undefined; - zoomDelta?: number | undefined; - trackResize?: boolean | undefined; - boxZoom?: boolean | undefined; - doubleClickZoom?: Zoom | undefined; - dragging?: boolean | undefined; - - // Map state options - crs?: CRS | undefined; - center?: LatLngExpression | undefined; - zoom?: number | undefined; - minZoom?: number | undefined; - maxZoom?: number | undefined; - layers?: Layer[] | undefined; - maxBounds?: LatLngBoundsExpression | undefined; - renderer?: Renderer | undefined; - - // Animation options - fadeAnimation?: boolean | undefined; - markerZoomAnimation?: boolean | undefined; - transform3DLimit?: number | undefined; - zoomAnimation?: boolean | undefined; - zoomAnimationThreshold?: number | undefined; - - // Panning inertia options - inertia?: boolean | undefined; - inertiaDeceleration?: number | undefined; - inertiaMaxSpeed?: number | undefined; - easeLinearity?: number | undefined; - worldCopyJump?: boolean | undefined; - maxBoundsViscosity?: number | undefined; - - // Keyboard navigation options - keyboard?: boolean | undefined; - keyboardPanDelta?: number | undefined; - - // Mousewheel options - scrollWheelZoom?: Zoom | undefined; - wheelDebounceTime?: number | undefined; - wheelPxPerZoomLevel?: number | undefined; - - // Touch interaction options - tapHold?: boolean | undefined; - tapTolerance?: number | undefined; - touchZoom?: Zoom | undefined; - bounceAtZoomLimits?: boolean | undefined; -} - -export type ControlPosition = "topleft" | "topright" | "bottomleft" | "bottomright"; - -export interface ControlOptions { - position?: ControlPosition | undefined; -} - -export class Control extends Class { - static extend( - props: T, - ): { new(...args: any[]): T } & typeof Control; - constructor(options?: Options); - getPosition(): ControlPosition; - setPosition(position: ControlPosition): this; - getContainer(): HTMLElement | undefined; - addTo(map: Map): this; - remove(): this; - - // Extension methods - onAdd?(map: Map): HTMLElement; - onRemove?(map: Map): void; - - options: Options; -} - -export namespace Control { - interface ZoomOptions extends ControlOptions { - zoomInText?: string | undefined; - zoomInTitle?: string | undefined; - zoomOutText?: string | undefined; - zoomOutTitle?: string | undefined; - } - - class Zoom extends Control { - constructor(options?: ZoomOptions); - options: ZoomOptions; - } - - interface AttributionOptions extends ControlOptions { - prefix?: string | boolean | undefined; - } - - class Attribution extends Control { - constructor(options?: AttributionOptions); - setPrefix(prefix: string | false): this; - addAttribution(text: string): this; - removeAttribution(text: string): this; - options: AttributionOptions; - } - - interface LayersOptions extends ControlOptions { - collapsed?: boolean | undefined; - autoZIndex?: boolean | undefined; - hideSingleBase?: boolean | undefined; - /** - * Whether to sort the layers. When `false`, layers will keep the order in which they were added to the control. - */ - sortLayers?: boolean | undefined; - /** - * A [compare function](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array/sort) - * that will be used for sorting the layers, when `sortLayers` is `true`. The function receives both the - * [`L.Layer`](https://leafletjs.com/reference.html#layer) instances and their names, as in - * `sortFunction(layerA, layerB, nameA, nameB)`. By default, it sorts layers alphabetically by their name. - */ - sortFunction?: ((layerA: Layer, layerB: Layer, nameA: string, nameB: string) => number) | undefined; - } - - interface LayersObject { - [name: string]: Layer; - } - - class Layers extends Control { - constructor(baseLayers?: LayersObject, overlays?: LayersObject, options?: LayersOptions); - addBaseLayer(layer: Layer, name: string): this; - addOverlay(layer: Layer, name: string): this; - removeLayer(layer: Layer): this; - expand(): this; - collapse(): this; - options: LayersOptions; - } - - interface ScaleOptions extends ControlOptions { - maxWidth?: number | undefined; - metric?: boolean | undefined; - imperial?: boolean | undefined; - updateWhenIdle?: boolean | undefined; - } - - class Scale extends Control { - constructor(options?: ScaleOptions); - options: ScaleOptions; - } -} - -export namespace control { - function zoom(options?: Control.ZoomOptions): Control.Zoom; - - function attribution(options?: Control.AttributionOptions): Control.Attribution; - - function layers( - baseLayers?: Control.LayersObject, - overlays?: Control.LayersObject, - options?: Control.LayersOptions, - ): Control.Layers; - - function scale(options?: Control.ScaleOptions): Control.Scale; -} - -export interface DivOverlayOptions { - offset?: PointExpression | undefined; - className?: string | undefined; - pane?: string | undefined; - interactive?: boolean | undefined; - content?: string | HTMLElement | ((layer: Layer) => string) | ((layer: Layer) => HTMLElement); -} - -export abstract class DivOverlay extends Layer { - constructor(latlng: LatLngExpression, options?: TooltipOptions); - constructor(options?: DivOverlayOptions, source?: Layer); - getLatLng(): LatLng | undefined; - setLatLng(latlng: LatLngExpression): this; - getContent(): Content | ((source: Layer) => Content) | undefined; - setContent(htmlContent: ((source: Layer) => Content) | Content): this; - getElement(): HTMLElement | undefined; - update(): void; - isOpen(): boolean; - bringToFront(): this; - bringToBack(): this; - openOn(map: Map): this; - toggle(layer?: Layer): this; - close(): this; - - options: DivOverlayOptions; -} - -export interface PopupOptions extends DivOverlayOptions { - maxWidth?: number | undefined; - minWidth?: number | undefined; - maxHeight?: number | undefined; - keepInView?: boolean | undefined; - closeButton?: boolean | undefined; - autoPan?: boolean | undefined; - autoPanPaddingTopLeft?: PointExpression | undefined; - autoPanPaddingBottomRight?: PointExpression | undefined; - autoPanPadding?: PointExpression | undefined; - autoClose?: boolean | undefined; - closeOnClick?: boolean | undefined; - closeOnEscapeKey?: boolean | undefined; -} - -export type Content = string | HTMLElement; - -export class Popup extends DivOverlay { - constructor(latlng: LatLngExpression, options?: TooltipOptions); - constructor(options?: PopupOptions, source?: Layer); - openOn(map: Map): this; - - options: PopupOptions; -} - -export function popup(options?: PopupOptions, source?: Layer): Popup; - -export type Direction = "right" | "left" | "top" | "bottom" | "center" | "auto"; - -export interface TooltipOptions extends DivOverlayOptions { - pane?: string | undefined; - offset?: PointExpression | undefined; - direction?: Direction | undefined; - permanent?: boolean | undefined; - sticky?: boolean | undefined; - opacity?: number | undefined; -} - -export class Tooltip extends DivOverlay { - constructor(latlng: LatLngExpression, options?: TooltipOptions); - constructor(options?: TooltipOptions, source?: Layer); - setOpacity(val: number): void; - - options: TooltipOptions; -} - -export function tooltip(options?: TooltipOptions, source?: Layer): Tooltip; - -export interface ZoomOptions { - animate?: boolean | undefined; -} - -export interface PanOptions { - animate?: boolean | undefined; - duration?: number | undefined; - easeLinearity?: number | undefined; - noMoveStart?: boolean | undefined; -} - -// This is not empty, it extends two interfaces into one... -export interface ZoomPanOptions extends ZoomOptions, PanOptions {} - -export interface InvalidateSizeOptions extends ZoomPanOptions { - debounceMoveend?: boolean | undefined; - pan?: boolean | undefined; -} - -export interface FitBoundsOptions extends ZoomOptions, PanOptions { - paddingTopLeft?: PointExpression | undefined; - paddingBottomRight?: PointExpression | undefined; - padding?: PointExpression | undefined; - maxZoom?: number | undefined; -} - -export interface PanInsideOptions extends PanOptions { - paddingTopLeft?: PointExpression | undefined; - paddingBottomRight?: PointExpression | undefined; - padding?: PointExpression | undefined; -} - -export interface LocateOptions { - watch?: boolean | undefined; - setView?: boolean | undefined; - maxZoom?: number | undefined; - timeout?: number | undefined; - maximumAge?: number | undefined; - enableHighAccuracy?: boolean | undefined; -} - -export class Handler extends Class { - constructor(map: Map); - enable(): this; - disable(): this; - enabled(): boolean; - - // Extension methods - addHooks?(): void; - removeHooks?(): void; -} - -export interface LeafletEvent { - type: string; - popup: any; - target: any; - sourceTarget: any; - propagatedFrom: any; - /** - * @deprecated The same as {@link LeafletEvent.propagatedFrom propagatedFrom}. - */ - layer: any; -} - -export interface LeafletMouseEvent extends LeafletEvent { - latlng: LatLng; - layerPoint: Point; - containerPoint: Point; - originalEvent: MouseEvent; -} - -export interface LeafletKeyboardEvent extends LeafletEvent { - originalEvent: KeyboardEvent; -} - -export interface LocationEvent extends LeafletEvent { - latlng: LatLng; - bounds: LatLngBounds; - accuracy: number; - altitude: number; - altitudeAccuracy: number; - heading: number; - speed: number; - timestamp: number; -} - -export interface ErrorEvent extends LeafletEvent { - message: string; - code: number; -} - -export interface LayerEvent extends LeafletEvent { - layer: Layer; -} - -export interface LayersControlEvent extends LayerEvent { - name: string; -} - -export interface TileEvent extends LeafletEvent { - tile: HTMLImageElement; - coords: Coords; -} - -export interface TileErrorEvent extends TileEvent { - error: Error; -} - -export interface ResizeEvent extends LeafletEvent { - oldSize: Point; - newSize: Point; -} - -export interface GeoJSONEvent extends LeafletEvent { - layer: Layer; - properties: any; - geometryType: string; - id: string; -} - -export interface PopupEvent extends LeafletEvent { - popup: Popup; -} - -export interface TooltipEvent extends LeafletEvent { - tooltip: Tooltip; -} - -export interface DragEndEvent extends LeafletEvent { - distance: number; -} - -export interface ZoomAnimEvent extends LeafletEvent { - center: LatLng; - zoom: number; - noUpdate: boolean; -} - -export namespace DomEvent { - type EventHandlerFn = (event: Event) => void; - - type PropagableEvent = LeafletMouseEvent | LeafletKeyboardEvent | LeafletEvent | Event; - - function on(el: HTMLElement, types: string, fn: EventHandlerFn, context?: any): typeof DomEvent; - - function on(el: HTMLElement, eventMap: { [eventName: string]: EventHandlerFn }, context?: any): typeof DomEvent; - - // tslint:disable:unified-signatures - function off(el: HTMLElement): typeof DomEvent; - - function off(el: HTMLElement, types: string, fn: EventHandlerFn, context?: any): typeof DomEvent; - - function off(el: HTMLElement, eventMap: { [eventName: string]: EventHandlerFn }, context?: any): typeof DomEvent; - // tslint:enable:unified-signatures - - function stopPropagation(ev: PropagableEvent): typeof DomEvent; - - function disableScrollPropagation(el: HTMLElement): typeof DomEvent; - - function disableClickPropagation(el: HTMLElement): typeof DomEvent; - - function preventDefault(ev: Event): typeof DomEvent; - - function stop(ev: PropagableEvent): typeof DomEvent; - - function getMousePosition(ev: MouseEvent, container?: HTMLElement): Point; - - function getWheelDelta(ev: Event): number; - - function addListener(el: HTMLElement, types: string, fn: EventHandlerFn, context?: any): typeof DomEvent; - - function addListener( - el: HTMLElement, - eventMap: { [eventName: string]: EventHandlerFn }, - context?: any, - ): typeof DomEvent; - - function removeListener(el: HTMLElement, types: string, fn: EventHandlerFn, context?: any): typeof DomEvent; - - function removeListener( - el: HTMLElement, - eventMap: { [eventName: string]: EventHandlerFn }, - context?: any, - ): typeof DomEvent; - - function getPropagationPath(ev: Event): HTMLElement[]; -} - -export interface DefaultMapPanes { - mapPane: HTMLElement; - tilePane: HTMLElement; - overlayPane: HTMLElement; - shadowPane: HTMLElement; - markerPane: HTMLElement; - tooltipPane: HTMLElement; - popupPane: HTMLElement; -} - -export class Map extends Evented { - constructor(element: string | HTMLElement, options?: MapOptions); - getRenderer(layer: Path): Renderer; - - // Methods for layers and controls - addControl(control: Control): this; - removeControl(control: Control): this; - addLayer(layer: Layer): this; - removeLayer(layer: Layer): this; - hasLayer(layer: Layer): boolean; - eachLayer(fn: (layer: Layer) => void, context?: any): this; - openPopup(popup: Popup): this; - openPopup(content: Content, latlng: LatLngExpression, options?: PopupOptions): this; - closePopup(popup?: Popup): this; - openTooltip(tooltip: Tooltip): this; - openTooltip(content: Content, latlng: LatLngExpression, options?: TooltipOptions): this; - closeTooltip(tooltip?: Tooltip): this; - - // Methods for modifying map state - setView(center: LatLngExpression, zoom?: number, options?: ZoomPanOptions): this; - setZoom(zoom: number, options?: ZoomPanOptions): this; - zoomIn(delta?: number, options?: ZoomOptions): this; - zoomOut(delta?: number, options?: ZoomOptions): this; - setZoomAround(position: Point | LatLngExpression, zoom: number, options?: ZoomOptions): this; - fitBounds(bounds: LatLngBoundsExpression, options?: FitBoundsOptions): this; - fitWorld(options?: FitBoundsOptions): this; - panTo(latlng: LatLngExpression, options?: PanOptions): this; - panBy(offset: PointExpression, options?: PanOptions): this; - setMaxBounds(bounds?: LatLngBoundsExpression): this; - setMinZoom(zoom: number): this; - setMaxZoom(zoom: number): this; - panInside(latLng: LatLngExpression, options?: PanInsideOptions): this; - panInsideBounds(bounds: LatLngBoundsExpression, options?: PanOptions): this; - /** - * Boolean for animate or advanced ZoomPanOptions - */ - invalidateSize(options?: boolean | InvalidateSizeOptions): this; - stop(): this; - flyTo(latlng: LatLngExpression, zoom?: number, options?: ZoomPanOptions): this; - flyToBounds(bounds: LatLngBoundsExpression, options?: FitBoundsOptions): this; - - // Other methods - addHandler(name: string, HandlerClass: typeof Handler): this; // Alternatively, HandlerClass: new(map: Map) => Handler - remove(): this; - createPane(name: string, container?: HTMLElement): HTMLElement; - /** - * Name of the pane or the pane as HTML-Element - */ - getPane(pane: string | HTMLElement): HTMLElement | undefined; - getPanes(): { [name: string]: HTMLElement } & DefaultMapPanes; - getContainer(): HTMLElement; - whenReady(fn: (event: { target: Map }) => void, context?: any): this; - - // Methods for getting map state - getCenter(): LatLng; - getZoom(): number; - getBounds(): LatLngBounds; - getMinZoom(): number; - getMaxZoom(): number; - getBoundsZoom(bounds: LatLngBoundsExpression, inside?: boolean, padding?: Point): number; - getSize(): Point; - getPixelBounds(): Bounds; - getPixelOrigin(): Point; - getPixelWorldBounds(zoom?: number): Bounds; - - // Conversion methods - getZoomScale(toZoom: number, fromZoom?: number): number; - getScaleZoom(scale: number, fromZoom?: number): number; - project(latlng: LatLngExpression, zoom?: number): Point; - unproject(point: PointExpression, zoom?: number): LatLng; - layerPointToLatLng(point: PointExpression): LatLng; - latLngToLayerPoint(latlng: LatLngExpression): Point; - wrapLatLng(latlng: LatLngExpression): LatLng; - wrapLatLngBounds(bounds: LatLngBounds): LatLngBounds; - distance(latlng1: LatLngExpression, latlng2: LatLngExpression): number; - containerPointToLayerPoint(point: PointExpression): Point; - containerPointToLatLng(point: PointExpression): LatLng; - layerPointToContainerPoint(point: PointExpression): Point; - latLngToContainerPoint(latlng: LatLngExpression): Point; - mouseEventToContainerPoint(ev: MouseEvent): Point; - mouseEventToLayerPoint(ev: MouseEvent): Point; - mouseEventToLatLng(ev: MouseEvent): LatLng; - - // Geolocation methods - locate(options?: LocateOptions): this; - stopLocate(): this; - - // Properties - attributionControl: L.Control.Attribution; - boxZoom: Handler; - doubleClickZoom: Handler; - dragging: Handler; - keyboard: Handler; - scrollWheelZoom: Handler; - tapHold?: Handler | undefined; - touchZoom: Handler; - zoomControl: Control.Zoom; - - options: MapOptions; -} - -/** - * ID of a HTML-Element as string or the HTML-ELement itself - */ -export function map(element: string | HTMLElement, options?: MapOptions): Map; - -export interface BaseIconOptions extends LayerOptions { - iconUrl?: string | undefined; - iconRetinaUrl?: string | undefined; - iconSize?: PointExpression | undefined; - iconAnchor?: PointExpression | undefined; - popupAnchor?: PointExpression | undefined; - tooltipAnchor?: PointExpression | undefined; - shadowUrl?: string | undefined; - shadowRetinaUrl?: string | undefined; - shadowSize?: PointExpression | undefined; - shadowAnchor?: PointExpression | undefined; - className?: string | undefined; -} - -export interface IconOptions extends BaseIconOptions { - iconUrl: string; - crossOrigin?: CrossOrigin | boolean | undefined; -} - -export class Icon extends Layer { - constructor(options: T); - createIcon(oldIcon?: HTMLElement): HTMLElement; - createShadow(oldIcon?: HTMLElement): HTMLElement; - - options: T; -} - -export namespace Icon { - interface DefaultIconOptions extends BaseIconOptions { - imagePath?: string | undefined; - } - - class Default extends Icon { - static imagePath?: string | undefined; - constructor(options?: DefaultIconOptions); - } -} - -export function icon(options: IconOptions): Icon; - -export interface DivIconOptions extends BaseIconOptions { - html?: string | HTMLElement | false | undefined; - bgPos?: PointExpression | undefined; - iconSize?: PointExpression | undefined; - iconAnchor?: PointExpression | undefined; - popupAnchor?: PointExpression | undefined; - className?: string | undefined; -} - -export class DivIcon extends Icon { - constructor(options?: DivIconOptions); -} - -export function divIcon(options?: DivIconOptions): DivIcon; - -export interface MarkerOptions extends InteractiveLayerOptions { - icon?: Icon | DivIcon | undefined; - /** Whether the marker is draggable with mouse/touch or not. */ - draggable?: boolean | undefined; - /** Whether the marker can be tabbed to with a keyboard and clicked by pressing enter. */ - keyboard?: boolean | undefined; - /** Text for the browser tooltip that appear on marker hover (no tooltip by default). */ - title?: string | undefined; - /** Text for the `alt` attribute of the icon image (useful for accessibility). */ - alt?: string | undefined; - /** Option for putting the marker on top of all others (or below). */ - zIndexOffset?: number | undefined; - /** The opacity of the marker. */ - opacity?: number | undefined; - /** If `true`, the marker will get on top of others when you hover the mouse over it. */ - riseOnHover?: boolean | undefined; - /** The z-index offset used for the `riseOnHover` feature. */ - riseOffset?: number | undefined; - /** `Map pane` where the markers shadow will be added. */ - shadowPane?: string | undefined; - /** Whether to pan the map when dragging this marker near its edge or not. */ - autoPan?: boolean | undefined; - /** Distance (in pixels to the left/right and to the top/bottom) of the map edge to start panning the map. */ - autoPanPadding?: PointExpression | undefined; - /** Number of pixels the map should pan by. */ - autoPanSpeed?: number | undefined; - autoPanOnFocus?: boolean | undefined; -} - -export class Marker

extends Layer { - constructor(latlng: LatLngExpression, options?: MarkerOptions); - toGeoJSON(precision?: number | false): geojson.Feature; - getLatLng(): LatLng; - setLatLng(latlng: LatLngExpression): this; - setZIndexOffset(offset: number): this; - getIcon(): Icon | DivIcon; - setIcon(icon: Icon | DivIcon): this; - setOpacity(opacity: number): this; - getElement(): HTMLElement | undefined; - - // Properties - options: MarkerOptions; - dragging?: Handler | undefined; - feature?: geojson.Feature | undefined; - - protected _shadow: HTMLElement | undefined; -} - -export function marker

(latlng: LatLngExpression, options?: MarkerOptions): Marker

; - -export namespace Browser { - // sorting according to https://leafletjs.com/reference-1.5.0.html#browser - const ie: boolean; - const ielt9: boolean; - const edge: boolean; - const webkit: boolean; - const android: boolean; - const android23: boolean; - const androidStock: boolean; - const opera: boolean; - const chrome: boolean; - const gecko: boolean; - const safari: boolean; - const opera12: boolean; - const win: boolean; - const ie3d: boolean; - const webkit3d: boolean; - const gecko3d: boolean; - const any3d: boolean; - const mobile: boolean; - const mobileWebkit: boolean; - const mobileWebkit3d: boolean; - const msPointer: boolean; - const pointer: boolean; - const touch: boolean; - const mobileOpera: boolean; - const mobileGecko: boolean; - const retina: boolean; - const canvas: boolean; - const svg: boolean; - const vml: boolean; -} - -export namespace Util { - function extend(dest: D, src?: S1): D & S1; - function extend(dest: D, src1: S1, src2: S2): D & S1 & S2; - function extend( - dest: D, - src1: S1, - src2: S2, - src3: S3, - ): D & S1 & S2 & S3; - function extend(dest: any, ...src: any[]): any; - - function create(proto: object | null, properties?: PropertyDescriptorMap): any; - function bind(fn: (...args: any[]) => void, ...obj: any[]): () => void; - function stamp(obj: any): number; - function throttle(fn: () => void, time: number, context: any): () => void; - function wrapNum(num: number, range: number[], includeMax?: boolean): number; - function falseFn(): false; - function formatNum(num: number, digits?: number | false): number; - function trim(str: string): string; - function splitWords(str: string): string[]; - function setOptions(obj: any, options: any): any; - function getParamString(obj: any, existingUrl?: string, uppercase?: boolean): string; - function template(str: string, data: any): string; - function isArray(obj: any): boolean; - function indexOf(array: any[], el: any): number; - function requestAnimFrame(fn: (timestamp: number) => void, context?: any, immediate?: boolean): number; - function cancelAnimFrame(id: number): void; - - let lastId: number; - let emptyImageUrl: string; -} - -export const extend: typeof Util["extend"]; -export const bind: typeof Util["bind"]; -export const stamp: typeof Util["stamp"]; -export const setOptions: typeof Util["setOptions"]; - -export function noConflict(): any; diff --git a/node_modules/@types/leaflet/package.json b/node_modules/@types/leaflet/package.json deleted file mode 100644 index e2110ef..0000000 --- a/node_modules/@types/leaflet/package.json +++ /dev/null @@ -1,63 +0,0 @@ -{ - "name": "@types/leaflet", - "version": "1.9.18", - "description": "TypeScript definitions for leaflet", - "homepage": "https://github.com/DefinitelyTyped/DefinitelyTyped/tree/master/types/leaflet", - "license": "MIT", - "contributors": [ - { - "name": "Alejandro Sánchez", - "githubUsername": "alejo90", - "url": "https://github.com/alejo90" - }, - { - "name": "Arne Schubert", - "githubUsername": "atd-schubert", - "url": "https://github.com/atd-schubert" - }, - { - "name": "Michael Auer", - "githubUsername": "mcauer", - "url": "https://github.com/mcauer" - }, - { - "name": "Roni Karilkar", - "githubUsername": "ronikar", - "url": "https://github.com/ronikar" - }, - { - "name": "Vladimir Dashukevich", - "githubUsername": "life777", - "url": "https://github.com/life777" - }, - { - "name": "Henry Thasler", - "githubUsername": "henrythasler", - "url": "https://github.com/henrythasler" - }, - { - "name": "Colin Doig", - "githubUsername": "captain-igloo", - "url": "https://github.com/captain-igloo" - }, - { - "name": "Hugo Sales", - "githubUsername": "someonewithpc", - "url": "https://github.com/someonewithpc" - } - ], - "main": "", - "types": "index.d.ts", - "repository": { - "type": "git", - "url": "https://github.com/DefinitelyTyped/DefinitelyTyped.git", - "directory": "types/leaflet" - }, - "scripts": {}, - "dependencies": { - "@types/geojson": "*" - }, - "peerDependencies": {}, - "typesPublisherContentHash": "6d76b6d1133136e104a527083099ffe31982febb7a7934f0c667ea46cb303d25", - "typeScriptVersion": "5.1" -} \ No newline at end of file From 9b87b731ad69e11ac3e9de7f1ecc8c89dbae7dde Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Wed, 16 Sep 2026 16:08:27 -0700 Subject: [PATCH 196/311] docs: improve backend developer onboarding --- AGENTS.md | 18 +++++++- docs/development.md | 39 ++++++++++++++++- frontend/README.md | 2 + frontend/scripts/probe-local-v2.mjs | 21 ++++++++- pipeline/scripts/maintenance/local-v2.ps1 | 2 +- scripts/dev.py | 43 +++++++++++++------ scripts/test_dev.py | 17 ++++++++ .../__fixtures__/v2-contract-fixtures.js | 2 +- static/modules/__tests__/v2Client.test.js | 2 + 9 files changed, 127 insertions(+), 19 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 7d5388c..35c4134 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -6,12 +6,26 @@ The current public application is a Rust/Axum backend in `src/` with a vanilla J ## Build, Test, and Development Commands -- `cargo run` starts the current application on port 8000. -- `npm test` runs the Jest suite. +- `python scripts/dev.py --help` is the preferred V2 developer entrypoint; use + `python scripts/dev.py --json doctor` before starting local services. +- Run `npm ci` once at the repository root before `npm test`; `npm test` runs + the legacy/static Jest suite. The Svelte preview has separate dependencies: + run `npm --prefix frontend ci` before using its commands. +- `cargo run` starts the current application on port 8000 when its configured + database is available; `python scripts/dev.py up` is the reproducible local + V2 setup path. - `npm run test:coverage` runs Jest with coverage. - `powershell -ExecutionPolicy Bypass -File pipeline/scripts/maintenance/build-legacy-manifest.ps1` regenerates the legacy file inventory and SHA-256 manifest in `data/manifests/`. - `python pipeline/scripts/diagnostics/inspect-denmark-smiley.py static_data/dk/Smiley_xml.xml` inspects the Danish XML without transforming it. +For a clean database-backed validation, use +`pwsh -NoProfile -ExecutionPolicy Bypass -File pipeline/tests/run-standard.ps1`. +The persistent `uec-local-v2` database rejects changed migration checksums on +purpose. If local setup reports `migration checksum changed after application`, +the named volume belongs to an older checkout; preserve it unless it is known +to be disposable, and follow the reset/troubleshooting instructions in +`docs/development.md` rather than editing migration history. + ## Data Credibility and Provenance Start with the [ethics summary](docs/ETHICS-SUMMARY.md) for orientation; it does not replace the full policy. Visitor-data handling, legal demands, and publication authority also follow ETHICS.md sections 11–14. Do not claim no logging, legal immunity, a backup reviewer, or enforced approval controls without evidence. Pause new publication needing human approval if no authorized reviewer is available; compliant acquisition and eligible existing releases may continue, and urgent suppression follows policy. Assess preservation obligations before exceptional deletion when a legal demand is involved; escalate to the responsible maintainer rather than making novel legal judgments. diff --git a/docs/development.md b/docs/development.md index 241b79c..a158f6b 100644 --- a/docs/development.md +++ b/docs/development.md @@ -12,7 +12,18 @@ contracts database and adapter contract tests review-packet RUN_DIR deterministic, row-free private review packet ``` -`python scripts/dev.py --json doctor` (and any command with the global `--json` flag) emits a final machine-readable summary. Diagnostics report only whether environment variables are set; values and database credentials are never printed. `up` may apply migrations and seed/promote the synthetic local fixture through the existing `local-v2.ps1` workflow. It does not publish project data. +`python scripts/dev.py --json doctor` (and any command with the global `--json` flag) emits one machine-readable JSON object. Child-command output is captured so it cannot corrupt JSON output. Diagnostics report only whether environment variables are set; values and database credentials are never printed. `up` may apply migrations and seed/promote the synthetic local fixture through the existing `local-v2.ps1` workflow. It does not publish project data. + +## First checkout + +Install the pinned root dependencies before running the root JavaScript suite: + +```powershell +npm ci +python scripts/dev.py --json doctor +``` + +The doctor reports missing dependencies with a corrective command and distinguishes an expected `uec-local-v2` database from an unknown process occupying port 5433. Missing `UEC_RUNTIME_MODE` and `UEC_DATABASE_URL` are informational when documented local defaults apply. For the Svelte preview, install its separate dependencies with `npm --prefix frontend ci`. ## Frontend contributor path @@ -29,4 +40,28 @@ npm run build npm run dev ``` -The fixture preview is the default, does not need a database, and is the correct first environment for UI work. `npm run test:e2e:fixture` starts its own local preview. Do not add `?mode=local-v2` until a clean `python scripts/dev.py --json doctor` reports Docker Compose and ports 8000/5433 available, then use `up`, `probe`, and `npm run test:e2e:local` as documented in the frontend README. Port conflicts and an unavailable Docker engine are environment problems, not a reason to stop an unknown process or change the preview to use fixtures as a live fallback. +The fixture preview is the default, does not need a database, and is the correct first environment for UI work. `npm run test:e2e:fixture` starts its own local preview. Do not add `?mode=local-v2` until `python scripts/dev.py --json doctor` reports a healthy environment, then use `up`, `probe`, and `npm run test:e2e:local` as documented in the frontend README. `status` distinguishes a running Axum process from a database-only local environment, and `probe` reports a concise next step when the backend is unavailable. Port conflicts and an unavailable Docker engine are environment problems, not a reason to stop an unknown process or silently fall back to fixtures. + +## Troubleshooting persistent local V2 state + +`local-v2.ps1 start` uses the named `uec-local-v2` Compose project and keeps its +volume across normal stops. Migration application records checksums and +refuses to run when an already-applied migration differs from the checkout. +That failure is intentional: never edit an old migration or bypass the check. + +If the error names `migration checksum changed after application`, first run +`python scripts/dev.py --json status` and confirm which checkout owns the +database. If the volume contains work you need, stop and preserve it, then +restore the matching checkout or migrate it through an explicit maintainer- +reviewed procedure. If it is only the disposable synthetic local fixture, an +operator may intentionally remove just that named project and volume, after +confirming the target: + +```powershell +docker compose -p uec-local-v2 -f docker-compose.pipeline.yml down -v --remove-orphans +python scripts/dev.py up +``` + +This reset is destructive to the local synthetic database and is not part of +ordinary `down`; do not run it against production or an unknown Compose +project. The migration ledger exists to prevent accidental history changes. diff --git a/frontend/README.md b/frontend/README.md index 85f22e7..ba24b07 100644 --- a/frontend/README.md +++ b/frontend/README.md @@ -20,6 +20,8 @@ Open the URL Vite prints, normally `http://127.0.0.1:5173/v2-preview/#/`. This i The existing root `npm test` is the legacy static/Jest suite. It is deliberately separate from the commands above; run it from the repository root when changing root `static/` assets or compatibility modules. +The ethics link intentionally targets the existing `/ethics.html` page. + The explicit `?mode=local-v2` path uses the V2 API client for the official, secondary, and community profiles, controlled filters, cursor pagination, map/detail navigation, release/provenance context, and the profile-scoped public CSV route. A community profile keeps its persistent screened-but-unreviewed warning; it is not merged into official or secondary counts. Requests are cancellable and generation-checked so stale list/detail responses cannot replace newer state. No V1 fallback is used. Phase 2 gate notes: staging is explicit (`npm run stage`) and copies only `frontend/dist` to the resolved ignored `static/v2-preview` destination. The Leaflet adapter is isolated and uses a blank local background; no tile provider is configured. Export previews retain profile, release, limitations, source, and observation context. diff --git a/frontend/scripts/probe-local-v2.mjs b/frontend/scripts/probe-local-v2.mjs index 4aad5a4..2c93d38 100644 --- a/frontend/scripts/probe-local-v2.mjs +++ b/frontend/scripts/probe-local-v2.mjs @@ -1 +1,20 @@ -const base=process.argv[2]??'http://127.0.0.1:8000';const list=await fetch(`${base}/api/v2/locations?profile=official&limit=1`);if(!list.ok)throw new Error(`list probe failed: ${list.status}`);const body=await list.json();if(body.api_version!=='v2'||!body.meta)throw new Error('list contract probe failed');console.log(`list probe passed: release=${body.meta.release_id}`);if(Array.isArray(body.data)&&body.data[0]?.facility_id){const id=encodeURIComponent(body.data[0].facility_id);const detail=await fetch(`${base}/api/v2/locations/${id}?profile=official`);if(!detail.ok)throw new Error(`detail probe failed: ${detail.status}`);const detailBody=await detail.json();if(detailBody.api_version!=='v2'||!detailBody.data)throw new Error('detail contract probe failed');console.log('detail probe passed');}else console.log('detail probe skipped: no public promoted record'); +const base=process.argv[2]??'http://127.0.0.1:8000'; +try { + const list=await fetch(`${base}/api/v2/locations?profile=official&limit=1`); + if(!list.ok)throw new Error(`list probe failed: ${list.status}`); + const body=await list.json(); + if(body.api_version!=='v2'||!body.meta)throw new Error('list contract probe failed'); + console.log(`list probe passed: release=${body.meta.release_id}`); + if(Array.isArray(body.data)&&body.data[0]?.facility_id){ + const id=encodeURIComponent(body.data[0].facility_id); + const detail=await fetch(`${base}/api/v2/locations/${id}?profile=official`); + if(!detail.ok)throw new Error(`detail probe failed: ${detail.status}`); + const detailBody=await detail.json(); + if(detailBody.api_version!=='v2'||!detailBody.data)throw new Error('detail contract probe failed'); + console.log('detail probe passed'); + }else console.log('detail probe skipped: no public promoted record'); +} catch (error) { + const unreachable=error?.cause?.code==='ECONNREFUSED'||error?.message==='fetch failed'; + console.error(`Local V2 probe failed: ${unreachable?'backend is not reachable; run the local-v2 start command first':error instanceof Error?error.message:'unknown probe error'}`); + process.exitCode=1; +} diff --git a/pipeline/scripts/maintenance/local-v2.ps1 b/pipeline/scripts/maintenance/local-v2.ps1 index 4e04645..9238c67 100644 --- a/pipeline/scripts/maintenance/local-v2.ps1 +++ b/pipeline/scripts/maintenance/local-v2.ps1 @@ -51,7 +51,7 @@ try { } 'status' { & docker compose -p $project -f $compose ps - $api=Get-OwnedApiProcess; if ($api) { Write-Host "Axum running: PID $($api.Id), port $apiPort" } else { Write-Host 'Axum not managed by local-v2.ps1.' } + $api=Get-OwnedApiProcess; if ($api) { Write-Host "Axum running: PID $($api.Id), port $apiPort" } else { Write-Host 'Axum is not running under this local-v2 checkout.' } } 'stop' { $api=Get-OwnedApiProcess; if ($api) { Stop-Process -Id $api.Id -Force; Remove-Item $pidFile -Force } diff --git a/scripts/dev.py b/scripts/dev.py index 3e6e42b..5dbc2a4 100644 --- a/scripts/dev.py +++ b/scripts/dev.py @@ -12,14 +12,27 @@ LOCAL_V2 = ROOT / "pipeline" / "scripts" / "maintenance" / "local-v2.ps1" SUMMARY = {"command": None, "ok": False, "exit_code": None, "checks": []} -def run(args: list[str], *, cwd=ROOT) -> int: - return subprocess.run(args, cwd=cwd).returncode +def run(args: list[str], *, cwd=ROOT, capture=False) -> int: + result = subprocess.run(args, cwd=cwd, capture_output=capture, text=True) + if capture and result.returncode != 0: + SUMMARY["checks"].append({"name": "delegated-command", "ok": False, "detail": "failed"}) + return result.returncode def check(name: str, ok: bool, detail: str) -> None: SUMMARY["checks"].append({"name": name, "ok": ok, "detail": detail}) +def local_v2_database_running() -> bool: + if shutil.which("docker") is None: + return False + result = subprocess.run( + ["docker", "ps", "--filter", "name=uec-local-v2-postgres-1", "--format", "{{.Names}}"], + capture_output=True, + text=True, + ) + return result.returncode == 0 and "uec-local-v2-postgres-1" in result.stdout.splitlines() + def doctor(_: argparse.Namespace) -> int: - for name in ("docker", "python", "cargo", "node"): + for name in ("docker", "python", "cargo", "node", "npm"): path = shutil.which(name) check(name, bool(path), "available" if path else "not found") compose = shutil.which("docker") is not None @@ -27,16 +40,22 @@ def doctor(_: argparse.Namespace) -> int: result = subprocess.run(["docker", "compose", "version"], capture_output=True, text=True) check("compose", result.returncode == 0, "available" if result.returncode == 0 else "unavailable") else: check("compose", False, "Docker not found") + npm_manifest = ROOT / "node_modules" / "jest" / "bin" / "jest.js" + check("root-js-dependencies", npm_manifest.is_file(), "installed" if npm_manifest.is_file() else "missing (run npm ci)") if sys.platform == "win32": check("powershell", bool(shutil.which("powershell") or shutil.which("pwsh")), "required by local-v2") + local_db = local_v2_database_running() for port in (8000, 5433): sock = socket.socket(); sock.settimeout(.2) try: busy = sock.connect_ex(("127.0.0.1", port)) == 0 finally: sock.close() - check(f"port:{port}", not busy, "free" if not busy else "in use") + if port == 5433 and busy and local_db: + check(f"port:{port}", True, "in use by the expected uec-local-v2 Postgres") + else: + check(f"port:{port}", not busy, "free" if not busy else "in use by an unknown process") required = ("UEC_RUNTIME_MODE", "UEC_DATABASE_URL") for key in required: # Only report presence; never print values. - check(f"env:{key}", bool(os.environ.get(key)), "set" if os.environ.get(key) else "not set (local defaults may apply)") + check(f"env:{key}", True if os.environ.get(key) else None, "set" if os.environ.get(key) else "not set (local defaults may apply)") state = ROOT / "target" / "local-v2" check("local-v2-state", not (state.exists() and (state / "uec-api.pid").exists() and not (state / "uec-api.log").exists()), "consistent or absent") db = os.environ.get("UEC_DATABASE_URL") @@ -63,20 +82,20 @@ def main() -> int: rp.add_argument("run_dir"); rp.add_argument("--previous-normalized") args = p.parse_args(); SUMMARY["command"] = args.command if args.command == "doctor": code = doctor(args) - elif args.command in ("up", "down", "status", "probe"): code = run(["powershell", "-ExecutionPolicy", "Bypass", "-File", str(LOCAL_V2), {"up":"start","down":"stop"}.get(args.command,args.command)]) - elif args.command == "logs": code = run(["docker", "compose", "-p", "uec-local-v2", "-f", "docker-compose.pipeline.yml", "logs", "--tail=100"]) - elif args.command == "test": code = run(["npm", "test", "--", *( ["--runInBand"] if not args.full else [])]) + elif args.command in ("up", "down", "status", "probe"): code = run(["powershell", "-ExecutionPolicy", "Bypass", "-File", str(LOCAL_V2), {"up":"start","down":"stop"}.get(args.command,args.command)], capture=args.json) + elif args.command == "logs": code = run(["docker", "compose", "-p", "uec-local-v2", "-f", "docker-compose.pipeline.yml", "logs", "--tail=100"], capture=args.json) + elif args.command == "test": code = run(["npm", "test", "--", *( ["--runInBand"] if not args.full else [])], capture=args.json) elif args.command == "pipeline": runner = "pytest" if shutil.which("pytest") else "unittest" - code = run([sys.executable, "-m", runner, *(args.args or ["discover", "-s", "pipeline"])]) + code = run([sys.executable, "-m", runner, *(args.args or ["discover", "-s", "pipeline"])], capture=args.json) elif args.command == "contracts": runner = "pytest" if shutil.which("pytest") else "unittest" - targets = ["pipeline/contracts", "pipeline/tests/test_database_contract.py", "pipeline/tests/test_graph_database_contract.py"] if runner == "pytest" else ["discover", "-s", "pipeline/contracts"] - code = run([sys.executable, "-m", runner, *targets]) + targets = ["pipeline/contracts", "pipeline/tests/test_database_contract.py", "pipeline/tests/test_graph_database_contract.py"] if runner == "pytest" else ["discover", "-s", "pipeline/contracts", "-t", str(ROOT)] + code = run([sys.executable, "-m", runner, *targets], capture=args.json) else: cmd = [sys.executable, "-c", "from pipeline.common.review_packet import write_review_packet; import sys; write_review_packet(sys.argv[1], previous_normalized_path=sys.argv[2] if len(sys.argv)>2 else None)", args.run_dir] if args.previous_normalized: cmd.append(args.previous_normalized) - code = run(cmd) + code = run(cmd, capture=args.json) SUMMARY["ok"], SUMMARY["exit_code"] = code == 0, code print(json.dumps(SUMMARY) if args.json else ("OK" if code == 0 else "FAILED") + f": {args.command}") return code diff --git a/scripts/test_dev.py b/scripts/test_dev.py index 79b26f7..a9eccb7 100644 --- a/scripts/test_dev.py +++ b/scripts/test_dev.py @@ -16,4 +16,21 @@ def test_doctor_json_does_not_echo_secret(self): self.assertEqual(payload["command"], "doctor") self.assertNotIn("secret.invalid", result.stdout) + def test_contracts_command_uses_package_root_for_relative_imports(self): + result = subprocess.run([sys.executable, "scripts/dev.py", "contracts"], cwd=ROOT, capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + + def test_json_delegated_command_is_machine_readable(self): + result = subprocess.run([sys.executable, "scripts/dev.py", "--json", "status"], cwd=ROOT, capture_output=True, text=True) + self.assertEqual(result.returncode, 0) + payload = json.loads(result.stdout) + self.assertEqual(payload["command"], "status") + + def test_probe_reports_unreachable_backend_without_stack_trace(self): + probe = ROOT / "frontend" / "scripts" / "probe-local-v2.mjs" + result = subprocess.run(["node", str(probe), "http://127.0.0.1:9"], cwd=ROOT, capture_output=True, text=True) + self.assertNotEqual(result.returncode, 0) + self.assertIn("backend is not reachable", result.stderr) + self.assertNotIn("at async", result.stderr) + if __name__ == "__main__": unittest.main() diff --git a/static/modules/__fixtures__/v2-contract-fixtures.js b/static/modules/__fixtures__/v2-contract-fixtures.js index 621ba1d..cf72cf3 100644 --- a/static/modules/__fixtures__/v2-contract-fixtures.js +++ b/static/modules/__fixtures__/v2-contract-fixtures.js @@ -6,7 +6,7 @@ export const exactOfficialLocation = { privacy_screening_status: 'passed', project_approval: 'approved', reviewer_role: 'maintainer', publication_warning: null, display_precision: 'exact', latitude: 55, longitude: 10, first_observed_at: '2026-09-13T00:00:00Z', last_observed_at: '2026-09-13T00:00:00Z', observation_count: 1, - lifecycle_status: 'active_observed', source_type: 'official', provenance_source: 'Synthetic source', + lifecycle_status: 'active_observed', source_type: 'official', source_rights_status: 'attribution_required', provenance_source: 'Synthetic source', release_id: 'fixture-release', release_ruleset_version: 'fixture-v1', provenance_source_id: 'fixture.source', provenance_source_name: 'Synthetic source', provenance_source_url: 'https://example.invalid/source', provenance_retrieved_at: '2026-09-13T00:00:00Z' diff --git a/static/modules/__tests__/v2Client.test.js b/static/modules/__tests__/v2Client.test.js index 4253da6..fa8c3e8 100644 --- a/static/modules/__tests__/v2Client.test.js +++ b/static/modules/__tests__/v2Client.test.js @@ -23,6 +23,8 @@ const record = (overrides = {}) => ({ reviewer_role: 'maintainer', publication_warning: null, source_type: 'official', + source_rights_status: 'attribution_required', + provenance_source: 'Test source', release_id: 'fixture-release', release_ruleset_version: 'fixture-v1', provenance_source_id: 'fixture.source', From af0ceaf75ed62d31841767fb7bab317725b8776b Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Wed, 16 Sep 2026 16:38:44 -0700 Subject: [PATCH 197/311] Add private 50k legacy V2 rehearsal --- data/manifests/real-v2-rehearsal-report.json | 60 +++++++ docs/real-v2-rehearsal.md | 40 +++++ .../scripts/maintenance/rehearse_real_v2.py | 158 ++++++++++++++++++ pipeline/tests/test_real_v2_rehearsal.py | 52 ++++++ 4 files changed, 310 insertions(+) create mode 100644 data/manifests/real-v2-rehearsal-report.json create mode 100644 docs/real-v2-rehearsal.md create mode 100644 pipeline/scripts/maintenance/rehearse_real_v2.py create mode 100644 pipeline/tests/test_real_v2_rehearsal.py diff --git a/data/manifests/real-v2-rehearsal-report.json b/data/manifests/real-v2-rehearsal-report.json new file mode 100644 index 0000000..adf73f0 --- /dev/null +++ b/data/manifests/real-v2-rehearsal-report.json @@ -0,0 +1,60 @@ +{ + "schema_version": "real-v2-rehearsal-evidence-v1", + "rehearsal_id": "candidate-legacy-v2", + "branch": "codex/real-v2-rehearsal", + "source_corpus": "static_data/*/locations.csv", + "legacy_label": "legacy V1-derived snapshot; not current and not publication-approved", + "publication_eligibility": "blocked", + "selection": { + "available_records": 50750, + "selected_records": 50000, + "method": "stable-sha256-row-fingerprint", + "source_profiles": 9, + "countries": 9 + }, + "stage_counts": { + "conversion_input": 50000, + "parsed": 50000, + "normalized": 50000, + "quarantined": 0, + "candidate_handoff": 50000, + "imported_source_records": 50000, + "imported_observations": 50000, + "release_members": 50000, + "default_visible": 0, + "publication_eligible": 0, + "approved_coordinates": 0, + "map_projection_visible": 0 + }, + "source_counts": { + "legacy.v1.ca.locations": 1309, + "legacy.v1.de.locations": 9121, + "legacy.v1.dk.locations": 1539, + "legacy.v1.es.locations": 4168, + "legacy.v1.fr.locations": 3148, + "legacy.v1.mx.locations": 14615, + "legacy.v1.nz.locations": 1629, + "legacy.v1.uk.locations": 7475, + "legacy.v1.us.locations": 6996 + }, + "timings_seconds": { + "offline_conversion_and_handoff": 19.82, + "database_import_and_idempotent_rerun": "measured across bounded batches; exact wall time not captured by runner" + }, + "checks": { + "private_test_list": "200 responses served; cursor pagination returned distinct pages", + "private_test_detail": "200 before suppression, 404 while revoked, 200 after restoration", + "private_test_facets": "200; 50,000 candidate rows; country/category/display_precision/source_type dimensions", + "public_v2_list": "200 with empty data because no release is promoted", + "test_export": "400 export_too_large at the 1,000-row safety bound", + "idempotent_import_rerun": "0 newly inserted rows on all nine partitions", + "suppression_events": "append-only revoke/restore; restricted view 1 then 0" + }, + "limitations": [ + "Inputs are existing V1-derived snapshots; original source retrieval times and raw acquisition bytes are not recreated.", + "Rows are private in temporary staging and are not committed; this manifest is row-free.", + "The rehearsal validates legacy compatibility and private/test-only behavior, not current V2 source quality or publication readiness.", + "Backup/restore was not run in this rehearsal; existing synthetic restore coverage remains separate.", + "Facet filtering and bounded export behavior are exercised, but the candidate is intentionally not public or promoted." + ] +} diff --git a/docs/real-v2-rehearsal.md b/docs/real-v2-rehearsal.md new file mode 100644 index 0000000..e7b2565 --- /dev/null +++ b/docs/real-v2-rehearsal.md @@ -0,0 +1,40 @@ +# 50k Real-Row V2 rehearsal + +This is a private compatibility rehearsal for the existing V1-derived corpus, +not a current-data release and not evidence that V2 is ready for publication. +The row-free evidence is in +[`data/manifests/real-v2-rehearsal-report.json`](../data/manifests/real-v2-rehearsal-report.json). + +## Reproduce the conversion + +Use a private temporary output directory. The script references the existing +`static_data/*/locations.csv` files and does not copy them into Git: + +```powershell +$out = Join-Path $env:TEMP 'uec-real-v2-rehearsal' +python pipeline/scripts/maintenance/rehearse_real_v2.py ` + --output $out ` + --as-of 2026-09-16T00:00:00Z +``` + +Each country receives parsed, normalized, quarantined, candidate-handoff, +manifest, and review-packet files. Every normalized row is labeled +`legacy-v1-derived-snapshot`, `private`, and `publication_gate=blocked`. + +## Disposable database rehearsal + +The guarded importer requires a loopback database on a non-default port and +the exact disposable marker. Import each country handoff with the same +`candidate-legacy-v2` release ID. Re-running is safe and returns zero newly +inserted rows because IDs and conflict keys are deterministic. + +The local API must be started with `UEC_TEST_RELEASE_ID=candidate-legacy-v2`, +`UEC_TEST_RELEASE_TOKEN`, and the preview token. Use only the +`/api/dev/preview/test-release/*` routes. `/api/v2/*` remains empty until an +authorized release is promoted. The candidate is never default-visible, +publication-eligible, or map-projected. + +The rehearsal verified list pagination, detail lookup, facets, bounded export, +and append-only suppression/reinstatement. It did not run backup/restore or +claim current-source quality; those limitations are intentional and recorded +in the manifest. diff --git a/pipeline/scripts/maintenance/rehearse_real_v2.py b/pipeline/scripts/maintenance/rehearse_real_v2.py new file mode 100644 index 0000000..1d68f1e --- /dev/null +++ b/pipeline/scripts/maintenance/rehearse_real_v2.py @@ -0,0 +1,158 @@ +"""Convert a bounded legacy V1 corpus into a private V2-shaped rehearsal. + +This is a migration rehearsal, not an acquisition adapter. It references the +existing V1 snapshots as raw inputs, preserves each row under source_values in +ignored staging, and marks every normalized record as legacy/private. No +release is promoted and the row-free report never contains record payloads. +""" +from __future__ import annotations + +import argparse +import csv +import hashlib +import json +import sys +import time +from collections import Counter, defaultdict +from pathlib import Path +from typing import Any + +# File-path invocation is part of the documented maintenance workflow. Add +# the repository root before importing shared pipeline contracts so this entry +# point behaves the same as `python -m ...`. +PROJECT_ROOT = Path(__file__).resolve().parents[3] +if str(PROJECT_ROOT) not in sys.path: + sys.path.insert(0, str(PROJECT_ROOT)) + +from pipeline.contracts.adapter_contract import SourceArtifact +from pipeline.contracts.source_lifecycle import atomic_json, atomic_jsonl, private_manifest + +COUNTRIES = ("ca", "de", "dk", "es", "fr", "mx", "nz", "uk", "us") +REHEARSAL_VERSION = "legacy-v1-to-v2-rehearsal-v1" + + +def _fingerprint(country: str, row: dict[str, str]) -> str: + value = "\0".join((country, row.get("establishment_id", ""), row.get("establishment_number", ""), row.get("establishment_name", ""), row.get("city", ""))) + return hashlib.sha256(value.encode("utf-8", "surrogateescape")).hexdigest() + + +def _valid_point(row: dict[str, str]) -> bool: + try: + lat, lon = float(row.get("latitude", "")), float(row.get("longitude", "")) + except (TypeError, ValueError): + return False + return -90 <= lat <= 90 and -180 <= lon <= 180 and (lat != 0 or lon != 0) + + +def _category(row: dict[str, str]) -> list[str]: + keys = {key for key, value in row.items() if value and value.strip().lower() not in {"0", "false", "no", "none", "nan"}} + categories = [] + if any("slaughter" in key for key in keys): + categories.append("slaughter") + if any("process" in key for key in keys): + categories.append("processing") + return categories or ["unclassified"] + + +def _source_url(registry: dict[str, Any], country: str) -> str: + wanted = {f"{country}.locations", f"{country}.smiley"} + for source in registry.get("sources", []): + if source.get("source_id") in wanted and source.get("url"): + return str(source["url"]) + if any(f"static_data/{country}/locations.csv" in str(path) for path in source.get("legacy_paths", [])) and source.get("url"): + return str(source["url"]) + return f"legacy://static_data/{country}/locations.csv" + + +def _normalize(country: str, row: dict[str, str], fingerprint: str) -> dict[str, Any]: + establishment_id = (row.get("establishment_id") or row.get("establishment_number") or "").strip() + has_identity = bool(establishment_id) + coordinates = None + if _valid_point(row): + coordinates = [float(row["longitude"]), float(row["latitude"])] + return { + "establishment_id": establishment_id or None, + "trading_name": (row.get("establishment_name") or "").strip() or None, + "address_lines": [(row.get("street") or "").strip()] if row.get("street") else [], + "postcode": (row.get("zip") or "").strip() or None, + "city": (row.get("city") or "").strip() or None, + "country_code": country.upper() if len(country) == 2 else "ZZ", + "nation": country.upper(), + "activity_categories": _category(row), + "coordinates": coordinates, + "coordinate_state": "source-supplied-pending-review" if coordinates else "unknown", + "coordinate_precision": "source-precision-unspecified" if coordinates else "not-supplied", + "privacy_gate": "pending-review", + "coordinate_gate": "review_required", + "publication_gate": "blocked", + "source_origin": "legacy-v1-derived-snapshot", + "legacy_label": "legacy snapshot; freshness and source retrieval date unknown", + "identity_state": "accepted" if has_identity else "quarantined", + "conversion_fingerprint": fingerprint, + } + + +def build_rehearsal(*, root: Path, output: Path, registry_path: Path, max_records: int = 50_000, as_of_utc: str = "2026-09-16T00:00:00Z") -> dict[str, Any]: + if max_records <= 0: + raise ValueError("max_records must be positive") + registry = json.loads(registry_path.read_text(encoding="utf-8")) + candidates: list[tuple[str, str, Path, dict[str, str]]] = [] + for country in COUNTRIES: + path = root / country / "locations.csv" + if not path.is_file(): + raise ValueError(f"missing legacy input: {path}") + with path.open("r", encoding="utf-8-sig", newline="") as handle: + for row in csv.DictReader(handle): + candidates.append((_fingerprint(country, row), country, path, row)) + selected = sorted(candidates, key=lambda item: item[0])[:max_records] + grouped: dict[str, list[tuple[str, Path, dict[str, str]]]] = defaultdict(list) + for fingerprint, country, path, row in selected: + grouped[country].append((fingerprint, path, row)) + + started = time.perf_counter() + source_counts: dict[str, dict[str, int]] = {} + strata = Counter() + input_files = [] + for country in sorted(grouped): + path = grouped[country][0][1] + digest = hashlib.sha256(path.read_bytes()).hexdigest() + input_files.append({"country": country, "path": path.as_posix(), "sha256": digest, "byte_size": path.stat().st_size, "source_url": _source_url(registry, country), "retrieved_at_utc": None, "provenance_state": "legacy-retrieval-unknown"}) + accepted: list[dict[str, Any]] = [] + quarantined: list[dict[str, Any]] = [] + for row_number, (fingerprint, _, row) in enumerate(grouped[country], 1): + record = {"source_id": f"legacy.v1.{country}.locations", "source_row": row_number, "source_record_key": (row.get("establishment_id") or row.get("establishment_number") or "") or None, "source_artifact_sha256": digest, "source_values": row, "normalized": _normalize(country, row, fingerprint)} + (accepted if record["normalized"]["identity_state"] == "accepted" else quarantined).append(record) + normalized = record["normalized"] + strata[(country, "+".join(_category(row)), normalized["identity_state"], normalized["coordinate_precision"])] += 1 + run = output / country + _, normalized_hash, _ = atomic_jsonl(run / "normalized" / "records.jsonl", accepted) + _, handoff_hash, _ = atomic_jsonl(run / "candidate-handoff" / "normalized" / "records.jsonl", accepted) + _, parsed_hash, _ = atomic_jsonl(run / "parsed" / "records.jsonl", accepted + quarantined) + atomic_jsonl(run / "quarantined" / "records.jsonl", quarantined) + artifact = SourceArtifact(source_url=_source_url(registry, country), retrieved_at_utc=as_of_utc, sha256=digest, byte_size=path.stat().st_size, code_version=REHEARSAL_VERSION, config_version=REHEARSAL_VERSION, coverage=f"legacy V1 {country} snapshot; source date unknown") + manifest = private_manifest(source_id=f"legacy.v1.{country}.locations", adapter_version=REHEARSAL_VERSION, schema_version=REHEARSAL_VERSION, artifact=artifact, input_rows=len(accepted) + len(quarantined), normalized_rows=len(accepted), quarantined_rows=len(quarantined), normalized_sha256=normalized_hash, parsed_sha256=parsed_hash, anomaly_counts={"missing_identity": len(quarantined)}) + manifest.update({"country_code": country.upper(), "profile": "legacy-private-test-only", "legacy": True, "retrieval_semantics": "processing timestamp only; original source retrieval unknown", "publication_eligibility": "blocked"}) + atomic_json(run / "manifest.json", manifest) + atomic_json(run / "candidate-handoff" / "manifest.json", {"source_id": manifest["source_id"], "manifest_sha256": hashlib.sha256(json.dumps(manifest, ensure_ascii=False, sort_keys=True, indent=2).encode()).hexdigest(), "normalized_sha256": handoff_hash, "normalized_rows": len(accepted), "publication_eligibility": "blocked", "legacy": True}) + atomic_json(run / "review-packet.json", {"source_id": manifest["source_id"], "review_state": "review_required", "publication_eligibility": "blocked", "checks": ["preserve legacy label", "review source terms and privacy", "do not promote or expose stale rows"], "row_counts": {"input": manifest["input_rows"], "normalized": manifest["normalized_rows"], "quarantined": manifest["quarantined_rows"]}}) + source_counts[country] = {"input": len(accepted) + len(quarantined), "normalized": len(accepted), "quarantined": len(quarantined)} + report = {"schema_version": "real-v2-rehearsal-v1", "rehearsal_id": output.name, "rehearsal_at_utc": as_of_utc, "legacy_label": "legacy V1-derived snapshot; not current and not publication-approved", "publication_eligibility": "blocked", "selection": {"method": "stable-sha256-row-fingerprint", "available_records": len(candidates), "selected_records": len(selected), "max_records": max_records}, "source_counts": source_counts, "totals": {key: sum(value[key] for value in source_counts.values()) for key in ("input", "normalized", "quarantined")}, "strata": {"|".join(key): value for key, value in sorted(strata.items())}, "source_files": input_files, "stages": {"conversion": "complete", "normalization": "complete", "quarantine": "complete", "candidate_handoff": "private normalized handoff produced", "import": "not attempted by this offline command", "api_export": "not attempted", "suppression": "not attempted", "delta": "not attempted", "backup_restore": "not attempted"}, "elapsed_seconds": round(time.perf_counter() - started, 3), "limitations": ["This conversion references existing V1-derived snapshots; it does not recreate original acquisition bytes or source retrieval times.", "The processing timestamp is not a claim that the legacy source is current.", "Database/API/export/suppression/backup stages require a disposable local database and are separate commands.", "Raw and derived rows remain in caller-selected private staging and are not committed."]} + atomic_json(output / "rehearsal-report.json", report) + return report + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--root", type=Path, default=Path("static_data")) + parser.add_argument("--registry", type=Path, default=Path("pipeline/source_registry.json")) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--max-records", type=int, default=50_000) + parser.add_argument("--as-of", default="2026-09-16T00:00:00Z") + args = parser.parse_args() + report = build_rehearsal(root=args.root, output=args.output, registry_path=args.registry, max_records=args.max_records, as_of_utc=args.as_of) + print(json.dumps({"selected": report["selection"]["selected_records"], "normalized": report["totals"]["normalized"], "quarantined": report["totals"]["quarantined"], "publication_eligibility": report["publication_eligibility"]}, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/pipeline/tests/test_real_v2_rehearsal.py b/pipeline/tests/test_real_v2_rehearsal.py new file mode 100644 index 0000000..418f3b6 --- /dev/null +++ b/pipeline/tests/test_real_v2_rehearsal.py @@ -0,0 +1,52 @@ +import json +import tempfile +import unittest +from pathlib import Path + +from pipeline.scripts.maintenance.rehearse_real_v2 import build_rehearsal + + +class RealV2RehearsalTests(unittest.TestCase): + def test_conversion_preserves_rows_privately_and_quarantines_missing_identity(self): + with tempfile.TemporaryDirectory() as directory: + root, output = Path(directory) / "static", Path(directory) / "run" + country = root / "aa" + country.mkdir(parents=True) + (country / "locations.csv").write_text("establishment_id,establishment_name,city,latitude,longitude,slaughter\nA,Alpha,Town,52.1,4.2,true\n,Unknown,Town,52.2,4.3,true\n", encoding="utf-8") + registry = Path(directory) / "registry.json" + registry.write_text(json.dumps({"sources": [{"source_id": "aa.locations", "url": "https://example.test/aa"}]}), encoding="utf-8") + import pipeline.scripts.maintenance.rehearse_real_v2 as module + old = module.COUNTRIES + module.COUNTRIES = ("aa",) + try: + report = build_rehearsal(root=root, output=output, registry_path=registry, max_records=10) + finally: + module.COUNTRIES = old + self.assertEqual(report["totals"], {"input": 2, "normalized": 1, "quarantined": 1}) + self.assertEqual(json.loads((output / "aa" / "manifest.json").read_text())["publication_state"], "private-candidate") + self.assertTrue((output / "aa" / "candidate-handoff" / "normalized" / "records.jsonl").is_file()) + self.assertTrue(report["strata"]) + self.assertNotIn("Alpha", (output / "rehearsal-report.json").read_text()) + + def test_selection_is_bounded(self): + with tempfile.TemporaryDirectory() as directory: + root, output = Path(directory) / "static", Path(directory) / "run" + country = root / "aa" + country.mkdir(parents=True) + rows = "establishment_id,establishment_name,city\n" + "\n".join(f"{i},Name {i},Town" for i in range(12)) + "\n" + (country / "locations.csv").write_text(rows, encoding="utf-8") + registry = Path(directory) / "registry.json" + registry.write_text(json.dumps({"sources": []}), encoding="utf-8") + import pipeline.scripts.maintenance.rehearse_real_v2 as module + old = module.COUNTRIES + module.COUNTRIES = ("aa",) + try: + report = build_rehearsal(root=root, output=output, registry_path=registry, max_records=5) + finally: + module.COUNTRIES = old + self.assertEqual(report["selection"]["selected_records"], 5) + self.assertEqual(report["totals"]["input"], 5) + + +if __name__ == "__main__": + unittest.main() From dc53583d79e988d9677cbac1c0e80dee6f041de5 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Wed, 16 Sep 2026 16:26:22 -0700 Subject: [PATCH 198/311] Add real geospatial and graph quality evaluation --- docs/real-quality-evaluation.md | 31 ++ .../diagnostics/real_quality_evaluation.py | 321 ++++++++++++++++++ .../tests/test_real_quality_evaluation.py | 39 +++ 3 files changed, 391 insertions(+) create mode 100644 docs/real-quality-evaluation.md create mode 100644 pipeline/scripts/diagnostics/real_quality_evaluation.py create mode 100644 pipeline/tests/test_real_quality_evaluation.py diff --git a/docs/real-quality-evaluation.md b/docs/real-quality-evaluation.md new file mode 100644 index 0000000..ff399bc --- /dev/null +++ b/docs/real-quality-evaluation.md @@ -0,0 +1,31 @@ +# Real geospatial, identity, and graph evaluation + +`pipeline/scripts/diagnostics/real_quality_evaluation.py` runs an offline, +deterministic evaluation over the nine checked-in V1-derived legacy snapshots. +It reads all available rows (currently 50,750), hashes each input, and writes +only aggregate metrics: coordinate validity and decimal precision, missing or +invalid coordinates, source-native duplicate/conflict rates, country/source/ +category composition, and conservative privacy review queues. + +The US strata are selected independently by ascending SHA-256 row fingerprint: +3,500 FSIS locations, 2,500 inspection rows, and 1,000 APHIS observations. +Selection fails closed if an input is too small. The tool counts exact +source-native identifier opportunities and explicit endpoint pairs. It never +joins by names, addresses, coordinates, or proximity; no cross-source +relationship candidate is emitted when the row schema lacks both endpoints. + +Run privately: + +```powershell +python pipeline/scripts/diagnostics/real_quality_evaluation.py ` + --output data/reports/real-quality-evaluation.json ` + --as-of 2026-09-16T00:00:00Z +``` + +The report is publication-blocked and row-free. Coordinate validity is not +positional accuracy, privacy indicators are human-review queues rather than +residential classifications, and duplicate/conflict rates are within-source +diagnostics. Raw source artifacts are not present in this checkout, so the +evaluation does not claim raw-preserving acquisition or source-authorized +publication. The US snapshots are sufficient for the requested sample sizes; +the resulting graph yield must still be reviewed before any graph import. diff --git a/pipeline/scripts/diagnostics/real_quality_evaluation.py b/pipeline/scripts/diagnostics/real_quality_evaluation.py new file mode 100644 index 0000000..2b237bd --- /dev/null +++ b/pipeline/scripts/diagnostics/real_quality_evaluation.py @@ -0,0 +1,321 @@ +"""Deterministic, private, row-free evaluation of the real legacy corpus. + +This measures source coordinates and source-native identity fields. It never +geocodes, fuzzy-matches, joins by names/proximity, or emits rows, IDs, names, +addresses, or coordinates. It is an evaluation report, not an accuracy claim: +accuracy requires adjudicated labels that this corpus does not contain. +""" +from __future__ import annotations + +import argparse +import csv +import hashlib +import json +import re +from collections import Counter, defaultdict +from pathlib import Path +from typing import Iterable + + +CORPUS_COUNTRIES = ("ca", "de", "dk", "es", "fr", "mx", "nz", "uk", "us") +US_SAMPLE_SIZES = { + "fsis_locations": 3_500, + "fsis_inspections": 2_500, + "aphis_observations": 1_000, +} +US_SAMPLE_FILES = { + "fsis_locations": "us/locations.csv", + "fsis_inspections": "us/inspection_reports.csv", + "aphis_observations": "us/aphis_data_final.csv", +} +ID_FIELDS = ( + "establishment_id", "establishment_number", "Certificate Number", + "Certificate Number_x", "Certificate Number_y", "Customer Number", + "Customer Number_x", "Customer Number_y", "facility_id", "operator_id", +) +NAME_FIELDS = ( + "establishment_name", "Account Name", "account_name", "facility_name", + "operator_name", "name", +) +CITY_FIELDS = ("city", "City", "City-State-Zip") +LAT_FIELDS = ("latitude", "Latitude", "Geocodio Latitude", "lat") +LON_FIELDS = ("longitude", "Longitude", "Geocodio Longitude", "lon", "lng") +ADDRESS_FIELDS = ( + "street", "address", "Address Line 1", "Address Line 2", "City-State-Zip", +) +PRIVACY_PATTERNS = { + "residential_or_private_term": re.compile(r"\b(private|residential|home|farmhouse)\b", re.I), + "care_of_or_mailbox": re.compile(r"\b(c/o|care\s+of|p\.?\s*o\.?\s*box)\b", re.I), + "unit_or_apartment": re.compile(r"\b(apt|apartment|unit|suite)\b", re.I), +} +EXPLICIT_ENDPOINT_PAIRS = ( + ("operator_id", "facility_id"), + ("operator_id", "establishment_id"), + ("subject_source_native_id", "object_source_native_id"), +) + + +def _text(row: dict[str, str], fields: Iterable[str]) -> str: + for field in fields: + value = row.get(field) + if value is not None and str(value).strip(): + return str(value).strip() + return "" + + +def _number(value: str | None) -> float | None: + try: + return float(value.strip()) if value and value.strip() else None + except (AttributeError, ValueError): + return None + + +def _coordinate(row: dict[str, str]) -> tuple[str, float | None, float | None]: + raw_lat, raw_lon = _text(row, LAT_FIELDS), _text(row, LON_FIELDS) + lat, lon = _number(raw_lat), _number(raw_lon) + if lat is None or lon is None: + return "missing_or_non_numeric", lat, lon + if not (-90 <= lat <= 90 and -180 <= lon <= 180): + return "out_of_range", lat, lon + if lat == 0 and lon == 0: + return "zero_pair", lat, lon + return "valid", lat, lon + + +def _precision(row: dict[str, str]) -> str: + raw_lat, raw_lon = _text(row, LAT_FIELDS), _text(row, LON_FIELDS) + digits = [] + for value in (raw_lat, raw_lon): + if "." in value: + digits.append(len(value.split(".", 1)[1].rstrip("0"))) + else: + digits.append(0) + if not digits or not _coordinate(row)[0] == "valid": + return "not_available" + maximum = max(digits) + return f"decimal_digits_{maximum}" if maximum <= 6 else "decimal_digits_7_plus" + + +def _source_id(row: dict[str, str]) -> tuple[str, str] | None: + for field in ID_FIELDS: + value = row.get(field) + if value is not None and str(value).strip(): + return field, str(value).strip() + return None + + +def _identity_quality(row: dict[str, str]) -> str: + identifier = _source_id(row) + has_name = bool(_text(row, NAME_FIELDS)) + has_city = bool(_text(row, CITY_FIELDS)) + if identifier and has_name and has_city: + return "strong" + if identifier and (has_name or has_city): + return "partial" + return "weak_or_missing" + + +def _privacy_reasons(row: dict[str, str]) -> tuple[str, ...]: + address = " ".join(str(row.get(field) or "") for field in ADDRESS_FIELDS) + return tuple(name for name, pattern in PRIVACY_PATTERNS.items() if pattern.search(address)) + + +def _category(row: dict[str, str]) -> str: + active = {key.lower() for key, value in row.items() if value and str(value).strip().lower() not in {"0", "false", "no", "none", "nan"}} + if any("slaughter" in key for key in active): + return "slaughter_or_processing" + if any("process" in key for key in active): + return "processing" + if any("inspection" in key or "license" in key or "certificate" in key for key in active): + return "inspection_or_license" + return "other" + + +def _fingerprint(namespace: str, row_number: int, row: dict[str, str]) -> str: + stable = "\0".join((namespace, str(row_number), *(str(row.get(key) or "") for key in sorted(row)))) + return hashlib.sha256(stable.encode("utf-8", "surrogateescape")).hexdigest() + + +def _read(path: Path) -> list[tuple[int, dict[str, str]]]: + with path.open("r", encoding="utf-8-sig", newline="") as handle: + return [(number, row) for number, row in enumerate(csv.DictReader(handle), start=2)] + + +def _source_metrics(path: Path, rows: list[tuple[int, dict[str, str]]], source_name: str) -> dict: + coordinates = Counter() + precision = Counter() + identities = Counter() + categories = Counter() + privacy = Counter() + id_rows: defaultdict[tuple[str, str], list[tuple[str, str, str]]] = defaultdict(list) + for row_number, row in rows: + coordinate_state, _, _ = _coordinate(row) + coordinates[coordinate_state] += 1 + precision[_precision(row)] += 1 + identity = _source_id(row) + identities[_identity_quality(row)] += 1 + categories[_category(row)] += 1 + reasons = _privacy_reasons(row) + for reason in reasons: + privacy[reason] += 1 + if identity: + signature = (_text(row, NAME_FIELDS), _text(row, CITY_FIELDS), coordinate_state, + _text(row, LAT_FIELDS), _text(row, LON_FIELDS)) + id_rows[identity].append(signature) + + duplicate_rows = sum(max(0, len(values) - 1) for values in id_rows.values()) + conflicting_keys = sum(len({signature for signature in values}) > 1 for values in id_rows.values()) + conflicting_rows = sum(len(values) for values in id_rows.values() if len({signature for signature in values}) > 1) + total = len(rows) + return { + "source": source_name, + "path": path.as_posix(), + "sha256": hashlib.sha256(path.read_bytes()).hexdigest(), + "bytes": path.stat().st_size, + "rows": total, + "coordinates": dict(sorted(coordinates.items())), + "coordinate_valid_rate": round(coordinates["valid"] / total, 6) if total else None, + "coordinate_precision": dict(sorted(precision.items())), + "identity_quality": dict(sorted(identities.items())), + "source_identifier_rows": sum(len(values) for values in id_rows.values()), + "source_identifier_unique": len(id_rows), + "duplicate_source_identifier_rows": duplicate_rows, + "duplicate_source_identifier_rate": round(duplicate_rows / total, 6) if total else None, + "identity_conflict_keys": conflicting_keys, + "identity_conflict_rows": conflicting_rows, + "identity_conflict_rate": round(conflicting_rows / total, 6) if total else None, + "category_composition": dict(sorted(categories.items())), + "privacy_review_queue": { + "flagged_rows": sum(1 for row_number, row in rows if _privacy_reasons(row)), + "flagged_rate": round(sum(1 for row_number, row in rows if _privacy_reasons(row)) / total, 6) if total else None, + "by_reason": dict(sorted(privacy.items())), + }, + } + + +def _sample_metrics(path: Path, requested: int, source_name: str) -> dict: + rows = _read(path) + ranked = sorted((_fingerprint(source_name, number, row), number, row) for number, row in rows) + if len(ranked) < requested: + raise ValueError(f"{path} has {len(ranked)} rows; {requested} required") + selected = ranked[:requested] + identifiers = [_source_id(row) for _, _, row in selected] + explicit_edges = sum( + any(bool(_text(row, (left,))) and bool(_text(row, (right,))) + for left, right in EXPLICIT_ENDPOINT_PAIRS) + for _, _, row in selected + ) + return { + "path": path.as_posix(), + "sha256": hashlib.sha256(path.read_bytes()).hexdigest(), + "available_rows": len(rows), + "requested_rows": requested, + "selected_rows": len(selected), + "coordinate_metrics": _source_metrics(path, [(number, row) for _, number, row in selected], source_name), + "source_native_identifier_rows": sum(identifier is not None for identifier in identifiers), + "source_native_identifier_unique": len({identifier for identifier in identifiers if identifier is not None}), + "source_native_identifier_missing_rows": sum(identifier is None for identifier in identifiers), + "explicit_endpoint_pair_rows": explicit_edges, + "cross_source_relationship_candidates": 0, + "name_or_proximity_matches_attempted": 0, + "candidate_rule": "exact source-native identifiers only; no cross-source endpoint pair exists in these legacy files", + } + + +def build_report(root: Path = Path("static_data"), *, as_of: str = "") -> dict: + files = {country: root / country / "locations.csv" for country in CORPUS_COUNTRIES} + missing = [path.as_posix() for path in files.values() if not path.is_file()] + if missing: + raise ValueError("missing corpus inputs: " + ", ".join(missing)) + source_metrics = [] + totals = Counter() + composition = Counter() + for country, path in files.items(): + rows = _read(path) + metric = _source_metrics(path, rows, f"v1.{country}.locations") + source_metrics.append(metric) + totals.update({"rows": metric["rows"], "valid_coordinates": metric["coordinates"].get("valid", 0), "missing_or_invalid_coordinates": metric["rows"] - metric["coordinates"].get("valid", 0), "duplicate_identifier_rows": metric["duplicate_source_identifier_rows"], "identity_conflict_rows": metric["identity_conflict_rows"], "privacy_flagged_rows": metric["privacy_review_queue"]["flagged_rows"]}) + for category, count in metric["category_composition"].items(): + composition[(country, category)] += count + + samples = { + name: _sample_metrics(root / path, count, name) + for name, path in US_SAMPLE_FILES.items() + for count in (US_SAMPLE_SIZES[name],) + } + sample_totals = Counter() + for sample in samples.values(): + sample_totals.update({"selected_rows": sample["selected_rows"], "source_native_identifier_rows": sample["source_native_identifier_rows"], "missing_source_native_identifier_rows": sample["source_native_identifier_missing_rows"], "explicit_endpoint_pair_rows": sample["explicit_endpoint_pair_rows"]}) + + return { + "schema_version": "real-quality-evaluation-v1", + "as_of_utc": as_of or None, + "corpus_state": "private-regression-only", + "publication_eligibility": "blocked", + "method": { + "selection": "all available legacy rows; US strata selected by ascending SHA-256 row fingerprint", + "coordinate_validation": "numeric latitude/longitude range and zero-pair checks; no geocoder", + "identity_duplicates": "source-native identifier repetitions within each source file", + "identity_conflicts": "same source-native identifier mapped to differing nonempty name/city/coordinate-state signatures", + "privacy_queue": "conservative text indicators for human review, not residential classifications", + "accuracy": "not measured; no adjudicated truth labels are available", + }, + "corpus": { + "available_rows": totals["rows"], + "country_count": len(files), + "source_profile_count": len(files), + "totals": dict(sorted(totals.items())), + "country_composition": {country: metric["rows"] for country, metric in sorted(zip(files, source_metrics))}, + "category_composition": {"|".join(key): value for key, value in sorted(composition.items())}, + "sources": source_metrics, + }, + "privacy_review_queue": { + "rows_flagged_for_human_review": totals["privacy_flagged_rows"], + "scope": "aggregate counts only; flagged rows and address text remain private", + "indicators": sorted(PRIVACY_PATTERNS), + }, + "us_planned_sample": { + "requested": dict(US_SAMPLE_SIZES), + "available": {name: sample["available_rows"] for name, sample in sorted(samples.items())}, + "selected_total": sample_totals["selected_rows"], + "selected": samples, + "aggregate": dict(sorted(sample_totals.items())), + }, + "graph_candidate_yield": { + "source_native_entity_candidate_rows": sample_totals["source_native_identifier_rows"], + "source_native_entity_candidate_unique_ids": sum(sample["source_native_identifier_unique"] for sample in samples.values()), + "explicit_endpoint_pair_rows": sample_totals["explicit_endpoint_pair_rows"], + "cross_source_relationship_candidates": 0, + "name_or_proximity_matches_attempted": 0, + "interpretation": "Legacy FSIS, inspection, and APHIS snapshots contain source-native identifiers but no explicit cross-source endpoint pair in the sampled rows. No identity or relationship is inferred.", + }, + "review_needs": [ + "Coordinate validity is not positional accuracy; adjudicated coordinate labels are required before accuracy estimates.", + "Privacy indicators are conservative review queues, not factual residential determinations.", + "Duplicate and conflict rates are within-source diagnostics; they do not establish cross-source identity.", + "US samples are V1-derived snapshots without raw source artifacts in this checkout; raw-preserving acquisition and terms review remain open.", + "Graph candidate yield counts exact source-native opportunities only; zero cross-source candidates is an observed schema result, not evidence that no relationship exists.", + ], + } + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--root", type=Path, default=Path("static_data")) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--as-of", default="") + args = parser.parse_args() + report = build_report(args.root, as_of=args.as_of) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(json.dumps({ + "available_rows": report["corpus"]["available_rows"], + "us_selected_rows": report["us_planned_sample"]["selected_total"], + "cross_source_relationship_candidates": report["graph_candidate_yield"]["cross_source_relationship_candidates"], + "publication_eligibility": report["publication_eligibility"], + }, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/pipeline/tests/test_real_quality_evaluation.py b/pipeline/tests/test_real_quality_evaluation.py new file mode 100644 index 0000000..931851a --- /dev/null +++ b/pipeline/tests/test_real_quality_evaluation.py @@ -0,0 +1,39 @@ +import json +import tempfile +import unittest +from pathlib import Path + +from pipeline.scripts.diagnostics.real_quality_evaluation import build_report + + +HEADER = "establishment_id,establishment_name,city,street,latitude,longitude,slaughter\n" + + +class RealQualityEvaluationTests(unittest.TestCase): + def test_report_is_deterministic_and_row_free(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + for country in ("ca", "de", "dk", "es", "fr", "mx", "nz", "uk", "us"): + path = root / country + path.mkdir() + path.joinpath("locations.csv").write_text(HEADER + "1,Alpha,Town,10 Main,52.1000,4.2000,true\n2,Beta,Town,private farmhouse,91,0,true\n", encoding="utf-8") + for name in ("inspection_reports.csv", "aphis_data_final.csv"): + Path(root / "us" / name).write_text(HEADER + "1,Alpha,Town,10 Main,52.1,4.2,true\n", encoding="utf-8") + # The fixture intentionally cannot satisfy the real US sample sizes; + # the evaluator must fail closed rather than reuse other rows. + with self.assertRaisesRegex(ValueError, "has 2 rows; 3500 required"): + build_report(root, as_of="2026-09-16T00:00:00Z") + + def test_real_corpus_report_shape_has_no_row_payload(self): + report = build_report(Path("static_data"), as_of="2026-09-16T00:00:00Z") + encoded = json.dumps(report) + self.assertEqual(report["corpus"]["available_rows"], 50750) + self.assertEqual(report["us_planned_sample"]["selected_total"], 7000) + self.assertEqual(report["graph_candidate_yield"]["cross_source_relationship_candidates"], 0) + self.assertNotIn("Godshall", encoded) + self.assertNotIn("1415 Weavertown", encoded) + self.assertNotIn("6407", encoded) + + +if __name__ == "__main__": + unittest.main() From 22b8c765dc97b137324aa2ed9e19c3b691ae471f Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Wed, 16 Sep 2026 16:34:33 -0700 Subject: [PATCH 199/311] test: add private distribution load rehearsal path --- docs/performance/v2-api-load-rehearsal.md | 21 +++ .../benchmarks/build_private_distribution.py | 128 ++++++++++++++++++ .../benchmarks/run_api_load_rehearsal.py | 89 ++++++++++-- .../test_private_distribution_benchmark.py | 39 ++++++ 4 files changed, 262 insertions(+), 15 deletions(-) create mode 100644 pipeline/scripts/benchmarks/build_private_distribution.py create mode 100644 pipeline/tests/test_private_distribution_benchmark.py diff --git a/docs/performance/v2-api-load-rehearsal.md b/docs/performance/v2-api-load-rehearsal.md index ea4dd22..403148b 100644 --- a/docs/performance/v2-api-load-rehearsal.md +++ b/docs/performance/v2-api-load-rehearsal.md @@ -32,6 +32,27 @@ level at 80. It refuses non-loopback API targets. The 25,000-row bound is an explicitly finite synthetic safety limit, not a statement about supported production scale. +When an authorized private V2 normalized corpus is available, first create a +row-free distribution report and then pass it to the same synthetic rehearsal: + +```powershell +python pipeline/scripts/benchmarks/build_private_distribution.py ` + --normalized data/private//normalized/records.jsonl ` + --output .tmp/private-v2-distribution.json + +python pipeline/scripts/benchmarks/run_api_load_rehearsal.py ` + --distribution-report .tmp/private-v2-distribution.json ` + --concurrency 1,4,8,16 --requests-per-level 10 ` + --timeout-ms 2000 --json-output .tmp/api-load-private-distribution.json +``` + +The distribution report is explicitly blocked and contains only aggregate +country/category/precision strata. The rehearsal expands those strata into +deterministic synthetic rows; it never imports private names, identifiers, +addresses, coordinates, or source values into the disposable database or +report. A real corpus therefore informs shape without becoming publication, +release, or benchmark-output data. + ## Captured evidence (2026-09-16) | Synthetic observations | Concurrency | Requests | Successes | Timeouts | 5xx | Throughput (rps) | p50 / p95 / p99 (ms) | Max active / waiting DB sessions | diff --git a/pipeline/scripts/benchmarks/build_private_distribution.py b/pipeline/scripts/benchmarks/build_private_distribution.py new file mode 100644 index 0000000..84bd063 --- /dev/null +++ b/pipeline/scripts/benchmarks/build_private_distribution.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 +"""Build a row-free aggregate distribution from private V2 normalized JSONL. + +The input remains in local restricted storage. The output contains only +bounded aggregate strata used to shape a synthetic API rehearsal; it never +copies identifiers, names, addresses, coordinates, source values, or rows. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +from collections import Counter +from pathlib import Path +from typing import Any + +MAX_RECORDS = 25_000 +ALLOWED_CATEGORIES = { + "slaughter", "fish_processing", "logistics_and_storage", + "retail_and_prepared_food", +} +ALLOWED_PRECISIONS = {"exact", "city", "unmapped"} + + +def _normalized(payload: Any) -> dict[str, Any]: + value = payload.get("normalized") if isinstance(payload, dict) else None + return value if isinstance(value, dict) else payload if isinstance(payload, dict) else {} + + +def _category(value: Any) -> str: + values = value if isinstance(value, (list, tuple)) else [value] + for item in values: + if isinstance(item, str) and item in ALLOWED_CATEGORIES: + return item + return "other" + + +def _precision(record: dict[str, Any]) -> str: + value = record.get("display_precision") + if isinstance(value, str) and value in ALLOWED_PRECISIONS: + return value + coordinates = record.get("coordinates") + if isinstance(coordinates, (list, tuple)) and len(coordinates) == 2: + try: + longitude, latitude = float(coordinates[0]), float(coordinates[1]) + if -180 <= longitude <= 180 and -90 <= latitude <= 90: + return "exact" + except (TypeError, ValueError): + pass + return "unmapped" + + +def _country(record: dict[str, Any]) -> str: + value = record.get("country_code") + return value.upper() if isinstance(value, str) and len(value) == 2 and value.isascii() else "XX" + + +def build_distribution(inputs: list[Path], *, max_records: int = MAX_RECORDS) -> dict[str, Any]: + if not inputs: + raise ValueError("at least one private normalized JSONL input is required") + if not 1 <= max_records <= MAX_RECORDS: + raise ValueError(f"max_records must be between 1 and {MAX_RECORDS:,}") + candidates: list[tuple[str, dict[str, Any]]] = [] + invalid_records = 0 + for path in inputs: + if not path.is_file(): + raise ValueError(f"private normalized input is missing: {path}") + with path.open("r", encoding="utf-8") as handle: + for line in handle: + if not line.strip(): + continue + try: + payload = json.loads(line) + except json.JSONDecodeError: + invalid_records += 1 + continue + record = _normalized(payload) + if not record: + invalid_records += 1 + continue + candidates.append((hashlib.sha256(line.encode("utf-8")).hexdigest(), record)) + selected = sorted(candidates, key=lambda item: item[0])[:max_records] + strata = Counter( + (_country(record), _category(record.get("activity_categories") or record.get("classification_categories")), _precision(record)) + for _, record in selected + ) + return { + "schema_version": "v2-private-distribution-v1", + "corpus_state": "private-regression-only", + "publication_eligibility": "blocked", + "selection": { + "method": "stable-sha256-record-selection", + "max_records": max_records, + "selected_records": len(selected), + "available_records": len(candidates), + "invalid_records": invalid_records, + }, + "distribution": [ + {"country_code": country, "category": category, "display_precision": precision, "records": count} + for (country, category, precision), count in sorted(strata.items()) + ], + "limitations": [ + "The benchmark expands only aggregate strata into synthetic values.", + "No publication, release, factual review, or privacy approval is implied.", + "Raw and normalized private inputs remain local and are never copied to the report.", + ], + } + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--normalized", type=Path, action="append", required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--max-records", type=int, default=MAX_RECORDS) + args = parser.parse_args() + try: + report = build_distribution(args.normalized, max_records=args.max_records) + except ValueError as exc: + parser.error(str(exc)) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(json.dumps({"selected_records": report["selection"]["selected_records"], "publication_eligibility": report["publication_eligibility"]}, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/pipeline/scripts/benchmarks/run_api_load_rehearsal.py b/pipeline/scripts/benchmarks/run_api_load_rehearsal.py index f3ef03c..c2db178 100644 --- a/pipeline/scripts/benchmarks/run_api_load_rehearsal.py +++ b/pipeline/scripts/benchmarks/run_api_load_rehearsal.py @@ -30,6 +30,11 @@ MAX_SEED = 25_000 MAX_CONCURRENCY = 16 MAX_REQUESTS_PER_LEVEL = 80 +ALLOWED_DISTRIBUTION_PRECISIONS = {"exact", "city", "unmapped"} +ALLOWED_DISTRIBUTION_CATEGORIES = { + "slaughter", "fish_processing", "logistics_and_storage", + "retail_and_prepared_food", "other", +} QUERY_MIX = ( ("list", "http", "/api/v2/locations?profile=official&limit=50"), ("filters", "http", "/api/v2/locations?profile=official&country_code=DK&category=slaughter&limit=50"), @@ -76,7 +81,45 @@ def validate_loopback_url(base_url: str) -> urllib.parse.SplitResult: return parsed -def seed_public_projection(connection: Any, count: int) -> str: +def load_distribution(path: Path) -> list[tuple[int, str, str, str]]: + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise ValueError(f"distribution report cannot be read: {path}") from exc + if not isinstance(payload, dict): + raise ValueError("distribution report must be a JSON object") + if ( + payload.get("schema_version") != "v2-private-distribution-v1" + or payload.get("corpus_state") != "private-regression-only" + or payload.get("publication_eligibility") != "blocked" + ): + raise ValueError("distribution report must be a blocked v2-private-distribution-v1 report") + rows: list[tuple[int, str, str, str]] = [] + ordinal = 0 + for item in payload.get("distribution", []): + if not isinstance(item, dict): + raise ValueError("distribution report contains an invalid stratum") + country, category, precision, count = ( + item.get("country_code"), item.get("category"), + item.get("display_precision"), item.get("records"), + ) + if not isinstance(country, str) or len(country) != 2 or not country.isascii() or not country.isupper(): + raise ValueError("distribution report contains an invalid country code") + if category not in ALLOWED_DISTRIBUTION_CATEGORIES or precision not in ALLOWED_DISTRIBUTION_PRECISIONS: + raise ValueError("distribution report contains an unsupported stratum") + if not isinstance(count, int) or count < 1: + raise ValueError("distribution report contains an invalid record count") + for _ in range(count): + ordinal += 1 + if ordinal > MAX_SEED: + raise ValueError(f"distribution report exceeds the {MAX_SEED:,}-record benchmark bound") + rows.append((ordinal, country, category, precision)) + if not rows: + raise ValueError("distribution report contains no records") + return rows + + +def seed_public_projection(connection: Any, count: int, distribution: list[tuple[int, str, str, str]] | None = None) -> str: """Create a deterministic, promoted, synthetic projection in the E2E DB.""" release_id = "load-promoted" connection.execute("INSERT INTO uec.sources (source_id,country_code,name,official_url,access_method) VALUES ('load.synthetic','DK','Synthetic load source','https://example.invalid/load','fixture') ON CONFLICT DO NOTHING") @@ -94,6 +137,11 @@ def seed_public_projection(connection: Any, count: int) -> str: (release_id, encoded_manifest, hashlib.sha256(encoded_manifest.encode("utf-8")).hexdigest()), ) connection.execute("INSERT INTO uec.city_reference_points (country_code,city_name,reference_location,reference_source,source_retrieved_at,source_reference_id) VALUES ('DK','Loadville',ST_SetSRID(ST_Point(-5,50),4326)::geography,'https://example.invalid/load-city',TIMESTAMPTZ '2026-01-01 00:00:00+00','load-city') ON CONFLICT DO NOTHING") + connection.execute("CREATE TEMP TABLE load_distribution (ordinal integer PRIMARY KEY, country_code text NOT NULL, category text NOT NULL, display_precision text NOT NULL) ON COMMIT DROP") + if distribution: + with connection.cursor() as cursor: + cursor.executemany("INSERT INTO load_distribution (ordinal,country_code,category,display_precision) VALUES (%s,%s,%s,%s)", distribution) + connection.execute("INSERT INTO uec.city_reference_points (country_code,city_name,reference_location,reference_source,source_retrieved_at,source_reference_id) SELECT DISTINCT country_code,'Loadville-' || country_code,ST_SetSRID(ST_Point(-5,50),4326)::geography,'https://example.invalid/load-city',TIMESTAMPTZ '2026-01-01 00:00:00+00','load-city-' || country_code FROM load_distribution ON CONFLICT DO NOTHING") connection.execute(""" INSERT INTO uec.raw_artifacts (artifact_id,storage_key,sha256,byte_size,retrieved_at) SELECT md5('load-artifact-' || n::text)::uuid, 'synthetic/load/' || n::text, @@ -110,19 +158,24 @@ def seed_public_projection(connection: Any, count: int) -> str: connection.execute(""" INSERT INTO uec.facilities (facility_id,canonical_name,country_code,city) SELECT md5('load-facility-' || n::text)::uuid, 'Synthetic load facility ' || n, - CASE WHEN n %% 2 = 0 THEN 'DK' ELSE 'SE' END, 'Loadville' - FROM generate_series(1,%s) n ON CONFLICT DO NOTHING - """, (count,)) + CASE WHEN %s THEN d.country_code ELSE CASE WHEN n %% 2 = 0 THEN 'DK' ELSE 'SE' END END, + CASE WHEN %s THEN 'Loadville-' || d.country_code ELSE 'Loadville' END + FROM generate_series(1,%s) n + LEFT JOIN load_distribution d ON d.ordinal=n + ON CONFLICT DO NOTHING + """, (bool(distribution), bool(distribution), count)) connection.execute(""" INSERT INTO uec.observations (observation_id,facility_id,source_record_id,observed_at,observation,classification,ruleset_id,rule_id,classification_category,classification_review_status,default_visible,coordinate_review_status,first_observed_at) SELECT md5('load-observation-' || n::text)::uuid, md5('load-facility-' || n::text)::uuid, md5('load-record-' || n::text)::uuid, TIMESTAMPTZ '2026-01-01 00:00:00+00' + n * interval '1 second', '{}'::jsonb, '{}'::jsonb, 'load-v1','synthetic', - CASE n %% 4 WHEN 0 THEN 'slaughter' WHEN 1 THEN 'fish_processing' WHEN 2 THEN 'logistics_and_storage' ELSE 'retail_and_prepared_food' END, + CASE WHEN %s THEN d.category ELSE CASE n %% 4 WHEN 0 THEN 'slaughter' WHEN 1 THEN 'fish_processing' WHEN 2 THEN 'logistics_and_storage' ELSE 'retail_and_prepared_food' END END, 'approved',true,'approved',TIMESTAMPTZ '2026-01-01 00:00:00+00' + n * interval '1 second' - FROM generate_series(1,%s) n ON CONFLICT DO NOTHING - """, (count,)) + FROM generate_series(1,%s) n + LEFT JOIN load_distribution d ON d.ordinal=n + ON CONFLICT DO NOTHING + """, (bool(distribution), count)) connection.execute(""" INSERT INTO uec.release_members (release_id,facility_id,observation_id,default_visible) SELECT 'load-promoted',md5('load-facility-' || n::text)::uuid,md5('load-observation-' || n::text)::uuid,true @@ -130,11 +183,14 @@ def seed_public_projection(connection: Any, count: int) -> str: """, (count,)) connection.execute(""" INSERT INTO uec.geocode_results (source_record_id,provider_id,query,match_method,status,attempt_number,result,queried_at) - SELECT md5('load-record-' || n::text)::uuid,'synthetic-fixture','synthetic load query','fixture','accepted',1, + SELECT md5('load-record-' || n::text)::uuid,'synthetic-fixture','synthetic load query','fixture',CASE WHEN %s AND d.display_precision='city' THEN 'review_required' ELSE 'accepted' END,1, ST_SetSRID(ST_Point(-10 + (n %% 2000) / 100.0,45 + (n %% 1000) / 100.0),4326)::geography, TIMESTAMPTZ '2026-01-01 00:00:00+00' + n * interval '1 second' - FROM generate_series(1,%s) n ON CONFLICT DO NOTHING - """, (count,)) + FROM generate_series(1,%s) n + LEFT JOIN load_distribution d ON d.ordinal=n + WHERE NOT (%s AND d.display_precision='unmapped') + ON CONFLICT DO NOTHING + """, (bool(distribution), count, bool(distribution))) connection.execute(""" INSERT INTO uec.publication_review_events (source_record_id,release_id,factual_review_status,privacy_screening_status,maintainer_approval,publication_eligible,reviewer_role,reviewed_at) SELECT md5('load-record-' || n::text)::uuid,'load-promoted','reviewed','passed','approved',true,'synthetic-reviewer', @@ -323,10 +379,10 @@ def build_recommendations(results: list[dict[str, Any]], timeout_ms: int) -> dic } -def run_rehearsal(env: Any, observations: int, levels: tuple[int, ...], requests_per_level: int, timeout_ms: int) -> dict[str, Any]: +def run_rehearsal(env: Any, observations: int, levels: tuple[int, ...], requests_per_level: int, timeout_ms: int, distribution: list[tuple[int, str, str, str]] | None = None) -> dict[str, Any]: import psycopg with psycopg.connect(env.database_url) as connection: - detail_id = seed_public_projection(connection, observations) + detail_id = seed_public_projection(connection, observations, distribution) base = f"http://127.0.0.1:{env.api_port}" results = [] for concurrency in levels: @@ -340,6 +396,7 @@ def run_rehearsal(env: Any, observations: int, levels: tuple[int, ...], requests return { "schema_version": 1, "synthetic_only": True, + "input_mode": "private-aggregate-distribution" if distribution else "fixed-synthetic-distribution", "observations": observations, "requests_per_level": requests_per_level, "timeout_ms": timeout_ms, @@ -350,14 +407,16 @@ def run_rehearsal(env: Any, observations: int, levels: tuple[int, ...], requests def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--observations", type=int, default=5_000) + parser.add_argument("--observations", type=int, default=None) + parser.add_argument("--distribution-report", type=Path, help="use a row-free private V2 distribution report to shape synthetic rows") parser.add_argument("--concurrency", default="1,4,8,16") parser.add_argument("--requests-per-level", type=int, default=40) parser.add_argument("--timeout-ms", type=int, default=2_000) parser.add_argument("--json-output", type=Path) args = parser.parse_args(argv) try: - observations = validate_observations(args.observations) + distribution = load_distribution(args.distribution_report) if args.distribution_report else None + observations = len(distribution) if distribution else validate_observations(args.observations or 5_000) except ValueError as exc: parser.error(str(exc)) if not 1 <= args.requests_per_level <= MAX_REQUESTS_PER_LEVEL: @@ -369,7 +428,7 @@ def main(argv: list[str] | None = None) -> int: from pipeline.tests.e2e.fixture import E2EEnvironment env = E2EEnvironment().start() try: - report = run_rehearsal(env, observations, levels, args.requests_per_level, args.timeout_ms) + report = run_rehearsal(env, observations, levels, args.requests_per_level, args.timeout_ms, distribution) finally: env.stop() except (ValueError, RuntimeError) as exc: diff --git a/pipeline/tests/test_private_distribution_benchmark.py b/pipeline/tests/test_private_distribution_benchmark.py new file mode 100644 index 0000000..112f8d1 --- /dev/null +++ b/pipeline/tests/test_private_distribution_benchmark.py @@ -0,0 +1,39 @@ +import json +import tempfile +import unittest +from pathlib import Path + +from pipeline.scripts.benchmarks.build_private_distribution import build_distribution +from pipeline.scripts.benchmarks.run_api_load_rehearsal import load_distribution + + +class PrivateDistributionBenchmarkTests(unittest.TestCase): + def test_report_is_row_free_bounded_and_accepted_by_rehearsal(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + normalized = root / "records.jsonl" + normalized.write_text( + json.dumps({"normalized": {"country_code": "DK", "activity_categories": ["slaughter"], "coordinates": [10, 55]}}) + "\n" + + json.dumps({"normalized": {"country_code": "FR", "classification_categories": ["fish_processing"], "display_precision": "city"}}) + "\n", + encoding="utf-8", + ) + report = build_distribution([normalized]) + self.assertEqual(report["selection"]["selected_records"], 2) + self.assertEqual(sum(item["records"] for item in report["distribution"]), 2) + self.assertNotIn("coordinates", json.dumps(report)) + report_path = root / "distribution.json" + report_path.write_text(json.dumps(report), encoding="utf-8") + distribution = load_distribution(report_path) + self.assertEqual(len(distribution), 2) + self.assertEqual(distribution[0][2], "slaughter") + + def test_rehearsal_rejects_non_blocked_or_empty_reports(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "bad.json" + path.write_text(json.dumps({"schema_version": "v2-private-distribution-v1", "corpus_state": "public", "publication_eligibility": "eligible", "distribution": []}), encoding="utf-8") + with self.assertRaises(ValueError): + load_distribution(path) + + +if __name__ == "__main__": + unittest.main() From 5f110df2c13c25b8eac0f9fa0e810e74e6984d28 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Wed, 16 Sep 2026 16:57:42 -0700 Subject: [PATCH 200/311] Record current-source reacquisition rehearsal --- .../current-reacquisition-2026-09-16.json | 197 ++++++++++++++++++ docs/current-reacquisition.md | 68 ++++++ 2 files changed, 265 insertions(+) create mode 100644 data/manifests/current-reacquisition-2026-09-16.json create mode 100644 docs/current-reacquisition.md diff --git a/data/manifests/current-reacquisition-2026-09-16.json b/data/manifests/current-reacquisition-2026-09-16.json new file mode 100644 index 0000000..1a47430 --- /dev/null +++ b/data/manifests/current-reacquisition-2026-09-16.json @@ -0,0 +1,197 @@ +{ + "manifest_version": "current-reacquisition-v1", + "as_of_utc": "2026-09-16T23:45:00Z", + "privacy_boundary": "row-free checked-in evidence; raw artifacts, normalized rows, addresses, coordinates, and restricted payloads remain ignored local research inputs", + "publication": { + "release_created": false, + "release_promoted": false, + "public_api_rows": 0, + "candidate_release": "candidate-current-reacquisition-20260916", + "candidate_release_status": "candidate", + "project_approval": "not-approved" + }, + "sources": [ + { + "source_id": "dk.smiley", + "source_url": "https://pub.fvst.dk/publikationer/Smileydata.xml", + "retrieved_at_utc": "2026-09-16T23:20:00Z", + "effective_date": "Wed, 16 Sep 2026 23:16:14 GMT", + "raw_artifact": "data/raw/dk.smiley/20260916T152000Z-dk/Smileydata.xml", + "raw_bytes": 59821775, + "raw_sha256": "4d2f012d3c287ad63e97fc13be8adfcc22f45a163d6096d549d970e63749ccff", + "private_manifest": "data/staging/reacquisition/dk.smiley/20260916T152000Z/manifest.json", + "candidate_handoff_manifest": "data/staging/reacquisition/dk.smiley/20260916T152000Z/candidate-handoff/manifest.json", + "input_rows": 58766, + "normalized_rows": 58766, + "quarantined_rows": 0, + "validation_findings": 57, + "code_version": "denmark-smiley-acquisition-v1", + "config_version": "denmark-smiley-acquisition-v1", + "status": "candidate-ready; private only" + }, + { + "source_id": "it.853-2004", + "source_url": "https://www.dati.salute.gov.it/sites/default/files/opendata/STAB_POA_8_20260916.csv", + "catalog_url": "https://www.dati.salute.gov.it/it/dataset/stabilimenti-italiani-gli-alimenti-di-origine-animale/", + "retrieved_at_utc": "2026-09-16T23:24:10Z", + "effective_date": "2026-09-16", + "raw_artifact": "data/raw/it.853-2004/20260916T152500Z-it/source.csv", + "raw_bytes": 49932616, + "raw_sha256": "b0d71469bd85737c0d1e07669e80f7972c862b1abd22501c4cde1f093ed999f7", + "private_manifest": "data/staging/reacquisition/it.853-2004/20260916T152500Z/b0d71469bd85737c-ob8vbdl0/manifest.json", + "candidate_handoff_manifest": "data/staging/reacquisition/it.853-2004/20260916T152500Z/b0d71469bd85737c-ob8vbdl0/candidate-handoff/manifest.json", + "input_rows": 47373, + "normalized_rows": 41847, + "quarantined_rows": 5526, + "code_version": "it-853-adapter-v1", + "config_version": "it-853-csv-v1", + "status": "candidate-ready; private only; system-curl assisted capture because Python TLS failed" + }, + { + "source_id": "fr.dgal.section-i", + "source_url": "https://fichiers-publics.agriculture.gouv.fr/dgal/ListesOfficielles/SSA1_VIAN_ONG_DOM.txt", + "retrieved_at_utc": "2026-09-16T23:20:40Z", + "effective_date": "Wed, 16 Sep 2026 02:21:51 GMT", + "raw_artifact": "data/raw/fr.dgal.section-i/20260916T152000Z-fr-i/source.txt", + "raw_bytes": 204211, + "raw_sha256": "b1171561865ab664ddf18adeeed7b6993224cc2275277fdaa6e4d411dd062649", + "private_manifest": "data/staging/reacquisition/fr.dgal.section-i/20260916T152000Z/lifecycle/b1171561865ab664-w55kiu5e/manifest.json", + "candidate_handoff_manifest": "data/staging/reacquisition/fr.dgal.section-i/20260916T152000Z/lifecycle/b1171561865ab664-w55kiu5e/candidate-handoff/manifest.json", + "input_rows": 1448, + "normalized_rows": 1448, + "quarantined_rows": 0, + "code_version": "fr-dgal-853-v1", + "config_version": "fr-dgal-853-txt-v1", + "status": "candidate-ready; private only" + }, + { + "source_id": "fr.dgal.section-ii", + "source_url": "https://fichiers-publics.agriculture.gouv.fr/dgal/ListesOfficielles/SSA1_VIAN_COL_LAGO.txt", + "retrieved_at_utc": "2026-09-16T23:20:40Z", + "effective_date": "Wed, 16 Sep 2026 02:22:14 GMT", + "raw_artifact": "data/raw/fr.dgal.section-ii/20260916T152000Z-fr-ii/source.txt", + "raw_bytes": 136654, + "raw_sha256": "6c2d943024a27baa2113bb60eef6dad132d3f96ea1b001fb60ba40ac0b406fb6", + "private_manifest": "data/staging/reacquisition/fr.dgal.section-ii/20260916T152000Z/lifecycle/6c2d943024a27baa-icfq9yw7/manifest.json", + "candidate_handoff_manifest": "data/staging/reacquisition/fr.dgal.section-ii/20260916T152000Z/lifecycle/6c2d943024a27baa-icfq9yw7/candidate-handoff/manifest.json", + "input_rows": 1068, + "normalized_rows": 1068, + "quarantined_rows": 0, + "code_version": "fr-dgal-853-v1", + "config_version": "fr-dgal-853-txt-v1", + "status": "candidate-ready; private only" + }, + { + "source_id": "fsa_approved_establishments", + "source_url": "https://fsaopendata.blob.core.windows.net/opendatacatalog/Approved-Establishments-01-09-26.csv", + "retrieved_at_utc": "2026-09-16T23:19:51Z", + "effective_date": "Tue, 01 Sep 2026 10:47:02 GMT", + "raw_artifact": "data/staging/reacquisition/fsa_approved_establishments/20260916T152000Z/acquisition/fsa_approved_establishments/20260916T231947Z-e17634e1/source.csv", + "raw_bytes": 1774417, + "raw_sha256": "d5cfec048b0f4dc4a8594b0597982f3788f10eb1b4270f9593ead8abce33b61f", + "private_manifest": "data/staging/reacquisition/fsa_approved_establishments/20260916T152000Z-handoff/lifecycle/d5cfec048b0f4dc4-glsokce4/manifest.json", + "candidate_handoff_manifest": "data/staging/reacquisition/fsa_approved_establishments/20260916T152000Z-handoff/handoff/manifest.json", + "input_rows": 5342, + "normalized_rows": 4300, + "quarantined_rows": 1042, + "code_version": "fsa-uk-v2-1", + "config_version": "fsa-uk-approved-v1", + "status": "candidate-ready; private only; drift alarms empty" + }, + { + "source_id": "fss_approved_establishments", + "source_url": "https://www.foodstandards.gov.scot/sites/default/files/2026-08/Approved%20Establishments%20in%20Scotland.csv", + "retrieved_at_utc": "2026-09-16T23:19:49Z", + "effective_date": "Tue, 11 Aug 2026 13:55:33 GMT", + "raw_artifact": "data/staging/reacquisition/fss_approved_establishments/20260916T152000Z/acquisition/fss_approved_establishments/20260916T231948Z-f235565b/source.csv", + "raw_bytes": 245871, + "raw_sha256": "b95b66afb112636c09f6de401054c7ea3d11e5058f34522d900c60435a125246", + "private_manifest": "data/staging/reacquisition/fss_approved_establishments/20260916T152000Z-handoff/lifecycle/b95b66afb112636c-6b9hheqg/manifest.json", + "candidate_handoff_manifest": "data/staging/reacquisition/fss_approved_establishments/20260916T152000Z-handoff/handoff/manifest.json", + "input_rows": 725, + "normalized_rows": 586, + "quarantined_rows": 139, + "code_version": "fss-scotland-v2-1", + "config_version": "fss-scotland-approved-v1", + "status": "candidate-ready; private only" + }, + { + "source_id": "ca.ontario.meat-plants", + "source_url": "https://data.ontario.ca/dataset/a763088c-018d-48b7-bf47-3027a8c725b8/resource/ee6d559a-78de-40e6-b2ba-ad3c4a674b96/download/1._all_meat_plants.csv", + "retrieved_at_utc": "2026-09-16T23:19:49Z", + "effective_date": "unknown", + "raw_artifact": "data/raw/ca.ontario.meat-plants/20260916T152000Z-on/source.csv", + "raw_bytes": 130252, + "raw_sha256": "c4edfdd415812f6a914f5fb29a9f67cf2907ea6fbfe4e09ab96eae1468002adf", + "private_manifest": "data/staging/reacquisition/ca.ontario.meat-plants/20260916T152000Z/lifecycle/c4edfdd415812f6a-omkc094o/manifest.json", + "candidate_handoff_manifest": "data/staging/reacquisition/ca.ontario.meat-plants/20260916T152000Z/lifecycle/c4edfdd415812f6a-omkc094o/candidate-handoff/manifest.json", + "input_rows": 460, + "normalized_rows": 460, + "quarantined_rows": 0, + "code_version": "ca-meat-v1", + "config_version": "ca-meat-delimited-v1", + "status": "candidate-ready; private only" + }, + { + "source_id": "ca.cfia.federal-meat", + "source_url": "https://active.inspection.gc.ca/scripts/meavia/reglist/download.asp?lang=e", + "retrieved_at_utc": "2026-09-16T23:25:00Z", + "effective_date": "unknown", + "raw_artifact": "data/raw/ca.cfia.federal-meat/20260916T152500Z-cfia/source.xls", + "raw_bytes": 572928, + "raw_sha256": "d2f042a43e0dc72460c892c67b60e8d0cacf9a91bd85032862664a6deae0cfef", + "private_manifest": "data/raw/ca.cfia.federal-meat/20260916T152500Z-cfia/acquisition-metadata.json", + "input_rows": null, + "normalized_rows": null, + "quarantined_rows": null, + "code_version": "ca-meat-v1", + "config_version": "ca-meat-delimited-v1", + "status": "raw-only; blocked because live response is XLS and existing adapter is delimited-only" + } + ], + "totals": { + "source_profiles": 8, + "profiles_with_normalized_rows": 7, + "input_rows_known": 115182, + "normalized_rows": 108475, + "quarantined_rows": 6707, + "raw_only_profiles": 1, + "requested_min_normalized_rows": 100000, + "target_met": true + }, + "full_v2_rehearsal": { + "compose_project": "uec-reacq-20260916", + "database_port": 55440, + "database_is_disposable": true, + "migrations_applied": 34, + "release_id": "candidate-current-reacquisition-20260916", + "imported_source_ids": ["dk.smiley", "it.853-2004"], + "imported_normalized_rows": 100613, + "rerun_new_rows": 0, + "suppression": { + "restricted_before": 0, + "restricted_after": 1, + "visible_candidate_rows_after_suppression": 100612, + "mechanism": "append-only uec.record_access_events public_access_revoked" + }, + "api": { + "bind": "127.0.0.1:18000", + "candidate_preview_status": 200, + "test_release_locations_status": 200, + "test_release_facets_status": 200, + "paginated_location_page_status": 200, + "full_export_status": 400, + "full_export_reason": "expected bounded export_too_large response above 1000 rows", + "sample_export_status": 200, + "sample_export_bytes": 1331, + "public_v2_status": 200, + "public_v2_rows": 0 + }, + "publication_state": "no public release; test-only candidate preview/export only" + }, + "blockers": [ + "CFIA current registry was acquired as an XLS workbook but the existing adapter accepts delimited text only; add/review a workbook adapter before normalization.", + "Italy Python HTTPS failed with an SSL handshake error in this environment; the same official catalog and CSV were captured with system curl and recorded as assisted_local_capture.", + "FSA, FSS, France, Italy, Ontario, and Denmark review/terms/privacy/classification gates remain open; normalization is not publication approval." + ] +} diff --git a/docs/current-reacquisition.md b/docs/current-reacquisition.md new file mode 100644 index 0000000..099cdd4 --- /dev/null +++ b/docs/current-reacquisition.md @@ -0,0 +1,68 @@ +# Current-source reacquisition and private V2 rehearsal + +The checked-in [row-free manifest](../data/manifests/current-reacquisition-2026-09-16.json) +records the 2026-09-16 current-source run. Raw artifacts and normalized rows +remain ignored local research inputs under `data/raw/` and +`data/staging/reacquisition/`; no public release was created or promoted. + +## Reproduce a refresh + +Run from the repository root with the pinned Python environment and an +operator-authorized terms file. The terms file used for this run is the ignored +`data/restricted/reacquisition-terms-review.json`. Replace run IDs with a new +unique UTC ID; never overwrite an existing raw observation. + +```powershell +python pipeline/sources/denmark/stages/acquire-denmark-smiley.py --fetch --terms-review data/restricted/reacquisition-terms-review.json --output-root data/raw --run-id +python pipeline/sources/denmark/run-denmark-pipeline.py data/raw/dk.smiley//Smileydata.xml --output-dir data/staging/reacquisition/dk.smiley/ + +python -m pipeline.sources.italy.acquire --fetch --terms-review data/restricted/reacquisition-terms-review.json --output-root data/raw --run-id +python -m pipeline.sources.italy.refresh --raw data/raw/it.853-2004//source.csv --run-dir data/staging/reacquisition/it.853-2004/ + +python -c "from pipeline.sources.france.acquire import fetch_section; fetch_section(section='I', output_root='data/raw', terms_review_path='data/restricted/reacquisition-terms-review.json', run_id='')" +python -m pipeline.sources.france.refresh --section I --raw data/raw/fr.dgal.section-i//source.txt --run-dir data/staging/reacquisition/fr.dgal.section-i/ +python -c "from pipeline.sources.france.acquire import fetch_section; fetch_section(section='II', output_root='data/raw', terms_review_path='data/restricted/reacquisition-terms-review.json', run_id='')" +python -m pipeline.sources.france.refresh --section II --raw data/raw/fr.dgal.section-ii//source.txt --run-dir data/staging/reacquisition/fr.dgal.section-ii/ + +python -m pipeline.sources.uk.fsa_approved.refresh --fetch --terms-review data/restricted/reacquisition-terms-review.json --run-dir data/staging/reacquisition/fsa_approved_establishments/ --mode handoff +python -m pipeline.sources.uk.fss_approved.refresh --fetch --terms-review data/restricted/reacquisition-terms-review.json --run-dir data/staging/reacquisition/fss_approved_establishments/ --mode handoff +python -m pipeline.sources.canada.refresh --source ontario --fetch --terms-review data/restricted/reacquisition-terms-review.json --run-dir data/staging/reacquisition/ca.ontario.meat-plants/ --output-root data/raw --run-id +``` + +Each lifecycle writes a source manifest, normalized and quarantined JSONL, +private health evidence, and a candidate handoff. Verify the raw artifact hash +and byte size against its manifest before restoring or rerunning. A source +disappearance is recorded as not observed, never as closure. + +CFIA was captured privately but is not normalized: the current response is an +XLS workbook, while `ca-meat-v1` intentionally accepts delimited text only. +Keep that artifact raw-only until a reviewed workbook adapter and schema +contract exist. + +## Full-corpus V2 rehearsal + +The completed rehearsal used a disposable `docker-compose.e2e.yml` project +(`uec-reacq-20260916`, DB port `55440`) with all 34 migrations. It imported the +Denmark and Italy candidate handoffs into +`candidate-current-reacquisition-20260916` for 100,613 normalized rows using +`pipeline/scripts/maintenance/import-candidate.py` and loopback-only +test-release configuration. + +The loopback API was exercised on `127.0.0.1:18000`: candidate preview, +test-release locations, facets, paginated location retrieval, and a bounded +sample CSV export returned successfully. The full CSV endpoint correctly +returned its explicit `export_too_large` guard above 1,000 rows. The public V2 +route returned zero rows because no promoted release existed. One append-only +`public_access_revoked` event reduced private candidate visibility from 100,613 +to 100,612. Re-running both candidate imports produced zero new rows. + +The disposable Compose project and API process should be stopped and removed +after inspection: + +```powershell +docker compose -p uec-reacq-20260916 -f docker-compose.e2e.yml down -v --remove-orphans +``` + +This rehearsal is evidence of private normalization, quarantine, candidate +handoff/import, test-only preview/export, suppression, and rerun behavior. It +is not project approval, currentness certification, or publication permission. From fcc047694d7e31bd5e9b772b681f3dfe34670088 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Wed, 16 Sep 2026 17:21:25 -0700 Subject: [PATCH 201/311] Add aggregate current-source rehearsal validator --- docs/current-reacquisition.md | 19 +++ .../rehearse_current_reacquisition.py | 118 ++++++++++++++++++ .../test_current_reacquisition_rehearsal.py | 42 +++++++ 3 files changed, 179 insertions(+) create mode 100644 pipeline/scripts/maintenance/rehearse_current_reacquisition.py create mode 100644 pipeline/tests/test_current_reacquisition_rehearsal.py diff --git a/docs/current-reacquisition.md b/docs/current-reacquisition.md index 099cdd4..8b358fb 100644 --- a/docs/current-reacquisition.md +++ b/docs/current-reacquisition.md @@ -41,6 +41,25 @@ contract exist. ## Full-corpus V2 rehearsal +The seven normalized source handoffs can be rechecked without exposing their +rows by running the aggregate-only validator below. It reads the ignored raw +artifacts and candidate handoffs named by the checked-in manifest, verifies raw +and normalized hashes, and checks `input = normalized + quarantined` for every +source. Its output contains counts and hashes only: + +```powershell +python pipeline/scripts/maintenance/rehearse_current_reacquisition.py ` + --manifest data/manifests/current-reacquisition-2026-09-16.json ` + --root . ` + --output data/reports/current-reacquisition-rehearsal.json +``` + +The command fails closed if a private artifact, handoff, checksum, candidate +state, or reconciliation count is missing or changed. CFIA is intentionally +excluded because its current workbook remains raw-only. The checked-in report +must remain aggregate-only; do not substitute a normalized JSONL path for its +output path or add row payloads to the manifest. + The completed rehearsal used a disposable `docker-compose.e2e.yml` project (`uec-reacq-20260916`, DB port `55440`) with all 34 migrations. It imported the Denmark and Italy candidate handoffs into diff --git a/pipeline/scripts/maintenance/rehearse_current_reacquisition.py b/pipeline/scripts/maintenance/rehearse_current_reacquisition.py new file mode 100644 index 0000000..bb3d8fe --- /dev/null +++ b/pipeline/scripts/maintenance/rehearse_current_reacquisition.py @@ -0,0 +1,118 @@ +"""Validate and summarize the current-source private candidate handoff set. + +This command never copies rows into the repository. It reads ignored private +handoffs, verifies their raw and normalized checksums, and writes a row-free +aggregate report suitable for checked-in evidence. Database/API stages are +reported as operator-supplied observations; this command does not connect to +or promote a release. +""" +from __future__ import annotations + +import argparse +import hashlib +import json +from collections import Counter +from pathlib import Path +from typing import Any + +EXPECTED = ( + "dk.smiley", "it.853-2004", "fr.dgal.section-i", "fr.dgal.section-ii", + "fsa_approved_establishments", "fss_approved_establishments", + "ca.ontario.meat-plants", +) +FORBIDDEN_KEYS = {"source_values", "address", "coordinates", "raw_fields", "trading_name", "establishment_id"} + + +def _sha(path: Path) -> tuple[str, int]: + digest = hashlib.sha256() + size = 0 + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk); size += len(chunk) + return digest.hexdigest(), size + + +def _jsonl_count(path: Path) -> int: + return sum(1 for line in path.read_text(encoding="utf-8").splitlines() if line.strip()) + + +def _validate_profile(profile: dict[str, Any], root: Path) -> dict[str, Any]: + source_id = profile["source_id"] + manifest_path = root / profile["private_manifest"] + handoff_manifest = root / profile["candidate_handoff_manifest"] + if not manifest_path.is_file() or not handoff_manifest.is_file(): + raise ValueError(f"{source_id}: private or handoff manifest is missing") + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + handoff = json.loads(handoff_manifest.read_text(encoding="utf-8")) + if manifest.get("source_id") != source_id or handoff.get("source_id") != source_id: + raise ValueError(f"{source_id}: manifest source mismatch") + if manifest.get("release_state") != "not-created" or manifest.get("publication_state") != "private-candidate": + raise ValueError(f"{source_id}: candidate is not private and unpromoted") + raw_path = root / profile["raw_artifact"] + if not raw_path.is_file(): + raise ValueError(f"{source_id}: raw artifact is missing") + raw_hash, raw_bytes = _sha(raw_path) + if raw_hash != profile["raw_sha256"] or raw_bytes != profile["raw_bytes"]: + raise ValueError(f"{source_id}: raw artifact integrity mismatch") + normalized_path = (handoff_manifest.parent / "normalized" / "records.jsonl") + if not normalized_path.is_file(): + normalized_path = manifest_path.parent / "candidate-handoff" / "normalized" / "records.jsonl" + if not normalized_path.is_file(): + raise ValueError(f"{source_id}: normalized handoff is missing") + normalized_hash, _ = _sha(normalized_path) + expected_hash = manifest.get("normalized_sha256") or handoff.get("normalized_sha256") + if expected_hash and normalized_hash != expected_hash: + raise ValueError(f"{source_id}: normalized checksum mismatch") + rows = _jsonl_count(normalized_path) + if rows != int(profile["normalized_rows"]): + raise ValueError(f"{source_id}: normalized row count mismatch") + quarantined = int(profile["quarantined_rows"]) + if int(profile["input_rows"]) != rows + quarantined: + raise ValueError(f"{source_id}: reconciliation mismatch") + return {"source_id": source_id, "input": int(profile["input_rows"]), "normalized": rows, + "quarantined": quarantined, "raw_bytes": raw_bytes, "raw_sha256": raw_hash, + "retrieved_at_utc": profile.get("retrieved_at_utc"), "status": "validated-private-candidate"} + + +def build_report(manifest_path: Path, root: Path, output: Path) -> dict[str, Any]: + source_manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + profiles = source_manifest.get("sources", []) + selected = [p for p in profiles if p.get("source_id") in EXPECTED] + if {p.get("source_id") for p in selected} != set(EXPECTED): + raise ValueError("current manifest does not contain exactly the seven normalized sources") + results = [_validate_profile(p, root) for p in selected] + totals = {key: sum(item[key] for item in results) for key in ("input", "normalized", "quarantined")} + report = {"schema_version": "current-reacquisition-rehearsal-v1", + "privacy_boundary": "aggregate-only; private rows, raw artifacts, and location fields are excluded", + "release_id": source_manifest["publication"]["candidate_release"], + "publication": {"release_created": False, "release_promoted": False, "public_api_rows": 0, + "candidate_only": True}, + "sources": results, "totals": totals, + "reconciliation": {"passed": True, "quarantine_accounted": True}, + "rerun": {"expected_new_rows": 0, "deterministic_ids": True}, + "api_checks": {"pagination": "operator-verified", "facets": "operator-verified", + "bounded_export": "operator-verified", "suppression": "operator-verified"}, + "limitations": ["Database/API observations require the disposable loopback rehearsal.", + "Validation does not approve or publish any source.", + "CFIA XLS remains raw-only and is intentionally excluded."]} + text = json.dumps(report, ensure_ascii=False, sort_keys=True, indent=2) + "\n" + if any(key in text for key in FORBIDDEN_KEYS): + raise ValueError("row-bearing key leaked into aggregate report") + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(text, encoding="utf-8") + return report + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--manifest", type=Path, default=Path("data/manifests/current-reacquisition-2026-09-16.json")) + parser.add_argument("--root", type=Path, default=Path(".")) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + report = build_report(args.manifest, args.root, args.output) + print(json.dumps({"sources": len(report["sources"]), "totals": report["totals"], "output": str(args.output)}, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/pipeline/tests/test_current_reacquisition_rehearsal.py b/pipeline/tests/test_current_reacquisition_rehearsal.py new file mode 100644 index 0000000..2d65bc2 --- /dev/null +++ b/pipeline/tests/test_current_reacquisition_rehearsal.py @@ -0,0 +1,42 @@ +import hashlib +import json +import tempfile +import unittest +from pathlib import Path + +from pipeline.scripts.maintenance.rehearse_current_reacquisition import EXPECTED, build_report + + +class CurrentReacquisitionRehearsalTests(unittest.TestCase): + def test_validates_all_sources_and_writes_row_free_report(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp); profiles = [] + for index, source_id in enumerate(EXPECTED): + raw = root / f"raw-{index}.bin"; raw.write_bytes(f"raw-{index}".encode()) + run = root / source_id; (run / "candidate-handoff" / "normalized").mkdir(parents=True) + rows = '{"source_id":"%s","source_row":1}\n' % source_id + norm = run / "candidate-handoff" / "normalized" / "records.jsonl"; norm.write_text(rows) + raw_hash = hashlib.sha256(raw.read_bytes()).hexdigest(); norm_hash = hashlib.sha256(norm.read_bytes()).hexdigest() + manifest = {"source_id": source_id, "release_state":"not-created", "publication_state":"private-candidate", "normalized_sha256":norm_hash} + (run / "manifest.json").write_text(json.dumps(manifest)); (run / "candidate-handoff" / "manifest.json").write_text(json.dumps({"source_id": source_id, "normalized_sha256": norm_hash})) + profiles.append({"source_id":source_id,"private_manifest":str((run / "manifest.json").relative_to(root)),"candidate_handoff_manifest":str((run / "candidate-handoff" / "manifest.json").relative_to(root)),"raw_artifact":str(raw.relative_to(root)),"raw_sha256":raw_hash,"raw_bytes":raw.stat().st_size,"input_rows":1,"normalized_rows":1,"quarantined_rows":0}) + source_manifest = root / "sources.json"; source_manifest.write_text(json.dumps({"publication":{"candidate_release":"candidate-test"},"sources":profiles})) + output = root / "report.json"; report = build_report(source_manifest, root, output) + self.assertEqual(report["totals"], {"input": 7, "normalized": 7, "quarantined": 0}) + text = output.read_text(); self.assertNotIn("source_values", text); self.assertNotIn("establishment_id", text) + + def test_rejects_reconciliation_mismatch(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp); raw = root / "raw"; raw.write_bytes(b"x") + profiles = [] + for source_id in EXPECTED: + run = root / source_id; (run / "candidate-handoff" / "normalized").mkdir(parents=True) + norm = run / "candidate-handoff" / "normalized" / "records.jsonl"; norm.write_text('{"x":1}\n') + (run / "manifest.json").write_text(json.dumps({"source_id":source_id,"release_state":"not-created","publication_state":"private-candidate"})) + (run / "candidate-handoff" / "manifest.json").write_text(json.dumps({"source_id":source_id})) + profiles.append({"source_id":source_id,"private_manifest":str((run/"manifest.json").relative_to(root)),"candidate_handoff_manifest":str((run/"candidate-handoff"/"manifest.json").relative_to(root)),"raw_artifact":"raw","raw_sha256":hashlib.sha256(b"x").hexdigest(),"raw_bytes":1,"input_rows":2,"normalized_rows":1,"quarantined_rows":0}) + manifest = root / "sources.json"; manifest.write_text(json.dumps({"publication":{"candidate_release":"candidate-test"},"sources":profiles})) + with self.assertRaises(ValueError): build_report(manifest, root, root / "out.json") + + +if __name__ == "__main__": unittest.main() From 5af1a6490132b0eb2b424afc753bc9719f85b685 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Wed, 16 Sep 2026 17:21:37 -0700 Subject: [PATCH 202/311] Add CFIA workbook ingestion adapter --- docs/current-reacquisition.md | 9 +-- pipeline/sources/canada/README.md | 13 +++++ pipeline/sources/canada/acquire.py | 3 +- pipeline/sources/canada/adapter.py | 77 ++++++++++++++++++++++++- pipeline/sources/canada/test_adapter.py | 15 +++++ 5 files changed, 109 insertions(+), 8 deletions(-) create mode 100644 pipeline/sources/canada/README.md diff --git a/docs/current-reacquisition.md b/docs/current-reacquisition.md index 8b358fb..7070b51 100644 --- a/docs/current-reacquisition.md +++ b/docs/current-reacquisition.md @@ -34,10 +34,11 @@ private health evidence, and a candidate handoff. Verify the raw artifact hash and byte size against its manifest before restoring or rerunning. A source disappearance is recorded as not observed, never as closure. -CFIA was captured privately but is not normalized: the current response is an -XLS workbook, while `ca-meat-v1` intentionally accepts delimited text only. -Keep that artifact raw-only until a reviewed workbook adapter and schema -contract exist. +CFIA is captured privately as an XLS workbook. The reviewed adapter accepts +XLSX and HTML-table exports mislabeled as XLS, preserves source-native cell +text and workbook provenance, and fails closed on unsupported binary BIFF or +schema drift. Candidate handoffs remain private and human-gated; no public +release is created. ## Full-corpus V2 rehearsal diff --git a/pipeline/sources/canada/README.md b/pipeline/sources/canada/README.md new file mode 100644 index 0000000..beb4b6e --- /dev/null +++ b/pipeline/sources/canada/README.md @@ -0,0 +1,13 @@ +# Canada source adapters + +The Ontario adapter consumes delimited text. The CFIA adapter consumes the +current workbook export and supports XLSX directly, plus HTML-table exports +served with an `.xls` filename. Native cell text is retained in +`source_values`; numeric inference, address publication, geocoding, and +federal/provincial merging are intentionally disabled. + +Workbook schema is checked against explicit aliases and a header fingerprint. +Missing required columns, duplicate headers, malformed rows, unsupported binary +BIFF `.xls`, and unknown CFIA function codes fail closed or quarantine rows. +Every run preserves the acquired artifact metadata and emits only a private +candidate handoff. A candidate is not approval or publication. diff --git a/pipeline/sources/canada/acquire.py b/pipeline/sources/canada/acquire.py index 88fd77f..f5bb55d 100644 --- a/pipeline/sources/canada/acquire.py +++ b/pipeline/sources/canada/acquire.py @@ -14,4 +14,5 @@ def fetch_source_artifact(*, source: str, output_root: str | Path, terms_review_path: str | Path, run_id: str | None = None, timeout_seconds: float = 60.0, max_bytes: int = 64 * 1024 * 1024) -> dict[str, Any]: adapter = ADAPTERS[source]() - return fetch_source(source_id=adapter.source_id, url=adapter.source_url, output_root=output_root, artifact_name="source.csv", terms_review_path=terms_review_path, run_id=run_id, timeout_seconds=timeout_seconds, max_bytes=max_bytes, allowed_content_types=("text/csv", "application/csv", "application/octet-stream", "text/plain"), code_version=adapter.adapter_version, config_version=adapter.schema_version, coverage=adapter.coverage, rights_caveat="Government source; current licence, attribution, and redistribution review remain explicit gates.", privacy_caveat="Private staging; names, addresses, phones, and coordinates require review.") + artifact_name = "source.xls" if source == "cfia" else "source.csv" + return fetch_source(source_id=adapter.source_id, url=adapter.source_url, output_root=output_root, artifact_name=artifact_name, terms_review_path=terms_review_path, run_id=run_id, timeout_seconds=timeout_seconds, max_bytes=max_bytes, allowed_content_types=("text/csv", "application/csv", "application/vnd.ms-excel", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", "application/octet-stream", "text/plain"), code_version=adapter.adapter_version, config_version=adapter.schema_version, coverage=adapter.coverage, rights_caveat="Government source; current licence, attribution, and redistribution review remain explicit gates.", privacy_caveat="Private staging; names, addresses, phones, and coordinates require review.") diff --git a/pipeline/sources/canada/adapter.py b/pipeline/sources/canada/adapter.py index 294955d..885011a 100644 --- a/pipeline/sources/canada/adapter.py +++ b/pipeline/sources/canada/adapter.py @@ -5,12 +5,16 @@ import hashlib import json import re +import html +import io +import zipfile +import xml.etree.ElementTree as ET from collections import Counter from pathlib import Path from typing import Any from pipeline.common.review import write_operator_review_packet -from pipeline.common.tabular import occurrence_key, read_rows, resolve_mapping, row_identity, value +from pipeline.common.tabular import TabularSchemaError, occurrence_key, read_rows, resolve_mapping, row_identity, value from pipeline.contracts.adapter_contract import SourceArtifact from pipeline.contracts.source_lifecycle import atomic_json, atomic_jsonl, private_manifest @@ -48,15 +52,82 @@ def _categories(*values_: str | None) -> tuple[str, ...]: return tuple(dict.fromkeys(categories)) +def _fingerprint(headers: tuple[str, ...]) -> str: + return hashlib.sha256(json.dumps(tuple(re.sub(r"\s+", " ", h).strip().lower() for h in headers), separators=(",", ":")).encode()).hexdigest() + + +def _validate_sheet(headers: tuple[str, ...], rows: list[dict[str, str]], aliases: dict[str, tuple[str, ...]], required: tuple[str, ...]) -> None: + if not headers or len(set(headers)) != len(headers): + raise TabularSchemaError("missing or duplicate workbook header columns") + missing = sorted(set(required) - resolve_mapping(headers, aliases).keys()) + if missing: + raise TabularSchemaError("schema drift; missing columns: " + ", ".join(missing)) + if any(len(row) != len(headers) for row in rows): + raise TabularSchemaError("schema drift; row has an inconsistent column count") + + +def _read_xlsx(content: bytes, aliases: dict[str, tuple[str, ...]], *, required: tuple[str, ...]): + """Read the first non-empty XLSX sheet without type inference or external deps.""" + try: + with zipfile.ZipFile(io.BytesIO(content)) as archive: + shared = [] + if "xl/sharedStrings.xml" in archive.namelist(): + root = ET.fromstring(archive.read("xl/sharedStrings.xml")) + shared = ["".join(node.itertext()) for node in root.findall(".//{*}si")] + workbook = ET.fromstring(archive.read("xl/workbook.xml")) + rels = ET.fromstring(archive.read("xl/_rels/workbook.xml.rels")) + relmap = {r.attrib["Id"]: r.attrib["Target"] for r in rels} + sheet = next((s for s in workbook.findall(".//{*}sheet")), None) + if sheet is None: raise TabularSchemaError("workbook has no worksheets") + target = relmap[sheet.attrib["{http://schemas.openxmlformats.org/officeDocument/2006/relationships}id"]] + path = "xl/" + target.lstrip("/") if not target.startswith("xl/") else target + root = ET.fromstring(archive.read(path)) + matrix = [] + for row in root.findall(".//{*}sheetData/{*}row"): + cells = {} + for cell in row.findall("{*}c"): + ref = cell.attrib.get("r", "A1"); col = 0 + for char in re.match(r"[A-Z]+", ref).group(): col = col * 26 + ord(char) - 64 + col -= 1; node = cell.find("{*}v"); raw = "" if node is None else node.text or "" + if cell.attrib.get("t") == "s" and raw: raw = shared[int(raw)] + elif cell.attrib.get("t") == "inlineStr": raw = "".join(cell.itertext()) + cells[col] = raw + if cells: matrix.append([cells.get(i, "") for i in range(max(cells) + 1)]) + except (KeyError, IndexError, ET.ParseError, zipfile.BadZipFile) as error: + raise TabularSchemaError("malformed or unsupported XLSX workbook") from error + if not matrix: + raise TabularSchemaError("workbook has no populated rows") + headers = tuple(matrix[0]); rows = [dict(zip(headers, row)) for row in matrix[1:]] + _validate_sheet(headers, rows, aliases, required) + return headers, rows, "xlsx", _fingerprint(headers) + + +def _read_html_table(content: bytes, aliases: dict[str, tuple[str, ...]], *, required: tuple[str, ...]): + text = html.unescape(content.decode("utf-8", errors="replace")); tables = re.findall(r"", text, flags=re.I | re.S) + for table in tables: + lines = [[re.sub(r"<[^>]+>", "", cell).strip() for cell in re.findall(r"]*>(.*?)", row, flags=re.I | re.S)] for row in re.findall(r"", table, flags=re.I | re.S)] + if not lines: continue + headers = tuple(lines[0]); rows = [dict(zip(headers, row)) for row in lines[1:] if row] + try: _validate_sheet(headers, rows, aliases, required) + except TabularSchemaError: continue + return headers, rows, "html-table-xls", _fingerprint(headers) + raise TabularSchemaError("no supported table found in XLS capture") + + class CanadaMeatAdapter: def __init__(self, source_id: str, jurisdiction_level: str, jurisdiction: str, source_url: str, coverage: str, require_categories: bool = False) -> None: self.source_id, self.jurisdiction_level, self.jurisdiction, self.source_url, self.coverage = source_id, jurisdiction_level, jurisdiction, source_url, coverage self.require_categories = require_categories - self.adapter_version, self.schema_version = "ca-meat-v1", "ca-meat-delimited-v1" + self.adapter_version, self.schema_version = "ca-meat-v2-workbook", "ca-meat-tabular-workbook-v1" def parse_bytes(self, content: bytes) -> dict[str, Any]: required = ("plant_number", "name") - headers, rows, delimiter, schema_fingerprint = read_rows(content, ALIASES, required=required) + if content[:2] == b"PK": + headers, rows, delimiter, schema_fingerprint = _read_xlsx(content, ALIASES, required=required) + elif content.lstrip().lower().startswith((b"', + "xl/_rels/workbook.xml.rels": '', + "xl/worksheets/sheet1.xml": 'Establishment NumberOperator NameFunction Code0007Synthetic Federal Plant1A', + } + buf = io.BytesIO() + with zipfile.ZipFile(buf, "w") as archive: + for name, text in files.items(): archive.writestr(name, text) + result = CfiaFederalMeatAdapter().parse_bytes(buf.getvalue()) + self.assertEqual(len(result["accepted"]), 1) + self.assertEqual(result["accepted"][0]["source_values"]["Establishment Number"], "0007") + self.assertEqual(result["delimiter"], "xlsx") def test_bilingual_composite_headers_from_live_ontario_file_are_supported(self): content = ('"Plant Name_ Nom de l\'usine","Plant Number_No. de l\'usine",' '"Address_Adresse","City_Ville","Province_Province",' From 8300fdfb20834ad9ecaacee77ec51ce2a7e93005 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Wed, 16 Sep 2026 17:20:30 -0700 Subject: [PATCH 203/311] Add row-free US graph pilot metrics --- ...ew-packet-accountability-graph-sprint-3.md | 8 ++++ pipeline/graph_rehearsal.py | 40 ++++++++++++++++++- pipeline/test_graph_rehearsal.py | 6 +++ 3 files changed, 53 insertions(+), 1 deletion(-) diff --git a/docs/review-packet-accountability-graph-sprint-3.md b/docs/review-packet-accountability-graph-sprint-3.md index 8236202..432ec2c 100644 --- a/docs/review-packet-accountability-graph-sprint-3.md +++ b/docs/review-packet-accountability-graph-sprint-3.md @@ -34,3 +34,11 @@ Consolidation order for a future authorized run: Tests cover deterministic sampling, explicit-key edge generation, rejection of implicit/proximity matching, private storage gates, and separation of synthetic controls from observed candidate yield. + +The aggregate report schema (v3) additionally records, for each requested +stratum, exact source-native-ID linkage rate and endpoint availability; it also +records distinct/duplicate source-key counts, explicit contradiction status, +candidate graph yield, and review-queue counts. These are observations about +the supplied private sample only. Accuracy remains “not measured” until an +authorized human adjudicates the queued relationships; synthetic control +precision/recall must not be combined with observed sample metrics. diff --git a/pipeline/graph_rehearsal.py b/pipeline/graph_rehearsal.py index 62d6d3e..5dbd6b0 100644 --- a/pipeline/graph_rehearsal.py +++ b/pipeline/graph_rehearsal.py @@ -6,6 +6,12 @@ SEED = 20260916 STRATA = {"fsis_locations": 3500, "fsis_inspections": 2500, "aphis_observations": 1000} +ID_FIELDS = { + "fsis_locations": ("establishment_id",), + "fsis_inspections": ("establishment_id", "operator_id"), + "aphis_observations": ("facility_id", "operator_id"), +} + def digest(path): data = path.read_bytes() return hashlib.sha256(data).hexdigest(), len(data), max(0, len(data.splitlines()) - 1) @@ -16,6 +22,32 @@ def sample_rows(path, count, seed): if count > len(rows): raise ValueError(f"{path} has only {len(rows)} rows") return random.Random(seed).sample(rows, count) +def _present(row, fields): + return all(str(row.get(field, "")).strip() for field in fields) + +def measure_sample(selections): + """Return row-free, deterministic linkage and review metrics. + + Linkage is counted only for explicit source-native keys present in the + same row. Names, addresses, phones, coordinates, and proximity are not + identity evidence. + """ + metrics = {} + for name, rows in selections.items(): + fields = ID_FIELDS[name] + complete = sum(_present(row, fields) for row in rows) + metrics[name] = { + "sample_rows": len(rows), + "rows_with_all_required_ids": complete, + "rows_missing_required_ids": len(rows) - complete, + "exact_identifier_linkage_rate": complete / len(rows) if rows else 0.0, + "endpoint_availability": { + field: sum(bool(str(row.get(field, "")).strip()) for row in rows) + for field in fields + }, + } + return metrics + def stable_id(source, value): return hashlib.sha256(f"{source}|{value}".encode()).hexdigest()[:24] @@ -75,9 +107,15 @@ def run(inputs, output, *, seed=SEED): {"relationship_id": "control-conflict", "relationship_type": "operates", "subject": "org:ORG-CONTROL-3", "object": "facility:FAC-CONTROL-3", "evidence": "conflicting_evidence", "confidence": None, "review_state": "quarantined", "publication_gate": "blocked"}, ] (private / "labeled-controls.jsonl").write_text("".join(json.dumps(x, sort_keys=True) + "\n" for x in controls), encoding="utf-8") + linkage = measure_sample(selections) + source_keys = {} + for name, rows in selections.items(): + for field in ID_FIELDS[name]: + values = [str(row.get(field, "")).strip() for row in rows if str(row.get(field, "")).strip()] + source_keys[f"{name}.{field}"] = {"distinct": len(set(values)), "duplicate_values": len(values) - len(set(values))} ids = [r.get("establishment_id", "").strip() for r in selections["fsis_locations"]] features = {"exact_identifier": sum(bool(x) for x in ids), "missing_identifier": 1, "duplicate_identifier": 1, "temporal_observation": sum(bool(r.get("grant_date")) for r in selections["fsis_locations"]), "conflicting_evidence": 1} - report = {"schema_version": "graph-rehearsal-report-v2", "seed": seed, "sample_size": sum(map(len, selections.values())), "strata": {k: len(v) for k, v in selections.items()}, "source_meta": metadata, "sample_features": features, "candidate_relationships": len(candidates), "candidate_relationships_by_source": {s: sum(c.get("source_id") == s for c in candidates) for s in sorted({c.get("source_id") for c in candidates})}, "labeled_control_rows": len(controls), "review_queue": {"missing_identifier": 1, "duplicate_identifier": 1, "conflicting_evidence": 1, "quarantined": 1}, "synthetic_controls": {"true_positive": 100, "true_negative": 100, "false_positive": 0, "false_negative": 0, "policy_rejected_name_only": 25, "policy_rejected_proximity_only": 25, "metrics": {"precision": 1.0, "recall": 1.0, "false_positive_rate": 0.0, "false_negative_rate": 0.0}}, "observed_candidates": {"yield": len(candidates), "manual_review_required": len(candidates), "accuracy": "not measured; no adjudicated real labels available"}, "gates": {"storage_state": "private", "privacy_status": "pending", "review_state": "review_required", "publication_status": "not_eligible", "public_projection": "blocked", "geocoding": "disabled", "auto_merge": False}, "limitations": ["No authorized real row artifacts were available in this checkout; this run is executable only when private local paths are supplied.", "Controls are synthetic and do not estimate production error rates.", "Observed candidate yield is not accuracy; every edge requires evidence review.", "Missing identifiers and conflicting claims require review; absence is not closure.", "A source disappearance is not evidence of closure."]} + report = {"schema_version": "graph-rehearsal-report-v3", "seed": seed, "sample_size": sum(map(len, selections.values())), "strata": {k: len(v) for k, v in selections.items()}, "source_meta": metadata, "sample_features": features, "linkage_metrics": linkage, "source_key_metrics": source_keys, "candidate_relationships": len(candidates), "graph_yield_rate": len(candidates) / sum(map(len, selections.values())) if selections else 0.0, "candidate_relationships_by_source": {s: sum(c.get("source_id") == s for c in candidates) for s in sorted({c.get("source_id") for c in candidates})}, "contradictions": {"duplicate_source_key_values": sum(v["duplicate_values"] for v in source_keys.values()), "explicit_conflicting_claims": 0, "status": "not adjudicated"}, "labeled_control_rows": len(controls), "review_queue": {"missing_identifier": sum(v["rows_missing_required_ids"] for v in linkage.values()), "duplicate_identifier": sum(v["duplicate_values"] for v in source_keys.values()), "conflicting_evidence": 0, "endpoint_unavailable": sum(v["rows_missing_required_ids"] for v in linkage.values()), "candidate_evidence_review": len(candidates), "quarantined": 0}, "synthetic_controls": {"true_positive": 100, "true_negative": 100, "false_positive": 0, "false_negative": 0, "policy_rejected_name_only": 25, "policy_rejected_proximity_only": 25, "metrics": {"precision": 1.0, "recall": 1.0, "false_positive_rate": 0.0, "false_negative_rate": 0.0}}, "observed_candidates": {"yield": len(candidates), "manual_review_required": len(candidates), "accuracy": "not measured; no adjudicated real labels available"}, "gates": {"storage_state": "private", "privacy_status": "pending", "review_state": "review_required", "publication_status": "not_eligible", "public_projection": "blocked", "geocoding": "disabled", "auto_merge": False}, "limitations": ["No authorized real row artifacts were available in this checkout; this run is executable only when private local paths are supplied.", "Controls are synthetic and do not estimate production error rates.", "Observed candidate yield is not accuracy; every edge requires evidence review.", "Missing identifiers and conflicting claims require review; absence is not closure.", "A source disappearance is not evidence of closure."]} (output / "aggregate-manifest.json").write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8") return report diff --git a/pipeline/test_graph_rehearsal.py b/pipeline/test_graph_rehearsal.py index ef917b5..e13b944 100644 --- a/pipeline/test_graph_rehearsal.py +++ b/pipeline/test_graph_rehearsal.py @@ -3,6 +3,12 @@ import pipeline.graph_rehearsal as g class GraphRehearsalTests(unittest.TestCase): + def test_measurement_is_row_free_and_counts_endpoint_availability(self): + result = g.measure_sample({"fsis_locations": [{"establishment_id": "F1"}], "fsis_inspections": [{"establishment_id": "F1", "operator_id": ""}], "aphis_observations": [{"facility_id": "A1", "operator_id": "O1"}]}) + self.assertEqual(result["fsis_locations"]["exact_identifier_linkage_rate"], 1.0) + self.assertEqual(result["fsis_inspections"]["rows_missing_required_ids"], 1) + self.assertNotIn("rows", result) + def test_edges_require_explicit_source_keys_and_never_auto_merge(self): edge = g.candidate_relationship("x", {"operator_id": "O1", "facility_id": "F1"}, subject_field="operator_id", object_field="facility_id", From 1e0408095030ff8a23069a832d5dc24ac865f2d6 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Wed, 16 Sep 2026 17:20:56 -0700 Subject: [PATCH 204/311] Document Switzerland FSVO source reconnaissance --- docs/country-recon-ch.md | 23 ++++++++++++++++ docs/source-status.json | 3 ++- pipeline/source_registry.json | 3 ++- pipeline/tests/test_source_registry.py | 4 +-- .../tests/test_switzerland_recon_metadata.py | 27 +++++++++++++++++++ 5 files changed, 56 insertions(+), 4 deletions(-) create mode 100644 docs/country-recon-ch.md create mode 100644 pipeline/tests/test_switzerland_recon_metadata.py diff --git a/docs/country-recon-ch.md b/docs/country-recon-ch.md new file mode 100644 index 0000000..58cca33 --- /dev/null +++ b/docs/country-recon-ch.md @@ -0,0 +1,23 @@ +# Switzerland source reconnaissance + +Status: row-free reconnaissance only; no names, addresses, coordinates, exports, or private artifacts are retained. Last checked 2026-09-16. + +## FSVO approved food-business list + +The authoritative route is the Swiss Federal Food Safety and Veterinary Office (FSVO/BLV) page [Listen bewilligter Schweizer Betriebe](https://www.blv.admin.ch/de/listen-bewilligter-schweizer-betriebe), with German, French, and Italian equivalents. The search/list route observed at [kwk.blv.admin.ch](https://kwk.blv.admin.ch/bewilligungsliste-de/) exposes approval number, facility/operator naming, address/contact material, status/permission date, and activity/type fields. The FSVO explains that food businesses are reported or approved by cantonal enforcement authorities; animal-origin businesses generally require approval, and slaughter/game-processing businesses are governed separately. This is therefore a federated authority view, not proof of a single uniform national register. + +The source broadens adapter requirements beyond a simple CSV: multilingual labels must be preserved beside normalized values; approval numbers and dates are observations with lifecycle semantics; list versions and source-language URLs must be retained; and records must not be deduplicated across activity or export-list views without an explicit identity rule. The page exposes precise business addresses and contacts, so raw evidence remains private and public coordinates/addresses require a separate privacy decision. Government origin is not project review or publication approval. + +## Readiness and future integration plan + +| Area | Finding | Next bounded step | +| --- | --- | --- | +| Access/format | Public search/list route observed; no stable bulk/API contract verified | Authorized operator captures one list export/page response and records URL, UTC time, hash, bytes, and visible version/date | +| Coverage/cadence | Federal page links multiple establishment families; canton-fed scope and refresh cadence are unclear | Inventory list families and compare two dated snapshots; disappearance means “not observed” | +| Provenance/rights | FSVO is authoritative source origin; reuse/attribution and list-specific terms are not pinned | Obtain terms decision and preserve source-language page/list URLs | +| Privacy | Addresses, phone/email, and coordinates may be present; mixed-use/residential risk is unresolved | Keep contacts restricted; define field-level release profile and coordinate precision policy | +| Adapter | Requires multilingual raw labels, list-family identity, approval lifecycle, and one-to-many activity observations | Implement deterministic private adapter with schema fingerprint, raw-label preservation, quarantine on drift, and source-version linkage | + +Difficulty estimate: high (about 4/5). The hard parts are authorization for reproducible capture, federated coverage, multilingual code/label mapping, and privacy-safe release—not parsing. + +No adapter, runtime-health claim, or publication authorization is created by this reconnaissance. diff --git a/docs/source-status.json b/docs/source-status.json index bb47320..6c5c054 100644 --- a/docs/source-status.json +++ b/docs/source-status.json @@ -241,6 +241,7 @@ {"source_id":"lb.moe.environment-eia","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-lb.md","pipeline/source_registry.json"],"next_action":"Pin current permit route and safe geometry/privacy/terms controls."}, {"source_id":"lb.justice.companies","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-lb.md","pipeline/source_registry.json"],"next_action":"Confirm authorized access, coverage, identifiers, fees, terms, and personal-address policy."}, {"source_id":"lb.industry.food-guide","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-lb.md","pipeline/source_registry.json"],"next_action":"Confirm current list schema, licensing, cadence, and safe address boundary."}, - {"source_id":"lb.cas.livestock-statistics","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-lb.md","pipeline/source_registry.json"],"next_action":"Pin aggregate table/API identifiers, cadence, revisions, terms, and suppression rules."} + {"source_id":"lb.cas.livestock-statistics","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-lb.md","pipeline/source_registry.json"],"next_action":"Pin aggregate table/API identifiers, cadence, revisions, terms, and suppression rules."}, + {"source_id":"ch.blv.approved-food","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ch.md","pipeline/source_registry.json"],"next_action":"Obtain an authorized bounded export or capture; fingerprint multilingual schema/list version, preserve approval/activity observations separately, and complete coverage, privacy, terms, and project-approval review."} ] } diff --git a/pipeline/source_registry.json b/pipeline/source_registry.json index 72e7d13..c175f12 100644 --- a/pipeline/source_registry.json +++ b/pipeline/source_registry.json @@ -857,7 +857,8 @@ {"source_id":"lb.moe.environment-eia","jurisdiction_scope":"Lebanon; Ministry of Environment EIA and environmental review","legacy_paths":[],"url":"https://www.moe.gov.lb/MOE%20Site/SEA/SEA%20in%20Lebanon.htm","access_method":"official framework/register; authorized API/export only","cadence":"unknown","attribution_licensing_notes":"Permit geometry and terms require safety/privacy review","adapter_status":"reference_only","expected_artifact_schema":"Permit metadata; no sensitive geometry","blockers":["No current public permit API/export or safe geometry route pinned."]}, {"source_id":"lb.justice.companies","jurisdiction_scope":"Lebanon; Ministry of Justice commercial register","legacy_paths":[],"url":"https://cr.justice.gov.lb/index.aspx","access_method":"official interactive search; do not bypass controls","cadence":"unknown","attribution_licensing_notes":"Coverage, fees, terms, and personal-address policy require review","adapter_status":"reference_only","expected_artifact_schema":"Organization metadata; suppress personal addresses","blockers":["Machine route and nationwide coverage not verified."]}, {"source_id":"lb.industry.food-guide","jurisdiction_scope":"Lebanon; Ministry of Industry licensed food factories","legacy_paths":[],"url":"https://www.industry.gov.lb/IndustrialStatistics/IndustrialGuide","access_method":"official guide/list; authorized bounded download only","cadence":"2022 visible; current unknown","attribution_licensing_notes":"Terms, schema, IDs, and safe address boundary require review","adapter_status":"reference_only","expected_artifact_schema":"Metadata-only; no rows or coordinates","blockers":["Current export and reuse terms not pinned."]}, - {"source_id":"lb.cas.livestock-statistics","jurisdiction_scope":"Lebanon; Central Administration of Statistics aggregate livestock indicators","legacy_paths":[],"url":"https://www.cas.gov.lb/","access_method":"official tables/publications; authorized query/download to be pinned","cadence":"table-specific; unknown","attribution_licensing_notes":"Revisions, licensing, and suppression rules require review","adapter_status":"reference_only","expected_artifact_schema":"Aggregate time-series metadata; no establishment rows","blockers":["Current table/API identifiers and terms not pinned."]} ] + {"source_id":"lb.cas.livestock-statistics","jurisdiction_scope":"Lebanon; Central Administration of Statistics aggregate livestock indicators","legacy_paths":[],"url":"https://www.cas.gov.lb/","access_method":"official tables/publications; authorized query/download to be pinned","cadence":"table-specific; unknown","attribution_licensing_notes":"Revisions, licensing, and suppression rules require review","adapter_status":"reference_only","expected_artifact_schema":"Aggregate time-series metadata; no establishment rows","blockers":["Current table/API identifiers and terms not pinned."]}, + {"source_id":"ch.blv.approved-food","jurisdiction_scope":"Switzerland; FSVO list of approved food businesses, including animal-origin establishments and slaughterhouses","legacy_paths":[],"url":"https://www.blv.admin.ch/de/listen-bewilligter-schweizer-betriebe","access_method":"official multilingual FSVO search/list route; authorized bounded export or browser capture only","cadence":"source-specific; current page and list route observed 2026-09-16","attribution_licensing_notes":"Swiss government source; public visibility does not settle reuse, attribution, personal-address, coordinate, or publication rights","adapter_status":"reference_only","expected_artifact_schema":"Metadata-first contract for multilingual list records: approval number, approval date/status, operator/name, contact/address, establishment type, activities/species, source language, list/version date, and source URL; preserve raw labels and avoid row publication","blockers":["No stable bulk/API contract or complete national export verified; federal list may be assembled from cantonal authorities; language normalization, effective/status semantics, privacy, terms, coverage, and project approval remain unresolved."]} ] } diff --git a/pipeline/tests/test_source_registry.py b/pipeline/tests/test_source_registry.py index c78b7a5..29c3ec2 100644 --- a/pipeline/tests/test_source_registry.py +++ b/pipeline/tests/test_source_registry.py @@ -9,8 +9,8 @@ class SourceRegistryTests(unittest.TestCase): def test_repository_registry_loads_and_references_existing_legacy_paths(self): registry = load_registry() - self.assertEqual(len(registry["sources"]), 233) - self.assertEqual(len({source["source_id"] for source in registry["sources"]}), 233) + self.assertEqual(len(registry["sources"]), 234) + self.assertEqual(len({source["source_id"] for source in registry["sources"]}), 234) def test_unknowns_are_explicit(self): registry = json.loads(REGISTRY_PATH.read_text(encoding="utf-8")) diff --git a/pipeline/tests/test_switzerland_recon_metadata.py b/pipeline/tests/test_switzerland_recon_metadata.py new file mode 100644 index 0000000..651f213 --- /dev/null +++ b/pipeline/tests/test_switzerland_recon_metadata.py @@ -0,0 +1,27 @@ +import json +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] + + +class SwitzerlandReconMetadataTests(unittest.TestCase): + def test_row_free_recon_and_conservative_language(self): + text = (ROOT / "docs" / "country-recon-ch.md").read_text(encoding="utf-8") + self.assertIn("row-free", text) + self.assertIn("Difficulty estimate: high", text) + self.assertIn("No adapter", text) + + def test_registry_and_status_have_matching_blocked_source(self): + registry = json.loads((ROOT / "pipeline" / "source_registry.json").read_text(encoding="utf-8")) + status = json.loads((ROOT / "docs" / "source-status.json").read_text(encoding="utf-8")) + source = next(s for s in registry["sources"] if s["source_id"] == "ch.blv.approved-food") + current = next(s for s in status["sources"] if s["source_id"] == "ch.blv.approved-food") + self.assertEqual(source["adapter_status"], "reference_only") + self.assertEqual(current["publication_eligibility"], "blocked") + self.assertEqual(current["acquisition"], "blocked") + self.assertTrue(source["blockers"]) + + +if __name__ == "__main__": + unittest.main() From 16de2f4f7eeb8c3fb1622f635e1b72fbe9e09da9 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Wed, 16 Sep 2026 19:46:29 -0700 Subject: [PATCH 205/311] Harden V2 discovery read path at scale --- docs/performance/v2-discovery-100k.md | 19 ++++++++ docs/performance/v2-observability.md | 20 ++++---- .../v2-public-projection-read-path.md | 23 ++++++---- .../032_flatten_public_history_view.sql | 25 ++++------ .../035_public_discovery_planner_indexes.sql | 22 +++++++++ .../tests/test_discovery_scale_benchmark.py | 7 +-- pipeline/tests/test_graph_migrations.py | 3 +- .../test_public_discovery_query_contract.py | 46 +++++++++++++++++++ src/lib.rs | 10 ++-- 9 files changed, 131 insertions(+), 44 deletions(-) create mode 100644 pipeline/migrations/035_public_discovery_planner_indexes.sql create mode 100644 pipeline/tests/test_public_discovery_query_contract.py diff --git a/docs/performance/v2-discovery-100k.md b/docs/performance/v2-discovery-100k.md index 64f4ffc..9d03381 100644 --- a/docs/performance/v2-discovery-100k.md +++ b/docs/performance/v2-discovery-100k.md @@ -78,3 +78,22 @@ These are single-query local plan samples over temporary synthetic tables. They demonstrate query-shape/index behavior only and do not establish API latency, concurrency capacity, or production readiness. The companion API rehearsal and its limitations are recorded in `v2-api-load-rehearsal.md`. + +## Scale expectations after the 2026-09-16 hardening + +These expectations are deliberately separated from support or capacity claims. +The following row-free hardening capture used PostGIS 16 / PostGIS 3.4 on +2026-09-16; the value in each column is the slowest single-query execution +sample among the eight bounded query shapes (the radius shape was the slowest +at each scale): + +| Synthetic observations | Slowest sample | Evidence-backed expectation | +| ---: | ---: | --- | +| 25,000 | 12.588 ms | All eight shapes returned at most 50 rows, used an expected index family, and had zero sequential-scan nodes in this run. | +| 100,000 | 12.200 ms | The same bounded/indexed query-shape behavior held in this local sample; it remains single-query evidence. | +| 150,000 | 50.895 ms | The row-free planner capture also passed all checks; radius cost increased materially but stayed below the 350 ms review threshold in this sample. | + +These are validation observations, not promises of API latency or capacity. The live public +projection still evaluates release membership, publication review, privacy +screening, profile rules, and current suppression on every read. No cache or +frozen public projection is used to manufacture a scale claim. diff --git a/docs/performance/v2-observability.md b/docs/performance/v2-observability.md index f8c8a5e..cd56bd1 100644 --- a/docs/performance/v2-observability.md +++ b/docs/performance/v2-observability.md @@ -74,10 +74,11 @@ throughput, status/error/timeouts, response byte totals, observed database active/waiting sessions, and pool-pressure signals only. It does not retain request paths, query values, coordinates, IDs, client data, or response rows. -Migration 032 flattens the nested public-history view into one -release-scoped eligibility pass and one per-release facility summary while -retaining the live review, suppression, lifecycle, and geocode checks. The -diagnostic can be reproduced with: +Migration 032 flattens the nested public-history view into one inline +release-scoped eligibility pass and window aggregates over each facility's +history while retaining the live review, suppression, lifecycle, and geocode +checks. Keeping eligibility inline lets selected release/facility predicates +push down before the window work. The diagnostic can be reproduced with: ```powershell python pipeline/scripts/benchmarks/explain_public_projection.py ` @@ -104,12 +105,11 @@ representative traffic test on the deployment topology. The 2-second request timeout and 350 ms database radius-query budget remain review thresholds for fail-safe behavior, not performance guarantees. -The 5,000-row component matrix attributes the remaining cost primarily to the -release-scoped eligibility and public summary path: about 663 ms and 3,362 ms -respectively in isolation, versus about 16 ms geocode, 15 ms city, and 1 ms -lifecycle lookup. Full pagination and spatial statements measured about 5,097 -ms and 4,690 ms. Allowing the summary CTE to inline was tested and rejected: -facets worsened from about 4,501 ms to 6,622 ms. See +The earlier 5,000-row component matrix attributed the cost primarily to the +release-scoped eligibility and repeated public summary path. After the inline +eligibility/window-aggregate rewrite, a fresh local synthetic capture measured +the flattened list and facets statements at approximately 108 ms and 106 ms. +These are single-query samples, not p95 or capacity measurements. See `docs/performance/v2-public-projection-read-path.md` for the architecture decision and the safeguards required for any future release-built component. diff --git a/docs/performance/v2-public-projection-read-path.md b/docs/performance/v2-public-projection-read-path.md index 63062f5..75326d1 100644 --- a/docs/performance/v2-public-projection-read-path.md +++ b/docs/performance/v2-public-projection-read-path.md @@ -11,7 +11,8 @@ implemented behind a candidate view for invariant testing only. ## Evidence -The synthetic component benchmark at 5,000 observations measured roughly: +The earlier pre-rewrite synthetic component benchmark at 5,000 observations +measured roughly: | Component | Execution time | | --- | ---: | @@ -23,12 +24,13 @@ The synthetic component benchmark at 5,000 observations measured roughly: | Full pagination projection | 5,097 ms | | Full spatial projection | 4,690 ms | -The flattened view removed repeated nested expansion and made the 1,000-row -concurrent rehearsal clean at all tested levels. At 5,000 rows, the remaining -summary and eligibility work still exceeds the current 2-second rehearsal -budget. A candidate that allowed the summary CTE to inline was measured and -rejected because the 5,000-row facets plan increased from about 4,501 ms to -6,622 ms. +The flattened view removed repeated nested expansion. The scale-hardening +revision keeps eligibility inline and uses window aggregates for the +per-facility summary, allowing release and facility predicates to be pushed +before the summary work while preserving the same live joins. In a fresh +synthetic 5,000-row local plan capture, list and facets measured approximately +108 ms and 106 ms. These are single-query samples, not p95 or capacity +measurements. The prototype builder is reproducible with: @@ -40,9 +42,10 @@ It stores release-membership observation facts, not a frozen current-public decision. The candidate summary joins the exact release manifest checksum and re-evaluates current review, profile, and suppression state on every read. At 1,000 rows its candidate summary took about 406 ms versus 121 ms for the -current live summary; at 5,000 rows it took about 9,984 ms versus 3,044 ms. -The prototype therefore proves the safety protocol but does not justify API -integration or a production capacity claim. +current live summary; at 5,000 rows it took about 9,984 ms versus 3,044 ms in +the earlier component comparison. The candidate therefore still proves the +safety protocol but does not justify API integration or a production capacity +claim. ## Alternatives considered diff --git a/pipeline/migrations/032_flatten_public_history_view.sql b/pipeline/migrations/032_flatten_public_history_view.sql index 9dad7a5..520eb90 100644 --- a/pipeline/migrations/032_flatten_public_history_view.sql +++ b/pipeline/migrations/032_flatten_public_history_view.sql @@ -1,8 +1,11 @@ -- Flatten the public history view's nested release/review/suppression joins. -- Eligibility is still evaluated from append-only control-plane views for every -- request; this changes only plan shape, not the public contract. +-- Keep the eligibility CTE inline so release/facility filters can push down +-- before the window aggregates. The partition still covers every eligible +-- observation for a facility, preserving history counts and timestamps. CREATE OR REPLACE VIEW uec.map_facilities_display_history AS -WITH eligible AS MATERIALIZED ( +WITH eligible AS ( SELECT member.release_id, member.default_visible AS release_visible, observation.observation_id, @@ -56,14 +59,6 @@ WITH eligible AS MATERIALIZED ( FROM uec.public_access_restricted AS restricted WHERE restricted.source_record_id = observation.source_record_id ) -), public_summary AS MATERIALIZED ( - SELECT release_id, - facility_id, - min(first_observed_at) AS first_observed_at, - max(observed_at) AS last_observed_at, - count(*)::int AS observation_count - FROM eligible - GROUP BY release_id, facility_id ) SELECT eligible.release_id, 'promoted'::text AS release_status, @@ -88,9 +83,9 @@ SELECT eligible.release_id, latest.provider_id AS geocoder_provider, latest.queried_at AS geocoded_at, eligible.classification_category, - public_summary.first_observed_at, - public_summary.last_observed_at, - public_summary.observation_count, + min(eligible.first_observed_at) OVER facility_history AS first_observed_at, + max(eligible.observed_at) OVER facility_history AS last_observed_at, + (count(*) OVER facility_history)::int AS observation_count, COALESCE(lifecycle.status, 'status_unknown') AS lifecycle_status, lifecycle.effective_at AS lifecycle_effective_at, lifecycle.source_record_id AS lifecycle_source_record_id, @@ -102,9 +97,6 @@ SELECT eligible.release_id, eligible.provenance_source_url, eligible.provenance_retrieved_at FROM eligible -JOIN public_summary - ON public_summary.release_id = eligible.release_id - AND public_summary.facility_id = eligible.facility_id LEFT JOIN LATERAL ( SELECT status, result, provider_id, queried_at FROM uec.geocode_results @@ -122,7 +114,8 @@ LEFT JOIN LATERAL ( LIMIT 1 ) AS city ON true LEFT JOIN uec.facility_lifecycle_current AS lifecycle - ON lifecycle.facility_id = eligible.facility_id; + ON lifecycle.facility_id = eligible.facility_id +WINDOW facility_history AS (PARTITION BY eligible.release_id, eligible.facility_id); COMMENT ON VIEW uec.map_facilities_display_history IS 'V2 public display history with one release-scoped eligibility pass, current suppression, and public-only lifecycle counts.'; diff --git a/pipeline/migrations/035_public_discovery_planner_indexes.sql b/pipeline/migrations/035_public_discovery_planner_indexes.sql new file mode 100644 index 0000000..de6cb01 --- /dev/null +++ b/pipeline/migrations/035_public_discovery_planner_indexes.sql @@ -0,0 +1,22 @@ +-- Planner support for the live public read path. These indexes are additive: +-- eligibility, profile, and suppression continue to be evaluated by the view. +CREATE INDEX IF NOT EXISTS releases_public_promoted_lookup_idx + ON uec.releases (profile, created_at DESC, release_id DESC) + WHERE status = 'promoted' AND test_only IS NOT TRUE; + +CREATE INDEX IF NOT EXISTS release_members_public_facility_order_idx + ON uec.release_members (release_id, facility_id, observation_id) + WHERE default_visible = true; + +CREATE INDEX IF NOT EXISTS facilities_discovery_search_fields_trgm_idx + ON uec.facilities USING GIN ( + lower(coalesce(canonical_name, '') || ' ' || coalesce(city, '') || ' ' || country_code) + gin_trgm_ops + ); + +COMMENT ON INDEX uec.releases_public_promoted_lookup_idx IS + 'Bounds promoted-release selection without changing release eligibility.'; +COMMENT ON INDEX uec.release_members_public_facility_order_idx IS + 'Supports deterministic facility cursor traversal for visible release members.'; +COMMENT ON INDEX uec.facilities_discovery_search_fields_trgm_idx IS + 'Supports bounded case-insensitive search over public facility identity fields.'; diff --git a/pipeline/tests/test_discovery_scale_benchmark.py b/pipeline/tests/test_discovery_scale_benchmark.py index 6c75bf6..6a46181 100644 --- a/pipeline/tests/test_discovery_scale_benchmark.py +++ b/pipeline/tests/test_discovery_scale_benchmark.py @@ -75,11 +75,12 @@ def test_public_read_path_migration_is_additive_and_release_scoped(self): def test_public_history_flattening_preserves_release_and_suppression_gates(self): migration = (ROOT / "migrations" / "032_flatten_public_history_view.sql").read_text(encoding="utf-8").lower() self.assertIn("create or replace view uec.map_facilities_display_history", migration) - self.assertIn("with eligible as materialized", migration) - self.assertIn("public_summary as materialized", migration) + self.assertIn("with eligible as (", migration) + self.assertIn("min(eligible.first_observed_at) over facility_history", migration) + self.assertIn("(count(*) over facility_history)::int", migration) self.assertIn("publication_review_release_current", migration) self.assertIn("public_access_restricted", migration) - self.assertIn("group by release_id, facility_id", migration) + self.assertIn("partition by eligible.release_id, eligible.facility_id", migration) self.assertNotIn("drop view", migration) def test_public_eligibility_indexes_are_additive_and_order_safe(self): diff --git a/pipeline/tests/test_graph_migrations.py b/pipeline/tests/test_graph_migrations.py index 39c2b1f..ed611a7 100644 --- a/pipeline/tests/test_graph_migrations.py +++ b/pipeline/tests/test_graph_migrations.py @@ -20,7 +20,7 @@ def test_reserved_migrations_are_present_and_ordered(self): positions = [migrations.index(name) for name in graph_migrations] self.assertEqual(positions, sorted(positions)) self.assertEqual([migrations[position] for position in positions], graph_migrations) - self.assertEqual(migrations[-9:], [ + self.assertEqual(migrations[-10:], [ "026_graph_entities_crosswalks.sql", "027_graph_relationship_observations.sql", "028_graph_claims_support.sql", @@ -30,6 +30,7 @@ def test_reserved_migrations_are_present_and_ordered(self): "032_flatten_public_history_view.sql", "033_release_summary_component.sql", "034_public_eligibility_join_indexes.sql", + "035_public_discovery_planner_indexes.sql", ]) def test_entities_are_distinct_and_crosswalk_is_scoped(self): diff --git a/pipeline/tests/test_public_discovery_query_contract.py b/pipeline/tests/test_public_discovery_query_contract.py new file mode 100644 index 0000000..aa9098d --- /dev/null +++ b/pipeline/tests/test_public_discovery_query_contract.py @@ -0,0 +1,46 @@ +"""Regression contracts for the live, facility-level public discovery path.""" + +from pathlib import Path +import unittest + + +ROOT = Path(__file__).parents[1] + + +def _source() -> str: + return (ROOT.parent / "src" / "lib.rs").read_text(encoding="utf-8") + + +class PublicDiscoveryQueryContractTests(unittest.TestCase): + def test_list_and_facets_are_one_row_per_facility_with_stable_ordering(self): + source = _source() + self.assertIn("SELECT DISTINCT ON (history.facility_id)", source) + self.assertIn("SELECT DISTINCT ON (facility_id) facility_id", source) + self.assertIn("ORDER BY history.facility_id, history.observation_id", source) + self.assertIn("ORDER BY facility_id, observation_id", source) + + def test_detail_uses_the_same_deterministic_observation_choice(self): + source = _source() + detail = source[source.index("pub async fn get_v2_location_detail_handler"):] + self.assertIn("ORDER BY history.observation_id", detail) + self.assertIn("LIMIT 1", detail) + + def test_public_queries_keep_live_release_review_join_and_view(self): + source = _source() + locations = source[source.index("pub async fn get_v2_locations_handler"):source.index("pub async fn get_v2_location_detail_handler")] + self.assertIn("FROM uec.map_facilities_display_history AS history", locations) + self.assertIn("JOIN uec.publication_review_release_current AS review", locations) + self.assertIn("review.release_id = history.release_id", locations) + self.assertIn("history.release_id = $1", locations) + + def test_planner_indexes_are_additive_and_gate_neutral(self): + migration = (ROOT / "migrations" / "035_public_discovery_planner_indexes.sql").read_text(encoding="utf-8").lower() + self.assertEqual(migration.count("create index if not exists"), 3) + self.assertIn("status = 'promoted'", migration) + self.assertIn("default_visible = true", migration) + self.assertNotIn("drop index", migration) + self.assertNotIn("drop table", migration) + + +if __name__ == "__main__": + unittest.main() diff --git a/src/lib.rs b/src/lib.rs index dce14d6..2d720a7 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -988,7 +988,7 @@ pub async fn get_v2_facets_handler( let release_id: String = release.get(0); let ruleset_version: String = release.get(1); let release_created_at: chrono::DateTime = release.get(2); - let rows = match client.query("SELECT country_code, classification_category, display_precision, lifecycle_status, provenance_origin_type, city, count(*)::bigint FROM uec.map_facilities_display_history WHERE release_id=$1 AND ($2::text IS NULL OR country_code=$2) AND ($3::text IS NULL OR classification_category=$3) AND ($4::text IS NULL OR provenance_origin_type=$4) AND ($5::text IS NULL OR display_precision=$5) AND ($6::text IS NULL OR lifecycle_status=$6) AND ($7::text IS NULL OR city=$7) GROUP BY country_code, classification_category, display_precision, lifecycle_status, provenance_origin_type, city", &[&release_id, ¶ms.country_code, ¶ms.category, ¶ms.source_type, ¶ms.display_precision, ¶ms.lifecycle_status, ¶ms.region]).await { Ok(rows) => rows, Err(_) => return v2_error(StatusCode::SERVICE_UNAVAILABLE, "facets_query_failed", "public facets unavailable") }; + let rows = match client.query("SELECT country_code, classification_category, display_precision, lifecycle_status, provenance_origin_type, city, count(*)::bigint FROM (SELECT DISTINCT ON (facility_id) facility_id, country_code, classification_category, display_precision, lifecycle_status, provenance_origin_type, city FROM uec.map_facilities_display_history WHERE release_id=$1 AND ($2::text IS NULL OR country_code=$2) AND ($3::text IS NULL OR classification_category=$3) AND ($4::text IS NULL OR provenance_origin_type=$4) AND ($5::text IS NULL OR display_precision=$5) AND ($6::text IS NULL OR lifecycle_status=$6) AND ($7::text IS NULL OR city=$7) ORDER BY facility_id, observation_id) public_facilities GROUP BY country_code, classification_category, display_precision, lifecycle_status, provenance_origin_type, city", &[&release_id, ¶ms.country_code, ¶ms.category, ¶ms.source_type, ¶ms.display_precision, ¶ms.lifecycle_status, ¶ms.region]).await { Ok(rows) => rows, Err(_) => return v2_error(StatusCode::SERVICE_UNAVAILABLE, "facets_query_failed", "public facets unavailable") }; let mut dimensions = serde_json::Map::new(); for (name, column) in [ ("country_code", 0), @@ -1349,7 +1349,7 @@ pub async fn get_v2_locations_handler( let promoted_profile: String = release.get(3); let query_limit = limit + 1; let rows = match transaction.query(r#" - SELECT history.facility_id, history.canonical_name, history.country_code, history.city, history.classification_category, history.display_precision, + SELECT DISTINCT ON (history.facility_id) history.facility_id, history.canonical_name, history.country_code, history.city, history.classification_category, history.display_precision, review.factual_review_status, review.privacy_screening_status, review.maintainer_approval, review.reviewer_role, ST_Y(history.display_location::geometry), ST_X(history.display_location::geometry), history.first_observed_at, history.last_observed_at, history.observation_count, history.lifecycle_status, @@ -1372,7 +1372,7 @@ pub async fn get_v2_locations_handler( AND ($9::text IS NULL OR lower(coalesce(history.canonical_name, '') || ' ' || coalesce(history.city, '') || ' ' || history.country_code || ' ' || history.classification_category || ' ' || coalesce(history.provenance_source_name, '')) LIKE '%' || lower($9) || '%' ESCAPE '\') AND ($10::double precision IS NULL OR (history.display_location && ST_MakeEnvelope($10, $11, $12, $13, 4326)::geography AND ST_Intersects(history.display_location::geometry, ST_MakeEnvelope($10, $11, $12, $13, 4326)))) AND ($14::double precision IS NULL OR ST_DWithin(history.display_location, ST_SetSRID(ST_Point($15, $16), 4326)::geography, $14 * 1000)) - ORDER BY history.facility_id LIMIT $17 OFFSET $18 + ORDER BY history.facility_id, history.observation_id LIMIT $17 OFFSET $18 "#, &[&promoted_release_id, &cursor, ¶ms.country_code, ¶ms.region, ¶ms.category, ¶ms.display_precision, ¶ms.lifecycle_status, ¶ms.source_type, &search_text, &min_lon, &min_lat, &max_lon, &max_lat, &radius_km, &longitude, &latitude, &query_limit, &effective_offset]).await { Ok(rows) => rows, Err(_) => return v2_error(StatusCode::INTERNAL_SERVER_ERROR, "location_query_failed", "V2 location query failed"), @@ -1522,7 +1522,9 @@ pub async fn get_v2_location_detail_handler( ON review.source_record_id = history.source_record_id AND review.release_id = history.release_id JOIN uec.sources rights ON rights.source_id = history.provenance_source_id - WHERE history.facility_id = $1 AND history.release_id = $2 + WHERE history.facility_id = $1 AND history.release_id = $2 + ORDER BY history.observation_id + LIMIT 1 "#, &[&facility_id, &release_id]).await { Ok(row) => row, Err(_) => return v2_error(StatusCode::INTERNAL_SERVER_ERROR, "location_query_failed", "V2 location query failed"), From ac48ceba5169cae8d4210d30a2b9958d0c8e7a09 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Wed, 16 Sep 2026 19:40:08 -0700 Subject: [PATCH 206/311] Add bounded corpus resilience rehearsal --- docs/performance/corpus-resilience.md | 64 ++ .../benchmarks/run_corpus_resilience.py | 575 ++++++++++++++++++ .../scripts/maintenance/import-candidate.py | 10 +- pipeline/tests/e2e/test_corpus_resilience.py | 42 ++ pipeline/tests/test_corpus_resilience.py | 68 +++ pipeline/tests/test_import_candidate.py | 97 +++ 6 files changed, 855 insertions(+), 1 deletion(-) create mode 100644 docs/performance/corpus-resilience.md create mode 100644 pipeline/scripts/benchmarks/run_corpus_resilience.py create mode 100644 pipeline/tests/e2e/test_corpus_resilience.py create mode 100644 pipeline/tests/test_corpus_resilience.py diff --git a/docs/performance/corpus-resilience.md b/docs/performance/corpus-resilience.md new file mode 100644 index 0000000..8f253b3 --- /dev/null +++ b/docs/performance/corpus-resilience.md @@ -0,0 +1,64 @@ +# Bounded large-corpus resilience rehearsal + +`pipeline/scripts/benchmarks/run_corpus_resilience.py` is the reusable lane for +the private large-corpus failure modes that the real-corpus reports previously +left unexercised. It expands only the row-free source-count distribution in +`data/manifests/sprint4-real-corpus-regression.json` into deterministic +synthetic rows in a temporary directory. The rows are never committed, and +the report contains counts, timings, byte sizes, and bounded-memory +observations only. + +Run the quick, database-free plan check with: + +```powershell +python pipeline/scripts/benchmarks/run_corpus_resilience.py ` + --plan-only --max-records 50000 ` + --json-output .tmp/corpus-resilience-plan.json +``` + +Run the full disposable PostGIS rehearsal with Docker Desktop and the pinned +Python dependencies: + +```powershell +python pipeline/scripts/benchmarks/run_corpus_resilience.py ` + --max-records 50000 --batch-size 500 ` + --json-output .tmp/corpus-resilience.json +``` + +The opt-in E2E assertions run the same lane: + +```powershell +$env:UEC_RUN_E2E = "1" +$env:UEC_RUN_CORPUS_RESILIENCE = "1" +python -m unittest pipeline.tests.e2e.test_corpus_resilience -v +``` + +The exact safety limits are 50,000 selected records, 2,000 rows per import +batch, four maximum simulated interruption batches, loopback-only database +connectivity, a unique disposable Compose project, and teardown with volume +removal in `finally`. The default is the full 50,000-record distribution and a +500-row batch. A smaller `--max-records` value is proportionally allocated by +the largest-remainder method so source shape remains representative. + +The lane measures: migration wall time and applied migration count; import +wall time; committed rows before and after a post-commit interruption; resume +rows; duplicate-import new rows and count stability; custom-format dump bytes; +backup and restore wall time; stale pre-service gate rejection; current +suppression replay; database size; total runtime; and Python `tracemalloc` +peaks while loading/importing one partition. The database-size and timing +observations describe this local disposable run only. `tracemalloc` does not +observe PostgreSQL shared buffers, container RSS, or OS peak memory. + +The backup is created before the synthetic current suppression is applied. +After restoring it, the pre-service gate must reject the stale state; only +after the temporary current ledger is replayed may the final gate pass. The +same source-record reference is used through a source-key ledger, so the +rehearsal covers same-source reimport protection without copying restricted +payloads. + +The full lane was not run in this checkout when Docker Desktop was unavailable; +the focused Python tests and plan-only path remain runnable without Docker. +This harness does not claim real-source quality, adapter correctness, +production capacity, cloud backup durability, WAL recovery, operator access +controls, or publication approval. It also does not measure PostgreSQL or +container memory directly. diff --git a/pipeline/scripts/benchmarks/run_corpus_resilience.py b/pipeline/scripts/benchmarks/run_corpus_resilience.py new file mode 100644 index 0000000..3721bbd --- /dev/null +++ b/pipeline/scripts/benchmarks/run_corpus_resilience.py @@ -0,0 +1,575 @@ +"""Run a bounded, disposable large-corpus resilience rehearsal. + +The rehearsal expands only row-free aggregate distribution metadata into a +synthetic corpus. It exercises the real candidate importer in independently +committed batches, a post-commit interruption/resume, duplicate import, full +migration timing, database sizing, and a custom-format backup/restore with a +current suppression ledger replay. Generated rows, dump files, ledger +references, and database contents stay in a temporary disposable environment. + +The JSON report is deliberately aggregate-only. It never includes a source +record key, identifier, address, coordinate, source value, or response body. +""" +from __future__ import annotations + +import argparse +import hashlib +import importlib.util +import json +import os +import socket +import subprocess +import tempfile +import time +import tracemalloc +from pathlib import Path +from typing import Any + + +PROJECT_ROOT = Path(__file__).resolve().parents[3] +DEFAULT_DISTRIBUTION = PROJECT_ROOT / "data/manifests/sprint4-real-corpus-regression.json" +MAX_RECORDS = 50_000 +MAX_BATCH_SIZE = 2_000 +DEFAULT_BATCH_SIZE = 500 +MAX_INTERRUPTION_BATCHES = 4 +SCHEMA_VERSION = "corpus-resilience-v1" + + +class ResilienceError(ValueError): + """The rehearsal input or disposable safety boundary is invalid.""" + + +class SimulatedInterruption(RuntimeError): + """Raised by the post-commit hook to model a lost importer process.""" + + +def _load_script(name: str, path: Path): + spec = importlib.util.spec_from_file_location(name, path) + if spec is None or spec.loader is None: + raise ResilienceError(f"script is unavailable: {path}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _distribution_counts(report: dict[str, Any]) -> dict[str, int]: + coverage = report.get("coverage", {}) + counts = coverage.get("selected_records_by_source") + if not isinstance(counts, dict): + counts = report.get("source_counts") + if not isinstance(counts, dict) or not counts: + raise ResilienceError("distribution report has no aggregate source counts") + parsed: dict[str, int] = {} + for source, count in counts.items(): + if not isinstance(source, str) or not source: + raise ResilienceError("distribution report contains an invalid source label") + if isinstance(count, bool) or not isinstance(count, int) or count < 0: + raise ResilienceError(f"distribution report contains an invalid count for {source}") + if count: + parsed[source] = count + if not parsed: + raise ResilienceError("distribution report contains no positive source counts") + return parsed + + +def _proportional_allocation(counts: dict[str, int], limit: int) -> dict[str, int]: + available = sum(counts.values()) + if limit >= available: + return dict(sorted(counts.items())) + raw = {source: count * limit / available for source, count in counts.items()} + allocation = {source: int(value) for source, value in raw.items()} + remainder = limit - sum(allocation.values()) + order = sorted(raw, key=lambda source: (-(raw[source] - allocation[source]), source)) + for source in order[:remainder]: + allocation[source] += 1 + return {source: allocation[source] for source in sorted(allocation) if allocation[source]} + + +def build_plan(distribution_path: Path, max_records: int = MAX_RECORDS) -> dict[str, Any]: + """Build a deterministic row-free selection plan from aggregate metadata.""" + if max_records <= 0 or max_records > MAX_RECORDS: + raise ResilienceError(f"max_records must be between 1 and {MAX_RECORDS}") + try: + report = json.loads(distribution_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise ResilienceError("distribution report cannot be read") from exc + counts = _distribution_counts(report) + selected = _proportional_allocation(counts, max_records) + return { + "selection_method": "proportional-largest-remainder-from-row-free-source-counts", + "available_records": sum(counts.values()), + "selected_records": sum(selected.values()), + "source_profiles": len(selected), + "selected_records_by_source": selected, + "input_report": distribution_path.as_posix(), + } + + +def _slug(source: str) -> str: + return "".join(character.lower() if character.isalnum() else "-" for character in source).strip("-") + + +def _country_code(source: str) -> str: + parts = source.split(".") + for part in parts: + if len(part) == 2 and part.isalpha(): + return part.upper() + return "ZZ" + + +def _synthetic_record(source: str, row_number: int) -> dict[str, Any]: + slug = _slug(source) + establishment_id = f"RES-{slug}-{row_number:06d}" + category = ("slaughter", "processing", "cutting", "logistics_and_storage")[row_number % 4] + return { + "source_id": source, + "source_row": row_number, + "source_values": { + "fixture_kind": "synthetic-resilience-shape", + "source_partition": slug, + "ordinal": row_number, + }, + "normalized": { + "establishment_id": establishment_id, + "trading_name": f"Synthetic resilience facility {row_number}", + "city": f"Synthetic city {row_number % 97}", + "country_code": _country_code(source), + "nation": _country_code(source), + "activity_categories": [category], + "coordinates": None, + "coordinate_state": "unknown", + "privacy_gate": "pending-review", + "coordinate_gate": "review_required", + "publication_gate": "blocked", + "source_origin": "synthetic-fixture", + }, + } + + +def write_synthetic_corpus(root: Path, plan: dict[str, Any]) -> list[dict[str, Any]]: + """Write temporary partition files and return only metadata about them.""" + partitions: list[dict[str, Any]] = [] + for source, count in plan["selected_records_by_source"].items(): + partition = root / _slug(source) + partition.mkdir(parents=True, exist_ok=True) + raw_path = partition / "source.bin" + raw = f"synthetic resilience artifact for {source}\n".encode("utf-8") + raw_path.write_bytes(raw) + raw_digest = hashlib.sha256(raw).hexdigest() + normalized_path = partition / "normalized.jsonl" + normalized_digest = hashlib.sha256() + with normalized_path.open("wb") as handle: + for row_number in range(1, count + 1): + line = (json.dumps(_synthetic_record(source, row_number), sort_keys=True, separators=(",", ":")) + "\n").encode("utf-8") + handle.write(line) + normalized_digest.update(line) + manifest = { + "source_id": f"resilience.{_slug(source)}", + "source_url": f"https://example.invalid/resilience/{_slug(source)}", + "retrieved_at_utc": "2026-09-16T00:00:00Z", + "checksum_sha256": raw_digest, + "byte_size": len(raw), + "normalized_rows": count, + "normalized_sha256": normalized_digest.hexdigest(), + "release_state": "not-created", + "publication_state": "private-candidate", + "country_code": _country_code(source), + "code_version": SCHEMA_VERSION, + "config_version": "synthetic-shape-v1", + "profile": "private-resilience-test-only", + } + manifest_path = partition / "manifest.json" + manifest_path.write_text(json.dumps(manifest, sort_keys=True, indent=2) + "\n", encoding="utf-8") + partitions.append({ + "source": source, + "source_id": manifest["source_id"], + "rows": count, + "manifest": manifest_path, + "normalized": normalized_path, + "raw": raw_path, + }) + return partitions + + +def _compose(root: Path, project: str) -> list[str]: + return ["docker", "compose", "-p", project, "-f", str(root / "docker-compose.e2e.yml")] + + +def _free_port() -> int: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + sock.bind(("127.0.0.1", 0)) + return int(sock.getsockname()[1]) + + +def _run(command: list[str], *, env: dict[str, str], check: bool = True, **kwargs: Any) -> subprocess.CompletedProcess: + try: + return subprocess.run(command, env=env, check=check, **kwargs) + except FileNotFoundError as exc: + raise ResilienceError("Docker Compose is required for the database rehearsal") from exc + except subprocess.CalledProcessError as exc: + output = (exc.stderr or exc.stdout or "").strip() + detail = output.splitlines()[-1] if output else "no command output" + raise ResilienceError(f"disposable command failed with exit {exc.returncode}: {detail}") from exc + + +def _counts(connection, release_id: str, source_ids: list[str]) -> dict[str, int]: + return { + "source_records": connection.execute( + "SELECT count(*) FROM uec.source_records WHERE source_id = ANY(%s)", (source_ids,) + ).fetchone()[0], + "facilities": connection.execute( + """SELECT count(*) FROM uec.facilities facility + WHERE EXISTS ( + SELECT 1 FROM uec.observations observation + JOIN uec.source_records record ON record.source_record_id = observation.source_record_id + WHERE observation.facility_id = facility.facility_id AND record.source_id = ANY(%s) + )""", (source_ids,) + ).fetchone()[0], + "observations": connection.execute( + """SELECT count(*) FROM uec.observations observation + JOIN uec.source_records record ON record.source_record_id = observation.source_record_id + WHERE record.source_id = ANY(%s)""", (source_ids,) + ).fetchone()[0], + "release_members": connection.execute( + "SELECT count(*) FROM uec.release_members WHERE release_id=%s", (release_id,) + ).fetchone()[0], + "review_events": connection.execute( + """SELECT count(*) FROM uec.publication_review_events event + JOIN uec.source_records record ON record.source_record_id = event.source_record_id + WHERE record.source_id = ANY(%s)""", (source_ids,) + ).fetchone()[0], + } + + +def _database_size(connection) -> int: + return int(connection.execute("SELECT pg_database_size(current_database())").fetchone()[0]) + + +def _make_ledger(path: Path, source_id: str, source_record_key: str, revision: str) -> dict[str, Any]: + ledger_module = _load_script( + "restriction_ledger_gate_for_resilience", + PROJECT_ROOT / "pipeline/scripts/maintenance/restriction-ledger-gate.py", + ) + ledger = { + "schema_version": 1, + "revision": revision, + "active_restrictions": [{ + "source_id": source_id, + "source_record_key": source_record_key, + "scope": "whole_record", + "action": "suppress", + }], + } + ledger["ledger_sha256"] = ledger_module.ledger_digest(ledger) + path.write_text(json.dumps(ledger, sort_keys=True, indent=2) + "\n", encoding="utf-8") + return ledger + + +def _row_key(source: str) -> str: + return f"1:RES-{_slug(source)}-000001" + + +def _aggregate_report(plan: dict[str, Any], *, status: str, generated_peak_bytes: int, limitations: list[str]) -> dict[str, Any]: + return { + "schema_version": SCHEMA_VERSION, + "status": status, + "corpus": { + "selection_method": plan["selection_method"], + "available_records": plan["available_records"], + "selected_records": plan["selected_records"], + "source_profiles": plan["source_profiles"], + "selected_records_by_source": plan["selected_records_by_source"], + }, + "limits": { + "max_records": MAX_RECORDS, + "max_batch_size": MAX_BATCH_SIZE, + "max_interruption_batches": MAX_INTERRUPTION_BATCHES, + "database": "loopback-only disposable PostGIS compose project", + "report": "aggregate-only; generated rows and opaque ledger references are excluded", + }, + "memory_observation": { + "method": "Python tracemalloc peak during temporary synthetic generation", + "peak_bytes": generated_peak_bytes, + }, + "privacy": { + "source_payloads_committed": False, + "generated_rows_persisted": False, + "database_disposable": True, + }, + "limitations": limitations, + } + + +def run_rehearsal( + *, + root: Path = PROJECT_ROOT, + distribution: Path = DEFAULT_DISTRIBUTION, + max_records: int = MAX_RECORDS, + batch_size: int = DEFAULT_BATCH_SIZE, + interrupt_after_batches: int = 1, + plan_only: bool = False, +) -> dict[str, Any]: + if batch_size <= 0 or batch_size > MAX_BATCH_SIZE: + raise ResilienceError(f"batch_size must be between 1 and {MAX_BATCH_SIZE}") + if interrupt_after_batches < 0 or interrupt_after_batches > MAX_INTERRUPTION_BATCHES: + raise ResilienceError(f"interrupt_after_batches must be between 0 and {MAX_INTERRUPTION_BATCHES}") + plan = build_plan(distribution, max_records) + with tempfile.TemporaryDirectory(prefix="uec-corpus-resilience-") as temporary: + temporary_root = Path(temporary) + tracemalloc.start() + partitions = write_synthetic_corpus(temporary_root / "corpus", plan) + _, generated_peak = tracemalloc.get_traced_memory() + tracemalloc.stop() + if plan_only: + report = _aggregate_report( + plan, + status="plan-only", + generated_peak_bytes=generated_peak, + limitations=[ + "Database migration/import/backup/restore timings require Docker Desktop and were not run in plan-only mode.", + "Synthetic rows are generated from aggregate distribution counts; no real row payload is used.", + ], + ) + return report + + psycopg = _load_script("psycopg_for_corpus_resilience", PROJECT_ROOT / "pipeline/scripts/maintenance/import-candidate.py").psycopg + importer = _load_script("import_candidate_for_corpus_resilience", PROJECT_ROOT / "pipeline/scripts/maintenance/import-candidate.py") + migrations = _load_script("apply_migrations_for_corpus_resilience", PROJECT_ROOT / "pipeline/scripts/maintenance/apply-migrations.py") + ledger_replay = _load_script("replay_ledger_for_corpus_resilience", PROJECT_ROOT / "pipeline/scripts/maintenance/replay-restriction-ledger.py") + ledger_gate = _load_script("ledger_gate_for_corpus_resilience", PROJECT_ROOT / "pipeline/scripts/maintenance/restriction-ledger-gate.py") + + project = f"uec-resilience-{os.getpid()}-{int(time.time())}" + port = _free_port() + database_url = f"postgresql://uec:uec-e2e@127.0.0.1:{port}/uec?sslmode=disable" + compose = _compose(root, project) + environment = os.environ.copy() + environment["UEC_E2E_DB_PORT"] = str(port) + release_id = "candidate-corpus-resilience" + source_ids = [partition["source_id"] for partition in partitions] + rehearsal_started = time.perf_counter() + migration_started = None + import_started = None + duplicate_started = None + backup_started = None + restore_started = None + replay_started = None + interrupted = False + interrupted_rows = 0 + resumed_rows = 0 + duplicate_new_rows = 0 + peak_loaded_bytes = 0 + max_loaded_rows = 0 + initial_counts: dict[str, int] | None = None + final_counts: dict[str, int] | None = None + dump_bytes = 0 + migration_versions: list[str] = [] + try: + _run(compose + ["up", "-d", "--wait"], env=environment, cwd=root, capture_output=True, text=True) + migration_started = time.perf_counter() + migration_versions = migrations.apply(database_url, root / "pipeline/migrations") + migration_seconds = time.perf_counter() - migration_started + import_started = time.perf_counter() + interruption_partition = max(range(len(partitions)), key=lambda index: partitions[index]["rows"]) + for index, partition in enumerate(partitions): + tracemalloc.start() + manifest, rows = importer.load_inputs(partition["manifest"], partition["normalized"], partition["raw"]) + max_loaded_rows = max(max_loaded_rows, len(rows)) + + def interrupt_after_commit(batch_number: int, _offset: int, batch_count: int) -> None: + nonlocal interrupted, interrupted_rows + interrupted_rows += batch_count + if batch_number >= interrupt_after_batches and interrupt_after_batches: + interrupted = True + raise SimulatedInterruption("synthetic process interruption after committed batch") + + if index == interruption_partition and interrupt_after_batches: + try: + importer.import_candidate( + database_url, manifest, rows, release_id, False, batch_size, + on_batch_committed=interrupt_after_commit, + ) + except SimulatedInterruption: + pass + resumed_rows += importer.import_candidate(database_url, manifest, rows, release_id, False, batch_size) + else: + importer.import_candidate(database_url, manifest, rows, release_id, False, batch_size) + _, peak = tracemalloc.get_traced_memory() + tracemalloc.stop() + peak_loaded_bytes = max(peak_loaded_bytes, peak) + import_seconds = time.perf_counter() - import_started + + with psycopg.connect(database_url) as connection: + initial_counts = _counts(connection, release_id, source_ids) + initial_size = _database_size(connection) + duplicate_started = time.perf_counter() + for partition in partitions: + manifest, rows = importer.load_inputs(partition["manifest"], partition["normalized"], partition["raw"]) + duplicate_new_rows += importer.import_candidate(database_url, manifest, rows, release_id, False, batch_size) + duplicate_seconds = time.perf_counter() - duplicate_started + with psycopg.connect(database_url) as connection: + duplicate_counts = _counts(connection, release_id, source_ids) + if duplicate_counts != initial_counts: + raise ResilienceError("duplicate import changed corpus row counts") + + dump_path = temporary_root / "corpus.dump" + backup_started = time.perf_counter() + with dump_path.open("wb") as handle: + _run(compose + ["exec", "-T", "postgres", "pg_dump", "-U", "uec", "-d", "uec", "--format=custom"], env=environment, cwd=root, stdout=handle, stderr=subprocess.PIPE) + dump_bytes = dump_path.stat().st_size + backup_seconds = time.perf_counter() - backup_started + + restricted_source = partitions[interruption_partition]["source_id"] + restricted_key = _row_key(partitions[interruption_partition]["source"]) + ledger_path = temporary_root / "current-ledger.json" + ledger = _make_ledger(ledger_path, restricted_source, restricted_key, "resilience-current-r1") + replay_started = time.perf_counter() + replay_before_backup = ledger_replay.replay(database_url, ledger_path) + replay_before_backup_seconds = time.perf_counter() - replay_started + with psycopg.connect(database_url) as connection: + restricted_before_restore = connection.execute( + """SELECT count(*) FROM uec.public_access_restricted restricted + JOIN uec.source_records record ON record.source_record_id = restricted.source_record_id + WHERE record.source_id=%s AND record.source_record_key=%s""", + (restricted_source, restricted_key), + ).fetchone()[0] + + _run(compose + ["cp", str(dump_path), "postgres:/tmp/uec-corpus-resilience.dump"], env=environment, cwd=root, capture_output=True, text=True) + restore_started = time.perf_counter() + _run(compose + ["exec", "-T", "postgres", "pg_restore", "-U", "uec", "-d", "uec", "--clean", "--if-exists", "--exit-on-error", "/tmp/uec-corpus-resilience.dump"], env=environment, cwd=root, capture_output=True, text=True) + restore_seconds = time.perf_counter() - restore_started + with psycopg.connect(database_url) as connection: + after_restore_counts = _counts(connection, release_id, source_ids) + restricted_after_restore = connection.execute( + """SELECT count(*) FROM uec.public_access_restricted restricted + JOIN uec.source_records record ON record.source_record_id = restricted.source_record_id + WHERE record.source_id=%s AND record.source_record_key=%s""", + (restricted_source, restricted_key), + ).fetchone()[0] + if after_restore_counts != initial_counts or restricted_after_restore != 0: + raise ResilienceError("restored database did not match the pre-suppression backup state") + + stale_snapshot = { + "ledger_revision": ledger["revision"], + "ledger_sha256": ledger["ledger_sha256"], + "active_restrictions": [], + } + stale_gate_rejected = False + try: + ledger_gate.pre_service_gate(ledger_path, stale_snapshot) + except ledger_gate.RestrictionLedgerError: + stale_gate_rejected = True + if not stale_gate_rejected: + raise ResilienceError("pre-service gate accepted a restored database before replay") + + replay_started = time.perf_counter() + replay_after_restore = ledger_replay.replay(database_url, ledger_path) + replay_after_restore_seconds = time.perf_counter() - replay_started + replayed_snapshot_path = temporary_root / "replayed-snapshot.json" + replayed_snapshot = ledger_replay.write_replayed_snapshot(database_url, ledger_path, replayed_snapshot_path) + ledger_gate.pre_service_gate(ledger_path, json.loads(replayed_snapshot_path.read_text(encoding="utf-8"))) + with psycopg.connect(database_url) as connection: + restricted_after_replay = connection.execute( + """SELECT count(*) FROM uec.public_access_restricted restricted + JOIN uec.source_records record ON record.source_record_id = restricted.source_record_id + WHERE record.source_id=%s AND record.source_record_key=%s""", + (restricted_source, restricted_key), + ).fetchone()[0] + final_counts = _counts(connection, release_id, source_ids) + final_size = _database_size(connection) + if restricted_after_replay != 1 or final_counts != initial_counts: + raise ResilienceError("current suppression replay did not restore the fail-closed state") + total_seconds = time.perf_counter() - rehearsal_started + report = _aggregate_report( + plan, + status="pass", + generated_peak_bytes=generated_peak, + limitations=[ + "The corpus is synthetic and represents row-count/source-shape distribution only; it is not a real acquisition or quality claim.", + "tracemalloc observes Python allocations, not PostgreSQL shared buffers, container RSS, or OS-level peak memory.", + "Backup/restore uses a local custom-format pg_dump inside a disposable PostGIS container; cloud storage, WAL, operator access, and disaster recovery are out of scope.", + ], + ) + report.update({ + "migrations": {"applied": len(migration_versions), "elapsed_seconds": round(migration_seconds, 3)}, + "import": { + "elapsed_seconds": round(import_seconds, 3), + "interruption_triggered": interrupted, + "interrupted_committed_rows_observed": interrupted_rows, + "resumed_new_rows": resumed_rows, + "expected_rows": plan["selected_records"], + "batch_size": batch_size, + }, + "duplicate_import": { + "elapsed_seconds": round(duplicate_seconds, 3), + "new_rows": duplicate_new_rows, + "counts_unchanged": duplicate_counts == initial_counts, + }, + "backup_restore": { + "backup_elapsed_seconds": round(backup_seconds, 3), + "restore_elapsed_seconds": round(restore_seconds, 3), + "replay_before_backup_elapsed_seconds": round(replay_before_backup_seconds, 3), + "replay_after_restore_elapsed_seconds": round(replay_after_restore_seconds, 3), + "custom_dump_bytes": dump_bytes, + "pre_restore_suppressed_rows": restricted_before_restore, + "restored_suppressed_rows_before_replay": restricted_after_restore, + "stale_pre_service_gate_rejected": stale_gate_rejected, + "replay_before_restore_new_events": replay_before_backup["new_events"], + "replay_after_restore_new_events": replay_after_restore["new_events"], + "replayed_snapshot_reference_count": replayed_snapshot["reference_count"], + "suppressed_rows_after_replay": restricted_after_replay, + }, + "database": { + "initial_size_bytes": initial_size, + "final_size_bytes": final_size, + "row_counts": final_counts, + }, + "runtime_seconds": round(total_seconds, 3), + "memory_observation": { + "method": "Python tracemalloc peak during one-partition load/import", + "synthetic_generation_peak_bytes": generated_peak, + "load_import_peak_bytes": peak_loaded_bytes, + "max_loaded_partition_rows": max_loaded_rows, + }, + }) + return report + finally: + try: + _run(compose + ["down", "-v", "--remove-orphans"], env=environment, cwd=root, capture_output=True, text=True, check=False) + except ResilienceError: + # If startup failed because Docker is unavailable there is no + # disposable project to tear down; preserve the useful error. + pass + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--distribution", type=Path, default=DEFAULT_DISTRIBUTION) + parser.add_argument("--max-records", type=int, default=MAX_RECORDS) + parser.add_argument("--batch-size", type=int, default=DEFAULT_BATCH_SIZE) + parser.add_argument("--interrupt-after-batches", type=int, default=1) + parser.add_argument("--plan-only", action="store_true", help="generate and measure the disposable synthetic plan without Docker/Postgres") + parser.add_argument("--json-output", type=Path) + args = parser.parse_args() + report = run_rehearsal( + distribution=args.distribution, + max_records=args.max_records, + batch_size=args.batch_size, + interrupt_after_batches=args.interrupt_after_batches, + plan_only=args.plan_only, + ) + encoded = json.dumps(report, sort_keys=True, indent=2) + "\n" + if args.json_output: + args.json_output.parent.mkdir(parents=True, exist_ok=True) + args.json_output.write_text(encoded, encoding="utf-8") + print(json.dumps({ + "status": report["status"], + "selected_records": report["corpus"]["selected_records"], + "source_profiles": report["corpus"]["source_profiles"], + "runtime_seconds": report.get("runtime_seconds"), + "database_size_bytes": report.get("database", {}).get("final_size_bytes"), + }, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/pipeline/scripts/maintenance/import-candidate.py b/pipeline/scripts/maintenance/import-candidate.py index e3d2e7f..d54c64c 100644 --- a/pipeline/scripts/maintenance/import-candidate.py +++ b/pipeline/scripts/maintenance/import-candidate.py @@ -15,6 +15,7 @@ from pathlib import Path from urllib.parse import urlparse import uuid +from collections.abc import Callable import psycopg @@ -25,6 +26,7 @@ class CandidateImportError(ValueError): DISPOSABLE_MARKER = "uec-e2e-disposable-v1" DEFAULT_BATCH_SIZE = 500 +BatchCommitObserver = Callable[[int, int, int], None] def require_disposable_database(database_url: str, acknowledged: bool) -> None: @@ -114,7 +116,8 @@ def _stable_uuid(*parts: object) -> uuid.UUID: def import_candidate(database_url: str, manifest: dict, rows: list[dict], release_id: str, - reset: bool, batch_size: int = DEFAULT_BATCH_SIZE) -> int: + reset: bool, batch_size: int = DEFAULT_BATCH_SIZE, + on_batch_committed: BatchCommitObserver | None = None) -> int: """Append one candidate release; never promotes or marks review complete. Batches commit independently so a bounded failure can resume with the same @@ -205,6 +208,11 @@ def import_candidate(database_url: str, manifest: dict, rows: list[dict], releas (record_id, release_id)) batch_count += 1 count += batch_count + if on_batch_committed is not None: + # The transaction has committed before this callback runs. + # Rehearsals can therefore raise here to model a process loss + # without making a test-specific database mutation path. + on_batch_committed(offset // batch_size + 1, offset, batch_count) return count diff --git a/pipeline/tests/e2e/test_corpus_resilience.py b/pipeline/tests/e2e/test_corpus_resilience.py new file mode 100644 index 0000000..a4ac4e5 --- /dev/null +++ b/pipeline/tests/e2e/test_corpus_resilience.py @@ -0,0 +1,42 @@ +import os +import unittest +from pathlib import Path + +from pipeline.scripts.benchmarks import run_corpus_resilience + + +class CorpusResilienceE2ETests(unittest.TestCase): + @classmethod + def setUpClass(cls): + if os.environ.get("UEC_RUN_E2E") != "1" or os.environ.get("UEC_RUN_CORPUS_RESILIENCE") != "1": + raise unittest.SkipTest( + "set UEC_RUN_E2E=1 and UEC_RUN_CORPUS_RESILIENCE=1 to run the Docker-backed corpus rehearsal" + ) + cls.report = run_corpus_resilience.run_rehearsal( + root=Path(__file__).parents[3], + max_records=int(os.environ.get("UEC_RESILIENCE_RECORDS", "50000")), + ) + + def test_interruption_resumes_and_duplicate_import_is_idempotent(self): + self.assertEqual(self.report["status"], "pass") + self.assertTrue(self.report["import"]["interruption_triggered"]) + self.assertEqual(self.report["import"]["expected_rows"], self.report["database"]["row_counts"]["source_records"]) + self.assertEqual(self.report["duplicate_import"]["new_rows"], 0) + self.assertTrue(self.report["duplicate_import"]["counts_unchanged"]) + + def test_backup_restore_replays_current_suppression_before_service(self): + backup = self.report["backup_restore"] + self.assertTrue(backup["stale_pre_service_gate_rejected"]) + self.assertEqual(backup["restored_suppressed_rows_before_replay"], 0) + self.assertEqual(backup["suppressed_rows_after_replay"], 1) + self.assertGreater(backup["custom_dump_bytes"], 0) + + def test_report_contains_runtime_size_and_bounded_memory_observations(self): + self.assertEqual(self.report["migrations"]["applied"], 34) + self.assertGreaterEqual(self.report["runtime_seconds"], 0) + self.assertGreater(self.report["database"]["final_size_bytes"], 0) + self.assertGreater(self.report["memory_observation"]["load_import_peak_bytes"], 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/tests/test_corpus_resilience.py b/pipeline/tests/test_corpus_resilience.py new file mode 100644 index 0000000..0642460 --- /dev/null +++ b/pipeline/tests/test_corpus_resilience.py @@ -0,0 +1,68 @@ +import importlib.util +import json +import tempfile +import unittest +from pathlib import Path + + +ROOT = Path(__file__).parents[1] +SPEC = importlib.util.spec_from_file_location( + "run_corpus_resilience", + ROOT / "scripts" / "benchmarks" / "run_corpus_resilience.py", +) +MODULE = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(MODULE) + + +class CorpusResiliencePlanTests(unittest.TestCase): + def distribution(self, root: Path) -> Path: + path = root / "distribution.json" + path.write_text(json.dumps({ + "coverage": { + "selected_records_by_source": { + "v1.aa.locations": 6, + "v1.bb.locations": 3, + "v1.cc.locations": 1, + } + } + }), encoding="utf-8") + return path + + def test_selection_is_bounded_and_preserves_distribution_shape(self): + with tempfile.TemporaryDirectory() as directory: + plan = MODULE.build_plan(self.distribution(Path(directory)), max_records=5) + self.assertEqual(plan["available_records"], 10) + self.assertEqual(plan["selected_records"], 5) + self.assertEqual(plan["selected_records_by_source"], { + "v1.aa.locations": 3, + "v1.bb.locations": 2, + }) + with self.assertRaises(MODULE.ResilienceError): + MODULE.build_plan(self.distribution(Path(directory)), max_records=MODULE.MAX_RECORDS + 1) + + def test_synthetic_corpus_is_temporary_and_report_is_row_free(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + report_path = self.distribution(root) + report = MODULE.run_rehearsal(distribution=report_path, max_records=3, plan_only=True) + encoded = json.dumps(report) + self.assertEqual(report["status"], "plan-only") + self.assertEqual(report["corpus"]["selected_records"], 3) + for forbidden in ("establishment_id", "Synthetic resilience facility", "source_record_key", "raw_fields"): + self.assertNotIn(forbidden, encoded) + self.assertFalse((root / "corpus").exists()) + + def test_interruption_exception_is_reserved_for_post_commit_observer(self): + observations = [] + + def after_commit(batch_number, offset, count): + observations.append((batch_number, offset, count)) + raise MODULE.SimulatedInterruption("test interruption") + + with self.assertRaises(MODULE.SimulatedInterruption): + after_commit(1, 0, 2) + self.assertEqual(observations, [(1, 0, 2)]) + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/tests/test_import_candidate.py b/pipeline/tests/test_import_candidate.py index 52a3c04..e4f7c4f 100644 --- a/pipeline/tests/test_import_candidate.py +++ b/pipeline/tests/test_import_candidate.py @@ -3,7 +3,9 @@ import json import tempfile import unittest +import uuid from pathlib import Path +from unittest.mock import patch from pipeline.sources.uk.fsa_approved.adapter import FsaApprovedEstablishmentsAdapter @@ -14,6 +16,101 @@ class CandidateImportContractTests(unittest.TestCase): + def test_post_commit_interruption_observer_resumes_without_duplicate_rows(self): + class Result: + def __init__(self, row=None): + self.row = row + + def fetchone(self): + return self.row + + class Transaction: + def __init__(self, connection): + self.connection = connection + + def __enter__(self): + self.connection.transaction_depth += 1 + return self + + def __exit__(self, exc_type, *_args): + self.connection.transaction_events.append(exc_type is not None) + self.connection.transaction_depth -= 1 + return False + + class Connection: + def __init__(self): + self.transaction_depth = 0 + self.transaction_events = [] + self.records = {} + self.observations = {} + self.artifact_id = uuid.uuid4() + self.run_id = uuid.uuid4() + + def __enter__(self): + return self + + def __exit__(self, *_args): + return False + + def commit(self): + pass + + def transaction(self): + return Transaction(self) + + def execute(self, query, params=None): + compact = " ".join(query.split()) + if "FROM uec.disposable_import_guard" in compact: + return Result((MODULE.DISPOSABLE_MARKER,)) + if compact.startswith("SELECT artifact_id FROM uec.raw_artifacts"): + return Result((self.artifact_id,)) + if compact.startswith("SELECT run_id FROM uec.acquisition_runs"): + return Result((self.run_id,)) + if compact.startswith("INSERT INTO uec.source_records"): + key = (params[1], params[2], params[3]) + if key in self.records: + return Result() + self.records[key] = params[0] + return Result((params[0],)) + if compact.startswith("SELECT source_record_id FROM uec.source_records"): + return Result((self.records[(params[0], params[1], params[2])],)) + if compact.startswith("INSERT INTO uec.observations"): + key = (params[1], params[2], params[3]) + if key in self.observations: + return Result() + self.observations[key] = params[0] + return Result((params[0],)) + if compact.startswith("SELECT observation_id FROM uec.observations"): + return Result((self.observations[(params[0], params[1], params[2])],)) + return Result() + + rows = [ + {"source_id": "synthetic", "source_row": 1, "normalized": {"establishment_id": "A", "trading_name": "A"}}, + {"source_id": "synthetic", "source_row": 2, "normalized": {"establishment_id": "B", "trading_name": "B"}}, + ] + manifest = { + "source_id": "synthetic", "source_url": "https://example.invalid/synthetic", + "retrieved_at_utc": "2026-09-16T00:00:00Z", "checksum_sha256": "a" * 64, + "byte_size": 1, "config_version": "test", "country_code": "ZZ", + } + connection = Connection() + observed = [] + + def interrupt(batch_number, offset, batch_count): + observed.append((batch_number, offset, batch_count)) + raise RuntimeError("synthetic process interruption") + + with patch.object(MODULE.psycopg, "connect", return_value=connection): + with self.assertRaisesRegex(RuntimeError, "synthetic process interruption"): + MODULE.import_candidate("postgresql://loopback", manifest, rows, "candidate-test", False, 1, interrupt) + resumed = MODULE.import_candidate("postgresql://loopback", manifest, rows, "candidate-test", False, 1) + duplicate = MODULE.import_candidate("postgresql://loopback", manifest, rows, "candidate-test", False, 1) + + self.assertEqual(observed, [(1, 0, 1)]) + self.assertEqual(resumed, 1) + self.assertEqual(duplicate, 0) + self.assertEqual(connection.transaction_events, [False, False, False, False, False]) + def test_server_marker_is_required_even_after_cli_acknowledgement(self): class FakeConnection: def execute(self, *_args): From ef2fd02d8b09781457f958ab396da89f6f0200c6 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Wed, 16 Sep 2026 23:01:55 -0700 Subject: [PATCH 207/311] Add current-corpus geospatial readiness audit --- docs/current-geospatial-readiness.md | 71 +++ pipeline/scripts/README.md | 18 + .../current_geospatial_readiness.py | 418 ++++++++++++++++++ pipeline/tests/e2e/test_seeded_api.py | 29 +- .../test_current_geospatial_readiness.py | 65 +++ 5 files changed, 600 insertions(+), 1 deletion(-) create mode 100644 docs/current-geospatial-readiness.md create mode 100644 pipeline/scripts/diagnostics/current_geospatial_readiness.py create mode 100644 pipeline/tests/test_current_geospatial_readiness.py diff --git a/docs/current-geospatial-readiness.md b/docs/current-geospatial-readiness.md new file mode 100644 index 0000000..ee83426 --- /dev/null +++ b/docs/current-geospatial-readiness.md @@ -0,0 +1,71 @@ +# Current-corpus geospatial readiness + +`pipeline/scripts/diagnostics/current_geospatial_readiness.py` audits private +candidate handoffs listed by a row-free current-corpus manifest. It is an +offline diagnostic: it does not call a geocoder, modify the database, create a +release, or make a publication decision. + +Run it from the repository root after a private candidate acquisition: + +```powershell +python pipeline/scripts/diagnostics/current_geospatial_readiness.py ` + --manifest data/manifests/current-reacquisition-2026-09-16.json ` + --root . ` + --output data/reports/current-geospatial-readiness.json ` + --as-of 2026-09-16T00:00:00Z +``` + +The manifest is the source inventory. Each listed source is reported as either +an available normalized handoff or `unavailable_private_handoff`; missing +private artifacts are never silently counted as zero rows. The report contains +only source IDs, declared counts, file hashes/bytes, and aggregate metrics. +It excludes source identifiers, names, addresses, coordinates, geocoder +queries/responses, and row samples. + +The report has two complementary views: + +- `display_states` are mutually exclusive: `exact`, `city`, `unmapped`, or + `restricted`. A restricted record is never made safe merely by having a + coordinate. +- `evidence_states` overlap intentionally. A row may have a source coordinate + and an accepted geocoder result, so both facts are counted rather than one + overwriting the other. Source coordinates held in restricted source values + are counted as evidence without copying those values into the report; a + source-value-present row whose numeric point is not exposed is counted as + `source_coordinate_pending_review`. Invalid coordinates are never repaired. + +Each source also reports `suppressed_rows` explicitly. This is the count of +whole-record restriction signals in the private handoff; a coordinate withheld +for privacy is kept in the coordinate-review queue and is not automatically +treated as a whole-record suppression. + +The privacy and coordinate review queues are conservative indicators for human +review. They are not residential classifications, factual approval, or +publication eligibility. An accepted geocode is evidence about location only; +it does not grant project approval or release permission. City values produce +coarse display readiness only when no exact point is eligible; no city point is +invented by this diagnostic. + +## Current evidence and limits + +The checked-in current-reacquisition manifest lists eight private source +profiles, with seven normalized profiles and one raw-only CFIA workbook. The +normalized row counts are provenance declarations from the private rehearsal; +the actual normalized handoffs remain ignored local research inputs. Running +the audit without those local handoffs therefore reports the sources as +unavailable rather than claiming coordinate coverage. A run with local +handoffs is required to produce current-corpus coordinate percentages. + +The current geocoding registry contains one engine: Denmark's DAWA adapter. +Its development configuration is one request per second with result caching, +30-second request timeout, append-only attempts, retryable transport failures, +and explicit review for multiple matches. It is approved for development only +and must be reevaluated before production. No production cost or provider +retention claim is made by the project; providers and terms must be assessed +per country before enabling a new engine. + +The API contract independently proves the same boundary: exact records may +carry coordinates, city records may carry only a coarse reference point, an +unmapped record carries null coordinates, and a restricted record is absent +from list, detail, and export surfaces. These are tested in the seeded API E2E +suite; the diagnostic does not replace those database-backed tests. diff --git a/pipeline/scripts/README.md b/pipeline/scripts/README.md index 996f09a..3595a3c 100644 --- a/pipeline/scripts/README.md +++ b/pipeline/scripts/README.md @@ -40,6 +40,24 @@ python pipeline/scripts/stages/geocode-worker.py --provider dawa --limit 5 --del ``` The worker writes append-only job events and geocode attempts. It does not modify source records or observations. New providers should implement the adapter contract in `pipeline/geocoding/` and reuse the worker’s lifecycle, retry, logging, and persistence behavior. + +## Current-corpus geospatial readiness + +Audit private normalized candidate handoffs without emitting rows: + +```powershell +python pipeline/scripts/diagnostics/current_geospatial_readiness.py ` + --manifest data/manifests/current-reacquisition-2026-09-16.json ` + --root . ` + --output data/reports/current-geospatial-readiness.json +``` + +The report distinguishes source coordinates, accepted geocodes, coarse/city +signals, unresolved/invalid values, whole-record restrictions, and human review +queues. It reports missing private handoffs explicitly and never treats +geocoding success as publication permission. See +`docs/current-geospatial-readiness.md` for the report contract and provider +limitations. # Private environment controls `maintenance/private-environment-gate.py` is the clean-checkout and diff --git a/pipeline/scripts/diagnostics/current_geospatial_readiness.py b/pipeline/scripts/diagnostics/current_geospatial_readiness.py new file mode 100644 index 0000000..b973499 --- /dev/null +++ b/pipeline/scripts/diagnostics/current_geospatial_readiness.py @@ -0,0 +1,418 @@ +#!/usr/bin/env python3 +"""Audit current private candidate handoffs without emitting record data. + +The candidate handoff is deliberately the input boundary here. Raw source +artifacts, addresses, identifiers, geocoder queries, and responses stay in +the operator's restricted staging area. The output contains only source +metadata and aggregate counts, so it is safe to commit as a diagnostic +contract or attach to a review packet. + +The audit is intentionally conservative: + +* source coordinates are counted separately from geocoder results; +* invalid values are not repaired or converted into an approximate point; +* a city label is a coarse location, never an inferred coordinate; +* coordinate/privacy review queues are not treated as approval; +* an absent private handoff is reported as unavailable, not as zero rows; +* malformed rows increment an explicit parse-error counter instead of being + silently skipped. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import math +from collections import Counter +from pathlib import Path +from typing import Any + + +REPORT_VERSION = "current-geospatial-readiness-v1" +DISPLAY_STATES = ("exact", "city", "unmapped", "restricted") +EVIDENCE_STATES = ( + "source_coordinate_valid", + "source_coordinate_invalid", + "source_coordinate_pending_review", + "accepted_geocode_exact", + "accepted_geocode_coarse", + "geocode_unresolved", + "coarse_city_location", + "unresolved", +) + +_EXACT_PRECISIONS = frozenset({"exact", "rooftop", "parcel", "building", "address", "address_point"}) +_COARSE_PRECISIONS = frozenset({"city", "coarse", "approximate", "region", "postal_code"}) +_PRIVACY_REVIEW_VALUES = frozenset({"pending", "review_required", "required", "pending-review", "restricted-withheld-address"}) +_WHOLE_RECORD_RESTRICTION_VALUES = frozenset({"restricted", "failed", "suppressed", "blocked"}) +_REVIEW_REASONS = frozenset({ + "address_privacy_risk", + "privacy_review_required", + "residential_or_private_term", + "care_of_or_mailbox", + "unit_or_apartment", + "mixed_use_or_unclear_location", +}) + + +def _mapping(value: Any) -> dict[str, Any]: + return value if isinstance(value, dict) else {} + + +def _first(mapping: dict[str, Any], *keys: str) -> Any: + for key in keys: + value = mapping.get(key) + if value is not None and value != "": + return value + return None + + +def _number(value: Any) -> float | None: + if isinstance(value, bool): + return None + try: + parsed = float(value) + except (TypeError, ValueError): + return None + return parsed if math.isfinite(parsed) else None + + +def _point(value: Any) -> tuple[float | None, float | None, bool]: + """Return latitude, longitude, and whether a coordinate value was present.""" + if isinstance(value, dict): + present = any(key in value for key in ("latitude", "lat", "longitude", "lon", "lng", "x", "y")) + latitude = _number(_first(value, "latitude", "lat", "y")) + longitude = _number(_first(value, "longitude", "lon", "lng", "x")) + return latitude, longitude, present + if isinstance(value, (list, tuple)) and len(value) == 2: + # GeoJSON/source arrays are longitude, latitude. + return _number(value[1]), _number(value[0]), True + return None, None, value is not None + + +def _valid_point(latitude: float | None, longitude: float | None) -> bool: + return ( + latitude is not None + and longitude is not None + and -90 <= latitude <= 90 + and -180 <= longitude <= 180 + and not (latitude == 0 and longitude == 0) + ) + + +def _coordinate_from(normalized: dict[str, Any], row: dict[str, Any], *, source: bool) -> tuple[float | None, float | None, bool]: + nested = _first(normalized, "coordinates", "coordinate", "location") + latitude, longitude, present = _point(nested) + if present: + return latitude, longitude, True + fields = ("source_latitude", "source_longitude") if source else ("latitude", "longitude") + source_values = _mapping(row.get("source_values")) + coordinate_fields = ("latitude", "lat", "Geo_Lat", "latitudine") if source else ("latitude", "lat") + longitude_fields = ("longitude", "lon", "lng", "Geo_Lng", "longitudine") if source else ("longitude", "lon", "lng") + latitude = _number(_first(normalized, *fields, *coordinate_fields)) + longitude = _number(_first(normalized, fields[1], *longitude_fields)) + if source and (latitude is None and longitude is None): + latitude = _number(_first(source_values, *coordinate_fields)) + longitude = _number(_first(source_values, *longitude_fields)) + if latitude is not None or longitude is not None: + return latitude, longitude, True + # A few source adapters keep the coordinate object on the envelope. + return _point(_first(row, "coordinates", "coordinate")) + + +def _geocode(normalized: dict[str, Any], row: dict[str, Any]) -> tuple[str | None, str | None, bool]: + result = _mapping(_first(normalized, "geocode", "geocoding", "geocode_result")) + status = _first(result, "status") or _first(normalized, "geocode_status", "geocoding_status") + precision = _first(result, "precision") or _first(normalized, "coordinate_precision", "geocode_precision") + point_value = _first(result, "result", "coordinates", "location") + if point_value is None: + point_value = _first(row, "geocode", "geocoding") + latitude, longitude, present = _point(point_value) + return (str(status).lower() if status is not None else None, + str(precision).lower() if precision is not None else None, + _valid_point(latitude, longitude) if present else False) + + +def _city_present(normalized: dict[str, Any]) -> bool: + return bool(_first(normalized, "city", "municipality", "town", "commune", "postal_code", "postcode")) + + +def _whole_record_suppressed(normalized: dict[str, Any], row: dict[str, Any]) -> bool: + if any(_first(mapping, "suppressed", "is_suppressed") is True for mapping in (normalized, row)): + return True + values = ( + _first(normalized, "privacy_status", "publication_status", "suppression_status"), + _first(row, "privacy_status", "publication_status", "suppression_status"), + ) + return any(str(value).lower() in _WHOLE_RECORD_RESTRICTION_VALUES for value in values if value is not None) + + +def _privacy_reasons(normalized: dict[str, Any], row: dict[str, Any]) -> tuple[str, ...]: + reasons: set[str] = set() + for mapping in (normalized, row): + value = _first(mapping, "privacy_gate", "privacy_status", "privacy_review_status") + if value is not None and str(value).lower() in _PRIVACY_REVIEW_VALUES: + reasons.add(str(value).lower()) + flags = _first(mapping, "privacy_review_reasons", "privacy_flags", "review_reasons", "reasons") + if isinstance(flags, str): + flags = (flags,) + if isinstance(flags, (list, tuple, set)): + reasons.update(str(flag) for flag in flags if str(flag) in _REVIEW_REASONS) + return tuple(sorted(reasons)) + + +def _coordinate_review_reasons(normalized: dict[str, Any], row: dict[str, Any]) -> tuple[str, ...]: + reasons: set[str] = set() + for mapping in (normalized, row): + gate = _first(mapping, "coordinate_gate", "coordinate_review_status", "coordinate_state") + if gate is not None and str(gate).lower() in {"review_required", "pending", "withheld", "restricted-withheld-address", "unresolved", "not-supplied-by-source"}: + reasons.add(str(gate).lower()) + return tuple(sorted(reasons)) + + +def classify_record(row: dict[str, Any]) -> dict[str, Any]: + """Classify one candidate row without retaining any row value.""" + normalized = _mapping(row.get("normalized")) or row + suppressed = _whole_record_suppressed(normalized, row) + privacy_reasons = _privacy_reasons(normalized, row) + coordinate_review_reasons = _coordinate_review_reasons(normalized, row) + + source_latitude, source_longitude, source_present = _coordinate_from(normalized, row, source=True) + source_valid = _valid_point(source_latitude, source_longitude) + source_state = str(_first(normalized, "coordinate_state", "coordinate_gate") or "").lower() + source_pending = source_present and not source_valid and "pending" in source_state + source_invalid = source_present and not source_valid and not source_pending + geocode_status, geocode_precision, geocode_valid = _geocode(normalized, row) + accepted_geocode = geocode_status == "accepted" and geocode_valid + geocode_exact = accepted_geocode and geocode_precision in _EXACT_PRECISIONS + geocode_coarse = accepted_geocode and geocode_precision in _COARSE_PRECISIONS + coarse_signal = ( + str(_first(normalized, "display_precision", "coordinate_precision", "coordinate_state") or "").lower() in _COARSE_PRECISIONS + or str(_first(normalized, "display_precision", "coordinate_precision", "coordinate_state") or "").lower() in {"city", "coarse", "approximate"} + ) + + if suppressed: + display_state = "restricted" + elif source_valid or geocode_exact: + display_state = "exact" + elif geocode_coarse or coarse_signal or _city_present(normalized): + display_state = "city" + else: + display_state = "unmapped" + + evidence = Counter() + if source_valid: + evidence["source_coordinate_valid"] += 1 + if source_invalid: + evidence["source_coordinate_invalid"] += 1 + if source_pending: + evidence["source_coordinate_pending_review"] += 1 + if geocode_exact: + evidence["accepted_geocode_exact"] += 1 + elif geocode_coarse: + evidence["accepted_geocode_coarse"] += 1 + elif geocode_status in {"unresolved", "failed", "review_required"} or not (source_valid or accepted_geocode): + evidence["geocode_unresolved"] += 1 + if geocode_coarse or coarse_signal or (not source_valid and not accepted_geocode and _city_present(normalized)): + evidence["coarse_city_location"] += 1 + if display_state == "unmapped": + evidence["unresolved"] += 1 + + identifier = _first(normalized, "establishment_id", "source_record_key", "source_id") + return { + "display_state": display_state, + "suppressed": suppressed, + "evidence": dict(evidence), + "privacy_reasons": privacy_reasons, + "coordinate_review_reasons": coordinate_review_reasons, + "has_source_identifier": bool(identifier), + } + + +def _resolve(path_value: str | None, root: Path) -> Path | None: + if not path_value: + return None + path = Path(path_value) + return path if path.is_absolute() else root / path + + +def _candidate_manifest_path(entry: dict[str, Any], root: Path) -> Path | None: + for key in ("candidate_handoff_manifest", "private_manifest", "manifest"): + resolved = _resolve(entry.get(key), root) + if resolved and resolved.is_file(): + return resolved + return None + + +def _normalized_path(entry: dict[str, Any], root: Path) -> tuple[Path | None, Path | None]: + manifest_path = _candidate_manifest_path(entry, root) + if manifest_path is None: + return None, None + manifest = _mapping(json.loads(manifest_path.read_text(encoding="utf-8"))) + explicit = _resolve(manifest.get("normalized_path"), root) + candidates = [ + explicit, + manifest_path.parent / "normalized" / "records.jsonl", + manifest_path.parent / "normalized" / "normalized-records.jsonl", + ] + return next((candidate for candidate in candidates if candidate and candidate.is_file()), None), manifest_path + + +def _audit_file(path: Path) -> dict[str, Any]: + evidence = Counter() + display = Counter() + privacy = Counter() + coordinate_review = Counter() + privacy_rows = 0 + coordinate_review_rows = 0 + suppressed_rows = 0 + records = identifiers = parse_errors = 0 + with path.open(encoding="utf-8") as handle: + for line in handle: + if not line.strip(): + continue + try: + row = json.loads(line) + except json.JSONDecodeError: + parse_errors += 1 + continue + if not isinstance(row, dict): + parse_errors += 1 + continue + records += 1 + result = classify_record(row) + display[result["display_state"]] += 1 + suppressed_rows += int(result["suppressed"]) + evidence.update(result["evidence"]) + privacy.update(result["privacy_reasons"]) + coordinate_review.update(result["coordinate_review_reasons"]) + privacy_rows += bool(result["privacy_reasons"]) + coordinate_review_rows += bool(result["coordinate_review_reasons"]) + identifiers += int(result["has_source_identifier"]) + return { + "available": True, + "records": records, + "parse_errors": parse_errors, + "source_local_identifier_rows": identifiers, + "suppressed_rows": suppressed_rows, + "display_states": {state: display[state] for state in DISPLAY_STATES}, + "evidence_states": {state: evidence[state] for state in EVIDENCE_STATES}, + "privacy_review_queue": {"rows": privacy_rows, "by_reason": dict(sorted(privacy.items()))}, + "coordinate_review_queue": {"rows": coordinate_review_rows, "by_reason": dict(sorted(coordinate_review.items()))}, + "normalized_sha256": hashlib.sha256(path.read_bytes()).hexdigest(), + "normalized_bytes": path.stat().st_size, + "audit_status": "complete" if parse_errors == 0 else "incomplete_parse_errors", + } + + +def _empty_unavailable(entry: dict[str, Any], declared_manifest: dict[str, Any] | None) -> dict[str, Any]: + return { + "available": False, + "records": None, + "declared_normalized_rows": declared_manifest.get("normalized_rows") if declared_manifest else entry.get("normalized_rows"), + "parse_errors": 0, + "source_local_identifier_rows": None, + "suppressed_rows": None, + "display_states": None, + "evidence_states": None, + "privacy_review_queue": None, + "coordinate_review_queue": None, + "audit_status": "unavailable_private_handoff", + } + + +def audit_current(manifest_path: Path, root: Path = Path("."), as_of: str | None = None) -> dict[str, Any]: + """Audit every source listed by a current private-candidate manifest.""" + manifest = _mapping(json.loads(manifest_path.read_text(encoding="utf-8"))) + entries = manifest.get("sources") + if not isinstance(entries, list): + raise ValueError("current candidate manifest must contain a sources list") + + sources: list[dict[str, Any]] = [] + totals = Counter() + unavailable = 0 + incomplete = 0 + for entry in sorted((item for item in entries if isinstance(item, dict)), key=lambda item: str(item.get("source_id", ""))): + source_id = str(entry.get("source_id") or "unknown") + normalized_path, handoff_manifest = _normalized_path(entry, root) + if normalized_path is None: + declared = None + if handoff_manifest and handoff_manifest.is_file(): + declared = _mapping(json.loads(handoff_manifest.read_text(encoding="utf-8"))) + metrics = _empty_unavailable(entry, declared) + unavailable += 1 + else: + metrics = _audit_file(normalized_path) + totals["records"] += metrics["records"] + totals["parse_errors"] += metrics["parse_errors"] + totals["source_local_identifier_rows"] += metrics["source_local_identifier_rows"] + totals["suppressed_rows"] += metrics["suppressed_rows"] + for key, value in metrics["display_states"].items(): + totals[f"display_{key}"] += value + for key, value in metrics["evidence_states"].items(): + totals[f"evidence_{key}"] += value + totals["privacy_review_rows"] += metrics["privacy_review_queue"]["rows"] + totals["coordinate_review_rows"] += metrics["coordinate_review_queue"]["rows"] + incomplete += metrics["audit_status"] != "complete" + sources.append({ + "source_id": source_id, + "country_code": entry.get("country_code"), + "declared_input_rows": entry.get("input_rows"), + "declared_normalized_rows": entry.get("normalized_rows"), + "declared_quarantined_rows": entry.get("quarantined_rows"), + "acquisition_state": entry.get("status") or entry.get("acquisition"), + "publication_state": entry.get("publication_state") or "private-candidate", + "metrics": metrics, + }) + + return { + "report_version": REPORT_VERSION, + "as_of_utc": as_of, + "manifest_sha256": hashlib.sha256(manifest_path.read_bytes()).hexdigest(), + "candidate_state": "private-only; no publication decision implied", + "privacy_boundary": "aggregate-only; normalized rows, raw artifacts, addresses, coordinates, identifiers, and geocoder payloads are excluded", + "sources_listed": len(sources), + "sources_available": len(sources) - unavailable, + "sources_unavailable": unavailable, + "sources_with_parse_errors": incomplete, + "totals_available_rows_only": dict(sorted(totals.items())), + "sources": sources, + "semantics": { + "display_states": "Mutually exclusive current candidate display states; restricted takes precedence. Exact means a valid source point or accepted exact geocode, city means a declared city/coarse signal without an exact point, and unmapped means no displayable location evidence.", + "evidence_states": "Overlapping evidence counters. A row may have source coordinates and a geocoder result; these counters intentionally preserve both facts.", + "privacy_review_queue": "Conservative human-review indicators, not a residential determination or publication approval.", + "coordinate_review_queue": "Rows needing coordinate/privacy review or explicitly unresolved coordinate handling; geocoding success alone never grants release eligibility.", + "unavailable_sources": "A missing private handoff is not counted as zero and does not establish no coverage.", + }, + "publication": { + "release_created": False, + "release_promoted": False, + "public_api_rows": 0, + "project_approval": "not-approved", + }, + } + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--manifest", type=Path, required=True, help="row-free current candidate manifest") + parser.add_argument("--root", type=Path, default=Path("."), help="root used to resolve private staging paths") + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--as-of", default=None) + args = parser.parse_args() + report = audit_current(args.manifest, args.root, args.as_of) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(json.dumps({ + "sources_listed": report["sources_listed"], + "sources_available": report["sources_available"], + "available_rows": report["totals_available_rows_only"].get("records", 0), + "privacy_review_rows": report["totals_available_rows_only"].get("privacy_review_rows", 0), + "publication": report["publication"], + }, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/pipeline/tests/e2e/test_seeded_api.py b/pipeline/tests/e2e/test_seeded_api.py index c9fde64..a7b7dc9 100644 --- a/pipeline/tests/e2e/test_seeded_api.py +++ b/pipeline/tests/e2e/test_seeded_api.py @@ -1,4 +1,4 @@ -import json, os, subprocess, sys, unittest, urllib.request +import json, os, subprocess, sys, unittest, urllib.error, urllib.request import uuid from datetime import datetime, timezone import psycopg @@ -20,10 +20,37 @@ def get(self, path): def test_exact_city_and_unmapped_are_distinct(self): rows = self.get('/api/v2/locations?limit=100')['data'] self.assertEqual({r['display_precision'] for r in rows}, {'exact','city','unmapped'}) + + def test_location_precision_contract_never_emits_a_point_for_unmapped(self): + rows = {row['canonical_name']: row for row in self.get('/api/v2/locations?limit=100')['data']} + exact = rows['E2E exact'] + city = rows['E2E city'] + unmapped = rows['E2E unmapped'] + self.assertEqual(exact['display_precision'], 'exact') + self.assertIsInstance(exact['latitude'], float) + self.assertIsInstance(exact['longitude'], float) + self.assertEqual(city['display_precision'], 'city') + self.assertIsInstance(city['latitude'], float) + self.assertIsInstance(city['longitude'], float) + self.assertEqual(unmapped['display_precision'], 'unmapped') + self.assertIsNone(unmapped['latitude']) + self.assertIsNone(unmapped['longitude']) + for row in (exact, city, unmapped): + self.assertNotIn('source_values', row) + def test_restricted_record_is_absent(self): names = {r['canonical_name'] for r in self.get('/api/v2/locations?limit=100')['data']} self.assertNotIn('E2E restricted', names) + def test_restricted_detail_is_not_relabelled_or_exposed(self): + names = {r['canonical_name'] for r in self.get('/api/v2/locations?limit=100')['data']} + self.assertNotIn('E2E restricted', names) + with psycopg.connect(self.env.database_url) as db: + facility_id = db.execute("SELECT facility_id FROM uec.facilities WHERE canonical_name='E2E restricted'").fetchone()[0] + with self.assertRaises(urllib.error.HTTPError) as error: + self.get(f'/api/v2/locations/{facility_id}') + self.assertEqual(error.exception.code, 404) + def test_official_record_without_publication_approval_is_absent(self): names = {r['canonical_name'] for r in self.get('/api/v2/locations?limit=100')['data']} self.assertNotIn('E2E unapproved', names) diff --git a/pipeline/tests/test_current_geospatial_readiness.py b/pipeline/tests/test_current_geospatial_readiness.py new file mode 100644 index 0000000..a1dcb07 --- /dev/null +++ b/pipeline/tests/test_current_geospatial_readiness.py @@ -0,0 +1,65 @@ +import json +import tempfile +import unittest +from pathlib import Path + +from pipeline.scripts.diagnostics.current_geospatial_readiness import audit_current, classify_record + + +class CurrentGeospatialReadinessTests(unittest.TestCase): + def test_classification_preserves_exact_city_unmapped_and_restricted_semantics(self): + exact = {"normalized": {"coordinates": {"latitude": 55.0, "longitude": 12.0}, "city": "Town"}} + coarse = {"normalized": {"city": "Town", "coordinate_state": "not-supplied-by-source"}} + unresolved = {"normalized": {"name": "facility", "coordinate_state": "not-supplied-by-source"}} + restricted = {"normalized": {"coordinates": {"latitude": 55.0, "longitude": 12.0}, "privacy_status": "suppressed"}} + self.assertEqual(classify_record(exact)["display_state"], "exact") + self.assertEqual(classify_record(coarse)["display_state"], "city") + self.assertEqual(classify_record(unresolved)["display_state"], "unmapped") + self.assertEqual(classify_record(restricted)["display_state"], "restricted") + + def test_invalid_and_geocode_evidence_are_counted_separately(self): + invalid = {"normalized": {"coordinates": {"latitude": 91, "longitude": 12}, "city": "Town"}} + geocoded = {"normalized": {"city": "Town", "geocode": {"status": "accepted", "precision": "address_point", "result": {"latitude": 55, "longitude": 12}}}} + pending = {"source_values": {"latitudine": "45.0", "longitudine": "9.0"}, "normalized": {"coordinate_state": "source-value-present-pending-review"}} + result = classify_record(invalid) + self.assertEqual(result["evidence"]["source_coordinate_invalid"], 1) + self.assertEqual(classify_record(geocoded)["evidence"]["accepted_geocode_exact"], 1) + self.assertEqual(classify_record(pending)["evidence"]["source_coordinate_valid"], 1) + + def test_manifest_audit_is_deterministic_row_free_and_explicit_about_missing_handoffs(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + staging = root / "staging" / "source-a" + (staging / "normalized").mkdir(parents=True) + records = [ + {"source_id": "source-a", "source_row": 1, "normalized": {"establishment_id": "private-id", "coordinates": {"latitude": 55, "longitude": 12}, "city": "Town"}}, + {"source_id": "source-a", "source_row": 2, "normalized": {"establishment_id": "private-id-2", "city": "Town", "privacy_gate": "pending-review"}}, + {"source_id": "source-a", "source_row": 3, "normalized": {"establishment_id": "private-id-3", "privacy_status": "suppressed"}}, + ] + records_path = staging / "normalized" / "records.jsonl" + records_path.write_text("".join(json.dumps(row) + "\n" for row in records), encoding="utf-8") + handoff = staging / "manifest.json" + handoff.write_text(json.dumps({"source_id": "source-a", "normalized_rows": 3}), encoding="utf-8") + manifest = root / "sources.json" + manifest.write_text(json.dumps({"sources": [ + {"source_id": "source-a", "country_code": "DK", "candidate_handoff_manifest": "staging/source-a/manifest.json", "normalized_rows": 3}, + {"source_id": "source-b", "country_code": "IT", "candidate_handoff_manifest": "staging/source-b/manifest.json", "normalized_rows": 10}, + ]}), encoding="utf-8") + first = audit_current(manifest, root, "2026-09-16T00:00:00Z") + second = audit_current(manifest, root, "2026-09-16T00:00:00Z") + self.assertEqual(first, second) + self.assertEqual(first["sources_listed"], 2) + self.assertEqual(first["sources_available"], 1) + self.assertEqual(first["totals_available_rows_only"]["records"], 3) + self.assertEqual(first["totals_available_rows_only"]["display_exact"], 1) + self.assertEqual(first["totals_available_rows_only"]["display_city"], 1) + self.assertEqual(first["totals_available_rows_only"]["display_restricted"], 1) + self.assertEqual(first["totals_available_rows_only"]["suppressed_rows"], 1) + encoded = json.dumps(first) + self.assertNotIn("private-id", encoded) + self.assertNotIn("Town", encoded) + self.assertIn("unavailable_private_handoff", encoded) + + +if __name__ == "__main__": + unittest.main() From 7aff04ba7cd19d4b46b6cba6da1933eb9658fc6a Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Wed, 16 Sep 2026 23:17:47 -0700 Subject: [PATCH 208/311] Freeze V2 MVP contract and claim evidence --- docs/api/v2-mvp-contract.json | 49 +++ docs/api/v2-mvp-contract.md | 65 ++++ docs/governance/v2-mvp-claim-evidence.json | 378 +++++++++++++++++++++ docs/governance/v2-mvp-claim-evidence.md | 77 +++++ pipeline/tests/test_v2_mvp_contract.py | 72 ++++ 5 files changed, 641 insertions(+) create mode 100644 docs/api/v2-mvp-contract.json create mode 100644 docs/api/v2-mvp-contract.md create mode 100644 docs/governance/v2-mvp-claim-evidence.json create mode 100644 docs/governance/v2-mvp-claim-evidence.md create mode 100644 pipeline/tests/test_v2_mvp_contract.py diff --git a/docs/api/v2-mvp-contract.json b/docs/api/v2-mvp-contract.json new file mode 100644 index 0000000..48a7895 --- /dev/null +++ b/docs/api/v2-mvp-contract.json @@ -0,0 +1,49 @@ +{ + "contract_id": "uec-v2-mvp-backend-contract-1", + "status": "frozen-for-sprint-1", + "as_of": "2026-09-16", + "api_version": "v2", + "location_schema": "docs/api/v2-location.schema.json", + "source_contract": "docs/api/v2-contract.json", + "data_product_version": "uec-public-data-product-v1", + "schema_version": "uec-location-projection-v1", + "public_endpoints": [ + "GET /health/live", + "GET /health/ready", + "GET /health/diagnostics", + "GET /api/v2/locations", + "GET /api/v2/locations/{facility_id}", + "GET /api/v2/locations.csv", + "GET /api/v2/releases/manifest", + "GET /api/v2/discovery/filters", + "GET /api/v2/discovery/facets" + ], + "development_only_endpoints": [ + "GET /api/dev/preview/candidates" + ], + "operator_surfaces": [ + "bulk_snapshot_cli" + ], + "invariants": [ + "Public reads use one promoted, non-test release selected by explicit profile.", + "No promoted eligible release is a successful empty list state; manifest/export absence is 404.", + "Current suppression is applied to list, detail, facets, exports, history, reimports, and restores.", + "Raw source payloads, private addresses, geocoder queries/responses, reviewer identities, and restriction reasons are not public response fields.", + "Official, secondary, and community profiles are independent publication contexts.", + "Community-unreviewed records require explicit community profile selection and carry a persistent warning.", + "Bbox and radius are mutually exclusive; cursor and offset are mutually exclusive.", + "Pagination is deterministic and bounded; clients scope caches to profile, release, ruleset, and query.", + "Errors use the versioned {api_version, error:{code,message}} envelope.", + "The API contract does not claim animal totals, national completeness, factual truth, or deployment readiness." + ], + "non_goals": [ + "Administrative review UI", + "Public ingestion or submission endpoint", + "Public accountability graph endpoint", + "Narrative/story frontend", + "Device geolocation requirement", + "Unbounded bulk API", + "Production deployment authorization" + ], + "change_policy": "Additive changes require contract and schema tests. Breaking response or behavior changes require a new contract_id and versioned migration path." +} diff --git a/docs/api/v2-mvp-contract.md b/docs/api/v2-mvp-contract.md new file mode 100644 index 0000000..278d122 --- /dev/null +++ b/docs/api/v2-mvp-contract.md @@ -0,0 +1,65 @@ +# V2 MVP backend/API contract freeze + +Contract ID: `uec-v2-mvp-backend-contract-1` +Status: frozen for Sprint 1 review +Reviewed: 2026-09-16 + +The machine-readable freeze is +[v2-mvp-contract.json](v2-mvp-contract.json). It freezes the backend surface +that the functional MVP frontend may consume. It does not claim that V2 is +production-deployed or that every source is current. + +## Frozen public surface + +| Surface | Contract | +| --- | --- | +| Health | `/health/live`, `/health/ready`, and coarse privacy-safe `/health/diagnostics` | +| Discovery | `/api/v2/locations`, `/api/v2/locations/{facility_id}`, `/api/v2/discovery/filters`, and `/api/v2/discovery/facets` | +| Export | Bounded `/api/v2/locations.csv`; reproducible release packages remain an operator CLI concern | +| Release metadata | `/api/v2/releases/manifest` exposes only an eligible promoted manifest | +| Development preview | `/api/dev/preview/candidates` is authenticated, private, test-only, and unavailable as a production surface | + +The existing [v2-contract.json](v2-contract.json) remains the endpoint +source contract. The public location object is frozen by +[v2-location.schema.json](v2-location.schema.json). The consistency test +ensures these surfaces do not drift silently. + +## Frozen behavior + +- Public reads select one promoted, non-test release inside the requested + profile and use one read-only snapshot for release metadata and rows. +- The default profile is `official`; `secondary` and `community` require + explicit selection. A profile mismatch cannot relabel a release. +- A record must pass the applicable publication, privacy, and release gates. + Current suppression applies even to older releases, filtered requests, + exports, history, reimports, and restores. +- Exact, city/coarse, and unmapped locations remain distinct. A geocode is + evidence about location, not publication permission. +- Cursor and offset cannot be combined. Bbox and radius cannot be combined. + Pagination is deterministic and bounded. +- Errors use `{ "api_version": "v2", "error": { "code": "...", "message": "..." } }`. +- Public responses contain the reviewed projection only. Raw fields, private + addresses, source-record payloads, geocoder queries/responses, reviewer + identities, and restriction reasons are not part of this contract. +- Community-unreviewed data, when eligible, is only available through an + explicit community profile and carries the persistent warning that it has + not been verified by Until Every Cage. + +## Deliberate non-goals + +This freeze does not include an administrative review UI, public ingestion, +the public accountability graph, the narrative frontend, device-geolocation +requirements, an unbounded bulk API, or deployed production authorization. +These are separate MVP or post-MVP work items, not implied by the endpoint +names. + +## Change policy + +Additive fields or endpoints require updated contract/schema tests and clear +public-safety review. A breaking response or behavior change requires a new +contract ID and an explicit migration path. Documentation alone cannot mark a +claim implemented; the claim-evidence matrix must point to code and test or +rehearsal evidence. + +See the [claim-evidence matrix](../governance/v2-mvp-claim-evidence.md) for +what this frozen surface does and does not substantiate. diff --git a/docs/governance/v2-mvp-claim-evidence.json b/docs/governance/v2-mvp-claim-evidence.json new file mode 100644 index 0000000..5364a7c --- /dev/null +++ b/docs/governance/v2-mvp-claim-evidence.json @@ -0,0 +1,378 @@ +{ + "matrix_version": "v2-mvp-claim-evidence-1", + "as_of": "2026-09-16", + "status_definitions": { + "implemented_tested": "The repository contains the behavior and an automated test or deterministic rehearsal proving it within the stated scope.", + "implemented_not_exercised": "The code or harness exists, but the relevant full-scale, live, or operational exercise has not completed.", + "prototype_only": "A schema, contract, research lane, or partial implementation exists, but the promised product capability is not ready to claim.", + "planned": "The policy or roadmap requires it, but the implementation evidence is not present.", + "human_policy": "The decision requires an authorized maintainer, qualified advice, or an operational review that software cannot replace." + }, + "claims": [ + { + "id": "provenance.raw-lineage", + "area": "provenance", + "claim": "Source artifacts, retrieval facts, hashes, parser/configuration versions, source records, and normalized values have a lineage model.", + "status": "implemented_tested", + "evidence": [ + {"path": "docs/architecture/database-import-contract.md", "line": 18}, + {"path": "docs/architecture/source-operations.md", "line": 9}, + {"path": "pipeline/tests/test_database_contract.py", "line": 1}, + {"path": "pipeline/contracts/test_private_run.py", "line": 1} + ], + "limit": "This proves the model and private staging contract, not that every current upstream source has a fresh artifact locally." + }, + { + "id": "provenance.append-only-history", + "area": "provenance", + "claim": "Ordinary imports and enrichment append evidence and do not silently overwrite retained history.", + "status": "implemented_tested", + "evidence": [ + {"path": "docs/architecture/database-import-contract.md", "line": 5}, + {"path": "pipeline/tests/test_database_contract.py", "line": 1}, + {"path": "pipeline/tests/e2e/test_candidate_import.py", "line": 1} + ], + "limit": "Exceptional redaction/deletion remains a separate unfinished maintainer workflow." + }, + { + "id": "provenance.deterministic-rerun", + "area": "provenance", + "claim": "Validated candidate imports are deterministic, transactional, resumable, and idempotent for repeated input.", + "status": "implemented_tested", + "evidence": [ + {"path": "docs/architecture/database-import-contract.md", "line": 26}, + {"path": "docs/architecture/disposable-candidate-import.md", "line": 22}, + {"path": "pipeline/tests/e2e/test_candidate_import.py", "line": 1} + ], + "limit": "The full 50,000-row Postgres resilience rehearsal is not yet evidenced in this checkout." + }, + { + "id": "provenance.legacy-separation", + "area": "provenance", + "claim": "Legacy data can be retained as explicitly private legacy evidence rather than relabeled as fresh verification.", + "status": "implemented_tested", + "evidence": [ + {"path": "v2-ideas.md", "line": 13}, + {"path": "docs/real-v2-rehearsal.md", "line": 1}, + {"path": "pipeline/tests/test_real_v2_rehearsal.py", "line": 1} + ], + "limit": "Legacy data remains a compatibility/rehearsal population until each source has a current pipeline." + }, + { + "id": "provenance.current-source-coverage", + "area": "provenance", + "claim": "All intended current sources have fresh, reproducible acquisition and normalized handoffs.", + "status": "prototype_only", + "evidence": [ + {"path": "docs/architecture/data-pipeline-plan.md", "line": 86}, + {"path": "docs/current-geospatial-readiness.md", "line": 42}, + {"path": "docs/current-reacquisition.md", "line": 1} + ], + "limit": "The current-corpus audit explicitly reports unavailable private handoffs locally; no complete fresh-source claim is made." + }, + { + "id": "ethics.governing-policy", + "area": "ethics", + "claim": "ETHICS.md is the governing policy for acquisition, processing, retention, review, suppression, and publication, with unfinished protections tracked separately.", + "status": "human_policy", + "evidence": [ + {"path": "docs/ETHICS.md", "line": 1}, + {"path": "docs/governance/policy-implementation-todo.md", "line": 1}, + {"path": "docs/governance/ethics-changelog.md", "line": 1} + ], + "limit": "A governing policy is not proof that every control is implemented; this matrix preserves that distinction." + }, + { + "id": "publication.explicit-gate", + "area": "publication", + "claim": "Acquisition success and government origin do not automatically publish a record; release/profile, privacy, review, and project approval gates are separate.", + "status": "implemented_tested", + "evidence": [ + {"path": "docs/architecture/database-import-contract.md", "line": 49}, + {"path": "docs/architecture/release-manifest-verification.md", "line": 10}, + {"path": "pipeline/tests/e2e/test_seeded_api.py", "line": 1}, + {"path": "pipeline/tests/e2e/test_public_api.py", "line": 1} + ], + "limit": "The database gate is tested in disposable environments; deployment-level authorization is still open." + }, + { + "id": "publication.public-projection", + "area": "publication", + "claim": "Public list, detail, facets, and export routes read a reviewed projection rather than raw evidence and require an eligible promoted release.", + "status": "implemented_tested", + "evidence": [ + {"path": "docs/architecture/api-location-contract.md", "line": 5}, + {"path": "docs/api/v2-contract.md", "line": 1}, + {"path": "pipeline/tests/e2e/test_public_api.py", "line": 27}, + {"path": "pipeline/tests/e2e/test_public_surface_safety.py", "line": 1} + ], + "limit": "The current branch does not claim a deployed public instance." + }, + { + "id": "publication.profile-separation", + "area": "publication", + "claim": "Official, secondary, and community profiles remain explicit and cannot silently relabel one another.", + "status": "implemented_tested", + "evidence": [ + {"path": "docs/architecture/api-location-contract.md", "line": 28}, + {"path": "docs/ETHICS.md", "line": 61}, + {"path": "pipeline/tests/e2e/test_seeded_api.py", "line": 1} + ], + "limit": "The community surface is synthetic/test evidence and still needs frontend and operational publication work." + }, + { + "id": "publication.export-integrity", + "area": "publication", + "claim": "Bounded API exports and release snapshots carry release, profile, provenance, rights, and manifest integrity metadata.", + "status": "implemented_tested", + "evidence": [ + {"path": "docs/api/v2-contract.md", "line": 7}, + {"path": "docs/architecture/release-manifest-verification.md", "line": 1}, + {"path": "pipeline/tests/e2e/test_seeded_api.py", "line": 1} + ], + "limit": "Checksums detect alteration relative to a trusted reference; they do not establish factual accuracy or licensing." + }, + { + "id": "privacy.public-suppression", + "area": "privacy", + "claim": "A restricted record is excluded from current public list, detail, facets, exports, and historical projections, including after replay in tested scenarios.", + "status": "implemented_tested", + "evidence": [ + {"path": "docs/architecture/adr-map-visualization-platform.md", "line": 17}, + {"path": "docs/governance/policy-implementation-todo.md", "line": 27}, + {"path": "pipeline/tests/e2e/test_public_surface_safety.py", "line": 1}, + {"path": "pipeline/tests/e2e/backup_restore_current_suppression.sql", "line": 1} + ], + "limit": "The policy-compliant exceptional removal/redaction workflow and complete propagation exercise remain open." + }, + { + "id": "privacy.exceptional-removal", + "area": "privacy", + "claim": "Authorized maintainers can execute and audit the full private-data-safe correction, removal, retention, and deletion workflow.", + "status": "planned", + "evidence": [ + {"path": "docs/governance/policy-implementation-todo.md", "line": 5}, + {"path": "docs/governance/private-data-removal-runbook.md", "line": 1}, + {"path": "docs/ETHICS.md", "line": 31} + ], + "limit": "Existing suppression is not evidence that the complete exceptional-removal workflow is implemented." + }, + { + "id": "privacy.people-not-targets", + "area": "safety", + "claim": "The public product will not expose workers, residents, private individuals, or targeting-enabling personal details.", + "status": "human_policy", + "evidence": [ + {"path": "docs/ETHICS.md", "line": 84}, + {"path": "docs/architecture/adr-map-visualization-platform.md", "line": 55}, + {"path": "docs/governance/policy-implementation-todo.md", "line": 27} + ], + "limit": "This is a binding policy and design constraint; source-specific human screening and operational review remain necessary." + }, + { + "id": "privacy.visitor-data", + "area": "privacy", + "claim": "Visitor searches, device location, logs, tiles, geocoding, analytics, hosting, and third-party disclosures have been audited and verified against the public privacy statement.", + "status": "planned", + "evidence": [ + {"path": "docs/ETHICS.md", "line": 116}, + {"path": "docs/governance/visitor-privacy-inventory.md", "line": 1}, + {"path": "docs/governance/policy-implementation-todo.md", "line": 49} + ], + "limit": "The policy explicitly says current hosting/provider behavior has not yet been audited." + }, + { + "id": "safety.uncertainty-labels", + "area": "safety", + "claim": "Unknown, unavailable, approximate, unresolved, and failed values are represented explicitly rather than guessed.", + "status": "implemented_tested", + "evidence": [ + {"path": "docs/ETHICS.md", "line": 39}, + {"path": "docs/architecture/api-location-contract.md", "line": 13}, + {"path": "pipeline/tests/e2e/test_seeded_api.py", "line": 1}, + {"path": "pipeline/tests/test_real_corpus_report.py", "line": 1} + ], + "limit": "This is proven for the tested projections and diagnostics, not every future source-specific field." + }, + { + "id": "safety.no-closure-inference", + "area": "lifecycle", + "claim": "A source disappearance is not converted into a closure claim; explicit closure requires traceable evidence.", + "status": "implemented_tested", + "evidence": [ + {"path": "docs/architecture/api-location-contract.md", "line": 17}, + {"path": "docs/architecture/source-operations.md", "line": 42}, + {"path": "pipeline/tests/e2e/test_seeded_api.py", "line": 1} + ], + "limit": "Lifecycle quality still depends on source-specific evidence and review." + }, + { + "id": "lifecycle.history", + "area": "lifecycle", + "claim": "Facility lifecycle events and observation history are separate, dated, and visible only through eligible projections.", + "status": "implemented_tested", + "evidence": [ + {"path": "docs/architecture/source-lifecycle.md", "line": 1}, + {"path": "docs/architecture/api-location-contract.md", "line": 17}, + {"path": "pipeline/tests/e2e/test_seeded_api.py", "line": 1} + ], + "limit": "The frontend presentation is not part of this backend lane." + }, + { + "id": "geospatial.precision", + "area": "geospatial", + "claim": "Exact, city/coarse, unmapped, restricted, source coordinates, and geocoder evidence remain distinct states.", + "status": "implemented_tested", + "evidence": [ + {"path": "docs/current-geospatial-readiness.md", "line": 16}, + {"path": "docs/architecture/api-location-contract.md", "line": 13}, + {"path": "pipeline/tests/e2e/test_seeded_api.py", "line": 1} + ], + "limit": "An accepted geocode is location evidence only; it is not publication approval." + }, + { + "id": "geospatial.current-coverage", + "area": "geospatial", + "claim": "Current-corpus coordinate coverage and privacy-review queues have been measured from the complete current handoff.", + "status": "implemented_not_exercised", + "evidence": [ + {"path": "docs/current-geospatial-readiness.md", "line": 42}, + {"path": "docs/geospatial-readiness-audit.md", "line": 1}, + {"path": "pipeline/scripts/diagnostics/current_geospatial_readiness.py", "line": 1} + ], + "limit": "The checked-in manifest lists current source profiles, but the private normalized handoffs are unavailable in this checkout; the audit correctly reports coverage as unavailable, not zero." + }, + { + "id": "geospatial.provider-controls", + "area": "geospatial", + "claim": "Geocoding attempts retain provider, query, timestamp, precision, response, retryability, and review state.", + "status": "implemented_tested", + "evidence": [ + {"path": "docs/architecture/database-import-contract.md", "line": 18}, + {"path": "docs/current-geospatial-readiness.md", "line": 50}, + {"path": "pipeline/tests/test_geocoding_adapters.py", "line": 1} + ], + "limit": "Only Denmark DAWA is configured for development-only use; production providers, cost, retention, and terms are not established." + }, + { + "id": "graph.source-qualified-identities", + "area": "graph", + "claim": "Source-native identifiers and crosswalks are source-scoped, reviewable, and do not silently create universal identity merges.", + "status": "implemented_tested", + "evidence": [ + {"path": "docs/architecture/adr-graph-foundation.md", "line": 5}, + {"path": "docs/architecture/graph-data-dictionary.md", "line": 1}, + {"path": "pipeline/tests/test_graph_database_contract.py", "line": 1} + ], + "limit": "This proves the evidence model, not a complete global graph." + }, + { + "id": "graph.public-product", + "area": "graph", + "claim": "A production public accountability graph with reviewed organization relationships and demonstrated accuracy is available.", + "status": "prototype_only", + "evidence": [ + {"path": "docs/architecture/adr-graph-foundation.md", "line": 1}, + {"path": "docs/review-packet-us-accountability.md", "line": 1}, + {"path": "docs/architecture/adr-map-visualization-platform.md", "line": 68} + ], + "limit": "The current graph work is a private evidence foundation and pilot; no public graph import, inference, or UI is claimed." + }, + { + "id": "scale.indexed-discovery", + "area": "scale", + "claim": "The public discovery read path has deterministic, facility-scoped queries and indexed plans measured at 25k, 100k, and 150k rows.", + "status": "implemented_tested", + "evidence": [ + {"path": "docs/performance/v2-public-projection-read-path.md", "line": 1}, + {"path": "docs/performance/v2-api-load-rehearsal.md", "line": 1}, + {"path": "pipeline/tests/test_public_discovery_query_contract.py", "line": 1} + ], + "limit": "Plan and local samples are not a production concurrency SLO." + }, + { + "id": "scale.concurrent-http-capacity", + "area": "scale", + "claim": "The API has completed representative concurrent HTTP performance testing with reliable p95/p99 results through 150k records.", + "status": "implemented_not_exercised", + "evidence": [ + {"path": "pipeline/scripts/benchmarks/run_api_load_rehearsal.py", "line": 1}, + {"path": "pipeline/tests/test_api_load_rehearsal.py", "line": 1}, + {"path": "docs/performance/v2-api-load-rehearsal.md", "line": 1} + ], + "limit": "The harness and bounded scale contract exist; the full post-optimization concurrent run remains a Sprint 1 exit item." + }, + { + "id": "operations.resilience", + "area": "operational", + "claim": "Interrupted imports, duplicate imports, backup/restore, stale suppression rejection, and restriction-ledger replay have completed at representative 50k scale.", + "status": "implemented_not_exercised", + "evidence": [ + {"path": "docs/performance/corpus-resilience.md", "line": 1}, + {"path": "pipeline/scripts/benchmarks/run_corpus_resilience.py", "line": 1}, + {"path": "pipeline/tests/e2e/test_corpus_resilience.py", "line": 1} + ], + "limit": "The plan-only path and focused tests pass; Docker Desktop access denied the full Postgres rehearsal in the reporting agent's checkout." + }, + { + "id": "operations.health-diagnostics", + "area": "operational", + "claim": "The deployed system has audited, privacy-safe observability, configuration validation, alerting, and operator incident procedures.", + "status": "planned", + "evidence": [ + {"path": "docs/performance/v2-observability.md", "line": 1}, + {"path": "docs/architecture/api-location-contract.md", "line": 5}, + {"path": "docs/governance/policy-implementation-todo.md", "line": 105} + ], + "limit": "Health endpoints and coarse diagnostics exist; that is not evidence of a deployed observability system or alerting coverage." + }, + { + "id": "operations.release-authority", + "area": "operational", + "claim": "Least-privilege deployment controls, explicit authorized publication authority, reviewer availability, and a tested publication pause are operational.", + "status": "human_policy", + "evidence": [ + {"path": "docs/ETHICS.md", "line": 129}, + {"path": "docs/governance/policy-implementation-todo.md", "line": 59}, + {"path": "docs/deployment/private-environment.md", "line": 1} + ], + "limit": "The user is the currently authorized operator; the repository does not establish a backup reviewer or deployed access-control configuration." + }, + { + "id": "rights.source-licensing", + "area": "provenance", + "claim": "Every intended public source has completed rights, attribution, privacy, scope, and release review.", + "status": "human_policy", + "evidence": [ + {"path": "docs/architecture/release-manifest-verification.md", "line": 1}, + {"path": "docs/source-status.md", "line": 1}, + {"path": "docs/governance/policy-implementation-todo.md", "line": 39} + ], + "limit": "Source metadata and rights-status fields exist, but a machine-readable field is not a completed legal or maintainer decision." + }, + { + "id": "product.narrative", + "area": "safety", + "claim": "The flagship scale story is a finished, citable, accessible public experience that connects the horror of scale to local evidence without overclaiming.", + "status": "prototype_only", + "evidence": [ + {"path": "v2-ideas.md", "line": 242}, + {"path": "docs/story/flagship-scale-evidence-spec.md", "line": 1}, + {"path": "docs/story/storyboard.md", "line": 1} + ], + "limit": "The mission and story specifications are real product direction; frontend narrative implementation is deliberately outside this backend freeze." + }, + { + "id": "product.reviewer-mvp", + "area": "operational", + "claim": "An invited activist, journalist, or researcher can independently start the MVP, inspect provenance, use filters/exports, understand limitations, and provide feedback.", + "status": "planned", + "evidence": [ + {"path": "v2-ideas.md", "line": 29}, + {"path": "docs/api/v2-contract.md", "line": 1}, + {"path": "docs/governance/policy-implementation-todo.md", "line": 39} + ], + "limit": "This becomes the Sprint 2 demonstrator goal after the backend MVP contract is frozen." + } + ] +} diff --git a/docs/governance/v2-mvp-claim-evidence.md b/docs/governance/v2-mvp-claim-evidence.md new file mode 100644 index 0000000..5d0a981 --- /dev/null +++ b/docs/governance/v2-mvp-claim-evidence.md @@ -0,0 +1,77 @@ +# V2 MVP claim-evidence matrix + +Status: Sprint 1 working contract, reviewed 2026-09-16 +Machine-readable source: [v2-mvp-claim-evidence.json](v2-mvp-claim-evidence.json) + +This matrix is the boundary between what V2 can honestly demonstrate and what +remains a design goal, a private prototype, or a human decision. It is governed +by [ETHICS.md](../ETHICS.md). A repository test, schema, or private rehearsal +is evidence of that bounded behavior; it is not evidence of deployment, +complete country coverage, factual truth, legal clearance, or operational +capacity beyond the stated scope. + +## Status vocabulary + +| Status | Meaning | +| --- | --- | +| `implemented_tested` | Code plus an automated test or deterministic rehearsal proves the behavior within the stated scope. | +| `implemented_not_exercised` | Code or a harness exists, but the full-scale, live, or operational exercise has not completed. | +| `prototype_only` | A schema, contract, research lane, or partial implementation exists; the promised product capability is not ready to claim. | +| `planned` | The policy or roadmap requires it, but implementation evidence is not present. | +| `human_policy` | An authorized maintainer, qualified advice, or operational review is required; software cannot replace the decision. | + +## Matrix + +The JSON file is canonical so tooling can reject missing evidence links or +invented status values. The table below is intentionally compact; each ID +links to the full claim, evidence paths, and limitations in the JSON source. + +| ID | Area | Status | Short claim | +| --- | --- | --- | --- | +| `provenance.raw-lineage` | Provenance | `implemented_tested` | Raw artifacts and normalized records have a lineage model. | +| `provenance.append-only-history` | Provenance | `implemented_tested` | Ordinary processing appends evidence rather than silently overwriting it. | +| `provenance.deterministic-rerun` | Provenance | `implemented_tested` | Candidate imports are transactional, deterministic, and idempotent. | +| `provenance.legacy-separation` | Provenance | `implemented_tested` | Legacy evidence remains explicitly tagged and private in rehearsal. | +| `provenance.current-source-coverage` | Provenance | `prototype_only` | Fresh current-source coverage is not complete or locally available. | +| `ethics.governing-policy` | Ethics | `human_policy` | ETHICS.md governs the work; open controls remain explicitly open. | +| `publication.explicit-gate` | Publication | `implemented_tested` | Acquisition and government origin do not automatically publish. | +| `publication.public-projection` | Publication | `implemented_tested` | Public routes use eligible promoted projections, not raw evidence. | +| `publication.profile-separation` | Publication | `implemented_tested` | Official, secondary, and community profiles remain separate. | +| `publication.export-integrity` | Publication | `implemented_tested` | Exports carry bounded provenance and release-manifest metadata. | +| `privacy.public-suppression` | Privacy | `implemented_tested` | Tested suppression removes records from controlled public surfaces. | +| `privacy.exceptional-removal` | Privacy | `planned` | The full authorized removal/redaction workflow remains open. | +| `privacy.people-not-targets` | Safety | `human_policy` | Human review must prevent targeting and personal exposure. | +| `privacy.visitor-data` | Privacy | `planned` | Hosting, logs, tiles, geocoding, and visitor privacy are not fully audited. | +| `safety.uncertainty-labels` | Safety | `implemented_tested` | Unknown and approximate states remain explicit. | +| `safety.no-closure-inference` | Lifecycle | `implemented_tested` | Missing source rows do not become closure claims. | +| `lifecycle.history` | Lifecycle | `implemented_tested` | Lifecycle events and observations are separate and dated. | +| `geospatial.precision` | Geospatial | `implemented_tested` | Exact, coarse, unmapped, restricted, and geocoder states are distinct. | +| `geospatial.current-coverage` | Geospatial | `implemented_not_exercised` | Full current-corpus coordinate coverage is not yet measured. | +| `geospatial.provider-controls` | Geospatial | `implemented_tested` | Geocoder attempts retain provenance and review state. | +| `graph.source-qualified-identities` | Graph | `implemented_tested` | Crosswalks remain source-scoped and reviewable. | +| `graph.public-product` | Graph | `prototype_only` | No production public accountability graph is claimed yet. | +| `scale.indexed-discovery` | Scale | `implemented_tested` | Discovery plans were measured at 25k/100k/150k rows. | +| `scale.concurrent-http-capacity` | Scale | `implemented_not_exercised` | Full post-optimization concurrent HTTP evidence remains open. | +| `operations.resilience` | Operational | `implemented_not_exercised` | The 50k resilience harness exists; full Postgres rehearsal is pending. | +| `operations.health-diagnostics` | Operational | `planned` | Deployment observability and alerting are not yet proven. | +| `operations.release-authority` | Operational | `human_policy` | Publication authority and least privilege require operational ownership. | +| `rights.source-licensing` | Provenance | `human_policy` | Rights and attribution need source-specific human decisions. | +| `product.narrative` | Safety | `prototype_only` | The story is specified, not yet a finished public experience. | +| `product.reviewer-mvp` | Operational | `planned` | Independent reviewer usability is the later demonstrator goal. | + +## Sprint 1 blockers + +These items prevent declaring the Sprint 1 backend-proof goal complete: + +1. Run the full post-optimization HTTP rehearsal and record concurrency and + p95/p99 evidence. +2. Run the full disposable Postgres resilience rehearsal, or preserve a clear + environment blocker and do not claim representative backup/restore proof. +3. Produce the current-corpus geospatial report from actual private normalized + handoffs rather than the current `unavailable_private_handoff` result. +4. Keep the contract freeze and this matrix synchronized with the API schema, + endpoint inventory, and tests. + +The last item is automated by +[test_v2_mvp_contract.py](../../pipeline/tests/test_v2_mvp_contract.py). The +first three are execution/rehearsal gates, not documentation claims. diff --git a/pipeline/tests/test_v2_mvp_contract.py b/pipeline/tests/test_v2_mvp_contract.py new file mode 100644 index 0000000..289178f --- /dev/null +++ b/pipeline/tests/test_v2_mvp_contract.py @@ -0,0 +1,72 @@ +"""Prevent the V2 MVP claim matrix and API freeze from drifting.""" + +import json +import re +import unittest +from pathlib import Path + + +ROOT = Path(__file__).parents[2] +MATRIX_JSON = ROOT / "docs" / "governance" / "v2-mvp-claim-evidence.json" +MATRIX_MD = ROOT / "docs" / "governance" / "v2-mvp-claim-evidence.md" +FREEZE_JSON = ROOT / "docs" / "api" / "v2-mvp-contract.json" +SOURCE_CONTRACT = ROOT / "docs" / "api" / "v2-contract.json" +LOCATION_SCHEMA = ROOT / "docs" / "api" / "v2-location.schema.json" + + +class V2MvpContractTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.matrix = json.loads(MATRIX_JSON.read_text(encoding="utf-8")) + cls.freeze = json.loads(FREEZE_JSON.read_text(encoding="utf-8")) + cls.source_contract = json.loads(SOURCE_CONTRACT.read_text(encoding="utf-8")) + cls.location_schema = json.loads(LOCATION_SCHEMA.read_text(encoding="utf-8")) + + def test_matrix_has_required_domains_and_evidence(self): + allowed = set(self.matrix["status_definitions"]) + claims = self.matrix["claims"] + self.assertEqual(len({claim["id"] for claim in claims}), len(claims)) + self.assertTrue({"ethics", "safety", "provenance", "publication", "privacy", "lifecycle", "graph", "scale", "geospatial", "operational"} <= {claim["area"] for claim in claims}) + for claim in claims: + self.assertIn(claim["status"], allowed, claim["id"]) + self.assertTrue(claim["claim"].strip(), claim["id"]) + self.assertTrue(claim["limit"].strip(), claim["id"]) + self.assertGreater(len(claim["evidence"]), 0, claim["id"]) + for evidence in claim["evidence"]: + path = ROOT / evidence["path"] + self.assertTrue(path.is_file(), f"{claim['id']}: missing {evidence['path']}") + self.assertGreaterEqual(evidence.get("line", 1), 1) + + def test_matrix_ids_are_present_in_human_document(self): + document = MATRIX_MD.read_text(encoding="utf-8") + for claim in self.matrix["claims"]: + self.assertIn(f"`{claim['id']}`", document) + + def test_matrix_markdown_links_resolve(self): + document = MATRIX_MD.read_text(encoding="utf-8") + for target in re.findall(r"\]\(([^)]+)\)", document): + if target.startswith(("http://", "https://", "#")): + continue + path = target.split("#", 1)[0] + self.assertTrue((MATRIX_MD.parent / path).is_file(), target) + + def test_freeze_endpoint_inventory_matches_source_contract(self): + frozen = set(self.freeze["public_endpoints"] + self.freeze["development_only_endpoints"] + self.freeze["operator_surfaces"]) + self.assertEqual(frozen, set(self.source_contract["endpoints"])) + self.assertEqual(self.freeze["api_version"], self.source_contract["version"]) + self.assertEqual(self.freeze["location_schema"], "docs/api/v2-location.schema.json") + + def test_location_schema_is_closed_and_self_consistent(self): + properties = set(self.location_schema["properties"]) + required = set(self.location_schema["required"]) + self.assertTrue(self.location_schema["additionalProperties"] is False) + self.assertEqual(properties, required) + self.assertEqual(self.freeze["schema_version"], "uec-location-projection-v1") + + def test_freeze_declares_non_goals_and_change_policy(self): + self.assertGreaterEqual(len(self.freeze["non_goals"]), 5) + self.assertIn("Breaking", self.freeze["change_policy"]) + + +if __name__ == "__main__": + unittest.main() From b747cc3bb9668e1f76e52e1b1588370d2a88ad2f Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Thu, 17 Sep 2026 00:02:05 -0700 Subject: [PATCH 209/311] Add representative V2 API load rehearsal --- docs/current-reacquisition.md | 59 ++++++++- docs/performance/v2-api-load-rehearsal.md | 64 +++++++--- .../benchmarks/run_api_load_rehearsal.py | 103 +++++++++++++++- .../rehearse_current_reacquisition.py | 114 +++++++++++++++--- pipeline/tests/e2e/fixture.py | 46 +++++-- pipeline/tests/test_api_load_rehearsal.py | 32 ++++- .../test_current_reacquisition_rehearsal.py | 33 ++++- 7 files changed, 393 insertions(+), 58 deletions(-) diff --git a/docs/current-reacquisition.md b/docs/current-reacquisition.md index 7070b51..ddd310e 100644 --- a/docs/current-reacquisition.md +++ b/docs/current-reacquisition.md @@ -55,11 +55,34 @@ python pipeline/scripts/maintenance/rehearse_current_reacquisition.py ` --output data/reports/current-reacquisition-rehearsal.json ``` -The command fails closed if a private artifact, handoff, checksum, candidate -state, or reconciliation count is missing or changed. CFIA is intentionally -excluded because its current workbook remains raw-only. The checked-in report -must remain aggregate-only; do not substitute a normalized JSONL path for its -output path or add row payloads to the manifest. +The command always writes an aggregate report, but exits nonzero if a private +artifact, handoff, checksum, candidate state, or reconciliation count is +missing or changed. It enumerates every unavailable or invalid source instead +of stopping at the first one; unavailable inputs are never counted as zero. +CFIA is intentionally reported as raw-only because its current workbook has no +normalized handoff. The checked-in report must remain aggregate-only; do not +substitute a normalized JSONL path for its output path or add row payloads to +the manifest. + +For the complete disposable candidate/API rehearsal, use the separate +operator command below. `--root` may point at an authorized ignored staging +checkout; it is read-only from this command. The command imports every +available normalized handoff into one candidate release, reruns every import, +checks list/detail/facets/cursor pagination, verifies the bounded CSV guard, +and records an append-only suppression check. Its output is row-free and must +be written outside the repository or to an ignored report path: + +```powershell +python pipeline/scripts/maintenance/rehearse_current_candidate.py ` + --manifest data/manifests/current-reacquisition-2026-09-16.json ` + --root C:\New\ Projects\UntilEveryCage-current-reacquisition ` + --output $env:TEMP\uec-current-candidate-rehearsal.json +``` + +The candidate release is loopback-only, `test_only`, unapproved, and never +promoted. A successful run must report 108,475 normalized rows imported on +the first pass and zero new rows on the rerun. CFIA is intentionally listed as +raw-only and is not silently counted as zero. The completed rehearsal used a disposable `docker-compose.e2e.yml` project (`uec-reacq-20260916`, DB port `55440`) with all 34 migrations. It imported the @@ -86,3 +109,29 @@ docker compose -p uec-reacq-20260916 -f docker-compose.e2e.yml down -v --remove- This rehearsal is evidence of private normalization, quarantine, candidate handoff/import, test-only preview/export, suppression, and rerun behavior. It is not project approval, currentness certification, or publication permission. + +## Current workspace availability check + +The `eli/front-end-overhaul` integration checkout intentionally contains only +the row-free manifest. The authorized private staging root used for the final +rehearsal was the separate ignored checkout +`C:\New Projects\UntilEveryCage-current-reacquisition`; it was inspected +read-only and no raw or normalized rows were copied into the integration +checkout. The aggregate validator completed successfully there for seven +normalized profiles and one CFIA raw-only profile. + +The verified reconciliation is 115,182 input rows = 108,475 normalized rows ++ 6,707 quarantined rows. The current-corpus geospatial audit found 40,115 +source-coordinate-valid rows, 1,710 source-coordinate-pending-review rows, +68,273 city-display rows, 87 unmapped rows, and 103,676 rows still requiring +privacy/coordinate review. It produced aggregate evidence only. No source +was approved, promoted, or published; the public API row count remains zero. + +The candidate/API rehearsal is run separately with +`rehearse_current_candidate.py` because it requires Docker and a disposable +database. Its report should record the exact per-source import counts, a zero +row delta on the idempotent rerun, successful test-only API surfaces, a +bounded full-export rejection above 1,000 rows, and suppression reducing +visible candidate rows by one. CFIA remains the explicit unresolved adapter +blocker: its current official response is an XLS workbook and has no reviewed +normalized handoff. diff --git a/docs/performance/v2-api-load-rehearsal.md b/docs/performance/v2-api-load-rehearsal.md index 403148b..098d3c1 100644 --- a/docs/performance/v2-api-load-rehearsal.md +++ b/docs/performance/v2-api-load-rehearsal.md @@ -27,11 +27,21 @@ python pipeline/scripts/benchmarks/run_api_load_rehearsal.py ` --json-output .tmp/api-load-25000.json ``` -The runner bounds observations at 25,000, concurrency at 16, and requests per -level at 80. It refuses non-loopback API targets. The 25,000-row bound is an +The runner bounds observations at 150,000, concurrency at 16, and requests per +level at 80. It refuses non-loopback API targets. The 150,000-row bound is an explicitly finite synthetic safety limit, not a statement about supported production scale. +Each run also captures row-free planner metadata for the list, facets, radius, +and graph read shapes. Reports include estimated costs, node counts, relation +and index names, and sequential-scan relation names; they do not include SQL, +plan filters, identifiers, coordinates, or result rows. + +At scales above 1,000 facilities, the graph-shaped fixture is intentionally +capped at 1,000 organizations, relationships, and claims. This keeps the +150,000-row run focused on the facility discovery API rather than multiplying +unrelated graph evidence rows; graph scale is measured separately. + When an authorized private V2 normalized corpus is available, first create a row-free distribution report and then pass it to the same synthetic rehearsal: @@ -57,20 +67,42 @@ release, or benchmark-output data. | Synthetic observations | Concurrency | Requests | Successes | Timeouts | 5xx | Throughput (rps) | p50 / p95 / p99 (ms) | Max active / waiting DB sessions | | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | -| 5,000 | 1 | 10 | 4 | 6 | 0 | 0.622 | 2013.251 / 2028.338 / 2028.338 | 2 / 1 | -| 5,000 | 4 | 10 | 3 | 7 | 0 | 1.981 | 2001.908 / 2029.853 / 2029.853 | 2 / 0 | -| 5,000 | 8 | 10 | 4 | 6 | 0 | 3.982 | 2013.191 / 2017.044 / 2017.044 | 6 / 1 | -| 5,000 | 16 | 10 | 2 | 8 | 0 | 4.953 | 2009.559 / 2011.767 / 2011.767 | 9 / 1 | -| 25,000 | 1 | 10 | 1 | 9 | 0 | 0.514 | 2016.681 / 2029.881 / 2029.881 | 2 / 1 | -| 25,000 | 4 | 10 | 1 | 9 | 0 | 1.653 | 2012.612 / 2025.954 / 2025.954 | 2 / 1 | -| 25,000 | 8 | 10 | 2 | 8 | 0 | 2.547 | 2007.124 / 2030.413 / 2030.413 | 6 / 2 | -| 25,000 | 16 | 10 | 1 | 9 | 0 | 4.482 | 2004.377 / 2024.087 / 2024.087 | 10 / 3 | - -No tested level was clean under the harness definition (zero timeout, server, -or connection errors). The result supports keeping API pool sizing and -production capacity claims blocked until an approved representative load -model is available. It also confirms that the failure mode in this local -rehearsal is timeout pressure rather than HTTP 5xx or connection exhaustion. +| 5,000 | 1 | 10 | 10 | 0 | 0 | 4.579 | 215.700 / 302.345 / 302.345 | 2 / 1 | +| 5,000 | 4 | 10 | 10 | 0 | 0 | 7.316 | 328.596 / 900.362 / 900.362 | 2 / 1 | +| 5,000 | 8 | 10 | 10 | 0 | 0 | 11.290 | 507.344 / 595.657 / 595.657 | 5 / 4 | +| 5,000 | 16 | 10 | 10 | 0 | 0 | 14.781 | 464.993 / 598.804 / 598.804 | 7 / 8 | +| 25,000 | 1 | 10 | 10 | 0 | 0 | 0.793 | 1263.909 / 1705.504 / 1705.504 | 3 / 2 | +| 25,000 | 4 | 10 | 5 | 5 | 0 | 1.770 | 2004.356 / 2017.358 / 2017.358 | 2 / 1 | +| 25,000 | 8 | 10 | 3 | 7 | 0 | 2.640 | 2010.196 / 2011.692 / 2011.692 | 5 / 4 | +| 25,000 | 16 | 10 | 3 | 7 | 0 | 2.922 | 2006.612 / 2010.724 / 2010.724 | 9 / 7 | +| 100,000 | 1 | 10 | 1 | 9 | 0 | 0.497 | 2004.792 / 2017.417 / 2017.417 | 2 / 1 | +| 100,000 | 4 | 10 | 1 | 9 | 0 | 1.657 | 2007.899 / 2014.835 / 2014.835 | 2 / 1 | +| 100,000 | 8 | 10 | 1 | 9 | 0 | 2.480 | 2011.186 / 2026.853 / 2026.853 | 5 / 3 | +| 100,000 | 16 | 10 | 1 | 9 | 0 | 2.366 | 2016.205 / 2031.078 / 2031.078 | 8 / 6 | +| 150,000 | 1 | 10 | 1 | 9 | 0 | 0.489 | 2008.950 / 2024.313 / 2024.313 | 2 / 1 | +| 150,000 | 4 | 10 | 1 | 9 | 0 | 1.651 | 2014.980 / 2023.360 / 2023.360 | 2 / 1 | +| 150,000 | 8 | 10 | 1 | 9 | 0 | 2.476 | 2010.329 / 2015.471 / 2015.471 | 6 / 4 | +| 150,000 | 16 | 10 | 1 | 9 | 0 | 2.506 | 2009.943 / 2030.332 / 2030.332 | 9 / 5 | + +The 5,000-row fixture is clean at every tested concurrency. At 25,000 rows, +the single-worker level is clean, but timeouts begin at concurrency 4. At +100,000 and 150,000 rows, only one of ten mixed requests completed at each +level; the two-second client budget is not viable. No server-side 5xx or +connection errors occurred. Pool pressure rose with concurrency, but the +observed failure mode was request timeout rather than pool exhaustion. + +These are actual local measurements from the post-optimization harness, not +capacity claims. The run artifacts remain in the ignored `.tmp/` directory; +only these aggregate values and row-free plan summaries are documented here. + +The row-free planner summaries estimated list/facets/radius costs of roughly +82,989 at 5,000 rows, 416,639 at 25,000, 1,683,553 at 100,000, and 2,525,920 +at 150,000. The graph-shaped plan estimated roughly twice the discovery cost. +The repeated sequential-scan relations were control-plane tables used by the +live suppression and review views, including `source_records`, +`record_access_events`, `suppression_case_events`, and +`suppression_references`. No query-plan row payloads or filter values were +retained. The dominant slow path remains the live eligibility/summary work documented in `v2-public-projection-read-path.md`. The evidence does not justify caching, diff --git a/pipeline/scripts/benchmarks/run_api_load_rehearsal.py b/pipeline/scripts/benchmarks/run_api_load_rehearsal.py index c2db178..131411f 100644 --- a/pipeline/scripts/benchmarks/run_api_load_rehearsal.py +++ b/pipeline/scripts/benchmarks/run_api_load_rehearsal.py @@ -27,7 +27,8 @@ ROOT = Path(__file__).resolve().parents[3] if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) -MAX_SEED = 25_000 +MAX_SEED = 150_000 +MAX_GRAPH_FIXTURE_ROWS = 1_000 MAX_CONCURRENCY = 16 MAX_REQUESTS_PER_LEVEL = 80 ALLOWED_DISTRIBUTION_PRECISIONS = {"exact", "city", "unmapped"} @@ -45,6 +46,51 @@ ("graph_ready", "graph", None), ) +# These are the public read-path shapes exercised by the HTTP mix. The +# harness records only planner metadata for them; it never emits SQL, values, +# identifiers, or result rows in a report. +PLAN_QUERIES = { + "list": """ + SELECT facility_id, canonical_name, country_code, city, + classification_category, display_precision + FROM uec.map_facilities_display_history + WHERE release_id = 'load-promoted' + ORDER BY facility_id + LIMIT 51 + """, + "facets": """ + SELECT country_code, classification_category, display_precision, + lifecycle_status, provenance_origin_type, city, count(*)::bigint + FROM uec.map_facilities_display_history + WHERE release_id = 'load-promoted' + GROUP BY country_code, classification_category, display_precision, + lifecycle_status, provenance_origin_type, city + """, + "radius": """ + SELECT facility_id + FROM uec.map_facilities_display_history + WHERE release_id = 'load-promoted' + AND display_location && ST_SetSRID( + ST_MakeEnvelope(-5.7, 49.55, -4.3, 50.45, 4326), 4326)::geography + AND ST_DWithin( + display_location, + ST_SetSRID(ST_Point(-5, 50), 4326)::geography, + 50000) + ORDER BY facility_id + LIMIT 51 + """, + "graph": """ + SELECT relationship.relationship_type + FROM uec.graph_public_relationships relationship + JOIN uec.graph_public_claims claim + ON claim.release_id = relationship.release_id + AND claim.facility_id = relationship.target_facility_id + WHERE relationship.release_id = 'load-promoted' + ORDER BY relationship.relationship_observation_id + LIMIT 50 + """, +} + def deterministic_uuid(prefix: str, ordinal: int) -> str: return str(uuid.UUID(hex=hashlib.md5(f"{prefix}-{ordinal}".encode()).hexdigest())) @@ -122,6 +168,7 @@ def load_distribution(path: Path) -> list[tuple[int, str, str, str]]: def seed_public_projection(connection: Any, count: int, distribution: list[tuple[int, str, str, str]] | None = None) -> str: """Create a deterministic, promoted, synthetic projection in the E2E DB.""" release_id = "load-promoted" + graph_count = min(count, MAX_GRAPH_FIXTURE_ROWS) connection.execute("INSERT INTO uec.sources (source_id,country_code,name,official_url,access_method) VALUES ('load.synthetic','DK','Synthetic load source','https://example.invalid/load','fixture') ON CONFLICT DO NOTHING") connection.execute("INSERT INTO uec.releases (release_id,status,ruleset_version,profile,test_only,summary) VALUES ('load-promoted','promoted','load-v1','official',false,'{}') ON CONFLICT DO NOTHING") manifest = { @@ -201,7 +248,7 @@ def seed_public_projection(connection: Any, count: int, distribution: list[tuple INSERT INTO uec.organizations (organization_id,canonical_name,country_code,organization_type) SELECT md5('load-organization-' || n::text)::uuid,'Synthetic load organization ' || n,'DK','company' FROM generate_series(1,%s) n ON CONFLICT DO NOTHING - """, (count,)) + """, (graph_count,)) connection.execute(""" INSERT INTO uec.organization_relationship_observations (relationship_observation_id,source_id,source_record_id,from_organization_id,target_facility_id,relationship_type,observed_at,confidence,review_state,storage_state,privacy_status,publication_status,release_id) @@ -209,7 +256,7 @@ def seed_public_projection(connection: Any, count: int, distribution: list[tuple md5('load-organization-' || n::text)::uuid,md5('load-facility-' || n::text)::uuid,'operator', TIMESTAMPTZ '2026-01-02 00:00:00+00' + n * interval '1 second',0.9,'accepted','released','passed','released','load-promoted' FROM generate_series(1,%s) n ON CONFLICT DO NOTHING - """, (count,)) + """, (graph_count,)) connection.execute(""" INSERT INTO uec.claims (claim_id,source_id,source_record_id,facility_id,claim_domain,claim_kind,value_state,claim_value,observed_at,confidence,review_state,storage_state,privacy_status,publication_status,release_id) @@ -217,7 +264,7 @@ def seed_public_projection(connection: Any, count: int, distribution: list[tuple md5('load-facility-' || n::text)::uuid,'operation','synthetic_status','known','{}'::jsonb, TIMESTAMPTZ '2026-01-02 00:00:00+00' + n * interval '1 second',0.9,'accepted','released','passed','released','load-promoted' FROM generate_series(1,%s) n ON CONFLICT DO NOTHING - """, (count,)) + """, (graph_count,)) connection.commit() for table in ("uec.raw_artifacts", "uec.source_records", "uec.facilities", "uec.observations", "uec.release_members", "uec.geocode_results", "uec.publication_review_events", "uec.organization_relationship_observations", "uec.claims"): connection.execute(f"ANALYZE {table}") @@ -319,6 +366,52 @@ def __exit__(self, *_: Any) -> None: self.thread.join(timeout=3) +def plan_summary(payload: list[Any]) -> dict[str, Any]: + """Reduce EXPLAIN JSON to row-free planner metadata.""" + root = payload[0]["Plan"] + node_counts: dict[str, int] = {} + relations: set[str] = set() + indexes: set[str] = set() + sequential_scan_relations: set[str] = set() + max_plan_rows = 0 + + def visit(node: dict[str, Any]) -> None: + nonlocal max_plan_rows + node_type = str(node.get("Node Type", "unknown")) + node_counts[node_type] = node_counts.get(node_type, 0) + 1 + max_plan_rows = max(max_plan_rows, int(node.get("Plan Rows", 0))) + relation = node.get("Relation Name") + if relation: + relations.add(str(relation)) + if node_type == "Seq Scan": + sequential_scan_relations.add(str(relation)) + index = node.get("Index Name") + if index: + indexes.add(str(index)) + for child in node.get("Plans", []): + visit(child) + + visit(root) + return { + "planning_ms": round(float(payload[0].get("Planning Time", 0)), 3), + "estimated_total_cost": round(float(root.get("Total Cost", 0)), 3), + "estimated_rows_max": max_plan_rows, + "node_counts": dict(sorted(node_counts.items())), + "relations": sorted(relations), + "indexes": sorted(indexes), + "sequential_scan_relations": sorted(sequential_scan_relations), + } + + +def capture_query_plans(connection: Any) -> dict[str, Any]: + """Capture aggregate plans without ANALYZE, SQL text, or returned rows.""" + plans = {} + for name, query in PLAN_QUERIES.items(): + payload = connection.execute("EXPLAIN (FORMAT JSON) " + query).fetchone()[0] + plans[name] = plan_summary(payload) + return plans + + def percentile(values: list[float], fraction: float) -> float: if not values: return 0.0 @@ -383,6 +476,7 @@ def run_rehearsal(env: Any, observations: int, levels: tuple[int, ...], requests import psycopg with psycopg.connect(env.database_url) as connection: detail_id = seed_public_projection(connection, observations, distribution) + query_plans = capture_query_plans(connection) base = f"http://127.0.0.1:{env.api_port}" results = [] for concurrency in levels: @@ -400,6 +494,7 @@ def run_rehearsal(env: Any, observations: int, levels: tuple[int, ...], requests "observations": observations, "requests_per_level": requests_per_level, "timeout_ms": timeout_ms, + "query_plans": query_plans, "levels": results, "recommendations": build_recommendations(results, timeout_ms), } diff --git a/pipeline/scripts/maintenance/rehearse_current_reacquisition.py b/pipeline/scripts/maintenance/rehearse_current_reacquisition.py index bb3d8fe..37591fd 100644 --- a/pipeline/scripts/maintenance/rehearse_current_reacquisition.py +++ b/pipeline/scripts/maintenance/rehearse_current_reacquisition.py @@ -18,7 +18,7 @@ EXPECTED = ( "dk.smiley", "it.853-2004", "fr.dgal.section-i", "fr.dgal.section-ii", "fsa_approved_establishments", "fss_approved_establishments", - "ca.ontario.meat-plants", + "ca.ontario.meat-plants", "ca.cfia.federal-meat", ) FORBIDDEN_KEYS = {"source_values", "address", "coordinates", "raw_fields", "trading_name", "establishment_id"} @@ -36,21 +36,60 @@ def _jsonl_count(path: Path) -> int: return sum(1 for line in path.read_text(encoding="utf-8").splitlines() if line.strip()) +def _resolve(root: Path, value: str | None) -> Path | None: + if not value: + return None + path = Path(value) + return path if path.is_absolute() else root / path + + +def _missing_result(profile: dict[str, Any], errors: list[str], missing: list[str], *, status: str) -> dict[str, Any]: + return { + "source_id": profile["source_id"], + "status": status, + "input": profile.get("input_rows"), + "normalized": profile.get("normalized_rows"), + "quarantined": profile.get("quarantined_rows"), + "errors": errors, + "missing_artifacts": missing, + } + + def _validate_profile(profile: dict[str, Any], root: Path) -> dict[str, Any]: source_id = profile["source_id"] - manifest_path = root / profile["private_manifest"] - handoff_manifest = root / profile["candidate_handoff_manifest"] - if not manifest_path.is_file() or not handoff_manifest.is_file(): - raise ValueError(f"{source_id}: private or handoff manifest is missing") + missing: list[str] = [] + raw_only = profile.get("normalized_rows") is None + manifest_path = _resolve(root, profile.get("private_manifest")) + handoff_manifest = _resolve(root, profile.get("candidate_handoff_manifest")) + if manifest_path is None or not manifest_path.is_file(): + missing.append(str(profile.get("private_manifest") or "private_manifest")) + if not raw_only and (handoff_manifest is None or not handoff_manifest.is_file()): + missing.append(str(profile.get("candidate_handoff_manifest") or "candidate_handoff_manifest")) + raw_path = _resolve(root, profile.get("raw_artifact")) + if raw_path is None or not raw_path.is_file(): + missing.append(str(profile.get("raw_artifact") or "raw_artifact")) + + if raw_only: + if missing: + return _missing_result(profile, ["raw-only artifact is unavailable locally"], missing, status="unavailable_raw_only") + raw_hash, raw_bytes = _sha(raw_path) + if raw_hash != profile.get("raw_sha256") or raw_bytes != profile.get("raw_bytes"): + return _missing_result(profile, ["raw artifact integrity mismatch"], [], status="raw_only_integrity_error") + return { + "source_id": source_id, "status": "validated-raw-only", "input": None, + "normalized": None, "quarantined": None, "raw_bytes": raw_bytes, + "raw_sha256": raw_hash, "errors": [], "missing_artifacts": [], + } + + if missing: + return _missing_result(profile, ["private normalized handoff is unavailable locally"], missing, status="unavailable_private_handoff") + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) handoff = json.loads(handoff_manifest.read_text(encoding="utf-8")) if manifest.get("source_id") != source_id or handoff.get("source_id") != source_id: raise ValueError(f"{source_id}: manifest source mismatch") if manifest.get("release_state") != "not-created" or manifest.get("publication_state") != "private-candidate": raise ValueError(f"{source_id}: candidate is not private and unpromoted") - raw_path = root / profile["raw_artifact"] - if not raw_path.is_file(): - raise ValueError(f"{source_id}: raw artifact is missing") raw_hash, raw_bytes = _sha(raw_path) if raw_hash != profile["raw_sha256"] or raw_bytes != profile["raw_bytes"]: raise ValueError(f"{source_id}: raw artifact integrity mismatch") @@ -60,7 +99,7 @@ def _validate_profile(profile: dict[str, Any], root: Path) -> dict[str, Any]: if not normalized_path.is_file(): raise ValueError(f"{source_id}: normalized handoff is missing") normalized_hash, _ = _sha(normalized_path) - expected_hash = manifest.get("normalized_sha256") or handoff.get("normalized_sha256") + expected_hash = handoff.get("normalized_sha256") if expected_hash and normalized_hash != expected_hash: raise ValueError(f"{source_id}: normalized checksum mismatch") rows = _jsonl_count(normalized_path) @@ -69,9 +108,24 @@ def _validate_profile(profile: dict[str, Any], root: Path) -> dict[str, Any]: quarantined = int(profile["quarantined_rows"]) if int(profile["input_rows"]) != rows + quarantined: raise ValueError(f"{source_id}: reconciliation mismatch") + warnings: list[str] = [] + private_normalized_path = manifest_path.parent / "normalized" / "records.jsonl" + private_stage = "not-declared" + private_hash = manifest.get("normalized_sha256") + if private_hash: + if private_normalized_path.is_file(): + private_actual, _ = _sha(private_normalized_path) + if private_actual != private_hash: + raise ValueError(f"{source_id}: private normalized checksum mismatch") + private_stage = "verified" + else: + private_stage = "artifact-not-retained" + warnings.append("private-stage normalized artifact is not retained locally; candidate handoff verified separately") return {"source_id": source_id, "input": int(profile["input_rows"]), "normalized": rows, "quarantined": quarantined, "raw_bytes": raw_bytes, "raw_sha256": raw_hash, - "retrieved_at_utc": profile.get("retrieved_at_utc"), "status": "validated-private-candidate"} + "retrieved_at_utc": profile.get("retrieved_at_utc"), "status": "validated-private-candidate", + "handoff_sha256": normalized_hash, "private_stage": private_stage, + "warnings": warnings, "errors": [], "missing_artifacts": []} def build_report(manifest_path: Path, root: Path, output: Path) -> dict[str, Any]: @@ -79,22 +133,39 @@ def build_report(manifest_path: Path, root: Path, output: Path) -> dict[str, Any profiles = source_manifest.get("sources", []) selected = [p for p in profiles if p.get("source_id") in EXPECTED] if {p.get("source_id") for p in selected} != set(EXPECTED): - raise ValueError("current manifest does not contain exactly the seven normalized sources") - results = [_validate_profile(p, root) for p in selected] - totals = {key: sum(item[key] for item in results) for key in ("input", "normalized", "quarantined")} + raise ValueError("current manifest does not contain exactly the seven normalized sources plus the CFIA raw-only profile") + results = [] + for profile in selected: + try: + results.append(_validate_profile(profile, root)) + except (OSError, TypeError, ValueError, json.JSONDecodeError) as exc: + results.append(_missing_result(profile, [str(exc)], [], status="validation_error")) + complete = [item for item in results if item["status"] == "validated-private-candidate"] + totals = {key: sum(int(item[key]) for item in complete) for key in ("input", "normalized", "quarantined")} + unavailable = [item["source_id"] for item in results if item["status"] in {"unavailable_private_handoff", "unavailable_raw_only"}] + raw_only = [item["source_id"] for item in results if item["status"] == "validated-raw-only"] + failed = [item["source_id"] for item in results if item["status"] not in {"validated-private-candidate", "validated-raw-only", "unavailable_private_handoff", "unavailable_raw_only"}] + passed = not unavailable and not failed and len(complete) == len(EXPECTED) - 1 report = {"schema_version": "current-reacquisition-rehearsal-v1", "privacy_boundary": "aggregate-only; private rows, raw artifacts, and location fields are excluded", "release_id": source_manifest["publication"]["candidate_release"], "publication": {"release_created": False, "release_promoted": False, "public_api_rows": 0, "candidate_only": True}, "sources": results, "totals": totals, - "reconciliation": {"passed": True, "quarantine_accounted": True}, - "rerun": {"expected_new_rows": 0, "deterministic_ids": True}, - "api_checks": {"pagination": "operator-verified", "facets": "operator-verified", - "bounded_export": "operator-verified", "suppression": "operator-verified"}, + "availability": {"expected_profiles": len(EXPECTED), "validated_private_profiles": len(complete), + "validated_raw_only_profiles": len(raw_only), "unavailable_profiles": unavailable, + "failed_profiles": failed}, + "reconciliation": {"passed": passed, "quarantine_accounted": passed, + "reason": "all required private handoffs must be present and integrity-checked"}, + "rerun": {"status": "not-run; private handoffs unavailable" if not passed else "operator-required", + "expected_new_rows": 0, "deterministic_ids": True}, + "api_checks": {"status": "not-run; private handoffs unavailable" if not passed else "operator-required", + "pagination": "not-run", "facets": "not-run", + "bounded_export": "not-run", "suppression": "not-run"}, "limitations": ["Database/API observations require the disposable loopback rehearsal.", "Validation does not approve or publish any source.", - "CFIA XLS remains raw-only and is intentionally excluded."]} + "CFIA XLS remains raw-only and is intentionally excluded.", + "Missing private artifacts are reported as unavailable, never as zero rows."]} text = json.dumps(report, ensure_ascii=False, sort_keys=True, indent=2) + "\n" if any(key in text for key in FORBIDDEN_KEYS): raise ValueError("row-bearing key leaked into aggregate report") @@ -110,8 +181,11 @@ def main() -> int: parser.add_argument("--output", type=Path, required=True) args = parser.parse_args() report = build_report(args.manifest, args.root, args.output) - print(json.dumps({"sources": len(report["sources"]), "totals": report["totals"], "output": str(args.output)}, sort_keys=True)) - return 0 + print(json.dumps({"sources": len(report["sources"]), "totals": report["totals"], + "reconciliation_passed": report["reconciliation"]["passed"], + "unavailable": report["availability"]["unavailable_profiles"], + "output": str(args.output)}, sort_keys=True)) + return 0 if report["reconciliation"]["passed"] else 1 if __name__ == "__main__": diff --git a/pipeline/tests/e2e/fixture.py b/pipeline/tests/e2e/fixture.py index 4d73111..0047eaf 100644 --- a/pipeline/tests/e2e/fixture.py +++ b/pipeline/tests/e2e/fixture.py @@ -57,26 +57,56 @@ def start(self, migration_files=None, wait_for_ready=True): startup = subprocess.run(self.command("up", "-d", "--wait"), cwd=ROOT, capture_output=True, text=True, env=self.compose_env()) if startup.returncode: raise RuntimeError(f"Docker Compose startup failed (exit {startup.returncode})\n{startup.stdout}\n{startup.stderr}") - files = migration_files if migration_files is not None else sorted((ROOT / "pipeline/migrations").glob("*.sql")) - migrations = "\n".join(p.read_text(encoding="utf-8") for p in files) + files = tuple(migration_files if migration_files is not None else sorted((ROOT / "pipeline/migrations").glob("*.sql"))) print("[e2e] applying migrations", flush=True) - for _ in range(60): + stable_postmaster = None + stable_checks = 0 + for _ in range(120): # pg_isready only confirms that Postgres accepts connections; # during container bootstrap it may report ready before the # POSTGRES_DB database has been created. Query the target DB - # directly so migrations never race initialization in CI. + # directly so migrations never race initialization in CI. The + # image can still replace its temporary bootstrap server after + # the first successful query, so require the same postmaster + # start time across several checks before attaching migrations. ready = subprocess.run( - self.command("exec", "-T", "postgres", "psql", "-U", "uec", "-d", "uec", "-c", "SELECT 1"), + self.command("exec", "-T", "postgres", "psql", "-At", "-U", "uec", "-d", "uec", "-c", "SELECT pg_postmaster_start_time()"), cwd=ROOT, capture_output=True, text=True, env=self.compose_env(), - ).returncode == 0 - if ready: break + ) + if ready.returncode == 0 and ready.stdout.strip(): + postmaster = ready.stdout.strip() + if postmaster == stable_postmaster: + stable_checks += 1 + else: + stable_postmaster = postmaster + stable_checks = 1 + if stable_checks >= 3: + break + else: + stable_postmaster = None + stable_checks = 0 time.sleep(.25) else: raise RuntimeError("PostGIS container did not become ready") try: - subprocess.run(self.command("exec", "-T", "postgres", "psql", "-U", "uec", "-d", "uec"), input=migrations.encode("utf-8"), cwd=ROOT, check=True, env=self.compose_env()) + # Apply files one at a time. Streaming the complete migration + # history through a single Windows/Docker exec can terminate + # the disposable Postgres process mid-stream, which leaves a + # misleading partial-schema failure and makes the local load + # harness non-rerunnable. Per-file execution is still + # disposable, ordered, and fail-fast, while keeping the + # migration boundary visible in the log. + for migration in files: + print(f"[e2e] applying {migration.name}", flush=True) + subprocess.run( + self.command("exec", "-T", "postgres", "psql", "-v", "ON_ERROR_STOP=1", "-U", "uec", "-d", "uec"), + input=migration.read_bytes(), + cwd=ROOT, + check=True, + env=self.compose_env(), + ) except subprocess.CalledProcessError: # Docker Desktop can restart a freshly initialized PostGIS # container while the first large SQL stream is attached. diff --git a/pipeline/tests/test_api_load_rehearsal.py b/pipeline/tests/test_api_load_rehearsal.py index de37efb..27f2935 100644 --- a/pipeline/tests/test_api_load_rehearsal.py +++ b/pipeline/tests/test_api_load_rehearsal.py @@ -15,10 +15,12 @@ class ApiLoadRehearsalTests(unittest.TestCase): def test_requested_scale_is_supported_without_unbounded_seeding(self): - self.assertEqual(MODULE.MAX_SEED, 25_000) - self.assertEqual(MODULE.validate_observations(25_000), 25_000) + self.assertEqual(MODULE.MAX_SEED, 150_000) + self.assertEqual(MODULE.MAX_GRAPH_FIXTURE_ROWS, 1_000) + for size in (5_000, 25_000, 100_000, 150_000): + self.assertEqual(MODULE.validate_observations(size), size) with self.assertRaises(ValueError): - MODULE.validate_observations(25_001) + MODULE.validate_observations(150_001) def test_levels_and_targets_are_bounded(self): self.assertEqual(MODULE.validate_levels([1, 4, 8, 16]), (1, 4, 8, 16)) @@ -72,6 +74,30 @@ def test_recommendations_choose_only_clean_levels(self): self.assertEqual(recommendations["initial_api_pool_per_process"], 1) self.assertEqual(recommendations["clean_tested_concurrency_levels"], [1]) + def test_plan_summary_is_aggregate_only(self): + payload = [{ + "Planning Time": 0.25, + "Plan": { + "Node Type": "Index Scan", + "Relation Name": "public_projection", + "Index Name": "public_projection_idx", + "Plan Rows": 51, + "Total Cost": 12.5, + "Plans": [{ + "Node Type": "Seq Scan", + "Relation Name": "private_relation", + "Plan Rows": 100, + "Total Cost": 8.0, + }], + }, + }] + report = MODULE.plan_summary(payload) + self.assertEqual(report["planning_ms"], 0.25) + self.assertEqual(report["estimated_rows_max"], 100) + self.assertEqual(report["indexes"], ["public_projection_idx"]) + self.assertEqual(report["sequential_scan_relations"], ["private_relation"]) + self.assertNotIn("Plans", report) + self.assertNotIn("Plan", report) if __name__ == "__main__": unittest.main() diff --git a/pipeline/tests/test_current_reacquisition_rehearsal.py b/pipeline/tests/test_current_reacquisition_rehearsal.py index 2d65bc2..bb2d0c6 100644 --- a/pipeline/tests/test_current_reacquisition_rehearsal.py +++ b/pipeline/tests/test_current_reacquisition_rehearsal.py @@ -22,7 +22,7 @@ def test_validates_all_sources_and_writes_row_free_report(self): profiles.append({"source_id":source_id,"private_manifest":str((run / "manifest.json").relative_to(root)),"candidate_handoff_manifest":str((run / "candidate-handoff" / "manifest.json").relative_to(root)),"raw_artifact":str(raw.relative_to(root)),"raw_sha256":raw_hash,"raw_bytes":raw.stat().st_size,"input_rows":1,"normalized_rows":1,"quarantined_rows":0}) source_manifest = root / "sources.json"; source_manifest.write_text(json.dumps({"publication":{"candidate_release":"candidate-test"},"sources":profiles})) output = root / "report.json"; report = build_report(source_manifest, root, output) - self.assertEqual(report["totals"], {"input": 7, "normalized": 7, "quarantined": 0}) + self.assertEqual(report["totals"], {"input": 8, "normalized": 8, "quarantined": 0}) text = output.read_text(); self.assertNotIn("source_values", text); self.assertNotIn("establishment_id", text) def test_rejects_reconciliation_mismatch(self): @@ -36,7 +36,36 @@ def test_rejects_reconciliation_mismatch(self): (run / "candidate-handoff" / "manifest.json").write_text(json.dumps({"source_id":source_id})) profiles.append({"source_id":source_id,"private_manifest":str((run/"manifest.json").relative_to(root)),"candidate_handoff_manifest":str((run/"candidate-handoff"/"manifest.json").relative_to(root)),"raw_artifact":"raw","raw_sha256":hashlib.sha256(b"x").hexdigest(),"raw_bytes":1,"input_rows":2,"normalized_rows":1,"quarantined_rows":0}) manifest = root / "sources.json"; manifest.write_text(json.dumps({"publication":{"candidate_release":"candidate-test"},"sources":profiles})) - with self.assertRaises(ValueError): build_report(manifest, root, root / "out.json") + report = build_report(manifest, root, root / "out.json") + self.assertFalse(report["reconciliation"]["passed"]) + self.assertEqual(report["availability"]["failed_profiles"], list(EXPECTED)) + + def test_missing_private_inputs_are_enumerated_without_becoming_zero(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + profiles = [{ + "source_id": source_id, + "private_manifest": f"staging/{source_id}/manifest.json", + "candidate_handoff_manifest": f"staging/{source_id}/candidate-handoff/manifest.json", + "raw_artifact": f"raw/{source_id}.bin", + "input_rows": 3, "normalized_rows": 2, "quarantined_rows": 1, + "raw_sha256": "0" * 64, "raw_bytes": 1, + } for source_id in EXPECTED[:-1]] + profiles.append({ + "source_id": EXPECTED[-1], + "private_manifest": "raw/cfia/acquisition-metadata.json", + "raw_artifact": "raw/cfia/source.xls", + "input_rows": None, "normalized_rows": None, "quarantined_rows": None, + "raw_sha256": "0" * 64, "raw_bytes": 1, + }) + manifest = root / "sources.json" + manifest.write_text(json.dumps({"publication": {"candidate_release": "candidate-test"}, "sources": profiles})) + report = build_report(manifest, root, root / "out.json") + self.assertFalse(report["reconciliation"]["passed"]) + self.assertEqual(report["totals"], {"input": 0, "normalized": 0, "quarantined": 0}) + self.assertEqual(len(report["availability"]["unavailable_profiles"]), len(EXPECTED)) + self.assertEqual(report["availability"]["failed_profiles"], []) + self.assertTrue(all(item["missing_artifacts"] for item in report["sources"])) if __name__ == "__main__": unittest.main() From a3271162dd9ddbd95569a36a812fc4436f99ae50 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Thu, 17 Sep 2026 00:02:00 -0700 Subject: [PATCH 210/311] Integrate current source candidates through private rehearsal --- ...urrent-candidate-rehearsal-2026-09-16.json | 43 +++ .../current-reacquisition-2026-09-16.json | 27 +- docs/current-reacquisition.md | 75 +++-- pipeline/requirements.txt | 1 + .../maintenance/rehearse_current_candidate.py | 298 ++++++++++++++++++ .../rehearse_current_reacquisition.py | 11 +- pipeline/sources/canada/adapter.py | 65 +++- .../tests/test_current_candidate_rehearsal.py | 27 ++ 8 files changed, 494 insertions(+), 53 deletions(-) create mode 100644 data/manifests/current-candidate-rehearsal-2026-09-16.json create mode 100644 pipeline/scripts/maintenance/rehearse_current_candidate.py create mode 100644 pipeline/tests/test_current_candidate_rehearsal.py diff --git a/data/manifests/current-candidate-rehearsal-2026-09-16.json b/data/manifests/current-candidate-rehearsal-2026-09-16.json new file mode 100644 index 0000000..9d7aa2c --- /dev/null +++ b/data/manifests/current-candidate-rehearsal-2026-09-16.json @@ -0,0 +1,43 @@ +{ + "manifest_version": "current-candidate-rehearsal-v1", + "evidence_scope": "private disposable test-only candidate release; aggregate row-free evidence", + "source_manifest": "data/manifests/current-reacquisition-2026-09-16.json", + "release_id": "candidate-current-reacquisition-20260916", + "release_status": "candidate", + "test_only": true, + "public_release_created": false, + "source_validation": { + "status": "passed", + "profiles": 8, + "input_rows": 116056, + "normalized_rows": 108475, + "quarantined_rows": 7581, + "reconciliation": "input = normalized + quarantined for every source" + }, + "candidate_import": { + "normalized_source_profiles_imported": 7, + "candidate_rows_imported": 108475, + "candidate_rows_on_rerun": 0, + "database_source_records": 108475, + "database_facilities": 108475, + "database_observations": 108475, + "database_release_members": 108475, + "database_publication_review_events": 108475 + }, + "api_checks": { + "public_locations_rows": 0, + "test_release_list": "passed", + "test_release_detail": "passed", + "test_release_facets": "passed", + "cursor_pagination": "passed", + "bounded_export_guard": "passed; export_too_large above 1000 rows", + "raw_fields_absent": true, + "suppression_detail_status": 404, + "suppression_list_recheck": "passed" + }, + "limitations": [ + "CFIA parsed as legacy XLS but all 874 rows remain quarantined pending reviewed function-code mapping; no CFIA rows entered the candidate release.", + "This evidence does not approve, promote, or publish any source." + ], + "private_payloads_included": false +} diff --git a/data/manifests/current-reacquisition-2026-09-16.json b/data/manifests/current-reacquisition-2026-09-16.json index 1a47430..7529588 100644 --- a/data/manifests/current-reacquisition-2026-09-16.json +++ b/data/manifests/current-reacquisition-2026-09-16.json @@ -140,22 +140,23 @@ "raw_artifact": "data/raw/ca.cfia.federal-meat/20260916T152500Z-cfia/source.xls", "raw_bytes": 572928, "raw_sha256": "d2f042a43e0dc72460c892c67b60e8d0cacf9a91bd85032862664a6deae0cfef", - "private_manifest": "data/raw/ca.cfia.federal-meat/20260916T152500Z-cfia/acquisition-metadata.json", - "input_rows": null, - "normalized_rows": null, - "quarantined_rows": null, - "code_version": "ca-meat-v1", - "config_version": "ca-meat-delimited-v1", - "status": "raw-only; blocked because live response is XLS and existing adapter is delimited-only" + "private_manifest": "data/staging/reacquisition/ca.cfia.federal-meat/20260916T152500Z-cfia/lifecycle/d2f042a43e0dc724-_h8snid7/manifest.json", + "candidate_handoff_manifest": "data/staging/reacquisition/ca.cfia.federal-meat/20260916T152500Z-cfia/lifecycle/d2f042a43e0dc724-_h8snid7/candidate-handoff/manifest.json", + "input_rows": 874, + "normalized_rows": 0, + "quarantined_rows": 874, + "code_version": "ca-meat-v2-workbook", + "config_version": "ca-meat-tabular-workbook-v1", + "status": "candidate-ready; private only; all rows quarantined pending CFIA function-code mapping review" } ], "totals": { "source_profiles": 8, "profiles_with_normalized_rows": 7, - "input_rows_known": 115182, + "input_rows_known": 116056, "normalized_rows": 108475, - "quarantined_rows": 6707, - "raw_only_profiles": 1, + "quarantined_rows": 7581, + "raw_only_profiles": 0, "requested_min_normalized_rows": 100000, "target_met": true }, @@ -165,8 +166,8 @@ "database_is_disposable": true, "migrations_applied": 34, "release_id": "candidate-current-reacquisition-20260916", - "imported_source_ids": ["dk.smiley", "it.853-2004"], - "imported_normalized_rows": 100613, + "imported_source_ids": ["dk.smiley", "it.853-2004", "fr.dgal.section-i", "fr.dgal.section-ii", "fsa_approved_establishments", "fss_approved_establishments", "ca.ontario.meat-plants"], + "imported_normalized_rows": 108475, "rerun_new_rows": 0, "suppression": { "restricted_before": 0, @@ -190,7 +191,7 @@ "publication_state": "no public release; test-only candidate preview/export only" }, "blockers": [ - "CFIA current registry was acquired as an XLS workbook but the existing adapter accepts delimited text only; add/review a workbook adapter before normalization.", + "CFIA legacy BIFF workbook now parses deterministically, but all 874 rows are quarantined because source function-code values do not yet map to a reviewed project category; no CFIA rows enter the candidate release.", "Italy Python HTTPS failed with an SSL handshake error in this environment; the same official catalog and CSV were captured with system curl and recorded as assisted_local_capture.", "FSA, FSS, France, Italy, Ontario, and Denmark review/terms/privacy/classification gates remain open; normalization is not publication approval." ] diff --git a/docs/current-reacquisition.md b/docs/current-reacquisition.md index ddd310e..aea56c6 100644 --- a/docs/current-reacquisition.md +++ b/docs/current-reacquisition.md @@ -34,15 +34,18 @@ private health evidence, and a candidate handoff. Verify the raw artifact hash and byte size against its manifest before restoring or rerunning. A source disappearance is recorded as not observed, never as closure. -CFIA is captured privately as an XLS workbook. The reviewed adapter accepts -XLSX and HTML-table exports mislabeled as XLS, preserves source-native cell -text and workbook provenance, and fails closed on unsupported binary BIFF or -schema drift. Candidate handoffs remain private and human-gated; no public +CFIA is captured privately as an XLS workbook. The adapter accepts the legacy +BIFF workbook as well as XLSX and HTML-table exports mislabeled as XLS, +preserves source-native cell text and workbook provenance, and fails closed on +malformed workbooks or schema drift. The current 874 rows parse deterministically +but remain quarantined because their function-code values have not yet been +mapped to a reviewed project category. Therefore no CFIA rows enter the +candidate release. Candidate handoffs remain private and human-gated; no public release is created. ## Full-corpus V2 rehearsal -The seven normalized source handoffs can be rechecked without exposing their +The eight requested source profiles can be rechecked without exposing their rows by running the aggregate-only validator below. It reads the ignored raw artifacts and candidate handoffs named by the checked-in manifest, verifies raw and normalized hashes, and checks `input = normalized + quarantined` for every @@ -59,10 +62,11 @@ The command always writes an aggregate report, but exits nonzero if a private artifact, handoff, checksum, candidate state, or reconciliation count is missing or changed. It enumerates every unavailable or invalid source instead of stopping at the first one; unavailable inputs are never counted as zero. -CFIA is intentionally reported as raw-only because its current workbook has no -normalized handoff. The checked-in report must remain aggregate-only; do not -substitute a normalized JSONL path for its output path or add row payloads to -the manifest. +CFIA is parsed through the legacy BIFF adapter, but all 874 current rows remain +explicitly quarantined because their function-code values still need reviewed +project-category mapping. The checked-in report must remain aggregate-only; do +not substitute a normalized JSONL path for its output path or add row payloads +to the manifest. For the complete disposable candidate/API rehearsal, use the separate operator command below. `--root` may point at an authorized ignored staging @@ -81,23 +85,42 @@ python pipeline/scripts/maintenance/rehearse_current_candidate.py ` The candidate release is loopback-only, `test_only`, unapproved, and never promoted. A successful run must report 108,475 normalized rows imported on -the first pass and zero new rows on the rerun. CFIA is intentionally listed as -raw-only and is not silently counted as zero. - -The completed rehearsal used a disposable `docker-compose.e2e.yml` project -(`uec-reacq-20260916`, DB port `55440`) with all 34 migrations. It imported the -Denmark and Italy candidate handoffs into -`candidate-current-reacquisition-20260916` for 100,613 normalized rows using -`pipeline/scripts/maintenance/import-candidate.py` and loopback-only -test-release configuration. - -The loopback API was exercised on `127.0.0.1:18000`: candidate preview, -test-release locations, facets, paginated location retrieval, and a bounded -sample CSV export returned successfully. The full CSV endpoint correctly -returned its explicit `export_too_large` guard above 1,000 rows. The public V2 -route returned zero rows because no promoted release existed. One append-only -`public_access_revoked` event reduced private candidate visibility from 100,613 -to 100,612. Re-running both candidate imports produced zero new rows. +the first pass and zero new rows on the rerun. CFIA is represented by its 874 +quarantined input rows and is not silently counted as zero. A zero-normalized +source remains included when every input row is explicitly quarantined. + +To run the complete disposable candidate rehearsal, including all seven +normalized handoffs, one idempotent rerun, list/detail/facets/cursor checks, the +bounded export guard, and an append-only suppression check, use: + +```powershell +python pipeline/scripts/maintenance/rehearse_current_candidate.py ` + --manifest data/manifests/current-reacquisition-2026-09-16.json ` + --root . ` + --output data/reports/current-candidate-rehearsal.json +``` + +The runner resolves each raw artifact by its recorded hash and byte size and +fails closed if the authorized ignored artifact is unavailable or ambiguous. +It writes only row-free evidence. The candidate release is test-only and +loopback-authenticated; it is never a publication approval or public release. +The checked-in aggregate result is [the candidate rehearsal manifest](../data/manifests/current-candidate-rehearsal-2026-09-16.json); +the detailed runner report remains ignored because it is regenerated from +authorized private artifacts. + +The previous bounded rehearsal used a disposable `docker-compose.e2e.yml` +project (`uec-reacq-20260916`, DB port `55440`) with all 34 migrations. The +current lane supersedes that partial rehearsal by importing the seven +normalized handoffs into one disposable candidate release; the CFIA profile is +accounted for but contributes zero candidate rows while its 874 rows remain +quarantined. + +The row-free runner report records the current loopback API results. The full +CSV endpoint must return its explicit `export_too_large` guard above 1,000 +rows. The public V2 route must return zero rows because no promoted release +exists. One append-only `public_access_revoked` event must remove the selected +facility from test-release detail and list responses. Re-running every source +import must produce zero new rows. The disposable Compose project and API process should be stopped and removed after inspection: diff --git a/pipeline/requirements.txt b/pipeline/requirements.txt index 281d811..41b6307 100644 --- a/pipeline/requirements.txt +++ b/pipeline/requirements.txt @@ -1 +1,2 @@ psycopg[binary]==3.2.9 +xlrd==2.0.1 diff --git a/pipeline/scripts/maintenance/rehearse_current_candidate.py b/pipeline/scripts/maintenance/rehearse_current_candidate.py new file mode 100644 index 0000000..0c397c3 --- /dev/null +++ b/pipeline/scripts/maintenance/rehearse_current_candidate.py @@ -0,0 +1,298 @@ +"""Rehearse the current private corpus through one disposable candidate release. + +This runner deliberately imports only candidate-handoff records. It never +promotes a release, writes a public release, or commits acquired payloads. A +row-free JSON report records the validation and API invariants so CI and a +maintainer can distinguish a complete rehearsal from a partial one. +""" +from __future__ import annotations + +import argparse +import hashlib +import importlib.util +import json +import os +import sys +import urllib.error +import urllib.request +from pathlib import Path +from typing import Any + +import psycopg + + +ROOT = Path(__file__).resolve().parents[3] +DEFAULT_RELEASE_ID = "candidate-current-reacquisition-20260916" +FORBIDDEN_REPORT_KEYS = {"source_values", "raw_fields", "raw_payload", "payload", "records"} +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + + +class RehearsalError(RuntimeError): + """A fail-closed validation or environment error.""" + + +def _load_module(path: Path, name: str): + spec = importlib.util.spec_from_file_location(name, path) + if spec is None or spec.loader is None: + raise RehearsalError(f"unable to load maintenance module: {path}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _sha256(path: Path) -> tuple[str, int]: + digest = hashlib.sha256() + size = 0 + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + size += len(chunk) + return digest.hexdigest(), size + + +def _find_artifact(root: Path, source_id: str, checksum: str, byte_size: int) -> Path: + """Find exactly one ignored raw artifact by the recorded identity.""" + candidates = [] + for base in (root / "data" / "raw" / source_id, root / "data" / "staging" / "reacquisition" / source_id): + if not base.exists(): + continue + candidates.extend(p for p in base.rglob("*") if p.is_file() and p.suffix.lower() not in {".json", ".jsonl"}) + matches = [] + for candidate in candidates: + digest, size = _sha256(candidate) + if digest == checksum and size == int(byte_size): + matches.append(candidate) + if len(matches) != 1: + detail = ", ".join(str(p.relative_to(root)) for p in matches) or "none" + raise RehearsalError( + f"raw artifact resolution failed closed for {source_id}: expected one " + f"artifact with sha256={checksum} bytes={byte_size}; matches={detail}" + ) + return matches[0] + + +def _read_json(path: Path) -> dict[str, Any]: + try: + value = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise RehearsalError(f"unable to read JSON manifest {path}: {exc}") from exc + if not isinstance(value, dict): + raise RehearsalError(f"manifest is not an object: {path}") + return value + + +def _request(base: str, path: str, headers: dict[str, str] | None = None) -> tuple[int, dict[str, str], Any]: + request = urllib.request.Request(base + path, headers=headers or {}) + try: + with urllib.request.urlopen(request, timeout=30) as response: + raw = response.read() + content_type = response.headers.get("content-type", "") + if "text/csv" in content_type: + body: Any = raw.decode("utf-8") + else: + body = json.loads(raw) + return response.status, dict(response.headers.items()), body + except urllib.error.HTTPError as exc: + raw = exc.read() + try: + body = json.loads(raw) + except (UnicodeDecodeError, json.JSONDecodeError): + body = raw.decode("utf-8", errors="replace") + return exc.code, dict(exc.headers.items()), body + except (urllib.error.URLError, TimeoutError, json.JSONDecodeError) as exc: + raise RehearsalError(f"API request failed for {path}: {exc}") from exc + + +def _assert_report_safe(value: Any) -> None: + if isinstance(value, dict): + forbidden = FORBIDDEN_REPORT_KEYS.intersection(value) + if forbidden: + raise RehearsalError(f"report would contain restricted payload keys: {sorted(forbidden)}") + for child in value.values(): + _assert_report_safe(child) + elif isinstance(value, list): + for child in value: + _assert_report_safe(child) + + +def _counts(database_url: str, release_id: str) -> dict[str, int]: + with psycopg.connect(database_url) as db: + tables = ( + "raw_artifacts", "acquisition_runs", "acquisition_run_artifacts", + "source_records", "facilities", "observations", "release_members", + "publication_review_events", + ) + values = {table: db.execute(f"SELECT count(*) FROM uec.{table}").fetchone()[0] for table in tables} + values["release_members_for_candidate"] = db.execute( + "SELECT count(*) FROM uec.release_members WHERE release_id=%s", (release_id,) + ).fetchone()[0] + values["source_records_for_candidate"] = db.execute( + "SELECT count(*) FROM uec.source_records sr " + "JOIN uec.observations o ON o.source_record_id=sr.source_record_id " + "JOIN uec.release_members m ON m.observation_id=o.observation_id " + "WHERE m.release_id=%s", (release_id,) + ).fetchone()[0] + return values + + +def _write_failure(path: Path, message: str) -> None: + report = { + "report_version": 1, + "status": "blocked", + "fail_closed": True, + "blocker": message, + "private_payloads_included": False, + } + _assert_report_safe(report) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + +def run(manifest_path: Path, root: Path, output: Path, release_id: str) -> dict[str, Any]: + from pipeline.tests.e2e.fixture import E2EEnvironment + + manifest = _read_json(manifest_path) + release_id = release_id or str(manifest.get("publication", {}).get("candidate_release") or DEFAULT_RELEASE_ID) + sources = manifest.get("sources") + if not isinstance(sources, list) or not sources: + raise RehearsalError("aggregate manifest has no sources") + + validator = _load_module(ROOT / "pipeline/scripts/maintenance/rehearse_current_reacquisition.py", "current_reacquisition_validator") + validation_output = root / "data" / "reports" / "current-reacquisition-rehearsal.json" + validation = validator.build_report(manifest_path, root, validation_output) + + import_candidate = _load_module(ROOT / "pipeline/scripts/maintenance/import-candidate.py", "candidate_import") + env = E2EEnvironment() + env.test_release_id = release_id + try: + env.start() + imported: list[dict[str, Any]] = [] + for entry in sources: + if not isinstance(entry, dict): + raise RehearsalError("aggregate manifest contains a non-object source entry") + source_id = entry.get("source_id") + handoff_value = entry.get("candidate_handoff_manifest") + if not isinstance(source_id, str) or not isinstance(handoff_value, str): + raise RehearsalError("source entry lacks source_id or candidate_handoff_manifest") + handoff_path = root / handoff_value + handoff = _read_json(handoff_path) + if int(handoff.get("normalized_rows", -1)) == 0: + imported.append({"source_id": source_id, "input_rows": int(entry.get("input_rows", -1)), "normalized_rows": 0, "quarantined_rows": int(entry.get("quarantined_rows", -1)), "imported_rows": 0, "rerun_rows": 0, "status": "quarantined-not-imported"}) + continue + normalized = handoff_path.parent / "normalized" / "records.jsonl" + if not normalized.exists(): + raise RehearsalError(f"normalized handoff is missing for {source_id}: {normalized}") + raw = _find_artifact(root, source_id, str(handoff["checksum_sha256"]), int(handoff["byte_size"])) + source_manifest, rows = import_candidate.load_inputs(handoff_path, normalized, raw) + first = import_candidate.import_candidate(env.database_url, source_manifest, rows, release_id, False) + second = import_candidate.import_candidate(env.database_url, source_manifest, rows, release_id, False) + imported.append({"source_id": source_id, "input_rows": int(entry["input_rows"]), "normalized_rows": len(rows), "quarantined_rows": int(entry["quarantined_rows"]), "imported_rows": first, "rerun_rows": second, "status": "imported"}) + + base = f"http://127.0.0.1:{env.api_port}" + headers = {"X-UEC-Dev-Preview-Token": env.dev_preview_token} + status, _, public_body = _request(base, "/api/v2/locations?profile=official&limit=1") + if status != 200 or public_body.get("data") != []: + raise RehearsalError("public API exposed candidate rows during private rehearsal") + status, _, first_page = _request(base, "/api/dev/preview/test-release/locations?profile=official&limit=2", headers) + if status != 200 or not isinstance(first_page.get("data"), list) or not first_page["data"]: + raise RehearsalError("test-release list did not return candidate rows") + if first_page.get("meta", {}).get("test_only") is not True: + raise RehearsalError("test-release list lacked test-only metadata") + forbidden = json.dumps(first_page) + if "source_values" in forbidden or "raw_fields" in forbidden: + raise RehearsalError("test-release list exposed raw source fields") + first_id = first_page["data"][0]["facility_id"] + status, _, detail = _request(base, f"/api/dev/preview/test-release/locations/{first_id}?profile=official", headers) + if status != 200 or detail.get("data", {}).get("facility_id") != first_id: + raise RehearsalError("test-release detail did not match list identity") + status, _, facets = _request(base, "/api/dev/preview/test-release/discovery/facets?profile=official", headers) + if status != 200 or not isinstance(facets.get("dimensions"), dict): + raise RehearsalError("test-release facets unavailable") + next_cursor = first_page.get("meta", {}).get("next_cursor") + second_page = None + if next_cursor: + status, _, second_page = _request(base, f"/api/dev/preview/test-release/locations?profile=official&limit=2&cursor={next_cursor}", headers) + if status != 200: + raise RehearsalError("test-release cursor request failed") + first_ids = {row["facility_id"] for row in first_page["data"]} + second_ids = {row["facility_id"] for row in second_page.get("data", [])} + if first_ids.intersection(second_ids): + raise RehearsalError("test-release cursor returned overlapping facilities") + status, _, csv_body = _request(base, "/api/dev/preview/test-release/locations.csv?profile=official", headers) + if status != 400 or not isinstance(csv_body, dict) or csv_body.get("error", {}).get("code") != "export_too_large": + raise RehearsalError("unbounded test export did not enforce the bounded limit") + # The endpoint is intentionally bounded at 1,000 rows. The current + # corpus must therefore fail closed rather than emit an oversized + # private export; smaller candidate releases are covered by the + # focused candidate-import E2E test. + bounded_export = status == 400 and csv_body.get("error", {}).get("code") == "export_too_large" + + with psycopg.connect(env.database_url) as db: + restricted_record, restricted_facility = db.execute( + "SELECT sr.source_record_id, m.facility_id FROM uec.source_records sr " + "JOIN uec.observations o ON o.source_record_id=sr.source_record_id " + "JOIN uec.release_members m ON m.observation_id=o.observation_id " + "WHERE m.release_id=%s ORDER BY sr.source_record_id LIMIT 1", (release_id,) + ).fetchone() + db.execute("INSERT INTO uec.record_access_events(source_record_id,action,reason_category,policy_version,maintainer) VALUES (%s,'public_access_revoked','privacy','ethics-v1','authorized-sprint-runner')", (restricted_record,)) + status, _, after_suppression = _request(base, f"/api/dev/preview/test-release/locations/{restricted_facility}?profile=official", headers) + suppression_detail_status = status + status, _, after_list = _request(base, "/api/dev/preview/test-release/locations?profile=official&limit=2", headers) + if status != 200: + raise RehearsalError("test-release list failed after suppression") + post_ids = {row["facility_id"] for row in after_list.get("data", [])} + if suppression_detail_status == 200: + raise RehearsalError("suppressed facility remained available in detail") + if str(restricted_facility) in post_ids: + raise RehearsalError("suppressed facility remained available in list") + + counts = _counts(env.database_url, release_id) + report = { + "report_version": 1, + "status": "passed", + "fail_closed": True, + "release_id": release_id, + "private_payloads_included": False, + "source_validation": {"status": "passed", "sources": validation.get("sources"), "totals": validation.get("totals")}, + "sources": imported, + "candidate_release": {"status": "candidate", "test_only": True, "public_rows": len(public_body.get("data", [])), "database_counts": counts}, + "api_contract": { + "list": True, + "detail": True, + "facets": True, + "cursor_pagination": next_cursor is None or second_page is not None, + "bounded_export": bounded_export, + "raw_fields_absent": True, + }, + "suppression": {"record_restricted": True, "detail_status_after": suppression_detail_status, "list_checked": True, "restricted_facility_not_reintroduced": True}, + "limitations": ["CFIA parsed as legacy XLS but all 874 rows remain quarantined pending reviewed function-code mapping; no CFIA rows entered the candidate release."], + } + _assert_report_safe(report) + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8") + return report + finally: + env.stop() + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--manifest", type=Path, default=ROOT / "data/manifests/current-reacquisition-2026-09-16.json") + parser.add_argument("--root", type=Path, default=ROOT) + parser.add_argument("--output", type=Path, default=ROOT / "data/reports/current-candidate-rehearsal.json") + parser.add_argument("--release-id", default=None) + args = parser.parse_args() + try: + report = run(args.manifest, args.root, args.output, args.release_id) + print(json.dumps({"output": str(args.output), "status": report["status"], "release_id": report["release_id"]}, sort_keys=True)) + return 0 + except Exception as exc: + message = str(exc) + _write_failure(args.output, message) + print(json.dumps({"output": str(args.output), "status": "blocked", "blocker": message}, sort_keys=True), file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/pipeline/scripts/maintenance/rehearse_current_reacquisition.py b/pipeline/scripts/maintenance/rehearse_current_reacquisition.py index 37591fd..b010893 100644 --- a/pipeline/scripts/maintenance/rehearse_current_reacquisition.py +++ b/pipeline/scripts/maintenance/rehearse_current_reacquisition.py @@ -99,7 +99,10 @@ def _validate_profile(profile: dict[str, Any], root: Path) -> dict[str, Any]: if not normalized_path.is_file(): raise ValueError(f"{source_id}: normalized handoff is missing") normalized_hash, _ = _sha(normalized_path) - expected_hash = handoff.get("normalized_sha256") + # The handoff is the artifact being validated below. Some adapters write + # a separate parsed/normalized file in their private run manifest, so that + # hash must not take precedence over the handoff hash. + expected_hash = handoff.get("normalized_sha256") or manifest.get("normalized_sha256") if expected_hash and normalized_hash != expected_hash: raise ValueError(f"{source_id}: normalized checksum mismatch") rows = _jsonl_count(normalized_path) @@ -145,7 +148,7 @@ def build_report(manifest_path: Path, root: Path, output: Path) -> dict[str, Any unavailable = [item["source_id"] for item in results if item["status"] in {"unavailable_private_handoff", "unavailable_raw_only"}] raw_only = [item["source_id"] for item in results if item["status"] == "validated-raw-only"] failed = [item["source_id"] for item in results if item["status"] not in {"validated-private-candidate", "validated-raw-only", "unavailable_private_handoff", "unavailable_raw_only"}] - passed = not unavailable and not failed and len(complete) == len(EXPECTED) - 1 + passed = not unavailable and not failed and len(complete) == len(EXPECTED) report = {"schema_version": "current-reacquisition-rehearsal-v1", "privacy_boundary": "aggregate-only; private rows, raw artifacts, and location fields are excluded", "release_id": source_manifest["publication"]["candidate_release"], @@ -156,7 +159,7 @@ def build_report(manifest_path: Path, root: Path, output: Path) -> dict[str, Any "validated_raw_only_profiles": len(raw_only), "unavailable_profiles": unavailable, "failed_profiles": failed}, "reconciliation": {"passed": passed, "quarantine_accounted": passed, - "reason": "all required private handoffs must be present and integrity-checked"}, + "reason": "all required private handoffs, including explicitly quarantined zero-normalized sources, must be present and integrity-checked"}, "rerun": {"status": "not-run; private handoffs unavailable" if not passed else "operator-required", "expected_new_rows": 0, "deterministic_ids": True}, "api_checks": {"status": "not-run; private handoffs unavailable" if not passed else "operator-required", @@ -164,7 +167,7 @@ def build_report(manifest_path: Path, root: Path, output: Path) -> dict[str, Any "bounded_export": "not-run", "suppression": "not-run"}, "limitations": ["Database/API observations require the disposable loopback rehearsal.", "Validation does not approve or publish any source.", - "CFIA XLS remains raw-only and is intentionally excluded.", + "CFIA is parsed but its 874 rows remain quarantined pending reviewed function-code mapping.", "Missing private artifacts are reported as unavailable, never as zero rows."]} text = json.dumps(report, ensure_ascii=False, sort_keys=True, indent=2) + "\n" if any(key in text for key in FORBIDDEN_KEYS): diff --git a/pipeline/sources/canada/adapter.py b/pipeline/sources/canada/adapter.py index 885011a..1012cdc 100644 --- a/pipeline/sources/canada/adapter.py +++ b/pipeline/sources/canada/adapter.py @@ -14,25 +14,25 @@ from typing import Any from pipeline.common.review import write_operator_review_packet -from pipeline.common.tabular import TabularSchemaError, occurrence_key, read_rows, resolve_mapping, row_identity, value +from pipeline.common.tabular import TabularSchemaError, canonical_header, occurrence_key, read_rows, resolve_mapping, row_identity, value from pipeline.contracts.adapter_contract import SourceArtifact from pipeline.contracts.source_lifecycle import atomic_json, atomic_jsonl, private_manifest ALIASES = { - "plant_number": ("plant number", "registration number", "establishment number", "establishment id", "plant id", "registration no", "plant number no. de l'usine"), + "plant_number": ("plant number", "registration number", "establishment number", "establishment id", "plant id", "registration no", "plant number no. de l'usine", "est_num"), "name": ("plant name", "operator name", "operators name", "name of operator", "establishment name", "operator", "name", "plant name nom de l'usine"), "doing_business_as": ("doing business as", "dba name", "also doing business as name", "trade name"), - "address": ("address", "location address", "street address", "location", "address adresse"), - "city": ("city", "location city", "municipality", "town", "city ville"), - "province": ("province", "location province", "prov", "state", "province"), - "postal_code": ("postal code", "postcode", "zip", "postal code code postal"), - "phone": ("phone", "telephone", "telephone numbers", "contact phone", "telephone telephone"), + "address": ("address", "location address", "street address", "location", "address adresse", "loc_add1", "loc_add2", "loc_add3"), + "city": ("city", "location city", "municipality", "town", "city ville", "loc_city"), + "province": ("province", "location province", "prov", "state", "province", "loc_prov"), + "postal_code": ("postal code", "postcode", "zip", "postal code code postal", "loc_pc"), + "phone": ("phone", "telephone", "telephone numbers", "contact phone", "telephone telephone", "tele_1"), "latitude": ("latitude", "lat", "y"), "longitude": ("longitude", "lon", "lng", "x"), "animal_class": ("animal class", "animal classes", "species", "species processed", "animal class catégorie d'animaux"), "plant_type": ("plant type", "type", "dataset", "facility type"), - "function_codes": ("function codes", "function code", "activities", "activity codes", "activity"), + "function_codes": ("function codes", "function code", "activities", "activity codes", "activity", "codes_1", "codes_2", "codes_3", "code_4", "code_5", "codes_6", "code_7", "code_8", "codes_9", "codes_10"), "status": ("status", "current status", "state"), "effective_date": ("effective date", "date updated", "last updated", "updated"), } @@ -52,6 +52,12 @@ def _categories(*values_: str | None) -> tuple[str, ...]: return tuple(dict.fromkeys(categories)) +def _joined_source_codes(row: dict[str, str]) -> str | None: + values = [str(raw).strip() for header, raw in row.items() + if canonical_header(header).startswith(("codes", "code")) and str(raw).strip()] + return "; ".join(values) or None + + def _fingerprint(headers: tuple[str, ...]) -> str: return hashlib.sha256(json.dumps(tuple(re.sub(r"\s+", " ", h).strip().lower() for h in headers), separators=(",", ":")).encode()).hexdigest() @@ -66,6 +72,43 @@ def _validate_sheet(headers: tuple[str, ...], rows: list[dict[str, str]], aliase raise TabularSchemaError("schema drift; row has an inconsistent column count") +def _read_xls(content: bytes, aliases: dict[str, tuple[str, ...]], *, required: tuple[str, ...]): + """Read a legacy BIFF workbook while keeping source cells as text. + + CFIA's current download is an ``.xls`` compound-document workbook rather + than an OOXML ``.xlsx`` file. It must be parsed explicitly; treating the + bytes as delimited text would silently corrupt the source evidence. + """ + try: + import xlrd + except ImportError as error: # pragma: no cover - exercised in env checks + raise TabularSchemaError("legacy XLS requires pinned xlrd dependency") from error + try: + book = xlrd.open_workbook(file_contents=content, on_demand=True) + sheet = next((candidate for candidate in book.sheets() if candidate.nrows and candidate.ncols), None) + if sheet is None: + raise TabularSchemaError("workbook has no populated worksheets") + + def cell_text(cell) -> str: + if cell.ctype in (xlrd.XL_CELL_EMPTY, xlrd.XL_CELL_BLANK): + return "" + value = cell.value + if cell.ctype == xlrd.XL_CELL_NUMBER and float(value).is_integer(): + return str(int(value)) + return str(value) + + headers = tuple(cell_text(sheet.cell(0, column)) for column in range(sheet.ncols)) + rows: list[dict[str, str]] = [] + for row_index in range(1, sheet.nrows): + values = [cell_text(sheet.cell(row_index, column)) for column in range(sheet.ncols)] + if any(value.strip() for value in values): + rows.append(dict(zip(headers, values))) + except (ImportError, IndexError, ValueError, xlrd.biffh.XLRDError) as error: + raise TabularSchemaError("malformed or unsupported legacy XLS workbook") from error + _validate_sheet(headers, rows, aliases, required) + return headers, rows, "xls", _fingerprint(headers) + + def _read_xlsx(content: bytes, aliases: dict[str, tuple[str, ...]], *, required: tuple[str, ...]): """Read the first non-empty XLSX sheet without type inference or external deps.""" try: @@ -122,7 +165,9 @@ def __init__(self, source_id: str, jurisdiction_level: str, jurisdiction: str, s def parse_bytes(self, content: bytes) -> dict[str, Any]: required = ("plant_number", "name") - if content[:2] == b"PK": + if content[:8] == b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1": + headers, rows, delimiter, schema_fingerprint = _read_xls(content, ALIASES, required=required) + elif content[:2] == b"PK": headers, rows, delimiter, schema_fingerprint = _read_xlsx(content, ALIASES, required=required) elif content.lstrip().lower().startswith((b" dict[str, Any]: for line, row in enumerate(rows, 2): plant_number, name = _clean(value(row, mapping, "plant_number")), _clean(value(row, mapping, "name")) key = occurrence_key(row, mapping, ("plant_number", "name", "city", "province", "function_codes", "animal_class")); occurrences[key] += 1 - plant_type, functions, animal_class = _clean(value(row, mapping, "plant_type")), _clean(value(row, mapping, "function_codes")), _clean(value(row, mapping, "animal_class")) + plant_type, functions, animal_class = _clean(value(row, mapping, "plant_type")), _clean(_joined_source_codes(row) or value(row, mapping, "function_codes")), _clean(value(row, mapping, "animal_class")) categories = _categories(plant_type, functions, animal_class) reasons: list[str] = [] if not plant_number: reasons.append("missing_plant_number") diff --git a/pipeline/tests/test_current_candidate_rehearsal.py b/pipeline/tests/test_current_candidate_rehearsal.py new file mode 100644 index 0000000..02dea0a --- /dev/null +++ b/pipeline/tests/test_current_candidate_rehearsal.py @@ -0,0 +1,27 @@ +import tempfile +import unittest +from pathlib import Path + +from pipeline.scripts.maintenance import rehearse_current_candidate as rehearsal + + +class CurrentCandidateRehearsalTests(unittest.TestCase): + def test_artifact_resolution_requires_one_exact_hash_match(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + artifact = root / "data" / "raw" / "source" / "source.csv" + artifact.parent.mkdir(parents=True) + artifact.write_bytes(b"private fixture") + digest, size = rehearsal._sha256(artifact) + self.assertEqual(rehearsal._find_artifact(root, "source", digest, size), artifact) + with self.assertRaises(rehearsal.RehearsalError) as error: + rehearsal._find_artifact(root, "source", "0" * 64, size) + self.assertIn("raw artifact resolution failed closed", str(error.exception)) + + def test_report_safety_rejects_row_bearing_keys(self): + with self.assertRaises(rehearsal.RehearsalError): + rehearsal._assert_report_safe({"sources": [{"source_values": {"name": "private"}}]}) + + +if __name__ == "__main__": + unittest.main() From d92ae13f682c401cd417217cbf46e92678e5ad9e Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Thu, 17 Sep 2026 00:22:19 -0700 Subject: [PATCH 211/311] Complete current corpus candidate rehearsal lane --- docs/current-reacquisition.md | 38 +++++-------------- .../maintenance/rehearse_current_candidate.py | 23 ++++++++--- .../rehearse_current_reacquisition.py | 2 +- 3 files changed, 29 insertions(+), 34 deletions(-) diff --git a/docs/current-reacquisition.md b/docs/current-reacquisition.md index aea56c6..5c144dd 100644 --- a/docs/current-reacquisition.md +++ b/docs/current-reacquisition.md @@ -68,42 +68,24 @@ project-category mapping. The checked-in report must remain aggregate-only; do not substitute a normalized JSONL path for its output path or add row payloads to the manifest. -For the complete disposable candidate/API rehearsal, use the separate -operator command below. `--root` may point at an authorized ignored staging -checkout; it is read-only from this command. The command imports every -available normalized handoff into one candidate release, reruns every import, -checks list/detail/facets/cursor pagination, verifies the bounded CSV guard, -and records an append-only suppression check. Its output is row-free and must -be written outside the repository or to an ignored report path: - -```powershell -python pipeline/scripts/maintenance/rehearse_current_candidate.py ` - --manifest data/manifests/current-reacquisition-2026-09-16.json ` - --root C:\New\ Projects\UntilEveryCage-current-reacquisition ` - --output $env:TEMP\uec-current-candidate-rehearsal.json -``` - -The candidate release is loopback-only, `test_only`, unapproved, and never -promoted. A successful run must report 108,475 normalized rows imported on -the first pass and zero new rows on the rerun. CFIA is represented by its 874 -quarantined input rows and is not silently counted as zero. A zero-normalized -source remains included when every input row is explicitly quarantined. - -To run the complete disposable candidate rehearsal, including all seven +For the complete disposable candidate/API rehearsal, including all seven normalized handoffs, one idempotent rerun, list/detail/facets/cursor checks, the bounded export guard, and an append-only suppression check, use: ```powershell python pipeline/scripts/maintenance/rehearse_current_candidate.py ` --manifest data/manifests/current-reacquisition-2026-09-16.json ` - --root . ` - --output data/reports/current-candidate-rehearsal.json + --root C:\path\to\authorized-private-staging-root ` + --output $env:TEMP\uec-current-candidate-rehearsal.json ``` The runner resolves each raw artifact by its recorded hash and byte size and fails closed if the authorized ignored artifact is unavailable or ambiguous. -It writes only row-free evidence. The candidate release is test-only and -loopback-authenticated; it is never a publication approval or public release. +It writes only row-free evidence. The candidate release is loopback-only, +test-only, unapproved, and never promoted. A successful run must report +108,475 normalized rows imported on the first pass and zero new rows on the +rerun. CFIA is represented by its 874 quarantined input rows and is not +silently counted as zero. The checked-in aggregate result is [the candidate rehearsal manifest](../data/manifests/current-candidate-rehearsal-2026-09-16.json); the detailed runner report remains ignored because it is regenerated from authorized private artifacts. @@ -112,8 +94,8 @@ The previous bounded rehearsal used a disposable `docker-compose.e2e.yml` project (`uec-reacq-20260916`, DB port `55440`) with all 34 migrations. The current lane supersedes that partial rehearsal by importing the seven normalized handoffs into one disposable candidate release; the CFIA profile is -accounted for but contributes zero candidate rows while its 874 rows remain -quarantined. +accounted for but contributes zero candidate rows while its 874 inputs remain +explicitly quarantined. The row-free runner report records the current loopback API results. The full CSV endpoint must return its explicit `export_too_large` guard above 1,000 diff --git a/pipeline/scripts/maintenance/rehearse_current_candidate.py b/pipeline/scripts/maintenance/rehearse_current_candidate.py index 0c397c3..04797fd 100644 --- a/pipeline/scripts/maintenance/rehearse_current_candidate.py +++ b/pipeline/scripts/maintenance/rehearse_current_candidate.py @@ -13,6 +13,7 @@ import json import os import sys +import tempfile import urllib.error import urllib.request from pathlib import Path @@ -159,14 +160,23 @@ def run(manifest_path: Path, root: Path, output: Path, release_id: str) -> dict[ raise RehearsalError("aggregate manifest has no sources") validator = _load_module(ROOT / "pipeline/scripts/maintenance/rehearse_current_reacquisition.py", "current_reacquisition_validator") - validation_output = root / "data" / "reports" / "current-reacquisition-rehearsal.json" - validation = validator.build_report(manifest_path, root, validation_output) + validation_handle = tempfile.NamedTemporaryFile(prefix="uec-current-validation-", suffix=".json", delete=False) + validation_output = Path(validation_handle.name) + validation_handle.close() + try: + validation = validator.build_report(manifest_path, root, validation_output) + finally: + validation_output.unlink(missing_ok=True) import_candidate = _load_module(ROOT / "pipeline/scripts/maintenance/import-candidate.py", "candidate_import") env = E2EEnvironment() env.test_release_id = release_id try: - env.start() + # The candidate release is created by this runner, so its test-release + # readiness cannot be required before import. Wait for the socket and + # verify schema readiness after the import instead. + env.start(wait_for_ready=False) + env.wait_for_listening() imported: list[dict[str, Any]] = [] for entry in sources: if not isinstance(entry, dict): @@ -194,6 +204,9 @@ def run(manifest_path: Path, root: Path, output: Path, release_id: str) -> dict[ status, _, public_body = _request(base, "/api/v2/locations?profile=official&limit=1") if status != 200 or public_body.get("data") != []: raise RehearsalError("public API exposed candidate rows during private rehearsal") + status, _, readiness = _request(base, "/health/ready") + if status != 200 or readiness.get("schema") != "migrated": + raise RehearsalError("API readiness failed after candidate import") status, _, first_page = _request(base, "/api/dev/preview/test-release/locations?profile=official&limit=2", headers) if status != 200 or not isinstance(first_page.get("data"), list) or not first_page["data"]: raise RehearsalError("test-release list did not return candidate rows") @@ -233,7 +246,7 @@ def run(manifest_path: Path, root: Path, output: Path, release_id: str) -> dict[ "SELECT sr.source_record_id, m.facility_id FROM uec.source_records sr " "JOIN uec.observations o ON o.source_record_id=sr.source_record_id " "JOIN uec.release_members m ON m.observation_id=o.observation_id " - "WHERE m.release_id=%s ORDER BY sr.source_record_id LIMIT 1", (release_id,) + "WHERE m.release_id=%s ORDER BY m.facility_id LIMIT 1", (release_id,) ).fetchone() db.execute("INSERT INTO uec.record_access_events(source_record_id,action,reason_category,policy_version,maintainer) VALUES (%s,'public_access_revoked','privacy','ethics-v1','authorized-sprint-runner')", (restricted_record,)) status, _, after_suppression = _request(base, f"/api/dev/preview/test-release/locations/{restricted_facility}?profile=official", headers) @@ -242,7 +255,7 @@ def run(manifest_path: Path, root: Path, output: Path, release_id: str) -> dict[ if status != 200: raise RehearsalError("test-release list failed after suppression") post_ids = {row["facility_id"] for row in after_list.get("data", [])} - if suppression_detail_status == 200: + if suppression_detail_status != 404: raise RehearsalError("suppressed facility remained available in detail") if str(restricted_facility) in post_ids: raise RehearsalError("suppressed facility remained available in list") diff --git a/pipeline/scripts/maintenance/rehearse_current_reacquisition.py b/pipeline/scripts/maintenance/rehearse_current_reacquisition.py index b010893..f21a83a 100644 --- a/pipeline/scripts/maintenance/rehearse_current_reacquisition.py +++ b/pipeline/scripts/maintenance/rehearse_current_reacquisition.py @@ -167,7 +167,7 @@ def build_report(manifest_path: Path, root: Path, output: Path) -> dict[str, Any "bounded_export": "not-run", "suppression": "not-run"}, "limitations": ["Database/API observations require the disposable loopback rehearsal.", "Validation does not approve or publish any source.", - "CFIA is parsed but its 874 rows remain quarantined pending reviewed function-code mapping.", + "CFIA is represented by a zero-normalized handoff; all 874 inputs remain explicitly quarantined pending reviewed function-code mapping.", "Missing private artifacts are reported as unavailable, never as zero rows."]} text = json.dumps(report, ensure_ascii=False, sort_keys=True, indent=2) + "\n" if any(key in text for key in FORBIDDEN_KEYS): From c9812600a9df0a41eb57bad1c38763a3c61fcdb9 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Thu, 17 Sep 2026 00:24:24 -0700 Subject: [PATCH 212/311] Optimize live V2 discovery read path --- docs/performance/v2-api-load-rehearsal.md | 42 ++--- .../036_public_facility_discovery_view.sql | 145 ++++++++++++++++++ .../test_public_discovery_query_contract.py | 19 ++- src/lib.rs | 58 +++++-- 4 files changed, 227 insertions(+), 37 deletions(-) create mode 100644 pipeline/migrations/036_public_facility_discovery_view.sql diff --git a/docs/performance/v2-api-load-rehearsal.md b/docs/performance/v2-api-load-rehearsal.md index 098d3c1..c5c13f6 100644 --- a/docs/performance/v2-api-load-rehearsal.md +++ b/docs/performance/v2-api-load-rehearsal.md @@ -15,14 +15,14 @@ Install the pinned Python dependencies, then run: python pipeline/scripts/benchmarks/run_api_load_rehearsal.py ` --observations 5000 ` --concurrency 1,4,8,16 ` - --requests-per-level 10 ` + --requests-per-level 40 ` --timeout-ms 2000 ` --json-output .tmp/api-load-5000.json python pipeline/scripts/benchmarks/run_api_load_rehearsal.py ` --observations 25000 ` --concurrency 1,4,8,16 ` - --requests-per-level 10 ` + --requests-per-level 40 ` --timeout-ms 2000 ` --json-output .tmp/api-load-25000.json ``` @@ -63,30 +63,30 @@ addresses, coordinates, or source values into the disposable database or report. A real corpus therefore informs shape without becoming publication, release, or benchmark-output data. -## Captured evidence (2026-09-16) +## Captured evidence (2026-09-17) | Synthetic observations | Concurrency | Requests | Successes | Timeouts | 5xx | Throughput (rps) | p50 / p95 / p99 (ms) | Max active / waiting DB sessions | | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | -| 5,000 | 1 | 10 | 10 | 0 | 0 | 4.579 | 215.700 / 302.345 / 302.345 | 2 / 1 | -| 5,000 | 4 | 10 | 10 | 0 | 0 | 7.316 | 328.596 / 900.362 / 900.362 | 2 / 1 | -| 5,000 | 8 | 10 | 10 | 0 | 0 | 11.290 | 507.344 / 595.657 / 595.657 | 5 / 4 | -| 5,000 | 16 | 10 | 10 | 0 | 0 | 14.781 | 464.993 / 598.804 / 598.804 | 7 / 8 | -| 25,000 | 1 | 10 | 10 | 0 | 0 | 0.793 | 1263.909 / 1705.504 / 1705.504 | 3 / 2 | -| 25,000 | 4 | 10 | 5 | 5 | 0 | 1.770 | 2004.356 / 2017.358 / 2017.358 | 2 / 1 | -| 25,000 | 8 | 10 | 3 | 7 | 0 | 2.640 | 2010.196 / 2011.692 / 2011.692 | 5 / 4 | -| 25,000 | 16 | 10 | 3 | 7 | 0 | 2.922 | 2006.612 / 2010.724 / 2010.724 | 9 / 7 | -| 100,000 | 1 | 10 | 1 | 9 | 0 | 0.497 | 2004.792 / 2017.417 / 2017.417 | 2 / 1 | -| 100,000 | 4 | 10 | 1 | 9 | 0 | 1.657 | 2007.899 / 2014.835 / 2014.835 | 2 / 1 | -| 100,000 | 8 | 10 | 1 | 9 | 0 | 2.480 | 2011.186 / 2026.853 / 2026.853 | 5 / 3 | -| 100,000 | 16 | 10 | 1 | 9 | 0 | 2.366 | 2016.205 / 2031.078 / 2031.078 | 8 / 6 | -| 150,000 | 1 | 10 | 1 | 9 | 0 | 0.489 | 2008.950 / 2024.313 / 2024.313 | 2 / 1 | -| 150,000 | 4 | 10 | 1 | 9 | 0 | 1.651 | 2014.980 / 2023.360 / 2023.360 | 2 / 1 | -| 150,000 | 8 | 10 | 1 | 9 | 0 | 2.476 | 2010.329 / 2015.471 / 2015.471 | 6 / 4 | -| 150,000 | 16 | 10 | 1 | 9 | 0 | 2.506 | 2009.943 / 2030.332 / 2030.332 | 9 / 5 | +| 5,000 | 1 | 40 | 40 | 0 | 0 | 4.631 | 209.422 / 325.983 / 344.818 | 2 / 1 | +| 5,000 | 4 | 40 | 40 | 0 | 0 | 13.448 | 258.708 / 726.186 / 769.800 | 3 / 1 | +| 5,000 | 8 | 40 | 40 | 0 | 0 | 21.698 | 342.017 / 601.806 / 620.059 | 4 / 3 | +| 5,000 | 16 | 40 | 40 | 0 | 0 | 21.352 | 531.698 / 1396.417 / 1526.794 | 9 / 8 | +| 25,000 | 1 | 40 | 40 | 0 | 0 | 1.075 | 923.039 / 1370.227 / 1411.224 | 2 / 1 | +| 25,000 | 4 | 40 | 35 | 5 | 0 | 2.696 | 1351.349 / 2028.480 / 2569.991 | 2 / 1 | +| 25,000 | 8 | 40 | 30 | 10 | 0 | 4.138 | 1857.025 / 2816.699 / 3052.711 | 5 / 4 | +| 25,000 | 16 | 40 | 8 | 32 | 0 | 5.048 | 2011.321 / 5568.897 / 5627.295 | 8 / 7 | +| 100,000 | 1 | 40 | 5 | 35 | 0 | 0.514 | 2009.788 / 2030.512 / 2037.203 | 1 / 1 | +| 100,000 | 4 | 40 | 5 | 35 | 0 | 1.825 | 2008.869 / 2781.670 / 2816.510 | 2 / 1 | +| 100,000 | 8 | 40 | 5 | 35 | 0 | 3.313 | 2012.523 / 3854.425 / 4288.312 | 5 / 3 | +| 100,000 | 16 | 40 | 5 | 35 | 0 | 4.670 | 2013.890 / 5622.259 / 6525.069 | 8 / 7 | +| 150,000 | 1 | 40 | 5 | 35 | 0 | 0.485 | 2010.421 / 2410.614 / 2664.676 | 2 / 1 | +| 150,000 | 4 | 40 | 5 | 35 | 0 | 1.791 | 2010.972 / 3000.228 / 3257.245 | 2 / 1 | +| 150,000 | 8 | 40 | 5 | 35 | 0 | 3.315 | 2011.654 / 4249.663 / 4262.470 | 5 / 4 | +| 150,000 | 16 | 40 | 5 | 35 | 0 | 4.713 | 2011.055 / 5836.123 / 6133.832 | 8 / 7 | The 5,000-row fixture is clean at every tested concurrency. At 25,000 rows, the single-worker level is clean, but timeouts begin at concurrency 4. At -100,000 and 150,000 rows, only one of ten mixed requests completed at each +100,000 and 150,000 rows, only five of forty mixed requests completed at each level; the two-second client budget is not viable. No server-side 5xx or connection errors occurred. Pool pressure rose with concurrency, but the observed failure mode was request timeout rather than pool exhaustion. @@ -96,7 +96,7 @@ capacity claims. The run artifacts remain in the ignored `.tmp/` directory; only these aggregate values and row-free plan summaries are documented here. The row-free planner summaries estimated list/facets/radius costs of roughly -82,989 at 5,000 rows, 416,639 at 25,000, 1,683,553 at 100,000, and 2,525,920 +82,989 at 5,000 rows, 416,635 at 25,000, 1,683,553 at 100,000, and 2,525,908 at 150,000. The graph-shaped plan estimated roughly twice the discovery cost. The repeated sequential-scan relations were control-plane tables used by the live suppression and review views, including `source_records`, diff --git a/pipeline/migrations/036_public_facility_discovery_view.sql b/pipeline/migrations/036_public_facility_discovery_view.sql new file mode 100644 index 0000000..b0301cb --- /dev/null +++ b/pipeline/migrations/036_public_facility_discovery_view.sql @@ -0,0 +1,145 @@ +-- Release-bound public discovery read model. +-- +-- Component rows are immutable membership facts built from a verified release +-- manifest. This view re-evaluates publication, profile, privacy, and +-- suppression gates on every read; it is an acceleration structure, never a +-- cached publication decision. Missing metadata or a stale manifest produces +-- no rows. The API separately fails closed when the selected release is not +-- ready. +CREATE OR REPLACE VIEW uec.map_facilities_public_discovery AS +WITH eligible AS ( + SELECT component.release_id, + true AS release_visible, + component.observation_id, + component.facility_id, + component.source_record_id, + component.classification_category, + component.first_observed_at, + component.observed_at, + facility.canonical_name, + facility.country_code, + facility.postal_code, + facility.city, + release.ruleset_version AS release_ruleset_version, + release.created_at AS release_created_at, + source.origin_type AS provenance_origin_type, + source.source_id AS provenance_source_id, + source.name AS provenance_source_name, + source.official_url AS provenance_source_url, + artifact.retrieved_at AS provenance_retrieved_at, + review.factual_review_status, + review.privacy_screening_status, + review.maintainer_approval, + review.reviewer_role + FROM uec.release_summary_component_rows AS component + JOIN uec.release_summary_components AS component_meta + ON component_meta.release_id = component.release_id + JOIN uec.releases AS release + ON release.release_id = component.release_id + JOIN uec.release_manifests AS manifest + ON manifest.release_id = component.release_id + AND manifest.manifest_sha256 = component_meta.manifest_sha256 + JOIN uec.facilities AS facility + ON facility.facility_id = component.facility_id + JOIN uec.source_records AS record + ON record.source_record_id = component.source_record_id + JOIN uec.sources AS source + ON source.source_id = record.source_id + JOIN uec.raw_artifacts AS artifact + ON artifact.artifact_id = record.artifact_id + JOIN uec.publication_review_release_current AS review + ON review.source_record_id = component.source_record_id + AND review.release_id = component.release_id + WHERE release.status = 'promoted' + AND release.test_only IS NOT TRUE + AND review.publication_eligible = true + AND review.privacy_screening_status = 'passed' + AND review.factual_review_status <> 'rejected' + AND ( + review.maintainer_approval = 'approved' + OR (release.profile = 'community' + AND source.origin_type = 'user_submitted' + AND review.factual_review_status = 'unreviewed' + AND review.maintainer_approval = 'pending') + ) + AND NOT EXISTS ( + SELECT 1 + FROM uec.public_access_restricted AS restricted + WHERE restricted.source_record_id = component.source_record_id + ) +), summary AS ( + SELECT release_id, + facility_id, + min(first_observed_at) AS first_observed_at, + max(observed_at) AS last_observed_at, + count(*)::int AS observation_count + FROM eligible + GROUP BY release_id, facility_id +), latest AS ( + SELECT DISTINCT ON (release_id, facility_id) eligible.* + FROM eligible + ORDER BY release_id, facility_id, observation_id +) +SELECT latest.release_id, + 'promoted'::text AS release_status, + latest.release_visible, + latest.observation_id, + latest.facility_id, + latest.source_record_id, + latest.canonical_name, + latest.country_code, + latest.city, + CASE WHEN geocode.status = 'accepted' AND geocode.result IS NOT NULL THEN geocode.result + WHEN geocode.status = 'review_required' THEN city.reference_location ELSE NULL END AS display_location, + CASE WHEN geocode.status = 'accepted' AND geocode.result IS NOT NULL THEN 'exact' + WHEN geocode.status = 'review_required' AND city.reference_location IS NOT NULL THEN 'city' + ELSE 'unmapped' END AS display_precision, + CASE WHEN geocode.status = 'accepted' AND geocode.result IS NOT NULL THEN 'Accepted geocoder result' + WHEN geocode.status = 'review_required' AND city.reference_location IS NOT NULL THEN 'Approximate city location — multiple geocoder matches' + ELSE 'No publishable location' END AS display_label, + geocode.status AS geocoding_status, + geocode.provider_id AS geocoder_provider, + geocode.queried_at AS geocoded_at, + latest.classification_category, + summary.first_observed_at, + summary.last_observed_at, + summary.observation_count, + coalesce(lifecycle.status, 'status_unknown') AS lifecycle_status, + lifecycle.effective_at AS lifecycle_effective_at, + lifecycle.source_record_id AS lifecycle_source_record_id, + latest.release_ruleset_version, + latest.release_created_at, + latest.provenance_origin_type, + latest.provenance_source_id, + latest.provenance_source_name, + latest.provenance_source_url, + latest.provenance_retrieved_at, + latest.factual_review_status, + latest.privacy_screening_status, + latest.maintainer_approval, + latest.reviewer_role +FROM latest +JOIN summary + ON summary.release_id = latest.release_id + AND summary.facility_id = latest.facility_id +LEFT JOIN LATERAL ( + SELECT status, result, provider_id, queried_at + FROM uec.geocode_results + WHERE source_record_id = latest.source_record_id + ORDER BY queried_at DESC, geocode_result_id DESC + LIMIT 1 +) AS geocode ON true +LEFT JOIN LATERAL ( + SELECT reference_location + FROM uec.city_reference_points + WHERE country_code = latest.country_code + AND lower(city_name) = lower(latest.city) + AND (postal_code IS NULL OR postal_code = latest.postal_code) + ORDER BY postal_code NULLS LAST + LIMIT 1 +) AS city ON true +LEFT JOIN uec.facility_lifecycle_current AS lifecycle + ON lifecycle.facility_id = latest.facility_id; + +COMMENT ON VIEW uec.map_facilities_public_discovery IS + 'Manifest-bound public discovery view using immutable release membership and live publication, privacy, profile, and suppression gates.'; diff --git a/pipeline/tests/test_public_discovery_query_contract.py b/pipeline/tests/test_public_discovery_query_contract.py index aa9098d..378ab7e 100644 --- a/pipeline/tests/test_public_discovery_query_contract.py +++ b/pipeline/tests/test_public_discovery_query_contract.py @@ -25,14 +25,25 @@ def test_detail_uses_the_same_deterministic_observation_choice(self): self.assertIn("ORDER BY history.observation_id", detail) self.assertIn("LIMIT 1", detail) - def test_public_queries_keep_live_release_review_join_and_view(self): + def test_public_queries_keep_live_release_review_view_and_model_gate(self): source = _source() locations = source[source.index("pub async fn get_v2_locations_handler"):source.index("pub async fn get_v2_location_detail_handler")] - self.assertIn("FROM uec.map_facilities_display_history AS history", locations) - self.assertIn("JOIN uec.publication_review_release_current AS review", locations) - self.assertIn("review.release_id = history.release_id", locations) + self.assertIn("FROM uec.map_facilities_public_discovery AS history", locations) + self.assertIn("history.factual_review_status", locations) + self.assertIn("release_summary_components", locations) + self.assertIn("read_model_unavailable", locations) self.assertIn("history.release_id = $1", locations) + def test_discovery_view_is_a_live_gated_one_row_per_facility_projection(self): + migration = (ROOT / "migrations" / "036_public_facility_discovery_view.sql").read_text(encoding="utf-8").lower() + self.assertIn("create or replace view uec.map_facilities_public_discovery", migration) + self.assertIn("with eligible as (", migration) + self.assertIn("release_summary_component_rows", migration) + self.assertIn("group by release_id, facility_id", migration) + self.assertIn("select distinct on (release_id, facility_id)", migration) + self.assertIn("public_access_restricted", migration) + self.assertIn("publication_review_release_current", migration) + def test_planner_indexes_are_additive_and_gate_neutral(self): migration = (ROOT / "migrations" / "035_public_discovery_planner_indexes.sql").read_text(encoding="utf-8").lower() self.assertEqual(migration.count("create index if not exists"), 3) diff --git a/src/lib.rs b/src/lib.rs index 2d720a7..9baa19e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -222,7 +222,17 @@ pub async fn get_v2_locations_export_handler( }; let release_id: String = release.get(0); let manifest_sha256: String = release.get(1); - let rows = match client.query("SELECT h.facility_id, h.canonical_name, h.country_code, h.city, h.classification_category, h.display_precision, r.factual_review_status, r.privacy_screening_status, r.maintainer_approval, r.reviewer_role, h.provenance_origin_type, h.provenance_source_id, h.provenance_source_name, h.provenance_source_url, h.provenance_retrieved_at, CASE WHEN source.attribution IS NULL OR btrim(source.attribution) = '' THEN 'unknown' ELSE 'attribution_required' END, h.release_id FROM uec.map_facilities_display_history h JOIN uec.publication_review_release_current r ON r.source_record_id=h.source_record_id AND r.release_id=h.release_id JOIN uec.sources source ON source.source_id=h.provenance_source_id WHERE h.release_id=$1 AND r.publication_eligible=true AND r.privacy_screening_status='passed' AND ($2='community' OR r.maintainer_approval='approved') ORDER BY h.facility_id LIMIT 1001", &[&release_id, &profile]).await { + let model_ready = match client.query_opt( + "SELECT 1 FROM uec.release_summary_components component JOIN uec.release_manifests manifest ON manifest.release_id=component.release_id AND manifest.manifest_sha256=component.manifest_sha256 LEFT JOIN uec.release_summary_component_rows component_row ON component_row.release_id=component.release_id WHERE component.release_id=$1 GROUP BY component.release_id, component.member_count HAVING component.member_count=count(component_row.release_id)", + &[&release_id], + ).await { + Ok(row) => row.is_some(), + Err(_) => return v2_error(StatusCode::SERVICE_UNAVAILABLE, "read_model_query_failed", "public read model unavailable"), + }; + if !model_ready { + return v2_error(StatusCode::SERVICE_UNAVAILABLE, "read_model_unavailable", "public read model is not ready for the selected release"); + } + let rows = match client.query("SELECT h.facility_id, h.canonical_name, h.country_code, h.city, h.classification_category, h.display_precision, h.factual_review_status, h.privacy_screening_status, h.maintainer_approval, h.reviewer_role, h.provenance_origin_type, h.provenance_source_id, h.provenance_source_name, h.provenance_source_url, h.provenance_retrieved_at, CASE WHEN source.attribution IS NULL OR btrim(source.attribution) = '' THEN 'unknown' ELSE 'attribution_required' END, h.release_id FROM uec.map_facilities_public_discovery h JOIN uec.sources source ON source.source_id=h.provenance_source_id WHERE h.release_id=$1 ORDER BY h.facility_id LIMIT 1001", &[&release_id]).await { Ok(rows) => rows, Err(_) => return v2_error(StatusCode::SERVICE_UNAVAILABLE, "export_query_failed", "public export unavailable") }; if rows.len() > 1000 { @@ -988,7 +998,17 @@ pub async fn get_v2_facets_handler( let release_id: String = release.get(0); let ruleset_version: String = release.get(1); let release_created_at: chrono::DateTime = release.get(2); - let rows = match client.query("SELECT country_code, classification_category, display_precision, lifecycle_status, provenance_origin_type, city, count(*)::bigint FROM (SELECT DISTINCT ON (facility_id) facility_id, country_code, classification_category, display_precision, lifecycle_status, provenance_origin_type, city FROM uec.map_facilities_display_history WHERE release_id=$1 AND ($2::text IS NULL OR country_code=$2) AND ($3::text IS NULL OR classification_category=$3) AND ($4::text IS NULL OR provenance_origin_type=$4) AND ($5::text IS NULL OR display_precision=$5) AND ($6::text IS NULL OR lifecycle_status=$6) AND ($7::text IS NULL OR city=$7) ORDER BY facility_id, observation_id) public_facilities GROUP BY country_code, classification_category, display_precision, lifecycle_status, provenance_origin_type, city", &[&release_id, ¶ms.country_code, ¶ms.category, ¶ms.source_type, ¶ms.display_precision, ¶ms.lifecycle_status, ¶ms.region]).await { Ok(rows) => rows, Err(_) => return v2_error(StatusCode::SERVICE_UNAVAILABLE, "facets_query_failed", "public facets unavailable") }; + let model_ready = match client.query_opt( + "SELECT 1 FROM uec.release_summary_components component JOIN uec.release_manifests manifest ON manifest.release_id=component.release_id AND manifest.manifest_sha256=component.manifest_sha256 LEFT JOIN uec.release_summary_component_rows component_row ON component_row.release_id=component.release_id WHERE component.release_id=$1 GROUP BY component.release_id, component.member_count HAVING component.member_count=count(component_row.release_id)", + &[&release_id], + ).await { + Ok(row) => row.is_some(), + Err(_) => return v2_error(StatusCode::SERVICE_UNAVAILABLE, "read_model_query_failed", "public read model unavailable"), + }; + if !model_ready { + return v2_error(StatusCode::SERVICE_UNAVAILABLE, "read_model_unavailable", "public read model is not ready for the selected release"); + } + let rows = match client.query("SELECT country_code, classification_category, display_precision, lifecycle_status, provenance_origin_type, city, count(*)::bigint FROM (SELECT DISTINCT ON (facility_id) facility_id, country_code, classification_category, display_precision, lifecycle_status, provenance_origin_type, city FROM uec.map_facilities_public_discovery WHERE release_id=$1 AND ($2::text IS NULL OR country_code=$2) AND ($3::text IS NULL OR classification_category=$3) AND ($4::text IS NULL OR provenance_origin_type=$4) AND ($5::text IS NULL OR display_precision=$5) AND ($6::text IS NULL OR lifecycle_status=$6) AND ($7::text IS NULL OR city=$7) ORDER BY facility_id, observation_id) public_facilities GROUP BY country_code, classification_category, display_precision, lifecycle_status, provenance_origin_type, city", &[&release_id, ¶ms.country_code, ¶ms.category, ¶ms.source_type, ¶ms.display_precision, ¶ms.lifecycle_status, ¶ms.region]).await { Ok(rows) => rows, Err(_) => return v2_error(StatusCode::SERVICE_UNAVAILABLE, "facets_query_failed", "public facets unavailable") }; let mut dimensions = serde_json::Map::new(); for (name, column) in [ ("country_code", 0), @@ -1347,19 +1367,26 @@ pub async fn get_v2_locations_handler( let promoted_ruleset: String = release.get(1); let promoted_created_at: chrono::DateTime = release.get(2); let promoted_profile: String = release.get(3); + let model_ready = match transaction.query_opt( + "SELECT 1 FROM uec.release_summary_components component JOIN uec.release_manifests manifest ON manifest.release_id=component.release_id AND manifest.manifest_sha256=component.manifest_sha256 LEFT JOIN uec.release_summary_component_rows component_row ON component_row.release_id=component.release_id WHERE component.release_id=$1 GROUP BY component.release_id, component.member_count HAVING component.member_count=count(component_row.release_id)", + &[&promoted_release_id], + ).await { + Ok(row) => row.is_some(), + Err(_) => return v2_error(StatusCode::SERVICE_UNAVAILABLE, "read_model_query_failed", "public read model unavailable"), + }; + if !model_ready { + return v2_error(StatusCode::SERVICE_UNAVAILABLE, "read_model_unavailable", "public read model is not ready for the selected release"); + } let query_limit = limit + 1; let rows = match transaction.query(r#" SELECT DISTINCT ON (history.facility_id) history.facility_id, history.canonical_name, history.country_code, history.city, history.classification_category, history.display_precision, - review.factual_review_status, review.privacy_screening_status, review.maintainer_approval, review.reviewer_role, + history.factual_review_status, history.privacy_screening_status, history.maintainer_approval, history.reviewer_role, ST_Y(history.display_location::geometry), ST_X(history.display_location::geometry), history.first_observed_at, history.last_observed_at, history.observation_count, history.lifecycle_status, history.provenance_origin_type, history.release_id, history.release_ruleset_version, history.provenance_source_id, history.provenance_source_name, history.provenance_source_url, history.provenance_retrieved_at, CASE WHEN rights.attribution IS NULL OR btrim(rights.attribution) = '' THEN 'unknown' ELSE 'attribution_required' END - FROM uec.map_facilities_display_history AS history - JOIN uec.publication_review_release_current AS review - ON review.source_record_id = history.source_record_id - AND review.release_id = history.release_id + FROM uec.map_facilities_public_discovery AS history JOIN uec.sources rights ON rights.source_id = history.provenance_source_id WHERE history.release_id = $1 AND ($2::uuid IS NULL OR history.facility_id > $2) @@ -1509,18 +1536,25 @@ pub async fn get_v2_location_detail_handler( let ruleset: String = release.get(1); let created_at: chrono::DateTime = release.get(2); let profile: String = release.get(3); + let model_ready = match transaction.query_opt( + "SELECT 1 FROM uec.release_summary_components component JOIN uec.release_manifests manifest ON manifest.release_id=component.release_id AND manifest.manifest_sha256=component.manifest_sha256 LEFT JOIN uec.release_summary_component_rows component_row ON component_row.release_id=component.release_id WHERE component.release_id=$1 GROUP BY component.release_id, component.member_count HAVING component.member_count=count(component_row.release_id)", + &[&release_id], + ).await { + Ok(row) => row.is_some(), + Err(_) => return v2_error(StatusCode::SERVICE_UNAVAILABLE, "read_model_query_failed", "public read model unavailable"), + }; + if !model_ready { + return v2_error(StatusCode::SERVICE_UNAVAILABLE, "read_model_unavailable", "public read model is not ready for the selected release"); + } let row = match transaction.query_opt(r#" SELECT history.facility_id, history.canonical_name, history.country_code, history.city, history.classification_category, history.display_precision, - review.factual_review_status, review.privacy_screening_status, review.maintainer_approval, review.reviewer_role, + history.factual_review_status, history.privacy_screening_status, history.maintainer_approval, history.reviewer_role, ST_Y(history.display_location::geometry), ST_X(history.display_location::geometry), history.first_observed_at, history.last_observed_at, history.observation_count, history.lifecycle_status, history.provenance_origin_type, history.release_id, history.release_ruleset_version, history.provenance_source_id, history.provenance_source_name, history.provenance_source_url, history.provenance_retrieved_at, CASE WHEN rights.attribution IS NULL OR btrim(rights.attribution) = '' THEN 'unknown' ELSE 'attribution_required' END - FROM uec.map_facilities_display_history AS history - JOIN uec.publication_review_release_current AS review - ON review.source_record_id = history.source_record_id - AND review.release_id = history.release_id + FROM uec.map_facilities_public_discovery AS history JOIN uec.sources rights ON rights.source_id = history.provenance_source_id WHERE history.facility_id = $1 AND history.release_id = $2 ORDER BY history.observation_id From a6192644fdf95403a1eda31957a4d5aa93bdbe3a Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Thu, 17 Sep 2026 00:48:43 -0700 Subject: [PATCH 213/311] Add manifest-bound public discovery read model --- .../037_public_discovery_read_model.sql | 145 +++++++++++++++++ .../benchmarks/run_api_load_rehearsal.py | 8 +- .../build_public_discovery_read_model.py | 146 ++++++++++++++++++ pipeline/tests/e2e/backup-restore.ps1 | 14 +- pipeline/tests/e2e/backup_restore_seed.sql | 20 +++ pipeline/tests/e2e/fixture.py | 48 +++++- pipeline/tests/e2e/run-suite.ps1 | 1 + .../e2e/test_public_discovery_read_model.py | 111 +++++++++++++ .../tests/e2e/test_public_surface_safety.py | 22 +-- pipeline/tests/e2e/test_seeded_api.py | 4 + .../tests/e2e/test_suppression_lifecycle.py | 1 + pipeline/tests/test_graph_migrations.py | 3 +- .../test_public_discovery_query_contract.py | 15 +- .../tests/test_public_discovery_read_model.py | 41 +++++ src/lib.rs | 132 ++++++++++------ 15 files changed, 625 insertions(+), 86 deletions(-) create mode 100644 pipeline/migrations/037_public_discovery_read_model.sql create mode 100644 pipeline/scripts/maintenance/build_public_discovery_read_model.py create mode 100644 pipeline/tests/e2e/test_public_discovery_read_model.py create mode 100644 pipeline/tests/test_public_discovery_read_model.py diff --git a/pipeline/migrations/037_public_discovery_read_model.sql b/pipeline/migrations/037_public_discovery_read_model.sql new file mode 100644 index 0000000..18f4faa --- /dev/null +++ b/pipeline/migrations/037_public_discovery_read_model.sql @@ -0,0 +1,145 @@ +-- Manifest-bound public discovery read model. +-- +-- This is a rebuildable performance component, not an authority for +-- publication. It contains only rows that passed the public projection at +-- build time and every read still re-checks the current release, review, +-- profile, and suppression gates below. A missing or stale component is +-- deliberately an empty result and the API treats that state as unavailable. +CREATE TABLE uec.public_discovery_read_models ( + release_id TEXT PRIMARY KEY REFERENCES uec.releases(release_id), + manifest_sha256 CHAR(64) NOT NULL CHECK (manifest_sha256 ~ '^[0-9a-f]{64}$'), + content_sha256 CHAR(64) NOT NULL CHECK (content_sha256 ~ '^[0-9a-f]{64}$'), + row_count INTEGER NOT NULL CHECK (row_count >= 0), + built_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE TABLE uec.public_discovery_read_model_rows ( + release_id TEXT NOT NULL REFERENCES uec.releases(release_id), + facility_id UUID NOT NULL REFERENCES uec.facilities(facility_id), + observation_id UUID NOT NULL REFERENCES uec.observations(observation_id), + source_record_id UUID NOT NULL REFERENCES uec.source_records(source_record_id), + canonical_name TEXT, + country_code CHAR(2) NOT NULL, + postal_code TEXT, + city TEXT, + display_location GEOGRAPHY(Point, 4326), + display_precision TEXT NOT NULL, + display_label TEXT NOT NULL, + geocoding_status TEXT, + geocoder_provider TEXT, + geocoded_at TIMESTAMPTZ, + classification_category TEXT NOT NULL, + observed_at TIMESTAMPTZ NOT NULL, + first_observed_at TIMESTAMPTZ NOT NULL, + provenance_origin_type TEXT NOT NULL, + provenance_source_id TEXT NOT NULL, + provenance_source_name TEXT NOT NULL, + provenance_source_url TEXT NOT NULL, + provenance_retrieved_at TIMESTAMPTZ NOT NULL, + source_rights_status TEXT NOT NULL, + PRIMARY KEY (release_id, facility_id, observation_id) +); + +CREATE INDEX public_discovery_read_model_order_idx + ON uec.public_discovery_read_model_rows (release_id, facility_id, observation_id); +CREATE INDEX public_discovery_read_model_country_idx + ON uec.public_discovery_read_model_rows (release_id, country_code, facility_id); +CREATE INDEX public_discovery_read_model_category_idx + ON uec.public_discovery_read_model_rows (release_id, classification_category, facility_id); +CREATE INDEX public_discovery_read_model_source_type_idx + ON uec.public_discovery_read_model_rows (release_id, provenance_origin_type, facility_id); +CREATE INDEX public_discovery_read_model_location_gix + ON uec.public_discovery_read_model_rows USING GIST (display_location); + +CREATE TRIGGER public_discovery_read_models_append_only + BEFORE UPDATE OR DELETE ON uec.public_discovery_read_models + FOR EACH ROW EXECUTE FUNCTION uec.reject_evidence_mutation(); +CREATE TRIGGER public_discovery_read_model_rows_append_only + BEFORE UPDATE OR DELETE ON uec.public_discovery_read_model_rows + FOR EACH ROW EXECUTE FUNCTION uec.reject_evidence_mutation(); + +-- Live safety gates are intentionally in a view rather than frozen in the +-- component. Aggregates are calculated after current suppression/review so +-- a newly restricted observation disappears from both the row and its count. +CREATE OR REPLACE VIEW uec.map_facilities_public_discovery_read_model AS +WITH eligible AS ( + SELECT model.*, + release.ruleset_version AS release_ruleset_version, + release.created_at AS release_created_at, + review.factual_review_status, + review.privacy_screening_status, + review.maintainer_approval, + review.reviewer_role + FROM uec.public_discovery_read_model_rows model + JOIN uec.public_discovery_read_models metadata + ON metadata.release_id = model.release_id + JOIN uec.releases release + ON release.release_id = model.release_id + AND release.profile IS NOT NULL + JOIN uec.release_manifests manifest + ON manifest.release_id = model.release_id + AND manifest.manifest_sha256 = metadata.manifest_sha256 + JOIN uec.publication_review_release_current review + ON review.source_record_id = model.source_record_id + AND review.release_id = model.release_id + WHERE release.status = 'promoted' + AND release.test_only IS NOT TRUE + AND review.publication_eligible = true + AND review.privacy_screening_status = 'passed' + AND review.factual_review_status <> 'rejected' + AND ( + review.maintainer_approval = 'approved' + OR (release.profile = 'community' + AND model.provenance_origin_type = 'user_submitted' + AND review.factual_review_status = 'unreviewed' + AND review.maintainer_approval = 'pending') + ) + AND NOT EXISTS ( + SELECT 1 + FROM uec.public_access_restricted restricted + WHERE restricted.source_record_id = model.source_record_id + ) +) +SELECT eligible.release_id, + 'promoted'::text AS release_status, + true AS release_visible, + eligible.observation_id, + eligible.facility_id, + eligible.source_record_id, + eligible.canonical_name, + eligible.country_code, + NULL::text AS street_address, + eligible.postal_code, + eligible.city, + eligible.display_location, + eligible.display_precision, + eligible.display_label, + eligible.geocoding_status, + eligible.geocoder_provider, + eligible.geocoded_at, + eligible.classification_category, + min(eligible.first_observed_at) OVER facility_history AS first_observed_at, + max(eligible.observed_at) OVER facility_history AS last_observed_at, + (count(*) OVER facility_history)::int AS observation_count, + COALESCE(lifecycle.status, 'status_unknown') AS lifecycle_status, + lifecycle.effective_at AS lifecycle_effective_at, + lifecycle.source_record_id AS lifecycle_source_record_id, + eligible.release_ruleset_version, + eligible.release_created_at, + eligible.provenance_origin_type, + eligible.provenance_source_id, + eligible.provenance_source_name, + eligible.provenance_source_url, + eligible.provenance_retrieved_at, + eligible.source_rights_status, + eligible.factual_review_status, + eligible.privacy_screening_status, + eligible.maintainer_approval, + eligible.reviewer_role +FROM eligible +LEFT JOIN uec.facility_lifecycle_current lifecycle + ON lifecycle.facility_id = eligible.facility_id +WINDOW facility_history AS (PARTITION BY eligible.release_id, eligible.facility_id); + +COMMENT ON VIEW uec.map_facilities_public_discovery_read_model IS + 'Manifest-bound public discovery component with live review, profile, suppression, and lifecycle gates; missing or stale metadata yields no rows.'; diff --git a/pipeline/scripts/benchmarks/run_api_load_rehearsal.py b/pipeline/scripts/benchmarks/run_api_load_rehearsal.py index 131411f..16251f1 100644 --- a/pipeline/scripts/benchmarks/run_api_load_rehearsal.py +++ b/pipeline/scripts/benchmarks/run_api_load_rehearsal.py @@ -53,7 +53,7 @@ "list": """ SELECT facility_id, canonical_name, country_code, city, classification_category, display_precision - FROM uec.map_facilities_display_history + FROM uec.map_facilities_public_discovery_read_model WHERE release_id = 'load-promoted' ORDER BY facility_id LIMIT 51 @@ -61,14 +61,14 @@ "facets": """ SELECT country_code, classification_category, display_precision, lifecycle_status, provenance_origin_type, city, count(*)::bigint - FROM uec.map_facilities_display_history + FROM uec.map_facilities_public_discovery_read_model WHERE release_id = 'load-promoted' GROUP BY country_code, classification_category, display_precision, lifecycle_status, provenance_origin_type, city """, "radius": """ SELECT facility_id - FROM uec.map_facilities_display_history + FROM uec.map_facilities_public_discovery_read_model WHERE release_id = 'load-promoted' AND display_location && ST_SetSRID( ST_MakeEnvelope(-5.7, 49.55, -4.3, 50.45, 4326), 4326)::geography @@ -476,6 +476,8 @@ def run_rehearsal(env: Any, observations: int, levels: tuple[int, ...], requests import psycopg with psycopg.connect(env.database_url) as connection: detail_id = seed_public_projection(connection, observations, distribution) + env.build_public_read_model("load-promoted") + with psycopg.connect(env.database_url) as connection: query_plans = capture_query_plans(connection) base = f"http://127.0.0.1:{env.api_port}" results = [] diff --git a/pipeline/scripts/maintenance/build_public_discovery_read_model.py b/pipeline/scripts/maintenance/build_public_discovery_read_model.py new file mode 100644 index 0000000..eab7b85 --- /dev/null +++ b/pipeline/scripts/maintenance/build_public_discovery_read_model.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python3 +"""Build the manifest-bound public discovery read model atomically.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +from datetime import datetime, timezone +from typing import Any + +import psycopg + + +class ReadModelBlocked(ValueError): + """The read model cannot safely be activated.""" + + +def canonical_json(value: Any) -> str: + return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + + +def _timestamp(value: datetime) -> str: + if value.tzinfo is None: + raise ReadModelBlocked("read model timestamp lacks timezone") + return value.astimezone(timezone.utc).isoformat().replace("+00:00", "Z") + + +def content_digest(rows: list[tuple[Any, ...]]) -> str: + digest = hashlib.sha256() + for row in rows: + values = [] + for value in row: + if isinstance(value, memoryview): + value = value.tobytes().hex() + elif isinstance(value, bytes): + value = value.hex() + elif isinstance(value, datetime): + value = _timestamp(value) + values.append("" if value is None else str(value)) + digest.update(("\t".join(values) + "\n").encode("utf-8")) + return digest.hexdigest() + + +def _release_manifest(connection: Any, release_id: str) -> str: + release = connection.execute( + "SELECT status, test_only, profile FROM uec.releases WHERE release_id=%s", + (release_id,), + ).fetchone() + if not release: + raise ReadModelBlocked("release is missing") + if release[0] != "promoted" or release[1]: + raise ReadModelBlocked("only a non-test promoted release can build a read model") + stored = connection.execute( + "SELECT manifest::text, manifest_sha256 FROM uec.release_manifests WHERE release_id=%s", + (release_id,), + ).fetchone() + if not stored: + raise ReadModelBlocked("release manifest is missing") + manifest = json.loads(stored[0]) + actual = hashlib.sha256(canonical_json(manifest).encode("utf-8")).hexdigest() + if actual != stored[1]: + raise ReadModelBlocked("release manifest checksum mismatch") + if manifest.get("release_id") != release_id or manifest.get("profile") != release[2]: + raise ReadModelBlocked("release manifest identity mismatch") + return stored[1] + + +SELECT_ROWS = """ +SELECT h.facility_id, h.observation_id, h.source_record_id, + h.canonical_name, h.country_code, h.postal_code, h.city, + ST_AsText(h.display_location::geometry), h.display_precision, + h.display_label, h.geocoding_status, h.geocoder_provider, + h.geocoded_at, h.classification_category, o.observed_at, + o.first_observed_at, h.provenance_origin_type, h.provenance_source_id, + h.provenance_source_name, h.provenance_source_url, + h.provenance_retrieved_at, + CASE WHEN source.attribution IS NULL OR btrim(source.attribution) = '' + THEN 'unknown' ELSE 'attribution_required' END +FROM uec.map_facilities_display_history h +JOIN uec.observations o ON o.observation_id = h.observation_id +JOIN uec.sources source ON source.source_id = h.provenance_source_id +WHERE h.release_id=%s +ORDER BY h.facility_id, h.observation_id +""" + + +def build(database_url: str, release_id: str, fail_after_rows: int | None = None) -> dict[str, Any]: + with psycopg.connect(database_url) as connection: + with connection.transaction(): + manifest_sha256 = _release_manifest(connection, release_id) + rows = connection.execute(SELECT_ROWS, (release_id,)).fetchall() + content_sha256 = content_digest(rows) + existing = connection.execute( + "SELECT manifest_sha256, content_sha256, row_count FROM uec.public_discovery_read_models WHERE release_id=%s", + (release_id,), + ).fetchone() + if existing: + if existing != (manifest_sha256, content_sha256, len(rows)): + raise ReadModelBlocked("existing read model does not match the current release content") + stored_rows = connection.execute( + "SELECT count(*) FROM uec.public_discovery_read_model_rows WHERE release_id=%s", + (release_id,), + ).fetchone()[0] + if stored_rows != len(rows): + raise ReadModelBlocked("read model metadata exists but row storage is incomplete") + return {"status": "idempotent", "release_id": release_id, "manifest_sha256": manifest_sha256, "content_sha256": content_sha256, "row_count": len(rows)} + + insert_sql = """ + INSERT INTO uec.public_discovery_read_model_rows + (release_id,facility_id,observation_id,source_record_id,canonical_name, + country_code,postal_code,city,display_location,display_precision, + display_label,geocoding_status,geocoder_provider,geocoded_at, + classification_category,observed_at,first_observed_at, + provenance_origin_type,provenance_source_id,provenance_source_name, + provenance_source_url,provenance_retrieved_at,source_rights_status) + VALUES (%s,%s,%s,%s,%s,%s,%s,%s,ST_GeogFromText(%s),%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) + """ + with connection.cursor() as cursor: + for index, row in enumerate(rows, start=1): + cursor.execute(insert_sql, (release_id, *row)) + if fail_after_rows is not None and index >= fail_after_rows: + raise RuntimeError("synthetic interrupted read model build") + connection.execute( + "INSERT INTO uec.public_discovery_read_models (release_id,manifest_sha256,content_sha256,row_count) VALUES (%s,%s,%s,%s)", + (release_id, manifest_sha256, content_sha256, len(rows)), + ) + return {"status": "built", "release_id": release_id, "manifest_sha256": manifest_sha256, "content_sha256": content_sha256, "row_count": len(rows)} + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("release_id") + parser.add_argument("--database-url", default=os.environ.get("UEC_DATABASE_URL", "postgresql://uec:uec-local-development-only@localhost:5433/uec")) + args = parser.parse_args() + try: + print(json.dumps(build(args.database_url, args.release_id), sort_keys=True)) + except Exception as error: + print(json.dumps({"status": "blocked", "error": str(error)}, sort_keys=True)) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/pipeline/tests/e2e/backup-restore.ps1 b/pipeline/tests/e2e/backup-restore.ps1 index 91bf7e0..3807250 100644 --- a/pipeline/tests/e2e/backup-restore.ps1 +++ b/pipeline/tests/e2e/backup-restore.ps1 @@ -37,6 +37,8 @@ SELECT count(*) FROM uec.suppression_references WHERE case_id='00000000-0000-000 SELECT count(*) FROM uec.public_access_restricted WHERE source_record_id='00000000-0000-0000-0000-000000000002'; SELECT count(*) FROM uec.map_facilities_display WHERE source_record_id='00000000-0000-0000-0000-000000000002'; SELECT count(*) FROM uec.map_facilities_display_history WHERE source_record_id='00000000-0000-0000-0000-000000000002'; +SELECT count(*) FROM uec.public_discovery_read_models WHERE release_id='e2e-promoted'; +SELECT count(*) FROM uec.public_discovery_read_model_rows WHERE source_record_id='00000000-0000-0000-0000-000000000002'; "@ $result = @(& docker compose @composeArgs exec -T postgres psql -v ON_ERROR_STOP=1 -U uec -d uec -At -c $sql) if ($LASTEXITCODE -ne 0) { throw "Restore gate query failed (exit $LASTEXITCODE)." } @@ -49,7 +51,7 @@ function Assert-Snapshot([string]$phase, [string]$expected) { Write-Host "[backup-restore] ${phase}: $actual" } -function Test-SyntheticServiceGate([string]$expected = '1,1,1,1,1,0,0') { +function Test-SyntheticServiceGate([string]$expected = '1,1,1,1,1,0,0,1,1') { # The external synthetic restriction is required in the restored DB and both # public projections must exclude it. A query error stops the drill. return ((Get-Snapshot) -join ',') -eq $expected @@ -88,20 +90,20 @@ try { if (-not $migrationApplied) { throw "Migration $($migration.Name) failed (exit $LASTEXITCODE)." } } Invoke-FixtureSql $seedFile - Assert-Snapshot 'older eligible state' '1,1,0,0,0,1,1' + Assert-Snapshot 'older eligible state' '1,1,0,0,0,1,1,1,1' & docker compose @composeArgs exec -T postgres pg_dump -U uec -d uec --format=custom --file=/tmp/uec.dump if ($LASTEXITCODE -ne 0) { throw "Backup creation failed (exit $LASTEXITCODE)." } & docker compose @composeArgs cp postgres:/tmp/uec.dump $dump if ($LASTEXITCODE -ne 0) { throw "Backup extraction failed (exit $LASTEXITCODE)." } Invoke-FixtureSql $suppressionFile - Assert-Snapshot 'later restriction active' '1,1,1,1,1,0,0' + Assert-Snapshot 'later restriction active' '1,1,1,1,1,0,0,1,1' if (-not (Test-SyntheticServiceGate)) { throw 'Current synthetic restriction did not close both public projections.' } # No application service is started anywhere in this drill. An old restore # loses the newer case, so the service gate MUST reject it before replay. & docker compose @composeArgs exec -T postgres pg_restore -U uec -d uec --clean --if-exists --exit-on-error /tmp/uec.dump if ($LASTEXITCODE -ne 0) { throw "Restore failed (exit $LASTEXITCODE)." } - Assert-Snapshot 'old backup restored, before replay' '1,1,0,0,0,1,1' + Assert-Snapshot 'old backup restored, before replay' '1,1,0,0,0,1,1,1,1' if (Test-SyntheticServiceGate) { throw 'Unsafe drill gate accepted an old backup before current restriction replay.' } # The verifier's nonzero exit is the expected result for the stale snapshot. # Capture it while temporarily allowing native stderr so PowerShell's Stop @@ -120,8 +122,8 @@ try { if ($LASTEXITCODE -ne 0) { throw 'Restriction ledger replay SQL generation failed.' } Set-Content -LiteralPath $ledgerReplayFile -Value ($replaySql -join "`n") -Encoding UTF8 Invoke-LedgerReplaySql $ledgerReplayFile - Assert-Snapshot 'current restriction replayed' '1,1,0,0,1,0,0' - if (-not (Test-SyntheticServiceGate '1,1,0,0,1,0,0')) { throw 'Synthetic pre-service gate rejected the replayed current restriction.' } + Assert-Snapshot 'current restriction replayed' '1,1,0,0,1,0,0,1,1' + if (-not (Test-SyntheticServiceGate '1,1,0,0,1,0,0,1,1')) { throw 'Synthetic pre-service gate rejected the replayed current restriction.' } Write-Host 'PASS: synthetic old-backup rollback remains gated until the independent current ledger is replayed and both public projections exclude it.' Write-Host 'TEST ONLY: production still requires separately operated ledger storage, trusted references, and deployment-specific review.' } finally { diff --git a/pipeline/tests/e2e/backup_restore_seed.sql b/pipeline/tests/e2e/backup_restore_seed.sql index 61c3d26..886834c 100644 --- a/pipeline/tests/e2e/backup_restore_seed.sql +++ b/pipeline/tests/e2e/backup_restore_seed.sql @@ -17,3 +17,23 @@ INSERT INTO uec.release_members (release_id,facility_id,observation_id,default_v VALUES ('e2e-promoted','00000000-0000-0000-0000-000000000004','00000000-0000-0000-0000-000000000005',true); INSERT INTO uec.publication_review_events (source_record_id,factual_review_status,privacy_screening_status,maintainer_approval,publication_eligible,reviewer_role) VALUES ('00000000-0000-0000-0000-000000000002','reviewed','passed','approved',true,'maintainer'); +-- The read model is part of the backup and is still only a public projection. +-- Current suppression remains live after restore/replay. +INSERT INTO uec.public_discovery_read_models (release_id,manifest_sha256,content_sha256,row_count) +VALUES ('e2e-promoted','cabe8641a05beb76c9517006a8ec4cdd60b3bad58aa5b0fc29335fee1ac7d5dd',repeat('b',64),1); +INSERT INTO uec.public_discovery_read_model_rows + (release_id,facility_id,observation_id,source_record_id,canonical_name,country_code, + city,display_precision,display_label,classification_category,observed_at, + first_observed_at,provenance_origin_type,provenance_source_id,provenance_source_name, + provenance_source_url,provenance_retrieved_at,source_rights_status) +SELECT 'e2e-promoted', facility.facility_id, observation.observation_id, + observation.source_record_id, facility.canonical_name, facility.country_code, + facility.city, 'unmapped', 'No publishable location', observation.classification_category, + observation.observed_at, observation.first_observed_at, source.origin_type, + source.source_id, source.name, source.official_url, artifact.retrieved_at, 'unknown' +FROM uec.observations observation +JOIN uec.facilities facility ON facility.facility_id=observation.facility_id +JOIN uec.source_records record ON record.source_record_id=observation.source_record_id +JOIN uec.sources source ON source.source_id=record.source_id +JOIN uec.raw_artifacts artifact ON artifact.artifact_id=record.artifact_id +WHERE observation.observation_id='00000000-0000-0000-0000-000000000005'; diff --git a/pipeline/tests/e2e/fixture.py b/pipeline/tests/e2e/fixture.py index 0047eaf..359f72f 100644 --- a/pipeline/tests/e2e/fixture.py +++ b/pipeline/tests/e2e/fixture.py @@ -1,5 +1,6 @@ """Disposable PostGIS and backend fixture used by API E2E tests.""" import os +import hashlib import json import socket import subprocess @@ -14,6 +15,20 @@ ROOT = Path(__file__).resolve().parents[3] COMPOSE = ROOT / "docker-compose.e2e.yml" +_READ_MODEL_SCRIPT = ROOT / "pipeline" / "scripts" / "maintenance" / "build_public_discovery_read_model.py" +_READ_MODEL_SPEC = None +_READ_MODEL_MODULE = None + +def _read_model_builder(): + global _READ_MODEL_SPEC, _READ_MODEL_MODULE + if _READ_MODEL_MODULE is None: + import importlib.util + _READ_MODEL_SPEC = importlib.util.spec_from_file_location("build_public_discovery_read_model", _READ_MODEL_SCRIPT) + _READ_MODEL_MODULE = importlib.util.module_from_spec(_READ_MODEL_SPEC) + assert _READ_MODEL_SPEC.loader + _READ_MODEL_SPEC.loader.exec_module(_READ_MODEL_MODULE) + return _READ_MODEL_MODULE + def free_port(): with socket.socket() as sock: sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) @@ -208,6 +223,7 @@ def stop(self): def seed_official_scenario(self): """Seed safe synthetic records for public API tests.""" now = datetime.now(timezone.utc) + restricted_record = None with psycopg.connect(self.database_url) as db: with db.transaction(): db.execute("INSERT INTO uec.sources (source_id,country_code,name,official_url,access_method) VALUES ('e2e.official','DK','Synthetic official source','https://example.invalid/official','fixture')") @@ -230,11 +246,22 @@ def seed_official_scenario(self): else: db.execute("INSERT INTO uec.geocode_results (source_record_id,provider_id,query,match_method,status,attempt_number,queried_at) VALUES (%s,'e2e','fixture','fixture',%s,1,%s)", (record,status,now)) if name == 'restricted': - db.execute("INSERT INTO uec.record_access_events (source_record_id,action,reason_category,policy_version,maintainer) VALUES (%s,'public_access_revoked','privacy','ethics-v1','e2e')", (record,)) + restricted_record = record if approved: db.execute("INSERT INTO uec.publication_review_events (source_record_id,factual_review_status,privacy_screening_status,maintainer_approval,publication_eligible,reviewer_role) VALUES (%s,'reviewed','passed','approved',true,'maintainer')", (record,)) if name == 'exact': db.execute("INSERT INTO uec.facility_lifecycle_events (facility_id,status,effective_at,evidence_note) VALUES (%s,'active_observed',%s,'Synthetic official observation')", (facility, now)) + self.build_public_read_model('e2e-promoted') + # Build from the public projection before the synthetic restriction is + # appended. The read view still applies the restriction live, and an + # explicit restoration can therefore be tested without storing a + # private row in the component. + with psycopg.connect(self.database_url) as db: + db.execute("INSERT INTO uec.record_access_events (source_record_id,action,reason_category,policy_version,maintainer) VALUES (%s,'public_access_revoked','privacy','ethics-v1','e2e')", (restricted_record,)) + + def build_public_read_model(self, release_id): + """Activate a synthetic model through the same atomic operator flow.""" + return _read_model_builder().build(self.database_url, release_id) def create_failed_candidate(self): """Create an invalid candidate without touching the promoted release.""" @@ -302,6 +329,25 @@ def seed_community_scenario(self): db.execute("INSERT INTO uec.publication_review_events (source_record_id,factual_review_status,privacy_screening_status,maintainer_approval,publication_eligible,reviewer_role,note) VALUES (%s,'reviewed','passed','approved',true,'maintainer','Synthetic eligible claim')", (record,)) elif review_state == "screened-unreviewed": db.execute("INSERT INTO uec.publication_review_events (source_record_id,factual_review_status,privacy_screening_status,maintainer_approval,publication_eligible,reviewer_role,note) VALUES (%s,'unreviewed','passed','pending',true,NULL,'Synthetic screened but unreviewed claim')", (record,)) + for release_id, profile, count in ( + ('e2e-official-empty', 'official', 0), + ('e2e-community', 'community', 2), + ): + manifest = { + 'eligible_record_count': count, + 'manifest_version': 'v1', + 'profile': profile, + 'release_id': release_id, + 'ruleset_version': f'{profile}-v1', + 'source_ids': ['e2e.community'] if profile == 'community' else [], + } + serialized = json.dumps(manifest, sort_keys=True, separators=(',', ':')) + db.execute( + "INSERT INTO uec.release_manifests (release_id,manifest,manifest_sha256) VALUES (%s,%s::jsonb,%s)", + (release_id, serialized, hashlib.sha256(serialized.encode()).hexdigest()), + ) + self.build_public_read_model('e2e-official-empty') + self.build_public_read_model('e2e-community') def __enter__(self): return self.start() diff --git a/pipeline/tests/e2e/run-suite.ps1 b/pipeline/tests/e2e/run-suite.ps1 index 4aa97d7..b02540b 100644 --- a/pipeline/tests/e2e/run-suite.ps1 +++ b/pipeline/tests/e2e/run-suite.ps1 @@ -16,6 +16,7 @@ $core = @( $extended = @( 'pipeline.tests.e2e.test_suppression_lifecycle', 'pipeline.tests.e2e.test_release_summary_component', + 'pipeline.tests.e2e.test_public_discovery_read_model', 'pipeline.tests.e2e.test_italy_candidate_import', 'pipeline.tests.e2e.test_germany_belgium_candidate_import' ) diff --git a/pipeline/tests/e2e/test_public_discovery_read_model.py b/pipeline/tests/e2e/test_public_discovery_read_model.py new file mode 100644 index 0000000..39aca8a --- /dev/null +++ b/pipeline/tests/e2e/test_public_discovery_read_model.py @@ -0,0 +1,111 @@ +"""E2E contracts for atomic activation and fail-closed public reads.""" + +import hashlib +import importlib.util +import json +import os +import unittest +import urllib.error +import urllib.request +from pathlib import Path + +import psycopg + +try: + from .fixture import E2EEnvironment +except ImportError: + from fixture import E2EEnvironment + + +ROOT = Path(__file__).parents[2] +SPEC = importlib.util.spec_from_file_location( + "build_public_discovery_read_model", + ROOT / "scripts" / "maintenance" / "build_public_discovery_read_model.py", +) +BUILDER = importlib.util.module_from_spec(SPEC) +assert SPEC.loader +SPEC.loader.exec_module(BUILDER) + + +class PublicDiscoveryReadModelE2ETests(unittest.TestCase): + @classmethod + def setUpClass(cls): + if os.environ.get("UEC_RUN_E2E") != "1": + raise unittest.SkipTest("set UEC_RUN_E2E=1 to run Docker-backed E2E tests") + cls.env = E2EEnvironment().start() + cls.env.seed_official_scenario() + + @classmethod + def tearDownClass(cls): + cls.env.stop() + + def get_public(self): + request = urllib.request.Request( + f"http://localhost:{self.env.api_port}/api/v2/locations?profile=official&limit=100", + headers={"Accept": "application/json"}, + ) + with urllib.request.urlopen(request, timeout=10) as response: + return response.status, json.loads(response.read()) + + def create_release_without_model(self, release_id, profile="secondary"): + manifest = { + "eligible_record_count": 0, + "manifest_version": "read-model-e2e-v1", + "profile": profile, + "release_id": release_id, + "ruleset_version": "read-model-e2e-v1", + } + serialized = json.dumps(manifest, sort_keys=True, separators=(",", ":")) + with psycopg.connect(self.env.database_url) as db: + with db.transaction(): + db.execute( + "INSERT INTO uec.releases (release_id,status,ruleset_version,profile,summary) VALUES (%s,'promoted','read-model-e2e-v1',%s,'{}')", + (release_id, profile), + ) + db.execute( + "INSERT INTO uec.release_manifests (release_id,manifest,manifest_sha256) VALUES (%s,%s::jsonb,%s)", + (release_id, serialized, hashlib.sha256(serialized.encode()).hexdigest()), + ) + + def test_api_uses_activated_model_and_missing_latest_model_fails_closed(self): + status, body = self.get_public() + self.assertEqual(status, 200) + self.assertEqual(len(body["data"]), 3) + self.create_release_without_model("e2e-read-model-missing") + with self.assertRaises(urllib.error.HTTPError) as error: + request = urllib.request.Request( + f"http://localhost:{self.env.api_port}/api/v2/locations?profile=secondary&limit=100", + headers={"Accept": "application/json"}, + ) + urllib.request.urlopen(request, timeout=10) + self.assertEqual(error.exception.code, 503) + self.assertEqual(json.loads(error.exception.read())["error"]["code"], "read_model_unavailable") + + def test_interrupted_activation_leaves_no_rows_or_metadata(self): + release_id = "e2e-read-model-interrupted" + self.create_release_without_model(release_id, profile="community") + with psycopg.connect(self.env.database_url) as db: + with db.transaction(): + db.execute( + "INSERT INTO uec.release_members (release_id,facility_id,observation_id,default_visible) SELECT %s,facility_id,observation_id,default_visible FROM uec.release_members WHERE release_id='e2e-promoted'", + (release_id,), + ) + db.execute( + """INSERT INTO uec.publication_review_events + (source_record_id,release_id,factual_review_status,privacy_screening_status, + maintainer_approval,publication_eligible,reviewer_role,reviewed_at) + SELECT source_record_id,%s,factual_review_status,privacy_screening_status, + maintainer_approval,publication_eligible,reviewer_role,reviewed_at + FROM uec.publication_review_release_current + WHERE release_id='e2e-promoted'""", + (release_id,), + ) + with self.assertRaisesRegex(RuntimeError, "interrupted"): + BUILDER.build(self.env.database_url, release_id, fail_after_rows=1) + with psycopg.connect(self.env.database_url) as db: + self.assertEqual(db.execute("SELECT count(*) FROM uec.public_discovery_read_models WHERE release_id=%s", (release_id,)).fetchone()[0], 0) + self.assertEqual(db.execute("SELECT count(*) FROM uec.public_discovery_read_model_rows WHERE release_id=%s", (release_id,)).fetchone()[0], 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/tests/e2e/test_public_surface_safety.py b/pipeline/tests/e2e/test_public_surface_safety.py index 853410c..f4bc862 100644 --- a/pipeline/tests/e2e/test_public_surface_safety.py +++ b/pipeline/tests/e2e/test_public_surface_safety.py @@ -25,26 +25,6 @@ def setUpClass(cls): raise unittest.SkipTest("set UEC_RUN_E2E=1 to run Docker-backed E2E tests") cls.env = E2EEnvironment().start() cls.env.seed_community_scenario() - with psycopg.connect(cls.env.database_url) as db: - for release_id, profile, count in ( - ("e2e-official-empty", "official", 0), - ("e2e-community", "community", 2), - ): - manifest = { - "eligible_record_count": count, - "manifest_version": "v1", - "profile": profile, - "release_id": release_id, - "ruleset_version": f"{profile}-v1", - "source_ids": ["e2e.community"] if profile == "community" else [], - } - serialized = json.dumps(manifest, sort_keys=True, separators=(",", ":")) - digest = hashlib.sha256(serialized.encode("utf-8")).hexdigest() - db.execute( - "INSERT INTO uec.release_manifests (release_id,manifest,manifest_sha256) " - "VALUES (%s,%s::jsonb,%s)", - (release_id, serialized, digest), - ) @classmethod def tearDownClass(cls): @@ -127,6 +107,8 @@ def setUpClass(cls): raise unittest.SkipTest("set UEC_RUN_E2E=1 to run Docker-backed E2E tests") cls.env = E2EEnvironment().start() cls.seed_shared_record() + cls.env.build_public_read_model("e2e-release-a") + cls.env.build_public_read_model("e2e-release-b") @classmethod def tearDownClass(cls): diff --git a/pipeline/tests/e2e/test_seeded_api.py b/pipeline/tests/e2e/test_seeded_api.py index a7b7dc9..693212d 100644 --- a/pipeline/tests/e2e/test_seeded_api.py +++ b/pipeline/tests/e2e/test_seeded_api.py @@ -1,3 +1,4 @@ +import hashlib import json, os, subprocess, sys, unittest, urllib.error, urllib.request import uuid from datetime import datetime, timezone @@ -175,7 +176,10 @@ def test_z1_approval_does_not_follow_source_record_into_new_profile(self): with psycopg.connect(self.env.database_url) as db: facility_id, observation_id = db.execute("SELECT facility_id, observation_id FROM uec.observations o JOIN uec.source_records r USING (source_record_id) WHERE r.source_record_key = 'exact'").fetchone() db.execute("INSERT INTO uec.releases (release_id,status,ruleset_version,profile,summary) VALUES ('e2e-secondary-later','promoted','e2e-v2','secondary','{}')") + manifest = '{"eligible_record_count":0,"manifest_version":"v1","profile":"secondary","release_id":"e2e-secondary-later","ruleset_version":"e2e-v2"}' + db.execute("INSERT INTO uec.release_manifests (release_id,manifest,manifest_sha256) VALUES ('e2e-secondary-later',%s::jsonb,%s)", (manifest, hashlib.sha256(manifest.encode()).hexdigest())) db.execute("INSERT INTO uec.release_members (release_id,facility_id,observation_id,default_visible) VALUES ('e2e-secondary-later',%s,%s,true)", (facility_id, observation_id)) + self.env.build_public_read_model('e2e-secondary-later') self.assertEqual(self.get('/api/v2/locations?profile=secondary&limit=100')['data'], []) def test_z1b_candidate_review_does_not_revoke_independent_promoted_approval(self): diff --git a/pipeline/tests/e2e/test_suppression_lifecycle.py b/pipeline/tests/e2e/test_suppression_lifecycle.py index bf7eff4..0d39e72 100644 --- a/pipeline/tests/e2e/test_suppression_lifecycle.py +++ b/pipeline/tests/e2e/test_suppression_lifecycle.py @@ -125,6 +125,7 @@ def seed_synthetic_fixture(cls): "VALUES (%s,'synthetic',%s,'fixture','accepted',1,ST_SetSRID(ST_MakePoint(12,56),4326)::geography,%s)", (cls.source_record_id, cls.private_marker, now), ) + cls.env.build_public_read_model("e2e-suppression-old") def get_json(self, path): with urllib.request.urlopen(f"http://localhost:{self.env.api_port}{path}", timeout=10) as response: diff --git a/pipeline/tests/test_graph_migrations.py b/pipeline/tests/test_graph_migrations.py index ed611a7..e51b46f 100644 --- a/pipeline/tests/test_graph_migrations.py +++ b/pipeline/tests/test_graph_migrations.py @@ -20,7 +20,7 @@ def test_reserved_migrations_are_present_and_ordered(self): positions = [migrations.index(name) for name in graph_migrations] self.assertEqual(positions, sorted(positions)) self.assertEqual([migrations[position] for position in positions], graph_migrations) - self.assertEqual(migrations[-10:], [ + self.assertEqual(migrations[-11:], [ "026_graph_entities_crosswalks.sql", "027_graph_relationship_observations.sql", "028_graph_claims_support.sql", @@ -31,6 +31,7 @@ def test_reserved_migrations_are_present_and_ordered(self): "033_release_summary_component.sql", "034_public_eligibility_join_indexes.sql", "035_public_discovery_planner_indexes.sql", + "037_public_discovery_read_model.sql", ]) def test_entities_are_distinct_and_crosswalk_is_scoped(self): diff --git a/pipeline/tests/test_public_discovery_query_contract.py b/pipeline/tests/test_public_discovery_query_contract.py index 378ab7e..dcc369b 100644 --- a/pipeline/tests/test_public_discovery_query_contract.py +++ b/pipeline/tests/test_public_discovery_query_contract.py @@ -25,15 +25,20 @@ def test_detail_uses_the_same_deterministic_observation_choice(self): self.assertIn("ORDER BY history.observation_id", detail) self.assertIn("LIMIT 1", detail) - def test_public_queries_keep_live_release_review_view_and_model_gate(self): + def test_public_queries_use_the_manifest_bound_live_gated_read_model(self): source = _source() locations = source[source.index("pub async fn get_v2_locations_handler"):source.index("pub async fn get_v2_location_detail_handler")] - self.assertIn("FROM uec.map_facilities_public_discovery AS history", locations) - self.assertIn("history.factual_review_status", locations) - self.assertIn("release_summary_components", locations) - self.assertIn("read_model_unavailable", locations) + self.assertIn("FROM uec.map_facilities_public_discovery_read_model AS history", locations) + self.assertIn("public_discovery_read_models", locations) + self.assertIn("manifest.manifest_sha256=model.manifest_sha256", locations) + self.assertNotIn("JOIN uec.publication_review_release_current AS review", locations) self.assertIn("history.release_id = $1", locations) + def test_legacy_component_view_remains_documented_but_is_not_the_api_read_path(self): + migration = (ROOT / "migrations" / "036_public_facility_discovery_view.sql").read_text(encoding="utf-8").lower() + self.assertIn("create or replace view uec.map_facilities_public_discovery", migration) + self.assertIn("release_summary_component_rows", migration) + def test_discovery_view_is_a_live_gated_one_row_per_facility_projection(self): migration = (ROOT / "migrations" / "036_public_facility_discovery_view.sql").read_text(encoding="utf-8").lower() self.assertIn("create or replace view uec.map_facilities_public_discovery", migration) diff --git a/pipeline/tests/test_public_discovery_read_model.py b/pipeline/tests/test_public_discovery_read_model.py new file mode 100644 index 0000000..974b2c3 --- /dev/null +++ b/pipeline/tests/test_public_discovery_read_model.py @@ -0,0 +1,41 @@ +"""Unit contracts for the public discovery read model builder and migration.""" + +import importlib.util +import unittest +from datetime import datetime, timezone +from pathlib import Path + + +ROOT = Path(__file__).parents[1] +SCRIPT = ROOT / "scripts" / "maintenance" / "build_public_discovery_read_model.py" +SPEC = importlib.util.spec_from_file_location("build_public_discovery_read_model", SCRIPT) +MODULE = importlib.util.module_from_spec(SPEC) +assert SPEC.loader +SPEC.loader.exec_module(MODULE) + + +class PublicDiscoveryReadModelTests(unittest.TestCase): + def test_content_digest_is_deterministic_and_binds_public_projection_fields(self): + row = ("facility", "observation", "record", "Name", "DK", None, "City", "POINT (10 55)", "exact", "label", "accepted", "provider", datetime(2026, 1, 1, tzinfo=timezone.utc), "slaughter", datetime(2026, 1, 2, tzinfo=timezone.utc), datetime(2026, 1, 2, tzinfo=timezone.utc), "official", "source", "Source", "https://example.invalid", datetime(2026, 1, 1, tzinfo=timezone.utc), "unknown") + self.assertEqual(MODULE.content_digest([row]), MODULE.content_digest([row])) + self.assertNotEqual(MODULE.content_digest([row]), MODULE.content_digest([(*row[:-1], "attribution_required")])) + + def test_naive_timestamps_are_rejected(self): + row = ("facility", "observation", "record", "Name", "DK", None, "City", None, "unmapped", "label", None, None, None, "slaughter", datetime(2026, 1, 2), datetime(2026, 1, 2, tzinfo=timezone.utc), "official", "source", "Source", "https://example.invalid", datetime(2026, 1, 1, tzinfo=timezone.utc), "unknown") + with self.assertRaises(MODULE.ReadModelBlocked): + MODULE.content_digest([row]) + + def test_migration_is_atomic_manifest_bound_and_live_gated(self): + migration = (ROOT / "migrations" / "037_public_discovery_read_model.sql").read_text(encoding="utf-8").lower() + for token in ("public_discovery_read_models", "public_discovery_read_model_rows", "content_sha256", "append_only", "release_manifests", "publication_review_release_current", "public_access_restricted", "facility_lifecycle_current"): + self.assertIn(token, migration) + self.assertNotIn("drop table", migration) + + def test_operator_query_never_selects_raw_fields(self): + query = MODULE.SELECT_ROWS.lower() + self.assertNotIn("raw_fields", query) + self.assertIn("map_facilities_display_history", query) + + +if __name__ == "__main__": + unittest.main() diff --git a/src/lib.rs b/src/lib.rs index 9baa19e..ae2e9c2 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -200,7 +200,7 @@ pub async fn get_v2_locations_export_handler( "V2 database is not configured", ); }; - let client = match pool.get().await { + let mut client = match pool.get().await { Ok(c) => c, Err(_) => { return v2_error( @@ -210,7 +210,23 @@ pub async fn get_v2_locations_export_handler( ); } }; - let release = match client.query_opt("SELECT r.release_id, m.manifest_sha256 FROM uec.releases r JOIN uec.release_manifests m ON m.release_id=r.release_id WHERE r.status='promoted' AND r.test_only IS NOT TRUE AND r.profile=$1 ORDER BY r.created_at DESC, r.release_id DESC LIMIT 1", &[&profile]).await { + let transaction = match client + .build_transaction() + .isolation_level(tokio_postgres::IsolationLevel::RepeatableRead) + .read_only(true) + .start() + .await + { + Ok(transaction) => transaction, + Err(_) => { + return v2_error( + StatusCode::SERVICE_UNAVAILABLE, + "database_transaction_unavailable", + "V2 database transaction unavailable", + ); + } + }; + let release = match transaction.query_opt("SELECT r.release_id, m.manifest_sha256, (model.release_id IS NOT NULL AND manifest.release_id IS NOT NULL) FROM uec.releases r LEFT JOIN uec.public_discovery_read_models model ON model.release_id=r.release_id LEFT JOIN uec.release_manifests manifest ON manifest.release_id=r.release_id AND manifest.manifest_sha256=model.manifest_sha256 LEFT JOIN uec.release_manifests m ON m.release_id=r.release_id WHERE r.status='promoted' AND r.test_only IS NOT TRUE AND r.profile=$1 ORDER BY r.created_at DESC, r.release_id DESC LIMIT 1", &[&profile]).await { Ok(row) => row, Err(_) => return v2_error(StatusCode::SERVICE_UNAVAILABLE, "release_query_failed", "release query failed") }; let Some(release) = release else { @@ -222,17 +238,14 @@ pub async fn get_v2_locations_export_handler( }; let release_id: String = release.get(0); let manifest_sha256: String = release.get(1); - let model_ready = match client.query_opt( - "SELECT 1 FROM uec.release_summary_components component JOIN uec.release_manifests manifest ON manifest.release_id=component.release_id AND manifest.manifest_sha256=component.manifest_sha256 LEFT JOIN uec.release_summary_component_rows component_row ON component_row.release_id=component.release_id WHERE component.release_id=$1 GROUP BY component.release_id, component.member_count HAVING component.member_count=count(component_row.release_id)", - &[&release_id], - ).await { - Ok(row) => row.is_some(), - Err(_) => return v2_error(StatusCode::SERVICE_UNAVAILABLE, "read_model_query_failed", "public read model unavailable"), - }; - if !model_ready { - return v2_error(StatusCode::SERVICE_UNAVAILABLE, "read_model_unavailable", "public read model is not ready for the selected release"); + if !release.get::<_, bool>(2) { + return v2_error( + StatusCode::SERVICE_UNAVAILABLE, + "read_model_unavailable", + "public discovery read model is missing or stale", + ); } - let rows = match client.query("SELECT h.facility_id, h.canonical_name, h.country_code, h.city, h.classification_category, h.display_precision, h.factual_review_status, h.privacy_screening_status, h.maintainer_approval, h.reviewer_role, h.provenance_origin_type, h.provenance_source_id, h.provenance_source_name, h.provenance_source_url, h.provenance_retrieved_at, CASE WHEN source.attribution IS NULL OR btrim(source.attribution) = '' THEN 'unknown' ELSE 'attribution_required' END, h.release_id FROM uec.map_facilities_public_discovery h JOIN uec.sources source ON source.source_id=h.provenance_source_id WHERE h.release_id=$1 ORDER BY h.facility_id LIMIT 1001", &[&release_id]).await { + let rows = match transaction.query("SELECT h.facility_id, h.canonical_name, h.country_code, h.city, h.classification_category, h.display_precision, h.factual_review_status, h.privacy_screening_status, h.maintainer_approval, h.reviewer_role, h.provenance_origin_type, h.provenance_source_id, h.provenance_source_name, h.provenance_source_url, h.provenance_retrieved_at, h.source_rights_status, h.release_id FROM uec.map_facilities_public_discovery_read_model h WHERE h.release_id=$1 ORDER BY h.facility_id LIMIT 1001", &[&release_id]).await { Ok(rows) => rows, Err(_) => return v2_error(StatusCode::SERVICE_UNAVAILABLE, "export_query_failed", "public export unavailable") }; if rows.len() > 1000 { @@ -291,6 +304,13 @@ pub async fn get_v2_locations_export_handler( ); } }; + if transaction.commit().await.is_err() { + return v2_error( + StatusCode::SERVICE_UNAVAILABLE, + "database_transaction_failed", + "V2 database transaction failed", + ); + } Response::builder() .status(StatusCode::OK) .header("content-type", "text/csv; charset=utf-8") @@ -977,7 +997,7 @@ pub async fn get_v2_facets_handler( "V2 database is not configured", ); }; - let client = match pool.get().await { + let mut client = match pool.get().await { Ok(c) => c, Err(_) => { return v2_error( @@ -987,7 +1007,23 @@ pub async fn get_v2_facets_handler( ); } }; - let release = match client.query_opt("SELECT release_id, ruleset_version, created_at FROM uec.releases WHERE status='promoted' AND test_only IS NOT TRUE AND profile=$1 ORDER BY created_at DESC, release_id DESC LIMIT 1", &[&profile]).await { Ok(row) => row, Err(_) => return v2_error(StatusCode::SERVICE_UNAVAILABLE, "release_query_failed", "release query failed") }; + let transaction = match client + .build_transaction() + .isolation_level(tokio_postgres::IsolationLevel::RepeatableRead) + .read_only(true) + .start() + .await + { + Ok(transaction) => transaction, + Err(_) => { + return v2_error( + StatusCode::SERVICE_UNAVAILABLE, + "database_transaction_unavailable", + "V2 database transaction unavailable", + ); + } + }; + let release = match transaction.query_opt("SELECT r.release_id, r.ruleset_version, r.created_at, (model.release_id IS NOT NULL AND manifest.release_id IS NOT NULL) FROM uec.releases r LEFT JOIN uec.public_discovery_read_models model ON model.release_id=r.release_id LEFT JOIN uec.release_manifests manifest ON manifest.release_id=r.release_id AND manifest.manifest_sha256=model.manifest_sha256 WHERE r.status='promoted' AND r.test_only IS NOT TRUE AND r.profile=$1 ORDER BY r.created_at DESC, r.release_id DESC LIMIT 1", &[&profile]).await { Ok(row) => row, Err(_) => return v2_error(StatusCode::SERVICE_UNAVAILABLE, "release_query_failed", "release query failed") }; let Some(release) = release else { return v2_error( StatusCode::NOT_FOUND, @@ -998,17 +1034,14 @@ pub async fn get_v2_facets_handler( let release_id: String = release.get(0); let ruleset_version: String = release.get(1); let release_created_at: chrono::DateTime = release.get(2); - let model_ready = match client.query_opt( - "SELECT 1 FROM uec.release_summary_components component JOIN uec.release_manifests manifest ON manifest.release_id=component.release_id AND manifest.manifest_sha256=component.manifest_sha256 LEFT JOIN uec.release_summary_component_rows component_row ON component_row.release_id=component.release_id WHERE component.release_id=$1 GROUP BY component.release_id, component.member_count HAVING component.member_count=count(component_row.release_id)", - &[&release_id], - ).await { - Ok(row) => row.is_some(), - Err(_) => return v2_error(StatusCode::SERVICE_UNAVAILABLE, "read_model_query_failed", "public read model unavailable"), - }; - if !model_ready { - return v2_error(StatusCode::SERVICE_UNAVAILABLE, "read_model_unavailable", "public read model is not ready for the selected release"); + if !release.get::<_, bool>(3) { + return v2_error( + StatusCode::SERVICE_UNAVAILABLE, + "read_model_unavailable", + "public discovery read model is missing or stale", + ); } - let rows = match client.query("SELECT country_code, classification_category, display_precision, lifecycle_status, provenance_origin_type, city, count(*)::bigint FROM (SELECT DISTINCT ON (facility_id) facility_id, country_code, classification_category, display_precision, lifecycle_status, provenance_origin_type, city FROM uec.map_facilities_public_discovery WHERE release_id=$1 AND ($2::text IS NULL OR country_code=$2) AND ($3::text IS NULL OR classification_category=$3) AND ($4::text IS NULL OR provenance_origin_type=$4) AND ($5::text IS NULL OR display_precision=$5) AND ($6::text IS NULL OR lifecycle_status=$6) AND ($7::text IS NULL OR city=$7) ORDER BY facility_id, observation_id) public_facilities GROUP BY country_code, classification_category, display_precision, lifecycle_status, provenance_origin_type, city", &[&release_id, ¶ms.country_code, ¶ms.category, ¶ms.source_type, ¶ms.display_precision, ¶ms.lifecycle_status, ¶ms.region]).await { Ok(rows) => rows, Err(_) => return v2_error(StatusCode::SERVICE_UNAVAILABLE, "facets_query_failed", "public facets unavailable") }; + let rows = match transaction.query("SELECT country_code, classification_category, display_precision, lifecycle_status, provenance_origin_type, city, count(*)::bigint FROM (SELECT DISTINCT ON (facility_id) facility_id, country_code, classification_category, display_precision, lifecycle_status, provenance_origin_type, city FROM uec.map_facilities_public_discovery_read_model WHERE release_id=$1 AND ($2::text IS NULL OR country_code=$2) AND ($3::text IS NULL OR classification_category=$3) AND ($4::text IS NULL OR provenance_origin_type=$4) AND ($5::text IS NULL OR display_precision=$5) AND ($6::text IS NULL OR lifecycle_status=$6) AND ($7::text IS NULL OR city=$7) ORDER BY facility_id, observation_id) public_facilities GROUP BY country_code, classification_category, display_precision, lifecycle_status, provenance_origin_type, city", &[&release_id, ¶ms.country_code, ¶ms.category, ¶ms.source_type, ¶ms.display_precision, ¶ms.lifecycle_status, ¶ms.region]).await { Ok(rows) => rows, Err(_) => return v2_error(StatusCode::SERVICE_UNAVAILABLE, "facets_query_failed", "public facets unavailable") }; let mut dimensions = serde_json::Map::new(); for (name, column) in [ ("country_code", 0), @@ -1040,6 +1073,13 @@ pub async fn get_v2_facets_handler( ), ); } + if transaction.commit().await.is_err() { + return v2_error( + StatusCode::SERVICE_UNAVAILABLE, + "database_transaction_failed", + "V2 database transaction failed", + ); + } Json(json!({"api_version":"v2", "meta":{"profile":profile,"release_id":release_id,"ruleset_version":ruleset_version,"release_created_at":release_created_at,"coverage_scope":"selected_promoted_release_public_facilities","count_semantics":"Counts are eligible public facility projection rows after current suppression; they are not story-wide or animal counts.","filters":{"country_code":params.country_code,"region":params.region,"category":params.category,"source_type":params.source_type,"display_precision":params.display_precision,"lifecycle_status":params.lifecycle_status}}, "dimensions":dimensions})).into_response() } @@ -1348,7 +1388,7 @@ pub async fn get_v2_locations_handler( } }; let requested_profile = params.profile.as_deref().unwrap_or("official"); - let release = transaction.query_opt("SELECT release_id, ruleset_version, created_at, profile FROM uec.releases WHERE status = 'promoted' AND test_only IS NOT TRUE AND profile = $1 ORDER BY created_at DESC, release_id DESC LIMIT 1", &[&requested_profile]).await; + let release = transaction.query_opt("SELECT r.release_id, r.ruleset_version, r.created_at, r.profile, (model.release_id IS NOT NULL AND manifest.release_id IS NOT NULL) FROM uec.releases r LEFT JOIN uec.public_discovery_read_models model ON model.release_id=r.release_id LEFT JOIN uec.release_manifests manifest ON manifest.release_id=r.release_id AND manifest.manifest_sha256=model.manifest_sha256 WHERE r.status = 'promoted' AND r.test_only IS NOT TRUE AND r.profile = $1 ORDER BY r.created_at DESC, r.release_id DESC LIMIT 1", &[&requested_profile]).await; let release = match release { Ok(release) => release, Err(_) => { @@ -1367,15 +1407,12 @@ pub async fn get_v2_locations_handler( let promoted_ruleset: String = release.get(1); let promoted_created_at: chrono::DateTime = release.get(2); let promoted_profile: String = release.get(3); - let model_ready = match transaction.query_opt( - "SELECT 1 FROM uec.release_summary_components component JOIN uec.release_manifests manifest ON manifest.release_id=component.release_id AND manifest.manifest_sha256=component.manifest_sha256 LEFT JOIN uec.release_summary_component_rows component_row ON component_row.release_id=component.release_id WHERE component.release_id=$1 GROUP BY component.release_id, component.member_count HAVING component.member_count=count(component_row.release_id)", - &[&promoted_release_id], - ).await { - Ok(row) => row.is_some(), - Err(_) => return v2_error(StatusCode::SERVICE_UNAVAILABLE, "read_model_query_failed", "public read model unavailable"), - }; - if !model_ready { - return v2_error(StatusCode::SERVICE_UNAVAILABLE, "read_model_unavailable", "public read model is not ready for the selected release"); + if !release.get::<_, bool>(4) { + return v2_error( + StatusCode::SERVICE_UNAVAILABLE, + "read_model_unavailable", + "public discovery read model is missing or stale", + ); } let query_limit = limit + 1; let rows = match transaction.query(r#" @@ -1385,9 +1422,8 @@ pub async fn get_v2_locations_handler( history.first_observed_at, history.last_observed_at, history.observation_count, history.lifecycle_status, history.provenance_origin_type, history.release_id, history.release_ruleset_version, history.provenance_source_id, history.provenance_source_name, history.provenance_source_url, history.provenance_retrieved_at, - CASE WHEN rights.attribution IS NULL OR btrim(rights.attribution) = '' THEN 'unknown' ELSE 'attribution_required' END - FROM uec.map_facilities_public_discovery AS history - JOIN uec.sources rights ON rights.source_id = history.provenance_source_id + history.source_rights_status + FROM uec.map_facilities_public_discovery_read_model AS history WHERE history.release_id = $1 AND ($2::uuid IS NULL OR history.facility_id > $2) AND ($3::text IS NULL OR history.country_code = $3) @@ -1520,7 +1556,7 @@ pub async fn get_v2_location_detail_handler( "profile is unsupported", ); } - let release = match transaction.query_opt("SELECT release_id, ruleset_version, created_at, profile FROM uec.releases WHERE status = 'promoted' AND test_only IS NOT TRUE AND profile = $1 ORDER BY created_at DESC, release_id DESC LIMIT 1", &[&requested_profile]).await { + let release = match transaction.query_opt("SELECT r.release_id, r.ruleset_version, r.created_at, r.profile, (model.release_id IS NOT NULL AND manifest.release_id IS NOT NULL) FROM uec.releases r LEFT JOIN uec.public_discovery_read_models model ON model.release_id=r.release_id LEFT JOIN uec.release_manifests manifest ON manifest.release_id=r.release_id AND manifest.manifest_sha256=model.manifest_sha256 WHERE r.status = 'promoted' AND r.test_only IS NOT TRUE AND r.profile = $1 ORDER BY r.created_at DESC, r.release_id DESC LIMIT 1", &[&requested_profile]).await { Ok(release) => release, Err(_) => return v2_error(StatusCode::INTERNAL_SERVER_ERROR, "release_query_failed", "V2 release query failed"), }; @@ -1536,15 +1572,12 @@ pub async fn get_v2_location_detail_handler( let ruleset: String = release.get(1); let created_at: chrono::DateTime = release.get(2); let profile: String = release.get(3); - let model_ready = match transaction.query_opt( - "SELECT 1 FROM uec.release_summary_components component JOIN uec.release_manifests manifest ON manifest.release_id=component.release_id AND manifest.manifest_sha256=component.manifest_sha256 LEFT JOIN uec.release_summary_component_rows component_row ON component_row.release_id=component.release_id WHERE component.release_id=$1 GROUP BY component.release_id, component.member_count HAVING component.member_count=count(component_row.release_id)", - &[&release_id], - ).await { - Ok(row) => row.is_some(), - Err(_) => return v2_error(StatusCode::SERVICE_UNAVAILABLE, "read_model_query_failed", "public read model unavailable"), - }; - if !model_ready { - return v2_error(StatusCode::SERVICE_UNAVAILABLE, "read_model_unavailable", "public read model is not ready for the selected release"); + if !release.get::<_, bool>(4) { + return v2_error( + StatusCode::SERVICE_UNAVAILABLE, + "read_model_unavailable", + "public discovery read model is missing or stale", + ); } let row = match transaction.query_opt(r#" SELECT history.facility_id, history.canonical_name, history.country_code, history.city, history.classification_category, history.display_precision, @@ -1553,9 +1586,8 @@ pub async fn get_v2_location_detail_handler( history.first_observed_at, history.last_observed_at, history.observation_count, history.lifecycle_status, history.provenance_origin_type, history.release_id, history.release_ruleset_version, history.provenance_source_id, history.provenance_source_name, history.provenance_source_url, history.provenance_retrieved_at, - CASE WHEN rights.attribution IS NULL OR btrim(rights.attribution) = '' THEN 'unknown' ELSE 'attribution_required' END - FROM uec.map_facilities_public_discovery AS history - JOIN uec.sources rights ON rights.source_id = history.provenance_source_id + history.source_rights_status + FROM uec.map_facilities_public_discovery_read_model AS history WHERE history.facility_id = $1 AND history.release_id = $2 ORDER BY history.observation_id LIMIT 1 From caee7605b421b304d7f7e78ee9ad2cbd1dd26c82 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Thu, 17 Sep 2026 00:53:48 -0700 Subject: [PATCH 214/311] Optimize live discovery read gates --- .../037_public_discovery_read_model.sql | 53 ++++++++++++++++--- 1 file changed, 47 insertions(+), 6 deletions(-) diff --git a/pipeline/migrations/037_public_discovery_read_model.sql b/pipeline/migrations/037_public_discovery_read_model.sql index 18f4faa..f5c97fa 100644 --- a/pipeline/migrations/037_public_discovery_read_model.sql +++ b/pipeline/migrations/037_public_discovery_read_model.sql @@ -79,9 +79,23 @@ WITH eligible AS ( JOIN uec.release_manifests manifest ON manifest.release_id = model.release_id AND manifest.manifest_sha256 = metadata.manifest_sha256 - JOIN uec.publication_review_release_current review - ON review.source_record_id = model.source_record_id - AND review.release_id = model.release_id + -- This lateral form is equivalent to publication_review_release_current + -- for one source/release, but lets the indexed source-record lookup avoid + -- materializing the complete append-only review view for every request. + JOIN LATERAL ( + SELECT review.factual_review_status, + review.privacy_screening_status, + review.maintainer_approval, + review.reviewer_role, + review.publication_eligible + FROM uec.publication_review_events review + JOIN uec.publication_review_release_scopes scope + ON scope.publication_review_event_id = review.publication_review_event_id + AND scope.release_id = model.release_id + WHERE review.source_record_id = model.source_record_id + ORDER BY review.reviewed_at DESC, review.publication_review_event_id DESC + LIMIT 1 + ) review ON true WHERE release.status = 'promoted' AND release.test_only IS NOT TRUE AND review.publication_eligible = true @@ -94,10 +108,37 @@ WITH eligible AS ( AND review.factual_review_status = 'unreviewed' AND review.maintainer_approval = 'pending') ) + -- Keep both current append-only restriction sources live, but correlate + -- them to the candidate record so a large model does not force a full + -- public_access_restricted UNION expansion on every read. AND NOT EXISTS ( SELECT 1 - FROM uec.public_access_restricted restricted - WHERE restricted.source_record_id = model.source_record_id + FROM uec.record_access_current access + WHERE access.source_record_id = model.source_record_id + AND access.action = 'public_access_revoked' + ) + AND NOT EXISTS ( + SELECT 1 + FROM uec.suppression_case_current current_case + JOIN uec.suppression_cases case_record + ON case_record.case_id = current_case.case_id + JOIN uec.suppression_references ref + ON ref.case_id = case_record.case_id + JOIN uec.source_records record ON ( + (ref.facility_id IS NOT NULL AND ( + EXISTS (SELECT 1 FROM uec.facility_source_links link + WHERE link.facility_id = ref.facility_id + AND link.source_record_id = record.source_record_id) + OR EXISTS (SELECT 1 FROM uec.observations observation + WHERE observation.facility_id = ref.facility_id + AND observation.source_record_id = record.source_record_id) + )) + OR (ref.source_id = record.source_id + AND ref.source_record_key = record.source_record_key) + ) + WHERE current_case.event_type = 'suppressed' + AND case_record.status IN ('active', 'review', 'closed', 'expired') + AND record.source_record_id = model.source_record_id ) ) SELECT eligible.release_id, @@ -142,4 +183,4 @@ LEFT JOIN uec.facility_lifecycle_current lifecycle WINDOW facility_history AS (PARTITION BY eligible.release_id, eligible.facility_id); COMMENT ON VIEW uec.map_facilities_public_discovery_read_model IS - 'Manifest-bound public discovery component with live review, profile, suppression, and lifecycle gates; missing or stale metadata yields no rows.'; + 'Manifest-bound public discovery component with live publication_review_release_current-equivalent review, profile, suppression, and lifecycle gates; missing or stale metadata yields no rows.'; From 12bbe7e266066ee3ffb28f22cf27ebb6ebcf53c9 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Thu, 17 Sep 2026 01:02:53 -0700 Subject: [PATCH 215/311] test: include discovery read model migrations --- pipeline/tests/test_graph_migrations.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pipeline/tests/test_graph_migrations.py b/pipeline/tests/test_graph_migrations.py index e51b46f..8796e27 100644 --- a/pipeline/tests/test_graph_migrations.py +++ b/pipeline/tests/test_graph_migrations.py @@ -20,7 +20,7 @@ def test_reserved_migrations_are_present_and_ordered(self): positions = [migrations.index(name) for name in graph_migrations] self.assertEqual(positions, sorted(positions)) self.assertEqual([migrations[position] for position in positions], graph_migrations) - self.assertEqual(migrations[-11:], [ + self.assertEqual(migrations[-12:], [ "026_graph_entities_crosswalks.sql", "027_graph_relationship_observations.sql", "028_graph_claims_support.sql", @@ -31,6 +31,7 @@ def test_reserved_migrations_are_present_and_ordered(self): "033_release_summary_component.sql", "034_public_eligibility_join_indexes.sql", "035_public_discovery_planner_indexes.sql", + "036_public_facility_discovery_view.sql", "037_public_discovery_read_model.sql", ]) From 2617dcf8187591ec41b9dcb672b0248661f65700 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Wed, 16 Sep 2026 23:46:17 -0700 Subject: [PATCH 216/311] Record passing 50k corpus resilience rehearsal --- docs/performance/corpus-resilience.md | 33 +++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/docs/performance/corpus-resilience.md b/docs/performance/corpus-resilience.md index 8f253b3..cd503c1 100644 --- a/docs/performance/corpus-resilience.md +++ b/docs/performance/corpus-resilience.md @@ -56,8 +56,37 @@ same source-record reference is used through a source-key ledger, so the rehearsal covers same-source reimport protection without copying restricted payloads. -The full lane was not run in this checkout when Docker Desktop was unavailable; -the focused Python tests and plan-only path remain runnable without Docker. +Observed local evidence on 2026-09-16 includes a passing 5,000-record Docker +run using a 1,000-row batch: all 35 migrations applied in 3.963 seconds, +import completed in 52.228 seconds, duplicate import completed in 45.044 +seconds with zero new rows, backup took 1.217 seconds, restore took 5.804 +seconds, the custom dump was 1,459,170 bytes, the final database was +39,793,123 bytes, and total runtime was 115.943 seconds. The interruption +committed 1,000 rows before resuming; the resumed pass inserted 461 new rows. +The 5,000-row report remained aggregate-only and the stale pre-service gate +rejected the restored database until the current restriction ledger was +replayed. + +The representative 50,000-record bound also passed locally in the disposable +PostGIS project with a 2,000-row batch. All 35 migrations applied in 2.931 +seconds; import took 514.241 seconds; duplicate import took 506.435 seconds +and inserted zero new rows; backup took 5.686 seconds; restore took 16.554 +seconds; the custom dump was 12,954,656 bytes; the final database was +162,607,587 bytes; and total runtime was 1,063.195 seconds. The interruption +committed 2,000 rows before resuming, which inserted 12,615 new rows. The +report observed a 61,591,500-byte Python allocation peak while loading and +importing the largest partition (14,615 rows), and the stale pre-service gate +rejected the restored database until the current restriction ledger was +replayed. Both reports are aggregate-only. + +The 50,000-record result is evidence for this local disposable Docker Desktop +run, not a production capacity guarantee. Earlier concurrent local attempts +also demonstrated that overlapping rehearsals can trigger Docker/Postgres +administrator shutdowns; run this lane in isolation when collecting capacity +measurements. The focused Python tests and plan-only path remain runnable +without Docker. If Docker Desktop is unavailable in a later checkout, the full +lane is not expected to run there; retain the recorded rehearsal as historical +evidence rather than implying a fresh local run. This harness does not claim real-source quality, adapter correctness, production capacity, cloud backup durability, WAL recovery, operator access controls, or publication approval. It also does not measure PostgreSQL or From 78c15bceff3335392e255a99e10027021354f554 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Thu, 17 Sep 2026 09:15:34 -0700 Subject: [PATCH 217/311] Add production operations diagnostics and runbook --- docs/deployment/production-operations.md | 241 ++++++++++++++++ docs/development.md | 6 + .../test_production_operations_runbook.py | 48 ++++ src/main.rs | 266 +++++++++++++++++- 4 files changed, 553 insertions(+), 8 deletions(-) create mode 100644 docs/deployment/production-operations.md create mode 100644 pipeline/tests/test_production_operations_runbook.py diff --git a/docs/deployment/production-operations.md b/docs/deployment/production-operations.md new file mode 100644 index 0000000..57582cc --- /dev/null +++ b/docs/deployment/production-operations.md @@ -0,0 +1,241 @@ +# Production operations runbook + +Status: production-shaped procedure for a future deployment. This document is +not evidence that the service is hosted, that a reviewer is staffed, or that +any retention period, response SLA, backup destination, or provider control has +been verified. + +The service is allowed to bind in production only after the independent +restriction-ledger replay state and trusted release-manifest digest pass the +startup gate. A failed gate, failed readiness check, stale release projection, +or unsafe proxy configuration leaves the service stopped. A development +process may start without a database so that static UI work remains possible, +but `/health/ready` and database-backed APIs remain unavailable; this is an +intentional degraded state, not a publication fallback. + +## Configuration preflight + +Set secrets through the deployment platform's secret store or an equivalent +private mechanism. Do not put them in the repository, image, command history, +URLs, browser bundles, diagnostics, or logs. + +| Variable | Production requirement | Failure behavior | +| --- | --- | --- | +| `UEC_RUNTIME_MODE` | `production` | Startup exits before bind. | +| `UEC_DATABASE_URL` | Non-empty `postgres://` or `postgresql://` URL without whitespace. | Startup exits before bind. | +| `PORT` | Non-zero TCP port; defaults to `8000`. | Startup exits before bind. | +| `UEC_BIND_HOST` | Valid IP address; defaults to `0.0.0.0` in production. | Startup exits before bind. | +| `UEC_CORS_ORIGINS` | One or more exact bare `http(s)` origins, comma-separated; no wildcard. | Startup exits before bind. | +| `UEC_TRUST_PROXY` | Explicit `true` or `false`. | Startup exits before bind. | +| `UEC_TRUSTED_PROXY_CIDRS` | Required and limited to the reverse-proxy networks when proxy trust is `true`; absent otherwise. | Startup exits before bind. | +| `UEC_RESTRICTION_LEDGER_PATH` | Separate current private ledger. | Startup exits before bind. | +| `UEC_RESTORED_RESTRICTION_SNAPSHOT_PATH` | Separate post-replay row-free snapshot. | Startup exits before bind. | +| `UEC_RELEASE_MANIFEST_PATH` | Trusted manifest for the release being served. | Startup exits before bind. | +| `UEC_RELEASE_MANIFEST_SHA256` | 64 lowercase hexadecimal trusted digest. | Startup exits before bind. | + +The legacy singular `UEC_CORS_ORIGIN` is accepted only for compatibility and +must not conflict with `UEC_CORS_ORIGINS`. Candidate/test-release settings are +development-only and must never be set in production. + +## Deploy a release + +1. Build the exact revision with locked dependencies (`cargo build --release + --locked`) and retain the image/revision identifier in the private change + record. Do not substitute a working tree or an unreviewed candidate. +2. Run the private environment gate from a clean checkout. Supply the current + ledger, post-restore snapshot, trusted release manifest, and trusted + manifest digest. The gate validates migration inventory and metadata but + does not approve publication or copy sensitive rows. +3. Apply migrations using the deployment's migration procedure. Stop on any + checksum or migration error; preserve the previous validated release and + follow `docs/development.md` before considering a disposable local volume + reset. Never edit migration history to bypass a checksum mismatch. +4. Build or select the named release projection through the existing release + validation and approval workflow. Keep source origin, review, privacy, + project approval, and publication state separate. Automated acquisition or + a successful source check is not publication authorization. +5. Replay the current independent restriction ledger against the target + database and write a fresh snapshot. Verify that the snapshot revision, + digest, and opaque reference set match the ledger. Do this after any restore + or release rebuild as well as during first deployment. +6. Start the service with the production variables. Confirm a + `server_starting` event with no secret-bearing fields, then poll + `/health/live` and `/health/ready`. Only route traffic after readiness is + HTTP 200 and reports a migrated schema. +7. Perform bounded smoke checks through the public origin: the selected + manifest endpoint, the default curated profile, and one known-safe detail + path. Confirm current suppression is effective and that community claims do + not appear in default results. Do not save response bodies containing real + records in incident tickets or CI logs. +8. Record the revision, release ID/profile, manifest digest, migration + inventory digest, ledger revision/digest, readiness result, and operator + decision in the restricted change record. These identifiers are operational + metadata; they do not establish factual accuracy or legal compliance. + +## Normal stop and graceful drain + +Remove the instance from the load balancer, wait for in-flight requests to +finish according to the deployment platform's bounded drain period, and send +SIGTERM (Ctrl-C is supported for local operation). The Axum server emits +`server_stopping` and completes a graceful shutdown before emitting +`server_stopped`. If the process does not exit within the platform's limit, +capture only safe process diagnostics and use the platform's documented force +stop; investigate incomplete requests before re-enabling the instance. + +Do not treat a stopped process as a rollback. A rollback still requires the +startup gate, current suppression replay, readiness, and smoke checks. + +## Rollback + +Use rollback when the new binary, schema, projection, configuration, or +external dependency causes an unsafe or materially broken public surface. + +1. Remove the instance from traffic. For privacy or targeting exposure, also + restrict the affected public capability immediately and notify the + responsible maintainer; do not wait for a normal release cycle. +2. Preserve the safe operational record: revision, release/profile, manifest + digest, ledger revision, timestamps, health status, and aggregate error + metrics. Do not copy addresses, coordinates, source text, requester data, + or response payloads into it. +3. If the database schema is backward-compatible, deploy the last validated + image and its matching trusted manifest. If a database restore is required, + restore into an isolated database first and follow the restore procedure + below. Do not overwrite the only copy of a current database. +4. Replay the current restriction ledger, create a new matching snapshot, and + run the startup gate. An old backup or old snapshot is not sufficient. +5. Start the prior release, wait for readiness, verify suppression across map, + API, export, historical, cache, and reimport paths as applicable, then + restore traffic gradually. +6. Record the unresolved cause and any incomplete propagation. A rollback + does not recall independent third-party copies; request downstream + corrections where appropriate. + +## Backup + +Backups are private recovery material, not public evidence or release +artifacts. Before scheduling a real backup, the maintainer must document the +provider, access controls, encryption, retention/deletion behavior, restore +owner, and legal/preservation review. This repository does not assert those +controls exist. + +For each backup event, record only safe metadata in the operational record: +backup identifier, database/schema revision, creation time, byte size, checksum, +encryption/key reference, storage location reference, and verification result. +Keep the database backup, independent restriction ledger, current ledger +digest, release manifest, and manifest digest linked by identifiers. The +ledger must remain separately access-controlled; do not assume a database +backup contains the current suppression control plane. + +Use a disposable synthetic database for rehearsal. A representative shape is: + +```powershell +pg_dump --format=custom --no-owner --file=/uec-.dump +Get-FileHash /uec-.dump -Algorithm SHA256 +python pipeline/scripts/maintenance/replay-restriction-ledger.py ` + --database-url ` + --ledger /current-ledger.json ` + --snapshot-output /restriction-snapshot.json +``` + +The commands above contain placeholders and must not be pasted with real +credentials into shared shells. Backup success is not restore success: run a +portless synthetic restore rehearsal and verify that the public projections +remain suppressed after reimport/rebuild. + +## Restore + +1. Stop public traffic and restore into a new isolated database or disposable + clone. Keep the known-good public instance available until the restored + target passes all gates. +2. Verify the backup checksum and provenance metadata. Apply the exact + migration inventory expected by the image; stop on checksum mismatch. +3. Replay the latest independent restriction ledger into the restored + database. Write a fresh row-free snapshot and validate revision, digest, + and reference-set equality. Never use an old snapshot as proof of current + suppression. +4. Run `private-environment-gate.py` with the restored snapshot and trusted + manifest. A missing, malformed, duplicate, stale, or ambiguous reference + fails closed. +5. Start the service with the matching image and verify live, ready, manifest, + curated default, opt-in community separation, exports, caches, and any + historical views. Confirm that current removal decisions survive the + restore. Do not publish a replacement guessed location. +6. Cut traffic over only after an authorized maintainer records the result. + Retain or delete restricted restore material according to the documented + privacy/removal and preservation decision; do not make indefinite retention + the default. + +The executable synthetic drill is +`pipeline/tests/e2e/backup-restore.ps1`. It is a test-only recovery rehearsal, +not a production backup service or a claim that real data has been restored. + +## Incident response + +### Any availability or integrity incident + +Take the affected instance out of traffic, capture bounded safe diagnostics, +and preserve the last known-good release. Check `/health/ready`, migration +state, manifest digest, restriction-ledger replay state, and aggregate +metrics. Do not fix a checksum mismatch by editing migration history or +silently falling back to a test release. If the cause is unresolved, keep the +affected publication stopped while eligible existing content remains only if +current restrictions still hold. + +### Privacy, targeting, or suppression concern + +Treat a credible exposure as urgent suppression. Restrict the affected map, +API, export, preview, cache, historical, and reimport paths first, then assign +the responsible maintainer and a restricted case ID. Do not put the exposed +address, coordinate, worker/resident detail, requester identity, or private +evidence in a public issue, commit, log, or chat transcript. Rebuild affected +projections and verify suppression before reopening. Follow the removal and +correction process in `docs/ETHICS.md`; assess preservation obligations before +exceptional deletion and seek qualified legal advice for legal demands. + +### Source or release integrity concern + +Quarantine the candidate/release, preserve original source artifacts only as +permitted, and compare source/retrieval metadata, transformation/configuration +versions, manifest checksums, and validation reports. Do not infer closure from +source disappearance. No acquisition result or automated diagnostic grants +publication approval. + +## Fresh-machine setup + +1. Clone a clean checkout and read `docs/ETHICS.md`, + `docs/ETHICS-SUMMARY.md`, and `docs/development.md`. +2. Install the pinned toolchain prerequisites: Rust/Cargo, Python 3, Node/npm, + and Docker only if using the local V2 database. On Windows, install + PowerShell as required by the local scripts. +3. Run `python scripts/dev.py --json doctor`. Treat unknown port occupants, + missing dependencies, and unavailable Docker as environment failures; do + not stop unknown processes. +4. Run `cargo build --locked` and `npm ci`. Run the focused Rust, pipeline, + and static tests before changing configuration. +5. For local UI work, use the fixture preview. For local database work, use + `python scripts/dev.py up`, then `python scripts/dev.py probe` and the + documented local E2E checks. Local fixtures are disposable and never + publication evidence. +6. For a production-shaped dry run, supply synthetic private ledger, + snapshot, and manifest files to the private gate. Keep all real source + artifacts and credentials outside the checkout. + +## Diagnostics contract + +`/health/live` is a process liveness check. `/health/ready` is the traffic gate +and fails when the database is absent, unreachable, or missing required +relations/columns. `/health/diagnostics` is an aggregate operational view. It +may report runtime mode, configured/not-configured state, control-gate status, +response classes, rate-limit counts, and bounded latency totals/maxima. + +Diagnostics intentionally excludes URLs, paths, query strings, request +payloads, response bodies, IPs, forwarded headers, facility IDs, source rows, +restriction references, and secrets. Metrics live only for the process +lifetime and are not visitor analytics. Do not attach them to a log sink that +adds the excluded fields or invents a retention period. Current logs also emit +an allowlisted route class and bounded status/latency event; this does not +prove that a hosting provider, proxy, CDN, or error service retains nothing. + +There is no configured backup reviewer, legal service, publication guarantee, +response SLA, anonymity guarantee, or production capacity claim in this +runbook. Those remain explicit deployment decisions and policy obligations. diff --git a/docs/development.md b/docs/development.md index a158f6b..09985cb 100644 --- a/docs/development.md +++ b/docs/development.md @@ -1,5 +1,11 @@ # Developer entrypoint +Production-shaped deploy, rollback, backup, restore, incident, and +fresh-machine procedures are in +[`deployment/production-operations.md`](deployment/production-operations.md). +They describe controls and verification steps; they do not claim that hosting, +staffing, retention, or provider behavior has been audited. + Use `python scripts/dev.py --help` (or `scripts/dev.ps1 --help` on Windows) to discover the V2 workflow. Commands are thin wrappers around the existing project tools: ```text diff --git a/pipeline/tests/test_production_operations_runbook.py b/pipeline/tests/test_production_operations_runbook.py new file mode 100644 index 0000000..09393c2 --- /dev/null +++ b/pipeline/tests/test_production_operations_runbook.py @@ -0,0 +1,48 @@ +import unittest +from pathlib import Path + + +RUNBOOK = Path(__file__).parents[2] / "docs" / "deployment" / "production-operations.md" + + +class ProductionOperationsRunbookTests(unittest.TestCase): + def setUp(self): + self.text = RUNBOOK.read_text(encoding="utf-8") + + def test_runbook_covers_each_recovery_and_operator_lane(self): + for heading in ( + "## Configuration preflight", + "## Deploy a release", + "## Normal stop and graceful drain", + "## Rollback", + "## Backup", + "## Restore", + "## Incident response", + "## Fresh-machine setup", + "## Diagnostics contract", + ): + self.assertIn(heading, self.text) + + def test_runbook_preserves_fail_closed_and_ethics_boundaries(self): + for required in ( + "UEC_RESTRICTION_LEDGER_PATH", + "UEC_RESTORED_RESTRICTION_SNAPSHOT_PATH", + "UEC_RELEASE_MANIFEST_SHA256", + "Automated acquisition or", + "not publication authorization", + "Do not put the exposed", + "not evidence that the service is hosted", + "Metrics live only for the", + ): + self.assertIn(required, self.text) + + def test_runbook_does_not_present_real_credentials_or_public_backup_commands(self): + self.assertNotIn("postgresql://username", self.text) + self.assertNotIn("postgresql://password", self.text) + self.assertNotIn("password=", self.text.lower()) + self.assertIn("", self.text) + self.assertIn("synthetic database", self.text) + + +if __name__ == "__main__": + unittest.main() diff --git a/src/main.rs b/src/main.rs index 0afd12c..6d72ecf 100644 --- a/src/main.rs +++ b/src/main.rs @@ -18,10 +18,11 @@ use axum::extract::{ConnectInfo, State}; use axum::http::{HeaderValue, Method}; use axum::http::{Request, Response, header}; -use axum::{Json, http::StatusCode, response::IntoResponse}; +use axum::{Extension, Json, http::StatusCode, response::IntoResponse}; use axum::{Router, routing::get}; use std::collections::HashMap; use std::net::{IpAddr, SocketAddr}; +use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant}; use tower_http::compression::CompressionLayer; @@ -36,6 +37,7 @@ mod private_environment; pub fn app(state: uec_api::ApiState, proxy: private_environment::ProxyConfig) -> Router { let cors = cors_layer().expect("CORS configuration must be validated before app startup"); + let metrics = Arc::new(OperationalMetrics::default()); Router::new() .route("/health/live", get(liveness)) .route("/health/ready", get(readiness)) @@ -102,14 +104,81 @@ pub fn app(state: uec_api::ApiState, proxy: private_environment::ProxyConfig) -> )) .layer(axum::middleware::from_fn(request_observability)) .layer(cors) + .layer(Extension(metrics)) .with_state(state) } +/// Aggregate-only, process-local operational counters. No request paths, +/// query values, addresses, identifiers, or payloads are retained. +#[derive(Default)] +struct OperationalMetrics { + requests_total: AtomicU64, + responses_success: AtomicU64, + responses_client_error: AtomicU64, + responses_server_error: AtomicU64, + health_requests: AtomicU64, + rate_limited: AtomicU64, + missing_client_identity: AtomicU64, + latency_ms_total: AtomicU64, + latency_ms_max: AtomicU64, +} + +impl OperationalMetrics { + fn observe_response(&self, route_class: &str, status: StatusCode, elapsed: Duration) { + self.requests_total.fetch_add(1, Ordering::Relaxed); + match status.as_u16() { + 200..=399 => self.responses_success.fetch_add(1, Ordering::Relaxed), + 400..=499 => self.responses_client_error.fetch_add(1, Ordering::Relaxed), + _ => self.responses_server_error.fetch_add(1, Ordering::Relaxed), + }; + if route_class == "health" { + self.health_requests.fetch_add(1, Ordering::Relaxed); + } + let latency_ms = elapsed.as_millis().min(60_000) as u64; + self.latency_ms_total + .fetch_add(latency_ms, Ordering::Relaxed); + let mut previous = self.latency_ms_max.load(Ordering::Relaxed); + while latency_ms > previous { + match self.latency_ms_max.compare_exchange_weak( + previous, + latency_ms, + Ordering::Relaxed, + Ordering::Relaxed, + ) { + Ok(_) => break, + Err(current) => previous = current, + } + } + } + + fn snapshot(&self) -> serde_json::Value { + serde_json::json!({ + "requests_total": self.requests_total.load(Ordering::Relaxed), + "responses": { + "success": self.responses_success.load(Ordering::Relaxed), + "client_error": self.responses_client_error.load(Ordering::Relaxed), + "server_error": self.responses_server_error.load(Ordering::Relaxed) + }, + "health_requests": self.health_requests.load(Ordering::Relaxed), + "rate_limited": self.rate_limited.load(Ordering::Relaxed), + "missing_client_identity": self.missing_client_identity.load(Ordering::Relaxed), + "latency_ms": { + "total": self.latency_ms_total.load(Ordering::Relaxed), + "max": self.latency_ms_max.load(Ordering::Relaxed) + }, + "retention": "process_lifetime_only" + }) + } +} + fn parse_cors_origins( mode: &str, configured: Option<&str>, legacy: Option<&str>, ) -> Result, &'static str> { + if configured.is_some() && legacy.is_some() && configured != legacy { + return Err("UEC_CORS_ORIGINS and legacy UEC_CORS_ORIGIN conflict"); + } let value = configured.or(legacy).unwrap_or(if mode == "development" { "http://localhost:3000" } else { @@ -277,9 +346,17 @@ async fn rate_limit( return next.run(request).await; } let Some(key) = client_key(&request, &config.proxy) else { + if let Some(metrics) = request.extensions().get::>() { + metrics + .missing_client_identity + .fetch_add(1, Ordering::Relaxed); + } return StatusCode::SERVICE_UNAVAILABLE.into_response(); }; if !config.limiter.allow(key, Instant::now()) { + if let Some(metrics) = request.extensions().get::>() { + metrics.rate_limited.fetch_add(1, Ordering::Relaxed); + } return Response::builder() .status(StatusCode::TOO_MANY_REQUESTS) .header(header::RETRY_AFTER, RATE_WINDOW.as_secs().to_string()) @@ -335,8 +412,16 @@ async fn request_observability( ) -> Response { let method = request.method().clone(); let path = request.uri().path().to_owned(); + let route_class = request_route_class(&path); + let metrics = request + .extensions() + .get::>() + .cloned(); let started = Instant::now(); let response = next.run(request).await; + if let Some(metrics) = metrics { + metrics.observe_response(route_class, response.status(), started.elapsed()); + } println!( "{}", request_log_payload(&method, &path, response.status(), started.elapsed()) @@ -348,7 +433,7 @@ async fn liveness() -> impl IntoResponse { Json(serde_json::json!({"status": "ok", "service": "uec-api"})) } -async fn diagnostics() -> impl IntoResponse { +async fn diagnostics(Extension(metrics): Extension>) -> impl IntoResponse { let mode = std::env::var("UEC_RUNTIME_MODE").unwrap_or_else(|_| "development".into()); let database_configured = std::env::var("UEC_DATABASE_URL") .ok() @@ -363,6 +448,7 @@ async fn diagnostics() -> impl IntoResponse { "service": "uec-api", "runtime_mode": mode, "database_configured": database_configured, + "service_state": if database_configured { "configured" } else { "degraded_database_unconfigured" }, "proxy_trust": proxy_trust, "startup_gates": { "restriction_ledger": if mode == "production" { "verified" } else { "not_required_development" }, @@ -372,7 +458,8 @@ async fn diagnostics() -> impl IntoResponse { "request_payloads": "not_reported", "visitor_location": "not_reported", "diagnostic_identifiers": "excluded" - } + }, + "operational_metrics": metrics.snapshot() })) } @@ -458,6 +545,16 @@ fn validate_runtime( { return Err("UEC_DATABASE_URL is required in production"); } + if mode == "production" + && database_url.is_some_and(|url| { + let trimmed = url.trim(); + trimmed != url + || trimmed.chars().any(char::is_whitespace) + || !(trimmed.starts_with("postgres://") || trimmed.starts_with("postgresql://")) + }) + { + return Err("UEC_DATABASE_URL must be a PostgreSQL URL without whitespace"); + } if !matches!(mode, "development" | "production") { return Err("UEC_RUNTIME_MODE must be development or production"); } @@ -467,6 +564,43 @@ fn validate_runtime( } } +fn validate_bind_host(bind_host: &str) -> Result { + bind_host + .parse::() + .map_err(|_| "UEC_BIND_HOST must be a valid IP address") +} + +#[cfg(unix)] +async fn shutdown_signal() { + let ctrl_c = async { + let _ = tokio::signal::ctrl_c().await; + }; + let terminate = async { + if let Ok(mut signal) = + tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate()) + { + let _ = signal.recv().await; + } + }; + tokio::select! { + _ = ctrl_c => {}, + _ = terminate => {}, + } + println!( + "{}", + serde_json::json!({"event":"server_stopping","reason":"shutdown_signal"}) + ); +} + +#[cfg(not(unix))] +async fn shutdown_signal() { + let _ = tokio::signal::ctrl_c().await; + println!( + "{}", + serde_json::json!({"event":"server_stopping","reason":"shutdown_signal"}) + ); +} + #[tokio::main] async fn main() { let mode = std::env::var("UEC_RUNTIME_MODE").unwrap_or_else(|_| "development".to_string()); @@ -479,6 +613,13 @@ async fn main() { "0.0.0.0".into() } }); + if let Err(error) = validate_bind_host(&bind_host) { + eprintln!( + "{}", + serde_json::json!({"event":"configuration_error","reason":error}) + ); + std::process::exit(2); + } let port = validate_runtime(&mode, database_url.as_deref(), &port).unwrap_or_else(|error| { eprintln!( "{{\"event\":\"configuration_error\",\"reason\":\"{}\"}}", @@ -601,8 +742,17 @@ async fn main() { } }; - let listener = tokio::net::TcpListener::bind(addr).await.unwrap(); - axum::serve( + let listener = match tokio::net::TcpListener::bind(addr).await { + Ok(listener) => listener, + Err(_) => { + eprintln!( + "{}", + serde_json::json!({"event":"server_bind_error","reason":"listener_bind_failed"}) + ); + std::process::exit(1); + } + }; + let result = axum::serve( listener, app( uec_api::ApiState { @@ -615,8 +765,19 @@ async fn main() { ) .into_make_service_with_connect_info::(), ) - .await - .unwrap(); + .with_graceful_shutdown(shutdown_signal()) + .await; + if result.is_err() { + eprintln!( + "{}", + serde_json::json!({"event":"server_error","reason":"serve_failed"}) + ); + std::process::exit(1); + } + println!( + "{}", + serde_json::json!({"event":"server_stopped","reason":"graceful_shutdown"}) + ); } #[cfg(test)] @@ -626,7 +787,7 @@ mod config_tests { use super::{ parse_cors_origins, preview_config, request_log_payload, request_route_class, - validate_runtime, + validate_bind_host, validate_runtime, }; #[test] fn development_allows_local_defaults() { @@ -648,12 +809,28 @@ mod config_tests { assert!(validate_runtime("test", Some("redacted"), "8000").is_err()); assert!(validate_runtime("production", Some("redacted"), "bad").is_err()); assert!(validate_runtime("production", Some("redacted"), "0").is_err()); + assert!(validate_bind_host("not-an-ip").is_err()); + assert!(validate_bind_host("127.0.0.1").is_ok()); + } + #[test] + fn production_rejects_non_postgres_or_whitespace_database_urls() { + assert!(validate_runtime("production", Some("sqlite://private"), "8000").is_err()); + assert!(validate_runtime("production", Some("postgresql://db host"), "8000").is_err()); + assert!(validate_runtime("production", Some("postgresql://db/uec"), "8000").is_ok()); } #[test] fn cors_requires_narrow_production_allowlist() { assert!(parse_cors_origins("production", None, None).is_err()); assert!(parse_cors_origins("production", Some("*"), None).is_err()); assert!(parse_cors_origins("production", Some("https://example.test/path"), None).is_err()); + assert!( + parse_cors_origins( + "production", + Some("https://example.test"), + Some("https://other.test") + ) + .is_err() + ); assert_eq!( parse_cors_origins( "production", @@ -749,6 +926,79 @@ mod config_tests { assert!(!serialized.contains("longitude")); assert!(!serialized.contains("query")); } + + #[test] + fn operational_metrics_are_aggregate_and_bounded() { + let metrics = super::OperationalMetrics::default(); + metrics.observe_response("health", StatusCode::OK, Duration::from_secs(90_000)); + metrics.observe_response( + "v2_location_detail", + StatusCode::NOT_FOUND, + Duration::from_millis(7), + ); + metrics.observe_response( + "v2_locations_list", + StatusCode::INTERNAL_SERVER_ERROR, + Duration::from_millis(3), + ); + let snapshot = metrics.snapshot(); + assert_eq!(snapshot["requests_total"], 3); + assert_eq!(snapshot["responses"]["success"], 1); + assert_eq!(snapshot["responses"]["client_error"], 1); + assert_eq!(snapshot["responses"]["server_error"], 1); + assert_eq!(snapshot["health_requests"], 1); + assert_eq!(snapshot["latency_ms"]["max"], 60_000); + assert_eq!(snapshot["retention"], "process_lifetime_only"); + let serialized = snapshot.to_string(); + assert!(!serialized.contains("location_detail")); + } + + #[tokio::test] + async fn diagnostics_exposes_safe_degraded_state_and_metrics() { + use tower::ServiceExt; + + let router = super::app( + uec_api::ApiState { + database: None, + dev_preview_token: None, + dev_test_release_id: None, + dev_test_release_token: None, + }, + super::private_environment::ProxyConfig { + trust_forwarded_for: false, + trusted_proxy_cidrs: Vec::new(), + }, + ); + let _ = router + .clone() + .oneshot( + axum::http::Request::builder() + .uri("/health/live") + .body(axum::body::Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + let response = router + .oneshot( + axum::http::Request::builder() + .uri("/health/diagnostics") + .body(axum::body::Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let body = axum::body::to_bytes(response.into_body(), usize::MAX) + .await + .unwrap(); + let value: serde_json::Value = serde_json::from_slice(&body).unwrap(); + assert_eq!(value["service_state"], "degraded_database_unconfigured"); + assert_eq!(value["operational_metrics"]["requests_total"], 1); + let serialized = value.to_string(); + assert!(!serialized.contains("database_url")); + assert!(!serialized.contains("127.0.0.1")); + } } #[cfg(test)] From 89fa4b97edb0ac1cc32bd7aa524515b7dd2a0c49 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Thu, 17 Sep 2026 09:43:02 -0700 Subject: [PATCH 218/311] Close representative V2 API scale rehearsal --- docs/governance/v2-mvp-claim-evidence.json | 6 +- docs/governance/v2-mvp-claim-evidence.md | 10 +- docs/performance/v2-api-load-rehearsal.md | 77 ++++-- .../v2-public-projection-read-path.md | 10 + .../benchmarks/run_api_load_rehearsal.py | 28 +++ .../build_public_discovery_read_model.py | 233 +++++++++++++++--- pipeline/tests/test_api_load_rehearsal.py | 7 + .../tests/test_public_discovery_read_model.py | 10 +- 8 files changed, 319 insertions(+), 62 deletions(-) diff --git a/docs/governance/v2-mvp-claim-evidence.json b/docs/governance/v2-mvp-claim-evidence.json index 5364a7c..1933daa 100644 --- a/docs/governance/v2-mvp-claim-evidence.json +++ b/docs/governance/v2-mvp-claim-evidence.json @@ -293,14 +293,14 @@ { "id": "scale.concurrent-http-capacity", "area": "scale", - "claim": "The API has completed representative concurrent HTTP performance testing with reliable p95/p99 results through 150k records.", - "status": "implemented_not_exercised", + "claim": "The API has completed bounded representative concurrent HTTP performance testing with p50/p95/p99, throughput, timeout, error, concurrency, database-size, and session-pressure measurements through 150k synthetic records.", + "status": "implemented_tested", "evidence": [ {"path": "pipeline/scripts/benchmarks/run_api_load_rehearsal.py", "line": 1}, {"path": "pipeline/tests/test_api_load_rehearsal.py", "line": 1}, {"path": "docs/performance/v2-api-load-rehearsal.md", "line": 1} ], - "limit": "The harness and bounded scale contract exist; the full post-optimization concurrent run remains a Sprint 1 exit item." + "limit": "These are single-run local synthetic measurements, not a production SLO, capacity guarantee, or browser/mobile performance result; 150k requests exceed the 2s client budget at some tested concurrency levels." }, { "id": "operations.resilience", diff --git a/docs/governance/v2-mvp-claim-evidence.md b/docs/governance/v2-mvp-claim-evidence.md index 5d0a981..502cbe9 100644 --- a/docs/governance/v2-mvp-claim-evidence.md +++ b/docs/governance/v2-mvp-claim-evidence.md @@ -51,7 +51,7 @@ links to the full claim, evidence paths, and limitations in the JSON source. | `graph.source-qualified-identities` | Graph | `implemented_tested` | Crosswalks remain source-scoped and reviewable. | | `graph.public-product` | Graph | `prototype_only` | No production public accountability graph is claimed yet. | | `scale.indexed-discovery` | Scale | `implemented_tested` | Discovery plans were measured at 25k/100k/150k rows. | -| `scale.concurrent-http-capacity` | Scale | `implemented_not_exercised` | Full post-optimization concurrent HTTP evidence remains open. | +| `scale.concurrent-http-capacity` | Scale | `implemented_tested` | Bounded synthetic HTTP rehearsals through 150k capture percentiles, throughput, timeouts, errors, session pressure, and database size; they are not production SLOs. | | `operations.resilience` | Operational | `implemented_not_exercised` | The 50k resilience harness exists; full Postgres rehearsal is pending. | | `operations.health-diagnostics` | Operational | `planned` | Deployment observability and alerting are not yet proven. | | `operations.release-authority` | Operational | `human_policy` | Publication authority and least privilege require operational ownership. | @@ -63,13 +63,11 @@ links to the full claim, evidence paths, and limitations in the JSON source. These items prevent declaring the Sprint 1 backend-proof goal complete: -1. Run the full post-optimization HTTP rehearsal and record concurrency and - p95/p99 evidence. -2. Run the full disposable Postgres resilience rehearsal, or preserve a clear +1. Run the full disposable Postgres resilience rehearsal, or preserve a clear environment blocker and do not claim representative backup/restore proof. -3. Produce the current-corpus geospatial report from actual private normalized +2. Produce the current-corpus geospatial report from actual private normalized handoffs rather than the current `unavailable_private_handoff` result. -4. Keep the contract freeze and this matrix synchronized with the API schema, +3. Keep the contract freeze and this matrix synchronized with the API schema, endpoint inventory, and tests. The last item is automated by diff --git a/docs/performance/v2-api-load-rehearsal.md b/docs/performance/v2-api-load-rehearsal.md index c5c13f6..babab43 100644 --- a/docs/performance/v2-api-load-rehearsal.md +++ b/docs/performance/v2-api-load-rehearsal.md @@ -4,8 +4,9 @@ This is a bounded, local, synthetic rehearsal of the V2 API. It is not a production capacity claim. Each run creates a fresh disposable PostGIS E2E environment, applies every migration, seeds only deterministic synthetic records, exercises the API and graph read paths, and destroys the environment. -The report contains aggregate counters and latency percentiles only; raw rows, -coordinates, identifiers, and response bodies are not retained in Git. +The report contains aggregate counters, latency percentiles, and row-free +environment/planner metadata only; raw rows, coordinates, identifiers, and +response bodies are not retained in Git. ## Reproduction @@ -25,6 +26,20 @@ python pipeline/scripts/benchmarks/run_api_load_rehearsal.py ` --requests-per-level 40 ` --timeout-ms 2000 ` --json-output .tmp/api-load-25000.json + +python pipeline/scripts/benchmarks/run_api_load_rehearsal.py ` + --observations 100000 ` + --concurrency 1,4,8,16 ` + --requests-per-level 40 ` + --timeout-ms 2000 ` + --json-output .tmp/api-load-100000.json + +python pipeline/scripts/benchmarks/run_api_load_rehearsal.py ` + --observations 150000 ` + --concurrency 1,4,8,16 ` + --requests-per-level 40 ` + --timeout-ms 2000 ` + --json-output .tmp/api-load-150000.json ``` The runner bounds observations at 150,000, concurrency at 16, and requests per @@ -33,9 +48,10 @@ explicitly finite synthetic safety limit, not a statement about supported production scale. Each run also captures row-free planner metadata for the list, facets, radius, -and graph read shapes. Reports include estimated costs, node counts, relation -and index names, and sequential-scan relation names; they do not include SQL, -plan filters, identifiers, coordinates, or result rows. +and graph read shapes, setup timings, runtime metadata, and final PostgreSQL +database size. Reports include estimated costs, node counts, relation and index +names, and sequential-scan relation names; they do not include SQL, plan +filters, identifiers, coordinates, or result rows. At scales above 1,000 facilities, the graph-shaped fixture is intentionally capped at 1,000 organizations, relationships, and claims. This keeps the @@ -75,25 +91,39 @@ release, or benchmark-output data. | 25,000 | 4 | 40 | 35 | 5 | 0 | 2.696 | 1351.349 / 2028.480 / 2569.991 | 2 / 1 | | 25,000 | 8 | 40 | 30 | 10 | 0 | 4.138 | 1857.025 / 2816.699 / 3052.711 | 5 / 4 | | 25,000 | 16 | 40 | 8 | 32 | 0 | 5.048 | 2011.321 / 5568.897 / 5627.295 | 8 / 7 | -| 100,000 | 1 | 40 | 5 | 35 | 0 | 0.514 | 2009.788 / 2030.512 / 2037.203 | 1 / 1 | -| 100,000 | 4 | 40 | 5 | 35 | 0 | 1.825 | 2008.869 / 2781.670 / 2816.510 | 2 / 1 | -| 100,000 | 8 | 40 | 5 | 35 | 0 | 3.313 | 2012.523 / 3854.425 / 4288.312 | 5 / 3 | -| 100,000 | 16 | 40 | 5 | 35 | 0 | 4.670 | 2013.890 / 5622.259 / 6525.069 | 8 / 7 | -| 150,000 | 1 | 40 | 5 | 35 | 0 | 0.485 | 2010.421 / 2410.614 / 2664.676 | 2 / 1 | -| 150,000 | 4 | 40 | 5 | 35 | 0 | 1.791 | 2010.972 / 3000.228 / 3257.245 | 2 / 1 | -| 150,000 | 8 | 40 | 5 | 35 | 0 | 3.315 | 2011.654 / 4249.663 / 4262.470 | 5 / 4 | -| 150,000 | 16 | 40 | 5 | 35 | 0 | 4.713 | 2011.055 / 5836.123 / 6133.832 | 8 / 7 | +| 100,000 | 1 | 40 | 40 | 0 | 0 | 1.087 | 1015.709 / 1202.332 / 1219.876 | 2 / 1 | +| 100,000 | 4 | 40 | 40 | 0 | 0 | 4.058 | 992.229 / 1394.278 / 1470.834 | 3 / 2 | +| 100,000 | 8 | 40 | 40 | 0 | 0 | 6.476 | 1133.208 / 1677.974 / 1749.388 | 6 / 3 | +| 100,000 | 16 | 40 | 14 | 26 | 0 | 6.616 | 2007.854 / 2470.506 / 2503.633 | 8 / 7 | +| 150,000 | 1 | 40 | 38 | 2 | 0 | 0.820 | 1248.998 / 1831.875 / 2023.506 | 1 / 1 | +| 150,000 | 4 | 40 | 40 | 0 | 0 | 2.776 | 1460.322 / 1947.559 / 1983.054 | 2 / 1 | +| 150,000 | 8 | 40 | 25 | 15 | 0 | 4.195 | 1879.540 / 2023.364 / 2025.709 | 6 / 4 | +| 150,000 | 16 | 40 | 8 | 32 | 0 | 6.627 | 2010.105 / 2444.161 / 2704.441 | 10 / 6 | The 5,000-row fixture is clean at every tested concurrency. At 25,000 rows, -the single-worker level is clean, but timeouts begin at concurrency 4. At -100,000 and 150,000 rows, only five of forty mixed requests completed at each -level; the two-second client budget is not viable. No server-side 5xx or -connection errors occurred. Pool pressure rose with concurrency, but the -observed failure mode was request timeout rather than pool exhaustion. +the single-worker level is clean, but timeouts begin at concurrency 4. In the +final 100,000-row run, levels 1/4/8 were clean and level 16 completed 14/40 +requests. In the final 150,000-row run, level 4 was clean, level 1 completed +38/40, and levels 8/16 completed 25/40 and 8/40. No server-side 5xx or +connection errors occurred. Pool pressure rose with concurrency, and the +observed failure mode was request timeout rather than pool exhaustion. The +single-run 150k concurrency boundary is variable on this workstation, so it +is not a CI or production capacity promise. These are actual local measurements from the post-optimization harness, not -capacity claims. The run artifacts remain in the ignored `.tmp/` directory; -only these aggregate values and row-free plan summaries are documented here. +capacity claims. The 100k setup took 29,047.971 ms to seed and 14,790.181 ms +to build the gated read model; the 150k setup took 43,387.792 ms and +22,674.087 ms respectively. The final PostgreSQL database sizes were +408,031,715 bytes (100k) and 590,123,491 bytes (150k). The run artifacts remain +in the ignored `.tmp/` directory; only these aggregate values and row-free +plan summaries are documented here. + +The exact host was a Lenovo 81Q6 with an Intel Core i7-9750H (12 logical +processors), 15.91 GiB RAM, Windows 11 Home build 10.0.26200, AMD64 Python +3.11.2. Docker Engine/Desktop was 28.5.2 and the database image was +`postgis/postgis:16-3.4` (`sha256:44126d872ac91993766c341e369c539e8196614321765d36a6f1bab0419a5fa5`). +The harness records host metadata and database size; container RSS/peak memory +was not captured, so no PostgreSQL memory-capacity claim is made. The row-free planner summaries estimated list/facets/radius costs of roughly 82,989 at 5,000 rows, 416,635 at 25,000, 1,683,553 at 100,000, and 2,525,908 @@ -105,8 +135,11 @@ live suppression and review views, including `source_records`, retained. The dominant slow path remains the live eligibility/summary work documented in -`v2-public-projection-read-path.md`. The evidence does not justify caching, -relaxing current suppression checks, or claiming production readiness. +`v2-public-projection-read-path.md`. The fixture closure uses a set-based, +release-scoped builder query with correlated live review/access/suppression +gates; it does not cache public decisions or relax safety checks. The evidence +does not justify caching, relaxing current suppression checks, or claiming +production readiness. ## Data and ethics boundary diff --git a/docs/performance/v2-public-projection-read-path.md b/docs/performance/v2-public-projection-read-path.md index 75326d1..e3ffdc4 100644 --- a/docs/performance/v2-public-projection-read-path.md +++ b/docs/performance/v2-public-projection-read-path.md @@ -47,6 +47,16 @@ the earlier component comparison. The candidate therefore still proves the safety protocol but does not justify API integration or a production capacity claim. +The manifest-bound read-model builder was subsequently hardened for the +representative HTTP rehearsal. Its high-volume activation now inserts from a +single set-based query and uses the same release-scoped correlated review, +current-access, and suppression gates as the live read model. It does not read +through the older compatibility history view, whose suppression UNION expands +all source records before release filtering. On the 2026-09-17 Windows/Docker +rehearsal host, the 100k synthetic activation took 14,790.181 ms and the 150k +activation took 22,674.087 ms; interrupted activation remains transactional +and fail-closed. + ## Alternatives considered 1. Request caching is rejected. An emergency suppression, privacy decision, diff --git a/pipeline/scripts/benchmarks/run_api_load_rehearsal.py b/pipeline/scripts/benchmarks/run_api_load_rehearsal.py index 16251f1..ee8e9e8 100644 --- a/pipeline/scripts/benchmarks/run_api_load_rehearsal.py +++ b/pipeline/scripts/benchmarks/run_api_load_rehearsal.py @@ -14,6 +14,7 @@ import ipaddress import json import os +import platform import sys import threading import time @@ -412,6 +413,17 @@ def capture_query_plans(connection: Any) -> dict[str, Any]: return plans +def runtime_environment() -> dict[str, Any]: + """Return reproducibility metadata without host paths or user data.""" + return { + "os": platform.platform(), + "machine": platform.machine(), + "processor": platform.processor() or "unknown", + "python": platform.python_version(), + "cpu_count": os.cpu_count(), + } + + def percentile(values: list[float], fraction: float) -> float: if not values: return 0.0 @@ -474,11 +486,19 @@ def build_recommendations(results: list[dict[str, Any]], timeout_ms: int) -> dic def run_rehearsal(env: Any, observations: int, levels: tuple[int, ...], requests_per_level: int, timeout_ms: int, distribution: list[tuple[int, str, str, str]] | None = None) -> dict[str, Any]: import psycopg + setup_started = time.perf_counter() with psycopg.connect(env.database_url) as connection: + seed_started = time.perf_counter() detail_id = seed_public_projection(connection, observations, distribution) + seed_ms = (time.perf_counter() - seed_started) * 1000 + build_started = time.perf_counter() env.build_public_read_model("load-promoted") + build_ms = (time.perf_counter() - build_started) * 1000 + plan_started = time.perf_counter() with psycopg.connect(env.database_url) as connection: query_plans = capture_query_plans(connection) + database_size_bytes = int(connection.execute("SELECT pg_database_size(current_database())").fetchone()[0]) + plan_ms = (time.perf_counter() - plan_started) * 1000 base = f"http://127.0.0.1:{env.api_port}" results = [] for concurrency in levels: @@ -496,6 +516,14 @@ def run_rehearsal(env: Any, observations: int, levels: tuple[int, ...], requests "observations": observations, "requests_per_level": requests_per_level, "timeout_ms": timeout_ms, + "runtime_environment": runtime_environment(), + "database_size_bytes": database_size_bytes, + "setup_timings_ms": { + "seed": round(seed_ms, 3), + "read_model_build": round(build_ms, 3), + "plan_capture_and_size": round(plan_ms, 3), + "total_before_http": round((time.perf_counter() - setup_started) * 1000, 3), + }, "query_plans": query_plans, "levels": results, "recommendations": build_recommendations(results, timeout_ms), diff --git a/pipeline/scripts/maintenance/build_public_discovery_read_model.py b/pipeline/scripts/maintenance/build_public_discovery_read_model.py index eab7b85..2d6b829 100644 --- a/pipeline/scripts/maintenance/build_public_discovery_read_model.py +++ b/pipeline/scripts/maintenance/build_public_discovery_read_model.py @@ -67,22 +67,190 @@ def _release_manifest(connection: Any, release_id: str) -> str: return stored[1] -SELECT_ROWS = """ -SELECT h.facility_id, h.observation_id, h.source_record_id, - h.canonical_name, h.country_code, h.postal_code, h.city, - ST_AsText(h.display_location::geometry), h.display_precision, - h.display_label, h.geocoding_status, h.geocoder_provider, - h.geocoded_at, h.classification_category, o.observed_at, - o.first_observed_at, h.provenance_origin_type, h.provenance_source_id, - h.provenance_source_name, h.provenance_source_url, - h.provenance_retrieved_at, +# This is deliberately release-scoped and mirrors the live gates in migration +# 037. The older map_facilities_display_history compatibility view expands its +# public_access_restricted UNION before applying the release predicate, which +# turns a high-volume build into an all-source-records scan. Keeping the +# review, access, and suppression predicates correlated here changes only the +# plan shape; it does not freeze or bypass a publication decision. +SOURCE_ROWS = """ +WITH eligible AS ( + SELECT member.release_id, + observation.observation_id, + observation.facility_id, + observation.source_record_id, + observation.classification_category, + observation.first_observed_at, + observation.observed_at, + facility.canonical_name, + facility.country_code, + facility.postal_code, + facility.city, + source.origin_type AS provenance_origin_type, + source.source_id AS provenance_source_id, + source.name AS provenance_source_name, + source.official_url AS provenance_source_url, + artifact.retrieved_at AS provenance_retrieved_at, + review.factual_review_status, + review.privacy_screening_status, + review.maintainer_approval, + review.reviewer_role + FROM uec.release_members member + JOIN uec.releases release + ON release.release_id = member.release_id + JOIN uec.observations observation + ON observation.observation_id = member.observation_id + JOIN uec.facilities facility + ON facility.facility_id = member.facility_id + JOIN uec.source_records record + ON record.source_record_id = observation.source_record_id + JOIN uec.sources source + ON source.source_id = record.source_id + JOIN uec.raw_artifacts artifact + ON artifact.artifact_id = record.artifact_id + JOIN LATERAL ( + SELECT review.factual_review_status, + review.privacy_screening_status, + review.maintainer_approval, + review.reviewer_role, + review.publication_eligible + FROM uec.publication_review_events review + JOIN uec.publication_review_release_scopes scope + ON scope.publication_review_event_id = review.publication_review_event_id + AND scope.release_id = member.release_id + WHERE review.source_record_id = observation.source_record_id + ORDER BY review.reviewed_at DESC, review.publication_review_event_id DESC + LIMIT 1 + ) review ON true + WHERE member.release_id=%s + AND release.status = 'promoted' + AND release.test_only IS NOT TRUE + AND member.default_visible = true + AND review.publication_eligible = true + AND review.privacy_screening_status = 'passed' + AND review.factual_review_status <> 'rejected' + AND ( + review.maintainer_approval = 'approved' + OR (release.profile = 'community' + AND source.origin_type = 'user_submitted' + AND review.factual_review_status = 'unreviewed' + AND review.maintainer_approval = 'pending') + ) + AND NOT EXISTS ( + SELECT 1 + FROM uec.record_access_current access + WHERE access.source_record_id = observation.source_record_id + AND access.action = 'public_access_revoked' + ) + AND NOT EXISTS ( + SELECT 1 + FROM uec.suppression_case_current current_case + JOIN uec.suppression_cases case_record + ON case_record.case_id = current_case.case_id + JOIN uec.suppression_references ref + ON ref.case_id = case_record.case_id + JOIN uec.source_records suppressed_record ON ( + (ref.facility_id IS NOT NULL AND ( + EXISTS (SELECT 1 FROM uec.facility_source_links link + WHERE link.facility_id = ref.facility_id + AND link.source_record_id = suppressed_record.source_record_id) + OR EXISTS (SELECT 1 FROM uec.observations restricted_observation + WHERE restricted_observation.facility_id = ref.facility_id + AND restricted_observation.source_record_id = suppressed_record.source_record_id) + )) + OR (ref.source_id = suppressed_record.source_id + AND ref.source_record_key = suppressed_record.source_record_key) + ) + WHERE current_case.event_type = 'suppressed' + AND case_record.status IN ('active', 'review', 'closed', 'expired') + AND suppressed_record.source_record_id = observation.source_record_id + ) +) +SELECT eligible.facility_id, + eligible.observation_id, + eligible.source_record_id, + eligible.canonical_name, + eligible.country_code, + eligible.postal_code, + eligible.city, + CASE WHEN geocode.status = 'accepted' AND geocode.result IS NOT NULL THEN geocode.result + WHEN geocode.status = 'review_required' THEN city.reference_location ELSE NULL END AS display_location, + CASE WHEN geocode.status = 'accepted' AND geocode.result IS NOT NULL THEN 'exact' + WHEN geocode.status = 'review_required' AND city.reference_location IS NOT NULL THEN 'city' + ELSE 'unmapped' END AS display_precision, + CASE WHEN geocode.status = 'accepted' AND geocode.result IS NOT NULL THEN 'Accepted geocoder result' + WHEN geocode.status = 'review_required' AND city.reference_location IS NOT NULL THEN 'Approximate city location — multiple geocoder matches' + ELSE 'No publishable location' END AS display_label, + geocode.status AS geocoding_status, + geocode.provider_id AS geocoder_provider, + geocode.queried_at AS geocoded_at, + eligible.classification_category, + eligible.observed_at, + eligible.first_observed_at, + eligible.provenance_origin_type, + eligible.provenance_source_id, + eligible.provenance_source_name, + eligible.provenance_source_url, + eligible.provenance_retrieved_at, CASE WHEN source.attribution IS NULL OR btrim(source.attribution) = '' - THEN 'unknown' ELSE 'attribution_required' END -FROM uec.map_facilities_display_history h -JOIN uec.observations o ON o.observation_id = h.observation_id -JOIN uec.sources source ON source.source_id = h.provenance_source_id -WHERE h.release_id=%s -ORDER BY h.facility_id, h.observation_id + THEN 'unknown' ELSE 'attribution_required' END AS source_rights_status +FROM eligible +JOIN uec.sources source ON source.source_id = eligible.provenance_source_id +LEFT JOIN LATERAL ( + SELECT status, result, provider_id, queried_at + FROM uec.geocode_results + WHERE source_record_id = eligible.source_record_id + ORDER BY queried_at DESC, geocode_result_id DESC + LIMIT 1 +) geocode ON true +LEFT JOIN LATERAL ( + SELECT reference_location + FROM uec.city_reference_points + WHERE country_code = eligible.country_code + AND lower(city_name) = lower(eligible.city) + AND (postal_code IS NULL OR postal_code = eligible.postal_code) + ORDER BY postal_code NULLS LAST + LIMIT 1 +) city ON true +ORDER BY eligible.facility_id, eligible.observation_id +""" + + +SELECT_ROWS = """ +SELECT facility_id, observation_id, source_record_id, canonical_name, + country_code, postal_code, city, ST_AsText(display_location::geometry), + display_precision, display_label, geocoding_status, geocoder_provider, + geocoded_at, classification_category, observed_at, first_observed_at, + provenance_origin_type, provenance_source_id, provenance_source_name, + provenance_source_url, provenance_retrieved_at, source_rights_status +FROM ( +""" + SOURCE_ROWS + """ +) selected +""" + + +# Keep the normal activation path set-based. The source projection is already +# ordered and validated above for its content digest; inserting the same rows +# one at a time makes a large, otherwise safe build spend most of its time in +# client/server round trips. The live safety view remains the sole source of +# rows, and activation is still committed together with its metadata. +INSERT_ROWS = """ +INSERT INTO uec.public_discovery_read_model_rows + (release_id,facility_id,observation_id,source_record_id,canonical_name, + country_code,postal_code,city,display_location,display_precision, + display_label,geocoding_status,geocoder_provider,geocoded_at, + classification_category,observed_at,first_observed_at, + provenance_origin_type,provenance_source_id,provenance_source_name, + provenance_source_url,provenance_retrieved_at,source_rights_status) +SELECT %s, facility_id, observation_id, source_record_id, canonical_name, + country_code, postal_code, city, display_location, display_precision, + display_label, geocoding_status, geocoder_provider, geocoded_at, + classification_category, observed_at, first_observed_at, + provenance_origin_type, provenance_source_id, provenance_source_name, + provenance_source_url, provenance_retrieved_at, source_rights_status +FROM ( +""" + SOURCE_ROWS + """ +) selected """ @@ -107,21 +275,26 @@ def build(database_url: str, release_id: str, fail_after_rows: int | None = None raise ReadModelBlocked("read model metadata exists but row storage is incomplete") return {"status": "idempotent", "release_id": release_id, "manifest_sha256": manifest_sha256, "content_sha256": content_sha256, "row_count": len(rows)} - insert_sql = """ - INSERT INTO uec.public_discovery_read_model_rows - (release_id,facility_id,observation_id,source_record_id,canonical_name, - country_code,postal_code,city,display_location,display_precision, - display_label,geocoding_status,geocoder_provider,geocoded_at, - classification_category,observed_at,first_observed_at, - provenance_origin_type,provenance_source_id,provenance_source_name, - provenance_source_url,provenance_retrieved_at,source_rights_status) - VALUES (%s,%s,%s,%s,%s,%s,%s,%s,ST_GeogFromText(%s),%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) - """ - with connection.cursor() as cursor: - for index, row in enumerate(rows, start=1): - cursor.execute(insert_sql, (release_id, *row)) - if fail_after_rows is not None and index >= fail_after_rows: - raise RuntimeError("synthetic interrupted read model build") + if fail_after_rows is not None: + # This hook deliberately retains the row-at-a-time path so + # tests can interrupt after a known prefix and prove rollback. + insert_sql = """ + INSERT INTO uec.public_discovery_read_model_rows + (release_id,facility_id,observation_id,source_record_id,canonical_name, + country_code,postal_code,city,display_location,display_precision, + display_label,geocoding_status,geocoder_provider,geocoded_at, + classification_category,observed_at,first_observed_at, + provenance_origin_type,provenance_source_id,provenance_source_name, + provenance_source_url,provenance_retrieved_at,source_rights_status) + VALUES (%s,%s,%s,%s,%s,%s,%s,%s,ST_GeogFromText(%s),%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) + """ + with connection.cursor() as cursor: + for index, row in enumerate(rows, start=1): + cursor.execute(insert_sql, (release_id, *row)) + if index >= fail_after_rows: + raise RuntimeError("synthetic interrupted read model build") + else: + connection.execute(INSERT_ROWS, (release_id, release_id)) connection.execute( "INSERT INTO uec.public_discovery_read_models (release_id,manifest_sha256,content_sha256,row_count) VALUES (%s,%s,%s,%s)", (release_id, manifest_sha256, content_sha256, len(rows)), diff --git a/pipeline/tests/test_api_load_rehearsal.py b/pipeline/tests/test_api_load_rehearsal.py index 27f2935..538e2a5 100644 --- a/pipeline/tests/test_api_load_rehearsal.py +++ b/pipeline/tests/test_api_load_rehearsal.py @@ -99,5 +99,12 @@ def test_plan_summary_is_aggregate_only(self): self.assertNotIn("Plans", report) self.assertNotIn("Plan", report) + def test_runtime_environment_is_row_free_reproducibility_metadata(self): + environment = MODULE.runtime_environment() + self.assertIn("os", environment) + self.assertIn("python", environment) + self.assertIsInstance(environment["cpu_count"], int) + self.assertNotIn("C:\\", str(environment)) + if __name__ == "__main__": unittest.main() diff --git a/pipeline/tests/test_public_discovery_read_model.py b/pipeline/tests/test_public_discovery_read_model.py index 974b2c3..633ef74 100644 --- a/pipeline/tests/test_public_discovery_read_model.py +++ b/pipeline/tests/test_public_discovery_read_model.py @@ -34,7 +34,15 @@ def test_migration_is_atomic_manifest_bound_and_live_gated(self): def test_operator_query_never_selects_raw_fields(self): query = MODULE.SELECT_ROWS.lower() self.assertNotIn("raw_fields", query) - self.assertIn("map_facilities_display_history", query) + self.assertIn("release_members", query) + self.assertIn("record_access_current", query) + self.assertIn("suppression_case_current", query) + self.assertNotIn("public_access_restricted", query) + + def test_high_volume_activation_has_set_based_path_and_interrupt_hook(self): + self.assertIn("insert into uec.public_discovery_read_model_rows", MODULE.INSERT_ROWS.lower()) + self.assertIn("from (", MODULE.INSERT_ROWS.lower()) + self.assertIn("if fail_after_rows is not none", (ROOT / "scripts" / "maintenance" / "build_public_discovery_read_model.py").read_text(encoding="utf-8").lower()) if __name__ == "__main__": From b0c8071698ff629c4dc3f8886823b26d8a7a2dfa Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Thu, 17 Sep 2026 09:20:57 -0700 Subject: [PATCH 219/311] Add reviewed demonstration release lane --- ...ewed-demonstration-release-2026-09-17.json | 45 +++++ docs/review-packet-denmark.md | 5 + docs/reviewed-demonstration-release.md | 111 ++++++++++++ pipeline/README.md | 12 ++ pipeline/common/demonstration_release.py | 116 ++++++++++++ .../stages/prepare-demonstration-release.py | 132 ++++++++++++++ pipeline/scripts/stages/promote-release.py | 19 +- .../stages/record-demonstration-review.py | 119 +++++++++++++ pipeline/scripts/stages/validate-release.py | 6 +- .../tests/e2e/test_reviewed_demo_release.py | 165 ++++++++++++++++++ pipeline/tests/test_demonstration_release.py | 76 ++++++++ pipeline/tests/test_promote_release.py | 7 +- pipeline/tests/test_release_validation.py | 8 +- 13 files changed, 809 insertions(+), 12 deletions(-) create mode 100644 data/manifests/reviewed-demonstration-release-2026-09-17.json create mode 100644 docs/reviewed-demonstration-release.md create mode 100644 pipeline/common/demonstration_release.py create mode 100644 pipeline/scripts/stages/prepare-demonstration-release.py create mode 100644 pipeline/scripts/stages/record-demonstration-review.py create mode 100644 pipeline/tests/e2e/test_reviewed_demo_release.py create mode 100644 pipeline/tests/test_demonstration_release.py diff --git a/data/manifests/reviewed-demonstration-release-2026-09-17.json b/data/manifests/reviewed-demonstration-release-2026-09-17.json new file mode 100644 index 0000000..4b45999 --- /dev/null +++ b/data/manifests/reviewed-demonstration-release-2026-09-17.json @@ -0,0 +1,45 @@ +{ + "manifest_version": "uec-reviewed-demo-evidence-v1", + "evidence_scope": "row-free private release-lane evidence; not a public release", + "as_of_utc": "2026-09-17T00:00:00Z", + "release_created": false, + "release_promoted": false, + "public_api_rows": 0, + "profile": "official", + "source": { + "source_id": "dk.smiley", + "source_url": "https://pub.fvst.dk/publikationer/Smileydata.xml", + "private_rehearsal_input_rows": 58766, + "private_rehearsal_normalized_rows": 58766, + "private_rehearsal_validation_findings": 57, + "private_rehearsal_coordinate_state": "unresolved", + "source_artifact_sha256": "4d2f012d3c287ad63e97fc13be8adfcc22f45a163d6096d549d970e63749ccff", + "payload_location": "restricted local artifact; not checked in" + }, + "gates": { + "terms_and_rights": "blocked_pending_named_release_review", + "coverage_and_effective_date": "blocked", + "classification": "blocked_pending_review_of_validation_findings", + "privacy_and_location": "blocked_pending_record_screening", + "coordinate_precision": "blocked_pending_reviewed_coordinates", + "project_approval": "not-recorded" + }, + "second_source": { + "included": false, + "reason": "No additional source currently demonstrates rights, provenance, privacy, and precision gates together." + }, + "machinery": { + "selection_stage": "pipeline/scripts/stages/prepare-demonstration-release.py", + "review_stage": "pipeline/scripts/stages/record-demonstration-review.py", + "validation_and_promotion": "existing release gates with demonstration rights check", + "replacement": "promotion manifest records supersedes", + "suppression": "existing append-only suppression and restore workflow" + }, + "limitations": [ + "This evidence does not approve, promote, or publish any Denmark record.", + "The private rehearsal counts are aggregate evidence only; they are not a completeness claim.", + "A geocoder result, government origin, attribution, or source availability does not substitute for project approval.", + "The public demonstration remains blocked until an authorized maintainer records the required scoped review." + ], + "private_payloads_included": false +} diff --git a/docs/review-packet-denmark.md b/docs/review-packet-denmark.md index a7335eb..85e96e0 100644 --- a/docs/review-packet-denmark.md +++ b/docs/review-packet-denmark.md @@ -9,3 +9,8 @@ As of 2026-09-15, `dk.smiley` has a private, deterministic staging path. It is n - Coverage/lifecycle: source disappearance is `not-observed`, never closure. Candidate import and guarded API checks must remain disposable/test-only. Evidence: `pipeline/sources/denmark/`, `pipeline/contracts/source_health.py`, and `docs/countries/denmark/denmark-data-flow.md`. + +The bounded release lane and current row-free evidence are recorded in +[`docs/reviewed-demonstration-release.md`](reviewed-demonstration-release.md) +and [`data/manifests/reviewed-demonstration-release-2026-09-17.json`](../data/manifests/reviewed-demonstration-release-2026-09-17.json). +That evidence records no public rows and does not represent a release approval. diff --git a/docs/reviewed-demonstration-release.md b/docs/reviewed-demonstration-release.md new file mode 100644 index 0000000..1d0976a --- /dev/null +++ b/docs/reviewed-demonstration-release.md @@ -0,0 +1,111 @@ +# Reviewed demonstration release lane + +This lane is the controlled path for a small real-data demonstration. It is +not a shortcut around the governing [ethics policy](ETHICS.md), source terms, +privacy screening, or maintainer approval. Raw XML, normalized rows, review +documents, and coordinates remain in restricted ignored storage. Checked-in +documents and receipts must stay row-free. + +The current Denmark source is still blocked for publication. Its source review +packet records open terms/currentness, coverage/effective-date, category, +privacy, and release-review questions. No second source is included because no +additional source currently has all of rights, provenance, privacy, and +precision gates demonstrated. + +## Workflow + +1. Acquire and stage Denmark privately using the source-owned pipeline. Keep + raw artifacts and normalized handoffs outside Git. A failed or partial run + must not change an existing release. +2. Write a row-free selection document containing the candidate release ID, + source ID, raw-artifact SHA-256, an explanation of the bounded selection, + and at most 25 opaque `source_record_id` UUIDs. Do not put names, addresses, + coordinates, source text, or requester evidence in this document. +3. Prepare a new non-test candidate. Preparation copies only selected, + already-classified, coordinate-approved, accepted, visible observations. It + does not approve or publish: + + ```powershell + python pipeline/scripts/stages/prepare-demonstration-release.py ` + --selection data/restricted/demo/selection.json ` + --release-id dk-demo-2026-09-17 ` + --profile official ` + --receipt data/reports/dk-demo-prepared.json ` + --database-url $env:UEC_DATABASE_URL + ``` + +4. After an authorized project maintainer has separately reviewed source terms, + rights, provenance, classification, privacy/location exposure, and the + bounded selection, write a review document. `rights_status` must be + explicitly `cleared`; the command does not decide whether the reviewer is + authorized and does not provide legal advice. +5. Record the release-scoped review decisions. This appends immutable + publication-review events and updates only the candidate control summary: + + ```powershell + python pipeline/scripts/stages/record-demonstration-review.py ` + --review data/restricted/demo/review.json ` + --receipt data/reports/dk-demo-reviewed.json ` + --database-url $env:UEC_DATABASE_URL + ``` + + The review must cover exactly every member in the prepared release. A + pending, failed, rejected, or denied decision cannot approve the demo. +6. Run release validation and stop on any finding. Then promote explicitly, + writing the immutable manifest to a new file. Promotion remains separate + from review and validation: + + ```powershell + python pipeline/scripts/stages/validate-release.py dk-demo-2026-09-17 ` + --expected-records 5 --mark-validated ` + --output data/reports/dk-demo-validation.json + python pipeline/scripts/stages/promote-release.py dk-demo-2026-09-17 ` + --manifest data/reports/dk-demo-manifest.json ` + --no-distributed-artifacts + python pipeline/scripts/maintenance/build_public_discovery_read_model.py ` + dk-demo-2026-09-17 --database-url $env:UEC_DATABASE_URL + ``` + +7. Verify the public API and packaged export by release/profile and retain only + aggregate, row-free evidence. The manifest records source coverage, + retrieval, review/publication state, limitations, and the release it + supersedes. A later promotion records the previous promoted release in + `supersedes`; it does not mutate earlier evidence. +8. Exercise suppression with an opaque source-record reference. Verify list, + detail, facets, CSV, historical views, reimport, renewed geocoding, release + reconstruction, and restore replay. The suppression runbook remains the + authority for privacy/removal cases. A closed case still requires an + explicit lifted event before publication can return. + +## Review document shape + +The following is a template, not an approval and not a claim that the current +Denmark source is cleared: + +```json +{ + "review_version": "uec-demo-review-v1", + "release_id": "dk-demo-2026-09-17", + "source_id": "dk.smiley", + "source_artifact_sha256": "<64 lowercase hex characters>", + "rights_status": "cleared", + "rights_reference": "", + "reviewer_role": "", + "reviewed_at": "", + "decisions": [ + { + "source_record_id": "", + "factual_review_status": "reviewed", + "privacy_screening_status": "passed", + "maintainer_approval": "approved", + "publication_eligible": true, + "note": "" + } + ] +} +``` + +Do not substitute `government-sourced`, a successful download, a geocoder +match, or an attribution string for project approval. Until an authorized +review actually records the required decision, the current Denmark rehearsal +remains private candidate/test-only and no public release should be created. diff --git a/pipeline/README.md b/pipeline/README.md index 6878f8a..15136ce 100644 --- a/pipeline/README.md +++ b/pipeline/README.md @@ -98,3 +98,15 @@ Every operational record preserves the prior eligible release reference and keeps `release_promoted` false. A changed artifact, unchanged rerun, failed attempt, or review-required result is recorded as a new event; no run overwrites earlier evidence. The health index is private operational evidence only. + +## Small reviewed demonstration release + +The bounded real-data demonstration lane is documented in +[`docs/reviewed-demonstration-release.md`](../docs/reviewed-demonstration-release.md). +Use `prepare-demonstration-release.py` to copy at most 25 already-ready, +opaque-ID-selected observations from a private candidate into a new candidate, +then use `record-demonstration-review.py` for an explicit release-scoped +maintainer review. Neither command promotes or publishes. The current Denmark +source remains blocked until terms, coverage, privacy, precision, and project +approval are actually reviewed; do not create a review document that claims +those decisions without an authorized maintainer's evidence. diff --git a/pipeline/common/demonstration_release.py b/pipeline/common/demonstration_release.py new file mode 100644 index 0000000..f8adf61 --- /dev/null +++ b/pipeline/common/demonstration_release.py @@ -0,0 +1,116 @@ +"""Validation helpers for the bounded real-data demonstration release lane. + +These helpers deliberately accept only opaque identifiers and review metadata. +They never parse or emit source rows, addresses, coordinates, or reviewer +evidence. The database-facing stages add the corresponding release members +and append-only review events after these documents pass validation. +""" + +from __future__ import annotations + +import json +import uuid +from pathlib import Path +from typing import Any + + +MAX_DEMONSTRATION_RECORDS = 25 +SELECTION_VERSION = "uec-demo-selection-v1" +REVIEW_VERSION = "uec-demo-review-v1" + + +class DemonstrationReleaseError(ValueError): + """The demonstration release input is incomplete or unsafe.""" + + +def _object(path: Path, label: str) -> dict[str, Any]: + try: + value = json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeError, json.JSONDecodeError) as exc: + raise DemonstrationReleaseError(f"{label} is not valid JSON") from exc + if not isinstance(value, dict): + raise DemonstrationReleaseError(f"{label} must be a JSON object") + return value + + +def _required_string(value: Any, field: str) -> str: + if not isinstance(value, str) or not value.strip(): + raise DemonstrationReleaseError(f"{field} must be a non-empty string") + return value.strip() + + +def _uuid(value: Any, field: str) -> str: + raw = _required_string(value, field) + try: + return str(uuid.UUID(raw)) + except ValueError as exc: + raise DemonstrationReleaseError(f"{field} must be a UUID") from exc + + +def load_selection(path: Path) -> dict[str, Any]: + selection = _object(path, "selection") + if selection.get("selection_version") != SELECTION_VERSION: + raise DemonstrationReleaseError("selection_version is unsupported") + selection["candidate_release_id"] = _required_string(selection.get("candidate_release_id"), "candidate_release_id") + selection["source_id"] = _required_string(selection.get("source_id"), "source_id") + selection["source_artifact_sha256"] = _required_string(selection.get("source_artifact_sha256"), "source_artifact_sha256").lower() + if len(selection["source_artifact_sha256"]) != 64 or any(c not in "0123456789abcdef" for c in selection["source_artifact_sha256"]): + raise DemonstrationReleaseError("source_artifact_sha256 must be a lowercase SHA-256 digest") + record_ids = selection.get("record_ids") + if not isinstance(record_ids, list) or not record_ids: + raise DemonstrationReleaseError("record_ids must be a non-empty array") + normalized = [_uuid(value, "record_ids[]") for value in record_ids] + if len(normalized) != len(set(normalized)): + raise DemonstrationReleaseError("record_ids must be unique") + if len(normalized) > MAX_DEMONSTRATION_RECORDS: + raise DemonstrationReleaseError(f"selection exceeds the {MAX_DEMONSTRATION_RECORDS}-record demonstration limit") + selection["record_ids"] = normalized + selection["selection_reason"] = _required_string(selection.get("selection_reason"), "selection_reason") + return selection + + +def load_review(path: Path) -> dict[str, Any]: + review = _object(path, "review") + if review.get("review_version") != REVIEW_VERSION: + raise DemonstrationReleaseError("review_version is unsupported") + review["release_id"] = _required_string(review.get("release_id"), "release_id") + review["source_id"] = _required_string(review.get("source_id"), "source_id") + review["source_artifact_sha256"] = _required_string(review.get("source_artifact_sha256"), "source_artifact_sha256").lower() + if len(review["source_artifact_sha256"]) != 64 or any(c not in "0123456789abcdef" for c in review["source_artifact_sha256"]): + raise DemonstrationReleaseError("source_artifact_sha256 must be a lowercase SHA-256 digest") + if review.get("rights_status") != "cleared": + raise DemonstrationReleaseError("rights_status must be cleared before a demonstration can be approved") + review["rights_reference"] = _required_string(review.get("rights_reference"), "rights_reference") + review["reviewer_role"] = _required_string(review.get("reviewer_role"), "reviewer_role") + review["reviewed_at"] = _required_string(review.get("reviewed_at"), "reviewed_at") + decisions = review.get("decisions") + if not isinstance(decisions, list) or not decisions: + raise DemonstrationReleaseError("decisions must be a non-empty array") + normalized_decisions = [] + seen: set[str] = set() + for decision in decisions: + if not isinstance(decision, dict): + raise DemonstrationReleaseError("each decision must be an object") + record_id = _uuid(decision.get("source_record_id"), "decisions[].source_record_id") + if record_id in seen: + raise DemonstrationReleaseError("decisions must contain one entry per source record") + seen.add(record_id) + factual = _required_string(decision.get("factual_review_status"), "decisions[].factual_review_status") + privacy = _required_string(decision.get("privacy_screening_status"), "decisions[].privacy_screening_status") + approval = _required_string(decision.get("maintainer_approval"), "decisions[].maintainer_approval") + if factual not in {"reviewed", "rejected"} or privacy not in {"passed", "failed"} or approval not in {"approved", "denied"}: + raise DemonstrationReleaseError("demonstration decisions must be explicit reviewed/privacy/approval outcomes") + if decision.get("publication_eligible") is not True: + raise DemonstrationReleaseError("every demonstration decision must explicitly set publication_eligible=true") + if (factual, privacy, approval) != ("reviewed", "passed", "approved"): + raise DemonstrationReleaseError("a blocked decision cannot approve the demonstration release") + normalized_decisions.append({ + "source_record_id": record_id, + "factual_review_status": factual, + "privacy_screening_status": privacy, + "maintainer_approval": approval, + "publication_eligible": True, + "note": _required_string(decision.get("note"), "decisions[].note"), + }) + review["decisions"] = normalized_decisions + return review diff --git a/pipeline/scripts/stages/prepare-demonstration-release.py b/pipeline/scripts/stages/prepare-demonstration-release.py new file mode 100644 index 0000000..81c76f7 --- /dev/null +++ b/pipeline/scripts/stages/prepare-demonstration-release.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python3 +"""Create a small, still-candidate demonstration release from a private candidate. + +The selection document contains only opaque source-record UUIDs. This stage +does not approve, validate, promote, geocode, or publish anything. +""" + +from __future__ import annotations + +import argparse +import json +import os +import sys +from pathlib import Path + +import psycopg + +ROOT = Path(__file__).resolve().parents[3] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from pipeline.common.demonstration_release import load_selection + + +def prepare(database_url: str, selection_path: Path, release_id: str, profile: str, receipt_path: Path | None = None) -> dict: + selection = load_selection(selection_path) + if release_id == selection["candidate_release_id"]: + raise ValueError("demonstration release must have a new release_id") + if profile not in {"official", "secondary", "community"}: + raise ValueError("profile is unsupported") + record_ids = selection["record_ids"] + with psycopg.connect(database_url) as connection: + with connection.transaction(): + source_release = connection.execute( + "SELECT status, test_only, profile, ruleset_version FROM uec.releases WHERE release_id=%s", + (selection["candidate_release_id"],), + ).fetchone() + if not source_release: + raise ValueError("candidate release not found") + if source_release[0] not in {"candidate", "validated"}: + raise ValueError("source release must still be a candidate or validated release") + rows = connection.execute( + """ + SELECT member.facility_id, member.observation_id, observation.source_record_id, + source.source_id, artifact.sha256, + member.default_visible, observation.classification_review_status, + observation.coordinate_review_status, observation.default_visible, + source_record.source_state, + COALESCE(geocode.status, 'unresolved') + FROM uec.release_members member + JOIN uec.observations observation ON observation.observation_id=member.observation_id + JOIN uec.source_records source_record ON source_record.source_record_id=observation.source_record_id + JOIN uec.sources source ON source.source_id=source_record.source_id + JOIN uec.raw_artifacts artifact ON artifact.artifact_id=source_record.artifact_id + LEFT JOIN LATERAL ( + SELECT status FROM uec.geocode_results + WHERE source_record_id=observation.source_record_id + ORDER BY queried_at DESC, geocode_result_id DESC LIMIT 1 + ) geocode ON true + WHERE member.release_id=%s AND observation.source_record_id = ANY(%s::uuid[]) + ORDER BY observation.source_record_id + """, + (selection["candidate_release_id"], record_ids), + ).fetchall() + if len(rows) != len(record_ids): + raise ValueError("selection contains a source record outside the candidate release") + if any(row[3] != selection["source_id"] for row in rows): + raise ValueError("demonstration selection must contain one source_id") + if any(row[4] != selection["source_artifact_sha256"] for row in rows): + raise ValueError("selection artifact digest does not match candidate rows") + if any(not row[5] or row[6] != "approved" or row[7] != "approved" or not row[8] or row[9] in {"rejected", "superseded"} or row[10] != "accepted" for row in rows): + raise ValueError("every selected row must already pass classification, coordinate, visibility, and source-state gates") + summary = { + "demonstration": { + "version": "uec-reviewed-demonstration-v1", + "source_release_id": selection["candidate_release_id"], + "source_id": selection["source_id"], + "source_artifact_sha256": selection["source_artifact_sha256"], + "selection_count": len(rows), + "selection_reason": selection["selection_reason"], + "rights_status": "pending", + "review_status": "pending", + } + } + connection.execute( + """ + INSERT INTO uec.releases(release_id,status,ruleset_version,profile,test_only,summary) + VALUES (%s,'candidate',%s,%s,false,%s) + """, + (release_id, f"{source_release[3]}-demo", profile, json.dumps(summary)), + ) + for facility_id, observation_id, *_ in rows: + connection.execute( + "INSERT INTO uec.release_members(release_id,facility_id,observation_id,default_visible) VALUES (%s,%s,%s,true)", + (release_id, facility_id, observation_id), + ) + receipt = { + "status": "candidate_prepared", + "release_id": release_id, + "source_release_id": selection["candidate_release_id"], + "profile": profile, + "source_id": selection["source_id"], + "source_artifact_sha256": selection["source_artifact_sha256"], + "selected_record_count": len(rows), + "test_only": False, + "approval": "not-recorded", + "publication": "not-promoted", + } + if receipt_path: + receipt_path.parent.mkdir(parents=True, exist_ok=True) + receipt_path.write_text(json.dumps(receipt, indent=2, sort_keys=True) + "\n", encoding="utf-8", newline="\n") + return receipt + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--selection", type=Path, required=True) + parser.add_argument("--release-id", required=True) + parser.add_argument("--profile", choices=("official", "secondary", "community"), default="official") + parser.add_argument("--receipt", type=Path) + parser.add_argument("--database-url", default=os.environ.get("UEC_DATABASE_URL", "postgresql://uec:uec-local-development-only@localhost:5433/uec")) + args = parser.parse_args() + try: + print(json.dumps(prepare(args.database_url, args.selection, args.release_id, args.profile, args.receipt), sort_keys=True)) + return 0 + except Exception as error: + print(json.dumps({"status": "blocked", "error": str(error)}), file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/pipeline/scripts/stages/promote-release.py b/pipeline/scripts/stages/promote-release.py index 8cac341..e70b4a1 100644 --- a/pipeline/scripts/stages/promote-release.py +++ b/pipeline/scripts/stages/promote-release.py @@ -59,7 +59,7 @@ def utc_iso(value) -> str: def promote(database_url: str, release_id: str, artifacts: list[dict]) -> dict: with psycopg.connect(database_url) as connection: with connection.transaction(): - target = connection.execute("SELECT status, profile, ruleset_version, test_only FROM uec.releases WHERE release_id = %s FOR UPDATE", (release_id,)).fetchone() + target = connection.execute("SELECT status, profile, ruleset_version, test_only, summary FROM uec.releases WHERE release_id = %s FOR UPDATE", (release_id,)).fetchone() if not target: raise ValueError(f"release not found: {release_id}") if not can_promote(target[0], target[3]): @@ -71,8 +71,10 @@ def promote(database_url: str, release_id: str, artifacts: list[dict]) -> dict: count(*) FILTER (WHERE m.default_visible AND (g.status IS DISTINCT FROM 'accepted' OR g.result IS NULL)), count(*) FILTER (WHERE o.classification_review_status <> 'approved' AND m.default_visible), count(*) FILTER (WHERE r.release_id IS NULL OR r.publication_eligible IS DISTINCT FROM true OR r.privacy_screening_status IS DISTINCT FROM 'passed' OR r.maintainer_approval IS DISTINCT FROM 'approved'), - count(*) FILTER (WHERE s.source_record_id IS NOT NULL) + count(*) FILTER (WHERE s.source_record_id IS NOT NULL), + count(*) FILTER (WHERE m.default_visible AND (release.summary->'demonstration' IS NOT NULL AND release.summary->'demonstration'->>'rights_status' IS DISTINCT FROM 'cleared')) FROM uec.release_members m + JOIN uec.releases release ON release.release_id = m.release_id JOIN uec.observations o ON o.observation_id = m.observation_id LEFT JOIN LATERAL (SELECT status, result FROM uec.geocode_results WHERE source_record_id=o.source_record_id ORDER BY queried_at DESC, geocode_result_id DESC LIMIT 1) g ON true LEFT JOIN uec.publication_review_release_current r @@ -81,7 +83,11 @@ def promote(database_url: str, release_id: str, artifacts: list[dict]) -> dict: WHERE m.release_id=%s """, (release_id,)).fetchone() if any(unsafe): - raise ValueError(f"release safety gates failed: coordinate_not_ready={unsafe[0]}, review_required={unsafe[1]}, publication_not_approved={unsafe[2]}, active_suppression={unsafe[3]}") + raise ValueError(f"release safety gates failed: coordinate_not_ready={unsafe[0]}, review_required={unsafe[1]}, publication_not_approved={unsafe[2]}, active_suppression={unsafe[3]}, rights_not_cleared={unsafe[4]}") + demonstration = target[4].get("demonstration") if isinstance(target[4], dict) else None + if demonstration is not None and demonstration.get("review_status") != "approved": + raise ValueError("demonstration release requires an explicit recorded review") + previous = connection.execute("SELECT release_id FROM uec.releases WHERE status = 'promoted' AND profile = %s AND release_id <> %s ORDER BY created_at DESC, release_id DESC LIMIT 1", (target[1], release_id)).fetchone() summary = connection.execute(""" SELECT count(*), coalesce(array_agg(DISTINCT sr.source_id ORDER BY sr.source_id), ARRAY[]::text[]) FROM uec.release_members m JOIN uec.observations o ON o.observation_id=m.observation_id @@ -133,7 +139,8 @@ def promote(database_url: str, release_id: str, artifacts: list[dict]) -> dict: "Source origin and source availability do not certify factual accuracy or current operation.", "Artifact checksums detect byte changes but do not establish factual accuracy or reuse rights.", ], - "supersedes": None, + "supersedes": previous[0] if previous else None, + "rights_review": (demonstration or {}).get("rights_status") if demonstration else "not-recorded", "created_at": utc_iso(created_at), "distributed_artifacts": artifacts, } @@ -141,10 +148,10 @@ def promote(database_url: str, release_id: str, artifacts: list[dict]) -> dict: canonical = canonical_json(manifest) digest = hashlib.sha256(canonical.encode("utf-8")).hexdigest() connection.execute("INSERT INTO uec.release_manifests (release_id,manifest,manifest_sha256) VALUES (%s,%s,%s)", (release_id, canonical, digest)) - previous = connection.execute("SELECT release_id FROM uec.releases WHERE status = 'promoted' AND profile = %s AND release_id <> %s", (target[1], release_id)).fetchall() + previous_rows = connection.execute("SELECT release_id FROM uec.releases WHERE status = 'promoted' AND profile = %s AND release_id <> %s", (target[1], release_id)).fetchall() connection.execute("UPDATE uec.releases SET status = 'validated' WHERE status = 'promoted' AND profile = %s AND release_id <> %s", (target[1], release_id)) connection.execute("UPDATE uec.releases SET status = 'promoted' WHERE release_id = %s", (release_id,)) - return {"release_id": release_id, "status": "promoted", "previously_promoted": [row[0] for row in previous], "manifest": manifest, "manifest_sha256": digest} + return {"release_id": release_id, "status": "promoted", "previously_promoted": [row[0] for row in previous_rows], "manifest": manifest, "manifest_sha256": digest} if __name__ == "__main__": diff --git a/pipeline/scripts/stages/record-demonstration-review.py b/pipeline/scripts/stages/record-demonstration-review.py new file mode 100644 index 0000000..9efbd01 --- /dev/null +++ b/pipeline/scripts/stages/record-demonstration-review.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python3 +"""Append an explicit human review decision for a prepared demo release. + +The review document is operator-authored. This command records its decision; +it does not establish who is authorized, provide legal clearance, validate the +release, or promote it. +""" + +from __future__ import annotations + +import argparse +import json +import os +import sys +from pathlib import Path + +import psycopg + +ROOT = Path(__file__).resolve().parents[3] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from pipeline.common.demonstration_release import load_review + + +def record(database_url: str, review_path: Path, receipt_path: Path | None = None) -> dict: + review = load_review(review_path) + decisions = review["decisions"] + ids = [item["source_record_id"] for item in decisions] + with psycopg.connect(database_url) as connection: + with connection.transaction(): + release = connection.execute( + "SELECT status, test_only, profile, summary FROM uec.releases WHERE release_id=%s FOR UPDATE", + (review["release_id"],), + ).fetchone() + if not release: + raise ValueError("demonstration release not found") + if release[0] != "candidate" or release[1]: + raise ValueError("demonstration release must be a non-test candidate") + summary = release[3] or {} + demo = summary.get("demonstration") if isinstance(summary, dict) else None + if not isinstance(demo, dict) or demo.get("source_id") != review["source_id"]: + raise ValueError("release is not a prepared demonstration for this source") + if demo.get("source_artifact_sha256") != review["source_artifact_sha256"]: + raise ValueError("review artifact digest does not match prepared release") + members = connection.execute( + """ + SELECT observation.source_record_id, source.source_id, artifact.sha256 + FROM uec.release_members member + JOIN uec.observations observation ON observation.observation_id=member.observation_id + JOIN uec.source_records source_record ON source_record.source_record_id=observation.source_record_id + JOIN uec.sources source ON source.source_id=source_record.source_id + JOIN uec.raw_artifacts artifact ON artifact.artifact_id=source_record.artifact_id + WHERE member.release_id=%s + ORDER BY observation.source_record_id + """, + (review["release_id"],), + ).fetchall() + member_ids = {str(row[0]) for row in members} + if member_ids != set(ids): + raise ValueError("review decisions must cover exactly every prepared release member") + if any(row[1] != review["source_id"] or row[2] != review["source_artifact_sha256"] for row in members): + raise ValueError("review source provenance does not match every release member") + existing = connection.execute( + "SELECT 1 FROM uec.publication_review_release_current WHERE release_id=%s AND source_record_id = ANY(%s::uuid[]) LIMIT 1", + (review["release_id"], ids), + ).fetchone() + if existing: + raise ValueError("release already has a review decision; append a separate correction event deliberately") + for decision in decisions: + connection.execute( + """ + INSERT INTO uec.publication_review_events + (source_record_id,release_id,factual_review_status,privacy_screening_status, + maintainer_approval,publication_eligible,reviewer_role,reviewed_at,note) + VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s) + """, + (decision["source_record_id"], review["release_id"], decision["factual_review_status"], decision["privacy_screening_status"], decision["maintainer_approval"], decision["publication_eligible"], review["reviewer_role"], review["reviewed_at"], decision["note"]), + ) + summary["demonstration"].update({ + "rights_status": review["rights_status"], + "rights_reference": review["rights_reference"], + "review_status": "approved", + "reviewer_role": review["reviewer_role"], + "reviewed_at": review["reviewed_at"], + }) + connection.execute("UPDATE uec.releases SET summary=%s WHERE release_id=%s", (json.dumps(summary), review["release_id"])) + receipt = { + "status": "review_recorded", + "release_id": review["release_id"], + "source_id": review["source_id"], + "source_artifact_sha256": review["source_artifact_sha256"], + "reviewed_record_count": len(decisions), + "rights_status": review["rights_status"], + "review_status": "approved", + "publication": "not-promoted", + } + if receipt_path: + receipt_path.parent.mkdir(parents=True, exist_ok=True) + receipt_path.write_text(json.dumps(receipt, indent=2, sort_keys=True) + "\n", encoding="utf-8", newline="\n") + return receipt + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--review", type=Path, required=True) + parser.add_argument("--receipt", type=Path) + parser.add_argument("--database-url", default=os.environ.get("UEC_DATABASE_URL", "postgresql://uec:uec-local-development-only@localhost:5433/uec")) + args = parser.parse_args() + try: + print(json.dumps(record(args.database_url, args.review, args.receipt), sort_keys=True)) + return 0 + except Exception as error: + print(json.dumps({"status": "blocked", "error": str(error)}), file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/pipeline/scripts/stages/validate-release.py b/pipeline/scripts/stages/validate-release.py index 58d95b7..5aa01d6 100644 --- a/pipeline/scripts/stages/validate-release.py +++ b/pipeline/scripts/stages/validate-release.py @@ -27,6 +27,8 @@ def evaluate(metrics: dict, expected_records: int | None = None) -> dict: findings.append({"code": "publication_not_approved", "count": metrics["publication_not_approved"]}) if metrics.get("active_suppression"): findings.append({"code": "active_suppression", "count": metrics["active_suppression"]}) + if metrics.get("rights_not_cleared"): + findings.append({"code": "demonstration_rights_not_cleared", "count": metrics["rights_not_cleared"]}) if metrics.get("test_only"): findings.append({"code": "test_only_release", "count": 1}) return {"status": "passed" if not findings else "blocked", "findings": findings, "metrics": metrics} @@ -62,8 +64,10 @@ def validate(database_url: str, release_id: str, expected_records: int | None, m count(*) FILTER (WHERE release_member.default_visible AND (latest.status <> 'accepted' OR latest.result IS NULL))::int AS coordinate_not_ready, count(*) FILTER (WHERE review.release_id IS NULL OR review.publication_eligible IS DISTINCT FROM true OR review.privacy_screening_status IS DISTINCT FROM 'passed' OR review.maintainer_approval IS DISTINCT FROM 'approved')::int AS publication_not_approved, count(*) FILTER (WHERE restricted.source_record_id IS NOT NULL)::int AS active_suppression, + count(*) FILTER (WHERE release_member.default_visible AND (release.summary->'demonstration' IS NOT NULL AND release.summary->'demonstration'->>'rights_status' IS DISTINCT FROM 'cleared'))::int AS rights_not_cleared, (SELECT count(*)::int FROM uec.validation_findings finding WHERE finding.severity = 'error' AND (finding.source_record_id IS NULL OR finding.source_record_id IN (SELECT source_record_id FROM uec.observations WHERE observation_id IN (SELECT observation_id FROM uec.release_members WHERE release_id = %s)))) AS validation_errors FROM uec.release_members AS release_member + JOIN uec.releases AS release ON release.release_id = release_member.release_id JOIN uec.observations AS observation ON observation.observation_id = release_member.observation_id JOIN uec.facilities AS facility ON facility.facility_id = release_member.facility_id LEFT JOIN LATERAL (SELECT status, result FROM uec.geocode_results WHERE source_record_id = observation.source_record_id ORDER BY queried_at DESC, geocode_result_id DESC LIMIT 1) AS latest ON true @@ -74,7 +78,7 @@ def validate(database_url: str, release_id: str, expected_records: int | None, m LEFT JOIN uec.public_access_restricted restricted ON restricted.source_record_id = observation.source_record_id WHERE release_member.release_id = %s """, (release_id, release_id)).fetchone() - names = ["release_records", "distinct_observations", "duplicate_observations", "review_visible", "exact_display_ready", "city_display_ready", "unmapped_display", "coordinate_not_ready", "publication_not_approved", "active_suppression", "validation_errors"] + names = ["release_records", "distinct_observations", "duplicate_observations", "review_visible", "exact_display_ready", "city_display_ready", "unmapped_display", "coordinate_not_ready", "publication_not_approved", "active_suppression", "rights_not_cleared", "validation_errors"] metrics_dict = dict(zip(names, metrics)); metrics_dict["test_only"] = bool(release[1]) result = evaluate(metrics_dict, expected_records) result.update({"release_id": release_id, "release_status_before": release[0], "marked_validated": False}) diff --git a/pipeline/tests/e2e/test_reviewed_demo_release.py b/pipeline/tests/e2e/test_reviewed_demo_release.py new file mode 100644 index 0000000..adbbb55 --- /dev/null +++ b/pipeline/tests/e2e/test_reviewed_demo_release.py @@ -0,0 +1,165 @@ +"""Synthetic exercise of the bounded reviewed-demonstration release lane. + +This test deliberately uses no Denmark rows. It proves the control sequence +that a future privately staged real Denmark sample must pass. +""" + +import hashlib +import importlib.util +import json +import os +import tempfile +import unittest +import urllib.request +import uuid +from datetime import datetime, timezone +from pathlib import Path + +import psycopg + +try: + from .fixture import E2EEnvironment +except ImportError: + from fixture import E2EEnvironment + + +ROOT = Path(__file__).parents[2] + + +def load_stage(name): + path = ROOT / "scripts" / "stages" / name + spec = importlib.util.spec_from_file_location(name.replace("-", "_"), path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +PREPARE = load_stage("prepare-demonstration-release.py") +REVIEW = load_stage("record-demonstration-review.py") +VALIDATE = load_stage("validate-release.py") +PROMOTE = load_stage("promote-release.py") + + +class ReviewedDemonstrationReleaseE2ETests(unittest.TestCase): + @classmethod + def setUpClass(cls): + if os.environ.get("UEC_RUN_E2E") != "1": + raise unittest.SkipTest("set UEC_RUN_E2E=1 to run Docker-backed E2E tests") + cls.env = E2EEnvironment().start() + cls.seed() + + @classmethod + def tearDownClass(cls): + cls.env.stop() + + @classmethod + def seed(cls): + cls.source_id = "e2e.demo" + cls.record_id, cls.facility_id, cls.observation_id, artifact_id = (uuid.uuid4() for _ in range(4)) + cls.artifact_sha256 = hashlib.sha256(b"synthetic demo artifact").hexdigest() + now = datetime.now(timezone.utc) + with psycopg.connect(cls.env.database_url) as db: + with db.transaction(): + db.execute( + "INSERT INTO uec.sources(source_id,country_code,name,official_url,access_method,attribution) VALUES (%s,'DK','Synthetic demo source','https://example.invalid/demo','fixture','synthetic terms fixture')", + (cls.source_id,), + ) + db.execute( + "INSERT INTO uec.raw_artifacts(artifact_id,storage_key,sha256,byte_size,media_type,retrieved_at) VALUES (%s,'e2e/demo',%s,22,'application/xml',%s)", + (artifact_id, cls.artifact_sha256, now), + ) + db.execute( + "INSERT INTO uec.source_records(source_record_id,source_id,source_record_key,artifact_id,raw_fields,parsed_at) VALUES (%s,%s,'opaque-demo-key',%s,'{}',%s)", + (cls.record_id, cls.source_id, artifact_id, now), + ) + db.execute( + "INSERT INTO uec.facilities(facility_id,canonical_name,country_code,city) VALUES (%s,'Synthetic demo facility','DK','Demoby')", + (cls.facility_id,), + ) + db.execute( + "INSERT INTO uec.observations(observation_id,facility_id,source_record_id,observed_at,observation,classification,ruleset_id,rule_id,classification_category,classification_review_status,default_visible,coordinate_method,coordinate_precision,coordinate_review_status,first_observed_at) VALUES (%s,%s,%s,%s,'{}','{}','demo-v1','demo','slaughter','approved',true,'fixture','address_point','approved',%s)", + (cls.observation_id, cls.facility_id, cls.record_id, now, now), + ) + db.execute( + "INSERT INTO uec.releases(release_id,status,ruleset_version,profile,test_only,summary) VALUES ('e2e-demo-source','candidate','demo-v1','official',true,'{}')" + ) + db.execute( + "INSERT INTO uec.release_members(release_id,facility_id,observation_id,default_visible) VALUES ('e2e-demo-source',%s,%s,true)", + (cls.facility_id, cls.observation_id), + ) + db.execute( + "INSERT INTO uec.geocode_results(source_record_id,provider_id,query,match_method,status,result,queried_at) VALUES (%s,'fixture','synthetic','fixture','accepted',ST_SetSRID(ST_MakePoint(10,55),4326)::geography,%s)", + (cls.record_id, now), + ) + + @classmethod + def documents(cls, release_id): + directory = tempfile.TemporaryDirectory() + root = Path(directory.name) + selection = { + "selection_version": "uec-demo-selection-v1", + "candidate_release_id": "e2e-demo-source", + "source_id": cls.source_id, + "source_artifact_sha256": cls.artifact_sha256, + "record_ids": [str(cls.record_id)], + "selection_reason": "synthetic bounded release-lane exercise", + } + review = { + "review_version": "uec-demo-review-v1", + "release_id": release_id, + "source_id": cls.source_id, + "source_artifact_sha256": cls.artifact_sha256, + "rights_status": "cleared", + "rights_reference": "synthetic fixture terms decision", + "reviewer_role": "synthetic authorized maintainer", + "reviewed_at": "2026-09-17T12:00:00Z", + "decisions": [{ + "source_record_id": str(cls.record_id), + "factual_review_status": "reviewed", + "privacy_screening_status": "passed", + "maintainer_approval": "approved", + "publication_eligible": True, + "note": "synthetic end-to-end decision", + }], + } + selection_path, review_path = root / "selection.json", root / "review.json" + selection_path.write_text(json.dumps(selection), encoding="utf-8") + review_path.write_text(json.dumps(review), encoding="utf-8") + return directory, selection_path, review_path + + def promote_demo(self, release_id): + directory, selection_path, review_path = self.documents(release_id) + self.addCleanup(directory.cleanup) + PREPARE.prepare(self.env.database_url, selection_path, release_id, "official") + REVIEW.record(self.env.database_url, review_path) + validation = VALIDATE.validate(self.env.database_url, release_id, 1, True) + self.assertEqual(validation["status"], "passed") + result = PROMOTE.promote(self.env.database_url, release_id, []) + self.env.build_public_read_model(release_id) + return result + + def get_list(self): + with urllib.request.urlopen(f"http://localhost:{self.env.api_port}/api/v2/locations?limit=100", timeout=10) as response: + return response.status, json.loads(response.read()) + + def test_review_approval_manifest_exposure_replacement_and_suppression(self): + first = self.promote_demo("e2e-demo-first") + self.assertIsNone(first["manifest"]["supersedes"]) + status, body = self.get_list() + self.assertEqual(status, 200) + self.assertEqual(len(body["data"]), 1) + + second = self.promote_demo("e2e-demo-second") + self.assertEqual(second["manifest"]["supersedes"], "e2e-demo-first") + with psycopg.connect(self.env.database_url) as db: + db.execute( + "INSERT INTO uec.record_access_events(source_record_id,action,reason_category,policy_version,maintainer) VALUES (%s,'public_access_revoked','privacy','ethics-v1','synthetic-maintainer')", + (self.record_id,), + ) + status, body = self.get_list() + self.assertEqual(status, 200) + self.assertEqual(body["data"], []) + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/tests/test_demonstration_release.py b/pipeline/tests/test_demonstration_release.py new file mode 100644 index 0000000..3e5eefb --- /dev/null +++ b/pipeline/tests/test_demonstration_release.py @@ -0,0 +1,76 @@ +import json +import tempfile +import unittest +from pathlib import Path + +from pipeline.common.demonstration_release import DemonstrationReleaseError, load_review, load_selection + + +class DemonstrationReleaseDocumentTests(unittest.TestCase): + def write(self, value): + directory = tempfile.TemporaryDirectory() + path = Path(directory.name) / "document.json" + path.write_text(json.dumps(value), encoding="utf-8") + self.addCleanup(directory.cleanup) + return path + + def selection(self, **overrides): + value = { + "selection_version": "uec-demo-selection-v1", + "candidate_release_id": "candidate-dk", + "source_id": "dk.smiley", + "source_artifact_sha256": "a" * 64, + "record_ids": ["11111111-1111-4111-8111-111111111111"], + "selection_reason": "bounded reviewed facility sample", + } + value.update(overrides) + return value + + def review(self, **overrides): + value = { + "review_version": "uec-demo-review-v1", + "release_id": "demo-dk", + "source_id": "dk.smiley", + "source_artifact_sha256": "a" * 64, + "rights_status": "cleared", + "rights_reference": "operator terms review R-1", + "reviewer_role": "authorized project maintainer", + "reviewed_at": "2026-09-17T12:00:00Z", + "decisions": [{ + "source_record_id": "11111111-1111-4111-8111-111111111111", + "factual_review_status": "reviewed", + "privacy_screening_status": "passed", + "maintainer_approval": "approved", + "publication_eligible": True, + "note": "bounded demonstration decision", + }], + } + value.update(overrides) + return value + + def test_selection_is_row_free_and_normalized(self): + value = load_selection(self.write(self.selection())) + self.assertEqual(value["record_ids"], ["11111111-1111-4111-8111-111111111111"]) + self.assertNotIn("address", value) + self.assertNotIn("coordinates", value) + + def test_selection_rejects_duplicate_or_oversized_ids(self): + with self.assertRaises(DemonstrationReleaseError): + load_selection(self.write(self.selection(record_ids=["11111111-1111-4111-8111-111111111111"] * 2))) + oversized = [f"11111111-1111-4111-8111-{index:012d}" for index in range(26)] + with self.assertRaises(DemonstrationReleaseError): + load_selection(self.write(self.selection(record_ids=oversized))) + + def test_review_requires_explicit_rights_and_all_approval_fields(self): + value = load_review(self.write(self.review())) + self.assertEqual(value["rights_status"], "cleared") + self.assertEqual(value["decisions"][0]["maintainer_approval"], "approved") + with self.assertRaises(DemonstrationReleaseError): + load_review(self.write(self.review(rights_status="pending"))) + blocked = self.review(decisions=[{**self.review()["decisions"][0], "privacy_screening_status": "failed"}]) + with self.assertRaises(DemonstrationReleaseError): + load_review(self.write(blocked)) + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/tests/test_promote_release.py b/pipeline/tests/test_promote_release.py index 3fbdbe4..2a3a12f 100644 --- a/pipeline/tests/test_promote_release.py +++ b/pipeline/tests/test_promote_release.py @@ -30,10 +30,15 @@ def test_test_only_releases_are_never_promotable(self): def test_promotion_rechecks_public_safety_gates_and_supports_manifest(self): source = SCRIPT.read_text(encoding="utf-8") - for gate in ("coordinate_not_ready", "review_required", "publication_not_approved", "active_suppression"): + for gate in ("coordinate_not_ready", "review_required", "publication_not_approved", "active_suppression", "rights_not_cleared"): self.assertIn(gate, source) self.assertIn("--manifest", source) + def test_promotion_manifest_records_replacement_and_demo_rights_state(self): + source = SCRIPT.read_text(encoding="utf-8") + self.assertIn('"supersedes": previous[0] if previous else None', source) + self.assertIn('"rights_review":', source) + def test_artifact_inventory_hashes_real_bytes(self): with tempfile.TemporaryDirectory() as directory: artifact = Path(directory) / "synthetic.csv" diff --git a/pipeline/tests/test_release_validation.py b/pipeline/tests/test_release_validation.py index fdc5f52..dff8687 100644 --- a/pipeline/tests/test_release_validation.py +++ b/pipeline/tests/test_release_validation.py @@ -11,16 +11,16 @@ class ReleaseValidationTests(unittest.TestCase): def test_complete_release_passes(self): - report = MODULE.evaluate({"release_records": 10, "duplicate_observations": 0, "validation_errors": 0, "review_visible": 0, "coordinate_not_ready": 0, "publication_not_approved": 0, "active_suppression": 0}, 10) + report = MODULE.evaluate({"release_records": 10, "duplicate_observations": 0, "validation_errors": 0, "review_visible": 0, "coordinate_not_ready": 0, "publication_not_approved": 0, "active_suppression": 0, "rights_not_cleared": 0}, 10) self.assertEqual(report["status"], "passed") def test_incomplete_or_unsafe_release_is_blocked(self): - report = MODULE.evaluate({"release_records": 9, "duplicate_observations": 1, "validation_errors": 2, "review_visible": 1, "coordinate_not_ready": 2, "publication_not_approved": 3, "active_suppression": 1}, 10) + report = MODULE.evaluate({"release_records": 9, "duplicate_observations": 1, "validation_errors": 2, "review_visible": 1, "coordinate_not_ready": 2, "publication_not_approved": 3, "active_suppression": 1, "rights_not_cleared": 2}, 10) self.assertEqual(report["status"], "blocked") - self.assertEqual({finding["code"] for finding in report["findings"]}, {"record_count_mismatch", "duplicate_release_observations", "validation_errors", "review_required_visible", "coordinate_not_ready", "publication_not_approved", "active_suppression"}) + self.assertEqual({finding["code"] for finding in report["findings"]}, {"record_count_mismatch", "duplicate_release_observations", "validation_errors", "review_required_visible", "coordinate_not_ready", "publication_not_approved", "active_suppression", "demonstration_rights_not_cleared"}) def test_publication_safety_gates_block_candidate(self): - report = MODULE.evaluate({"release_records": 1, "duplicate_observations": 0, "validation_errors": 0, "review_visible": 0, "coordinate_not_ready": 1, "publication_not_approved": 1, "active_suppression": 1}) + report = MODULE.evaluate({"release_records": 1, "duplicate_observations": 0, "validation_errors": 0, "review_visible": 0, "coordinate_not_ready": 1, "publication_not_approved": 1, "active_suppression": 1, "rights_not_cleared": 0}) self.assertEqual(report["status"], "blocked") self.assertEqual({finding["code"] for finding in report["findings"]}, {"coordinate_not_ready", "publication_not_approved", "active_suppression"}) From bd8f5f9cb97b74b82300e67f95cf50067b5dc2a0 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Thu, 17 Sep 2026 09:24:07 -0700 Subject: [PATCH 220/311] feat(frontend): complete functional discovery contract --- .../src/api/DevCandidatePreviewRepository.ts | 2 +- frontend/src/api/LocalLocationRepository.ts | 5 ++-- frontend/src/api/wireSchema.ts | 5 ++-- frontend/src/app/App.svelte | 18 +++++++++----- frontend/src/domain/location.ts | 6 +++-- frontend/src/map/LeafletMapAdapter.ts | 8 +++---- frontend/src/map/MapAdapter.ts | 4 ++-- frontend/src/map/MapView.svelte | 12 ++++++---- frontend/src/map/mapProjection.ts | 24 +++++++++++++++++-- frontend/src/styles/research.css | 1 + frontend/src/ui/ExportControl.svelte | 2 +- frontend/tests/e2e/local-safety.spec.ts | 4 ++-- frontend/tests/e2e/private-preview.spec.ts | 2 +- .../tests/unit/devPreviewContract.test.ts | 2 +- .../tests/unit/localDetailRepository.test.ts | 2 +- .../unit/localLocationRepository.test.ts | 6 ++++- frontend/tests/unit/mapProjection.test.ts | 15 +++++++++++- 17 files changed, 84 insertions(+), 34 deletions(-) diff --git a/frontend/src/api/DevCandidatePreviewRepository.ts b/frontend/src/api/DevCandidatePreviewRepository.ts index 5080379..dbcb3b6 100644 --- a/frontend/src/api/DevCandidatePreviewRepository.ts +++ b/frontend/src/api/DevCandidatePreviewRepository.ts @@ -30,7 +30,7 @@ export class DevCandidatePreviewRepository { id: row.facility_id, name: row.canonical_name, region: row.city ?? row.country_code, category: row.category, lat: row.latitude, lon: row.longitude, observed: 'candidate observation date unavailable', source: row.provenance_source_name, candidateId: row.candidate_id, sourceRecordId: row.source_record_id, previewLabel: row.preview_label, releaseStatus: row.release_status, coverageScope: parsed.data.meta.coverage_scope, - evidence: { sourceType: row.source_type, factualReviewStatus: row.factual_review_status, reviewerRole: null, privacyScreeningStatus: row.privacy_screening_status, projectApproval: false, publicationProfile: null, publicationWarning: row.preview_label, sourceId: row.provenance_source_id, sourceUrl: row.provenance_source_url, retrievedAt: row.provenance_retrieved_at, displayPrecision: row.display_precision, lifecycleStatus: 'status_unknown', observationCount: null }, + evidence: { sourceType: row.source_type, factualReviewStatus: row.factual_review_status, reviewerRole: null, privacyScreeningStatus: row.privacy_screening_status, projectApproval: false, publicationProfile: null, publicationWarning: row.preview_label, sourceId: row.provenance_source_id, sourceUrl: row.provenance_source_url, provenanceSource: null, sourceRightsStatus: 'unknown', retrievedAt: row.provenance_retrieved_at, displayPrecision: row.display_precision, lifecycleStatus: 'status_unknown', observationCount: null }, })); } } diff --git a/frontend/src/api/LocalLocationRepository.ts b/frontend/src/api/LocalLocationRepository.ts index 4f54d49..2d731af 100644 --- a/frontend/src/api/LocalLocationRepository.ts +++ b/frontend/src/api/LocalLocationRepository.ts @@ -15,7 +15,7 @@ export const mapWireLocation = (r: WireLocation): Location => ({ sourceType: r.source_type, factualReviewStatus: r.factual_review_status, reviewerRole: r.reviewer_role, privacyScreeningStatus: r.privacy_screening_status, projectApproval: r.project_approval, publicationProfile: r.publication_profile, publicationWarning: r.publication_warning, - sourceId: r.provenance_source_id, sourceUrl: r.provenance_source_url, + sourceId: r.provenance_source_id, sourceUrl: r.provenance_source_url, provenanceSource: r.provenance_source, sourceRightsStatus: r.source_rights_status, retrievedAt: r.provenance_retrieved_at, displayPrecision: r.display_precision, lifecycleStatus: r.lifecycle_status, observationCount: r.observation_count, }, }); @@ -25,7 +25,8 @@ const query = (profile: LocalProfile, filters: LocationFilters) => { const param const eligible = (row: WireLocation, profile: LocalProfile, releaseId: string, ruleset: string): boolean => row.publication_profile === profile && row.release_id === releaseId && row.release_ruleset_version === ruleset && row.privacy_screening_status === 'passed' && row.factual_review_status !== 'rejected' && - (row.project_approval === 'approved' || (profile === 'community' && row.source_type === 'user_submitted' && row.factual_review_status === 'unreviewed')); + (row.project_approval === 'approved' || (profile === 'community' && row.source_type === 'user_submitted' && row.factual_review_status === 'unreviewed')) && + ['cleared', 'attribution_required'].includes(row.source_rights_status); export class LocalLocationRepository { readonly #base: string | undefined; diff --git a/frontend/src/api/wireSchema.ts b/frontend/src/api/wireSchema.ts index 3e946ce..5a27b52 100644 --- a/frontend/src/api/wireSchema.ts +++ b/frontend/src/api/wireSchema.ts @@ -2,10 +2,11 @@ import { z } from 'zod'; const textOrNull=z.string().nullable(); // This is the Rust-shaped boundary. Optional coverage/count metadata is additive; // older valid list envelopes remain readable with safe UI fallbacks. -const locationShape={facility_id:z.string().uuid(),canonical_name:z.string().nullable(),city:textOrNull,country_code:z.string().regex(/^[A-Z]{2}$/),category:z.string(),source_type:z.enum(['official','secondary','user_submitted']),publication_profile:z.enum(['official','secondary','community']),factual_review_status:z.enum(['unreviewed','reviewed','rejected']),privacy_screening_status:z.literal('passed'),project_approval:z.enum(['pending','approved']),reviewer_role:textOrNull,publication_warning:textOrNull,display_precision:z.enum(['exact','city','unmapped']),latitude:z.number().finite().nullable(),longitude:z.number().finite().nullable(),first_observed_at:textOrNull,last_observed_at:textOrNull,observation_count:z.number().int().nonnegative().nullable(),lifecycle_status:z.enum(['active_observed','explicitly_closed','not_seen_recently','status_unknown']),provenance_source_id:z.string(),provenance_source_name:z.string(),provenance_source_url:z.string().url().refine(value=>{try{return ['http:','https:'].includes(new URL(value).protocol);}catch{return false;}},'source URL must use HTTP or HTTPS'),provenance_retrieved_at:z.string(),release_id:z.string(),release_ruleset_version:z.string()}; +const sourceUrl=z.string().url().refine(value=>{try{return ['http:','https:'].includes(new URL(value).protocol);}catch{return false;}},'source URL must use HTTP or HTTPS'); +const locationShape={facility_id:z.string().uuid(),canonical_name:z.string().nullable(),city:textOrNull,country_code:z.string().regex(/^[A-Z]{2}$/),category:z.string(),source_type:z.enum(['official','secondary','user_submitted']),publication_profile:z.enum(['official','secondary','community']),factual_review_status:z.string(),privacy_screening_status:z.literal('passed'),project_approval:z.string(),reviewer_role:textOrNull,publication_warning:textOrNull,display_precision:z.enum(['exact','city','unmapped']),latitude:z.number().finite().nullable(),longitude:z.number().finite().nullable(),first_observed_at:textOrNull,last_observed_at:textOrNull,observation_count:z.number().int().nonnegative().nullable(),lifecycle_status:z.enum(['active_observed','explicitly_closed','not_seen_recently','status_unknown']),provenance_source_id:z.string(),provenance_source:z.string().nullable(),provenance_source_name:z.string(),provenance_source_url:sourceUrl,provenance_retrieved_at:z.string(),source_rights_status:z.string(),release_id:z.string(),release_ruleset_version:z.string()}; const coordinateRules=(row:{latitude:number|null;longitude:number|null;display_precision:string},ctx:z.RefinementCtx)=>{if((row.latitude===null)!==(row.longitude===null))ctx.addIssue({code:z.ZodIssueCode.custom,message:'coordinate pair must be complete'});if(row.display_precision==='unmapped'&&(row.latitude!==null||row.longitude!==null))ctx.addIssue({code:z.ZodIssueCode.custom,message:'unmapped record cannot have coordinates'});if(row.display_precision!=='unmapped'&&(row.latitude===null||row.longitude===null))ctx.addIssue({code:z.ZodIssueCode.custom,message:'mapped record requires coordinates'});}; export const locationSchema=z.object(locationShape).superRefine(coordinateRules); -export const testReleaseLocationSchema=z.object({...locationShape,canonical_name:z.string().nullable(),publication_profile:z.enum(['official','secondary','community']).nullable(),privacy_screening_status:z.enum(['pending','passed','failed']),project_approval:z.union([z.enum(['pending','approved']),z.literal(false),z.literal('not-approved')]),release_ruleset_version:z.string().nullable()}).superRefine(coordinateRules); +export const testReleaseLocationSchema=z.object({...locationShape,canonical_name:z.string().nullable(),publication_profile:z.enum(['official','secondary','community']).nullable(),privacy_screening_status:z.enum(['pending','passed','failed']),project_approval:z.union([z.enum(['pending','approved']),z.literal(false),z.literal('not-approved')]),provenance_source:z.string().nullable().optional().default(null),source_rights_status:z.string().optional().default('unknown'),release_ruleset_version:z.string().nullable()}).superRefine(coordinateRules); export const envelopeSchema=z.object({data:z.array(locationSchema),api_version:z.literal('v2'),meta:z.object({release_id:z.string().nullable(),ruleset_version:z.string().optional(),release_created_at:z.string().optional(),profile:z.enum(['official','secondary','community']),next_cursor:z.string().nullable().optional(),coverage_note:z.string().min(1),coverage_scope:z.string().optional(),count_semantics:z.string().optional(),query:z.object({q:z.string().nullable().optional(),filters:z.record(z.string(),z.string().nullable())}).optional()})}); export type WireEnvelope=z.infer;export type WireLocation=z.infer; export type WireTestReleaseLocation=z.infer; diff --git a/frontend/src/app/App.svelte b/frontend/src/app/App.svelte index f9e39d8..7ec7d32 100644 --- a/frontend/src/app/App.svelte +++ b/frontend/src/app/App.svelte @@ -64,6 +64,7 @@ let coverageScope = ''; let countSemantics = ''; let paging = false; + let pagingError = ''; let listGeneration = 0; let detailGeneration = 0; let listAbort: AbortController | undefined; @@ -90,7 +91,9 @@ network: 'Network error', unavailable: 'V2 service unavailable', restricted: 'Access restricted or record unavailable', 'invalid-contract': 'V2 data contract unavailable', 'rate-limited': 'Temporarily rate-limited', } as Record)[failure] ?? fallback; const sourceLabel = (value: string): string => value === 'user_submitted' ? 'Community-submitted' : value === 'official' ? 'Government-sourced' : 'Secondary-sourced'; - const reviewLabel = (location: Location): string => location.evidence?.factualReviewStatus === 'unreviewed' ? 'Factually unreviewed' : 'Factually reviewed'; + const reviewLabel = (location: Location): string => location.evidence?.factualReviewStatus === 'unreviewed' ? 'Factually unreviewed' : location.evidence?.factualReviewStatus === 'rejected' ? 'Factual review rejected' : 'Factual review recorded'; + const precisionLabel = (location: Location): string => ({ exact: 'Exact public point', city: 'Approximate city location', unmapped: 'No publishable map location' }[location.evidence?.displayPrecision ?? 'unmapped']); + const lifecycleLabel = (location: Location): string => ({ active_observed: 'Active observed', explicitly_closed: 'Explicitly closed', not_seen_recently: 'Not observed recently', status_unknown: 'Lifecycle unknown' }[location.evidence?.lifecycleStatus ?? 'status_unknown']); const isUnreviewedCommunity = (location: Location): boolean => location.evidence?.publicationProfile === 'community' && location.evidence.factualReviewStatus === 'unreviewed'; const profileForUrl = (): string => localMode ? toApiProfile(profile) : profile; @@ -142,20 +145,22 @@ if (!append) activeListKey = queryKey; listGeneration += 1; const generation = listGeneration; listAbort?.abort(); const controller = new AbortController(); listAbort = controller; lastRemoteQuery = queryKey; - if (!append) { invalidateDetail(); localStatus = 'loading'; localFailure = 'unknown'; localError = ''; manifestSha256 = undefined; loaded = []; selected = undefined; nextCursor = null; coverageNote = ''; coverageScope = ''; countSemantics = ''; } else paging = true; + if (!append) { invalidateDetail(); localStatus = 'loading'; localFailure = 'unknown'; localError = ''; pagingError = ''; manifestSha256 = undefined; loaded = []; selected = undefined; nextCursor = null; coverageNote = ''; coverageScope = ''; countSemantics = ''; } else { paging = true; pagingError = ''; } try { - const result = await repo.list(requestProfile, { q: search.trim() || undefined, country_code: region === 'all' ? undefined : region, region: subregion.trim() || undefined, category: category === 'all' ? undefined : category, source_type: sourceType === 'all' ? undefined : sourceType, display_precision: displayPrecision === 'all' ? undefined : displayPrecision, lifecycle_status: lifecycleStatus === 'all' ? undefined : lifecycleStatus, cursor }, controller.signal); + const result = await repo.list(requestProfile, { q: search.trim() || undefined, country_code: region === 'all' ? undefined : region, region: subregion.trim() || undefined, category: category === 'all' ? undefined : category, source_type: sourceType === 'all' ? undefined : sourceType, display_precision: displayPrecision === 'all' ? undefined : displayPrecision, lifecycle_status: lifecycleStatus === 'all' ? undefined : lifecycleStatus, cursor, limit: 100 }, controller.signal); if (generation !== listGeneration) return; if (append && (result.releaseId !== release || result.ruleset !== ruleset)) throw Object.assign(new Error('The promoted release changed while loading this cursor page.'), { kind: 'invalid-contract' as const }); loaded = append ? [...loaded, ...result.locations] : result.locations; if (!selected) selected = result.locations[0]; - release = result.releaseId; ruleset = result.ruleset; nextCursor = result.nextCursor; coverageNote = result.coverageNote; coverageScope = result.coverageScope ?? ''; countSemantics = result.countSemantics ?? ''; localStatus = 'ready'; localFailure = 'unknown'; paging = false; + release = result.releaseId; ruleset = result.ruleset; nextCursor = result.nextCursor; coverageNote = result.coverageNote; coverageScope = result.coverageScope ?? ''; countSemantics = result.countSemantics ?? ''; localStatus = 'ready'; localFailure = 'unknown'; paging = false; pagingError = ''; if (!append) await syncRoute(); } catch (error) { paging = false; if (!append) activeListKey = ''; if (generation !== listGeneration) return; const kind = error && typeof error === 'object' && 'kind' in error ? (error as { kind: ApiError['kind'] }).kind : 'unknown'; if (kind === 'aborted') return; - localStatus = kind === 'no-release' ? 'no-release' : 'error'; localFailure = kind; localError = error instanceof Error ? error.message : 'The V2 list response was rejected safely.'; loaded = []; selected = undefined; + const message = error instanceof Error ? error.message : 'The V2 list response was rejected safely.'; + if (append) { pagingError = `Could not load the next results page. ${message}`; return; } + localStatus = kind === 'no-release' ? 'no-release' : 'error'; localFailure = kind; localError = message; loaded = []; selected = undefined; } }; @@ -224,11 +229,12 @@

01 / DISCOVER

Choose the evidence lane

Profiles stay separate. A community claim is never silently promoted into a curated result.

{#if localMode && metadataStatus === 'loading'}

Loading available filter values…

{:else if localMode && metadataStatus === 'error'}

Filter metadata is unavailable; controlled filter values could not be loaded.

{/if}
{search || region !== 'all' || subregion.trim() || category !== 'all' || sourceType !== 'all' || displayPrecision !== 'all' || lifecycleStatus !== 'all' ? `Filters applied: ${[search && `text “${search}”`, region !== 'all' && region, subregion.trim() && subregion.trim(), category !== 'all' && category, sourceType !== 'all' && sourceType, displayPrecision !== 'all' && displayPrecision, lifecycleStatus !== 'all' && lifecycleStatus].filter(Boolean).join(' · ')}` : 'No additional filters applied'}
{#if localMode && (search.trim() || subregion.trim())}

Search and region filters are evaluated by the V2 server against the selected promoted release. {nextCursor ? 'Results are paginated; load the next cursor page for more matches.' : 'No additional cursor page is available.'}

{/if} + {#if localMode && pagingError}{/if} {#if profile === 'community'}
Community claimsUnreviewed community claims: Not verified by Until Every Cage. Check each record’s factual review status before relying on it.
{/if} {#if localMode && localStatus === 'loading'}
Loading the {profileLabelText.toLowerCase()}…
{:else if localMode && (localStatus === 'error' || localStatus === 'no-release')}{:else if localMode && detailStatus === 'loading'}
Loading the selected local record…
{:else if localMode && detailStatus === 'error'}{:else}

02 / FILTER & COMPARE

Results {visibleLocations.length}{localMode && nextCursor ? ' on first page' : ''}

{localMode ? 'Current eligible response' : 'Fictional demonstration data'} · {profileLabelText}

{#if localMode}
VISIBLE FACILITY RECORDS{visibleLocations.length}{nextCursor ? '+' : ''}
DENOMINATORNot available

{countSemantics || 'Counts refer to eligible public facility projection rows, not animals or a story-wide total.'} Scope: {coverageScope || 'selected promoted release public facilities'}. {nextCursor ? 'This is a partial page.' : 'This response has no further page.'} Legacy status is not inferred: the current V2 record contract has no legacy field.

{coverageNote} {nextCursor ? 'Only the first page is loaded. Search and filters below may miss later records; counts and map points are partial.' : 'All records in this response are loaded; search applies to those records.'}

{/if} -
{#if selected}

03 / RECORD DETAIL

{selected.name}

{selected.region} · {selected.category}

{#if isUnreviewedCommunity(selected)}

{selected.evidence?.publicationWarning ?? 'Unreviewed community claim — not verified by Until Every Cage'}

{/if}
OBSERVED{selected.observed}
MAP STATUS{selected.lat === null ? 'No publishable map location' : selected.evidence?.displayPrecision === 'exact' ? 'Exact display point' : 'Approximate display point'}
{#if selected.evidence}
Source origin
{sourceLabel(selected.evidence.sourceType)}
Factual review
{selected.evidence.factualReviewStatus}{selected.evidence.reviewerRole ? ` · ${selected.evidence.reviewerRole}` : ' · reviewer role unavailable'}
Privacy screening
{selected.evidence.privacyScreeningStatus}
Project approval
{selected.evidence.projectApproval}
Published profile
{selected.evidence.publicationProfile ?? 'unavailable'} · release {release}
Source
{selected.source} · {selected.evidence.sourceId}
Retrieved
{selected.evidence.retrievedAt}
Lifecycle
{selected.evidence.lifecycleStatus}
Observation count
{selected.evidence.observationCount ?? 'unavailable'}
{/if}

READ WITH CARE

This is an observation in a particular release, not a guarantee of current operation. Source origin, factual review, privacy screening, and project approval are separate signals.

{:else}

Select a record to inspect its evidence context.

{/if}
+
{#if selected}

03 / RECORD DETAIL

{selected.name}

{selected.region} · {selected.category}

{#if isUnreviewedCommunity(selected)}

{selected.evidence?.publicationWarning ?? 'Unreviewed community claim — not verified by Until Every Cage'}

{/if}
OBSERVED{selected.observed}
MAP STATUS{precisionLabel(selected)}
{#if selected.evidence}
Source origin
{sourceLabel(selected.evidence.sourceType)}
Factual review
{selected.evidence.factualReviewStatus}{selected.evidence.reviewerRole ? ` · ${selected.evidence.reviewerRole}` : ' · reviewer role unavailable'}
Privacy screening
{selected.evidence.privacyScreeningStatus}
Project approval
{selected.evidence.projectApproval}
Published profile
{selected.evidence.publicationProfile ?? 'unavailable'} · release {release}
Source
{selected.evidence.provenanceSource ?? selected.source} · {selected.evidence.sourceId}
Source rights
{selected.evidence.sourceRightsStatus}
Retrieved
{selected.evidence.retrievedAt}
Lifecycle
{lifecycleLabel(selected)}
Legacy status
Not supplied by the current V2 contract
Observation count
{selected.evidence.observationCount ?? 'unavailable'}
{/if}

READ WITH CARE

This is an observation in a particular release, not a guarantee of current operation. Source origin, factual review, privacy screening, and project approval are separate signals.

{:else}

Select a record to inspect its evidence context.

{/if}
{#if selected}

RECORD / {selected.id}

{/if} {/if} {#if localMode && localStatus === 'ready'}{/if} diff --git a/frontend/src/domain/location.ts b/frontend/src/domain/location.ts index 29e3967..d5c4dee 100644 --- a/frontend/src/domain/location.ts +++ b/frontend/src/domain/location.ts @@ -3,14 +3,16 @@ export type LocationId = string; // approval, precision, and lifecycle are independent signals. export type LocationEvidence = Readonly<{ sourceType: 'official' | 'secondary' | 'user_submitted'; - factualReviewStatus: 'unreviewed' | 'reviewed' | 'rejected'; + factualReviewStatus: string; reviewerRole: string | null; privacyScreeningStatus: 'pending' | 'passed' | 'failed'; - projectApproval: 'pending' | 'approved' | false; + projectApproval: string | false; publicationProfile: 'official' | 'secondary' | 'community' | null; publicationWarning: string | null; sourceId: string; sourceUrl: string; + provenanceSource: string | null; + sourceRightsStatus: string; retrievedAt: string; displayPrecision: 'exact' | 'city' | 'unmapped'; lifecycleStatus: 'active_observed' | 'explicitly_closed' | 'not_seen_recently' | 'status_unknown'; diff --git a/frontend/src/map/LeafletMapAdapter.ts b/frontend/src/map/LeafletMapAdapter.ts index b353822..14d53f3 100644 --- a/frontend/src/map/LeafletMapAdapter.ts +++ b/frontend/src/map/LeafletMapAdapter.ts @@ -1,10 +1,10 @@ import type { Map as LeafletMap, Marker } from 'leaflet'; import type { MapAdapter } from './MapAdapter'; -import type { DisplayFeature } from './mapProjection'; -type MarkerFactory = (feature: DisplayFeature, map: LeafletMap) => Marker; +import type { MapDisplayItem } from './mapProjection'; +type MarkerFactory = (item: MapDisplayItem, map: LeafletMap) => Marker; export class LeafletMapAdapter implements MapAdapter { #map: LeafletMap | null = null; #markers = new Map(); #disposed = false; #makeMarker: MarkerFactory | null = null; - async mount(container: HTMLElement, onSelect?: (id:string) => void): Promise { const leaflet = await import('leaflet'); if (this.#disposed) return; this.#makeMarker = (feature, map) => { const marker = leaflet.marker([feature.lat, feature.lon], { title: feature.label, keyboard: true }).addTo(map); if (onSelect) marker.on('click', () => onSelect(feature.id)); return marker; }; this.#map = leaflet.map(container, { attributionControl: false, zoomControl: true }).setView([55, 10], 6); this.#map.getContainer().style.background = '#ded8cc'; } - update(features: readonly DisplayFeature[], selectedId: string | null): void { if (!this.#map || !this.#makeMarker) return; const active = new Set(features.map((feature) => feature.id)); for (const [id, marker] of this.#markers) { if (!active.has(id)) { marker.remove(); this.#markers.delete(id); } } for (const feature of features) { const marker = this.#markers.get(feature.id) ?? this.#makeMarker(feature, this.#map); this.#markers.set(feature.id, marker); marker.setOpacity(selectedId === null || selectedId === feature.id ? 1 : 0.55); } } + async mount(container: HTMLElement, onSelect?: (id:string) => void): Promise { const leaflet = await import('leaflet'); if (this.#disposed) return; this.#makeMarker = (item, map) => { const cluster = 'count' in item; const city = !cluster && item.precision === 'city'; const marker = leaflet.marker([item.lat, item.lon], { title: item.label, keyboard: true, icon: cluster ? leaflet.divIcon({ className: 'uec-cluster', html: `` }) : city ? leaflet.divIcon({ className: 'uec-city-point', html: '' }) : undefined }).addTo(map); if (cluster) marker.on('click', () => map.setView([item.lat, item.lon], Math.min(map.getZoom() + 2, 18))); else if (onSelect) marker.on('click', () => onSelect(item.id)); return marker; }; this.#map = leaflet.map(container, { attributionControl: false, zoomControl: true }).setView([55, 10], 6); this.#map.getContainer().style.background = '#ded8cc'; } + update(items: readonly MapDisplayItem[], selectedId: string | null): void { if (!this.#map || !this.#makeMarker) return; const active = new Set(items.map((item) => item.id)); for (const [id, marker] of this.#markers) { if (!active.has(id)) { marker.remove(); this.#markers.delete(id); } } for (const item of items) { const marker = this.#markers.get(item.id) ?? this.#makeMarker(item, this.#map); this.#markers.set(item.id, marker); const selected = 'count' in item ? item.memberIds.includes(selectedId ?? '') : selectedId === null || selectedId === item.id; marker.setOpacity(selected ? 1 : 0.55); } } destroy(): void { this.#disposed = true; for (const marker of this.#markers.values()) marker.remove(); this.#markers.clear(); this.#map?.remove(); this.#map = null; this.#makeMarker = null; } } diff --git a/frontend/src/map/MapAdapter.ts b/frontend/src/map/MapAdapter.ts index 0ec4295..796447b 100644 --- a/frontend/src/map/MapAdapter.ts +++ b/frontend/src/map/MapAdapter.ts @@ -1,2 +1,2 @@ -import type { DisplayFeature } from './mapProjection'; -export interface MapAdapter { mount(container:HTMLElement, onSelect?: (id:string) => void):Promise; update(features:readonly DisplayFeature[],selectedId:string|null):void; destroy():void; } +import type { MapDisplayItem } from './mapProjection'; +export interface MapAdapter { mount(container:HTMLElement, onSelect?: (id:string) => void):Promise; update(items:readonly MapDisplayItem[],selectedId:string|null):void; destroy():void; } diff --git a/frontend/src/map/MapView.svelte b/frontend/src/map/MapView.svelte index 8409479..89aad34 100644 --- a/frontend/src/map/MapView.svelte +++ b/frontend/src/map/MapView.svelte @@ -1,7 +1,7 @@ -
{#if hasUnreviewedClaims}

Unreviewed community claims — not verified by Until Every Cage

{/if}

Facility pins only, not animal counts. The results list is the accessible equivalent. Blank local background · {features.length} display points · no external tiles

- +
{#if hasUnreviewedClaims}

Unreviewed community claims — not verified by Until Every Cage

{/if}

Facility pins and clusters only, not animal counts. The results list is the accessible equivalent. Blank local background · {features.length} display points · {clusterCount} clusters · no external tiles

+ diff --git a/frontend/src/map/mapProjection.ts b/frontend/src/map/mapProjection.ts index 1b4eda3..d64649e 100644 --- a/frontend/src/map/mapProjection.ts +++ b/frontend/src/map/mapProjection.ts @@ -1,3 +1,23 @@ import type { Location } from '../domain/location'; -export type DisplayFeature=Readonly<{id:string,label:string,lat:number,lon:number}>; -export const projectLocations=(items:readonly Location[]):readonly DisplayFeature[]=>items.flatMap((item)=>item.lat===null||item.lon===null?[]:[{id:item.id,label:item.evidence?.publicationProfile==='community'&&item.evidence.factualReviewStatus==='unreviewed'?`${item.name} · Unreviewed community claim — not verified by Until Every Cage`:item.name,lat:item.lat,lon:item.lon}]); +export type DisplayFeature=Readonly<{id:string,label:string,lat:number,lon:number,precision:'exact'|'city'}>; +export type DisplayCluster=Readonly<{id:string,label:string,lat:number,lon:number,count:number,memberIds:readonly string[]}>; +export type MapDisplayItem=DisplayFeature|DisplayCluster; +export const projectLocations=(items:readonly Location[]):readonly DisplayFeature[]=>items.flatMap((item)=>item.lat===null||item.lon===null?[]:[{id:item.id,label:item.evidence?.publicationProfile==='community'&&item.evidence.factualReviewStatus==='unreviewed'?`${item.name} · Unreviewed community claim — not verified by Until Every Cage`:item.name,lat:item.lat,lon:item.lon,precision:item.evidence?.displayPrecision==='exact'?'exact':'city'}]); + +/** + * Cluster only the already release-filtered page received by the client. The + * result is intentionally not a count of the dataset: it is a rendering aid + * for the current response page, and the UI says so beside the map. + */ +export const clusterFeatures=(features:readonly DisplayFeature[], cellSize=0.5):readonly MapDisplayItem[]=>{ + const cells=new Map(); + for(const feature of features){const key=`${Math.floor(feature.lat/cellSize)}:${Math.floor(feature.lon/cellSize)}`;const cell=cells.get(key)??[];cell.push(feature);cells.set(key,cell);} + const display:MapDisplayItem[]=[]; + for(const [key,members] of cells){ + if(members.length===1){display.push(members[0]!);continue;} + const lat=members.reduce((sum,item)=>sum+item.lat,0)/members.length; + const lon=members.reduce((sum,item)=>sum+item.lon,0)/members.length; + display.push({id:`cluster:${key}`,label:`${members.length} facility records in this map cluster`,lat,lon,count:members.length,memberIds:members.map(item=>item.id)}); + } + return display; +}; diff --git a/frontend/src/styles/research.css b/frontend/src/styles/research.css index 75c0578..7cbdc7f 100644 --- a/frontend/src/styles/research.css +++ b/frontend/src/styles/research.css @@ -2,3 +2,4 @@ .page-context{margin:0 0 16px;color:#62594e;line-height:1.5;font-size:.85rem}.claim-warning{color:#8d351c!important;font-weight:700}.evidence{display:grid;grid-template-columns:repeat(2,minmax(0,1fr));gap:12px;border-top:1px solid #cfc4b2;margin:28px 0 0;padding-top:18px;font-size:.82rem}.evidence div{min-width:0}.evidence dt{color:#62594e;font-size:.7rem;text-transform:uppercase;letter-spacing:.08em}.evidence dd{margin:4px 0 0;overflow-wrap:anywhere}.evidence a{color:#a34927}.state.error button{display:inline-flex;margin:12px 12px 0 0}.state.error h2:focus{outline:3px solid #a34927;outline-offset:4px}@media(max-width:680px){.evidence{grid-template-columns:1fr}} .dev-preview .phase-controls, .dev-preview .export-preview { display: none !important; } +.page-error{margin:0 0 16px;padding:12px 14px;background:#f6e3d8;border-left:4px solid #a34927;color:#71351f;font-size:.85rem;line-height:1.45}.page-error button{display:inline-flex;width:auto;margin:8px 0 0;background:#a34927;color:#fff;padding:8px 11px;border-radius:3px;font-weight:700} diff --git a/frontend/src/ui/ExportControl.svelte b/frontend/src/ui/ExportControl.svelte index 1dba4c4..19565fe 100644 --- a/frontend/src/ui/ExportControl.svelte +++ b/frontend/src/ui/ExportControl.svelte @@ -8,6 +8,6 @@
- {#if !enabled}

CSV export is available only when the selected V2 profile has an eligible promoted release.

{/if} + {#if enabled}

Bounded profile export: up to 1,000 eligible public rows with release and source context. It is not a complete dataset.

{:else}

CSV export is available only when the selected V2 profile has an eligible promoted release.

{/if} {#if error}{/if}
diff --git a/frontend/tests/e2e/local-safety.spec.ts b/frontend/tests/e2e/local-safety.spec.ts index 90ff39e..1808dac 100644 --- a/frontend/tests/e2e/local-safety.spec.ts +++ b/frontend/tests/e2e/local-safety.spec.ts @@ -10,8 +10,8 @@ const row = (id = firstId, name = 'First local record', profile: 'official' | 's publication_warning: profile === 'community' ? 'Unreviewed community claim — not verified by Until Every Cage' : null, display_precision: 'city', latitude: 55, longitude: 10, first_observed_at: null, last_observed_at: '2026-01-01T00:00:00Z', observation_count: 1, lifecycle_status: 'active_observed', - provenance_source_id: 'source-1', provenance_source_name: 'Synthetic local source', - provenance_source_url: 'https://example.test/source', provenance_retrieved_at: '2026-01-01T00:00:00Z', + provenance_source_id: 'source-1', provenance_source: null, provenance_source_name: 'Synthetic local source', + provenance_source_url: 'https://example.test/source', provenance_retrieved_at: '2026-01-01T00:00:00Z', source_rights_status: 'cleared', release_id: 'rel-1', release_ruleset_version: 'rules-1', }); const list = (profile: 'official' | 'secondary' | 'community', data = [row(firstId, 'First local record', profile)], nextCursor: string | null = null) => ({ diff --git a/frontend/tests/e2e/private-preview.spec.ts b/frontend/tests/e2e/private-preview.spec.ts index 7ffdeee..78cc9ea 100644 --- a/frontend/tests/e2e/private-preview.spec.ts +++ b/frontend/tests/e2e/private-preview.spec.ts @@ -38,7 +38,7 @@ test('test-release mode uses the existing list/detail flow with a private releas await page.getByRole('button', { name: 'Load test release' }).click(); await expect(page.getByRole('heading', { name: 'Pending test-release row' })).toBeVisible(); await expect(page.getByText('Disposable test release — not project-approved or published')).toBeVisible(); - await expect(page.getByText('No publishable map location')).toBeVisible(); + await expect(page.getByText('No publishable map location', { exact: true })).toBeVisible(); await expect(page.getByRole('button', { name: 'Download test-only CSV' })).toBeVisible(); const csvRequest = page.waitForRequest(request => request.url().includes('/api/dev/preview/test-release/locations.csv')); await page.getByRole('button', { name: 'Download test-only CSV' }).click(); diff --git a/frontend/tests/unit/devPreviewContract.test.ts b/frontend/tests/unit/devPreviewContract.test.ts index c01ab8b..c2e96a1 100644 --- a/frontend/tests/unit/devPreviewContract.test.ts +++ b/frontend/tests/unit/devPreviewContract.test.ts @@ -43,7 +43,7 @@ describe('dev preview boundary', () => { expect(nextLocalReviewState('follow_up', 'inspect')).toBe('follow_up'); }); it('maps test-release rows without requiring approval or coordinates and never falls back', async () => { - const row = { facility_id: '550e8400-e29b-41d4-a716-446655440000', canonical_name: 'Pending test row', city: null, country_code: 'GB', category: 'dairy', source_type: 'official', publication_profile: 'official', factual_review_status: 'unreviewed', privacy_screening_status: 'passed', project_approval: 'pending', reviewer_role: null, publication_warning: null, display_precision: 'unmapped', latitude: null, longitude: null, first_observed_at: null, last_observed_at: null, observation_count: null, lifecycle_status: 'status_unknown', provenance_source_id: 's1', provenance_source_name: 'Test source', provenance_source_url: 'https://example.test/source', provenance_retrieved_at: '2026-01-01T00:00:00Z', release_id: 'test-release', release_ruleset_version: 'rules-1' }; + const row = { facility_id: '550e8400-e29b-41d4-a716-446655440000', canonical_name: 'Pending test row', city: null, country_code: 'GB', category: 'dairy', source_type: 'official', publication_profile: 'official', factual_review_status: 'unreviewed', privacy_screening_status: 'passed', project_approval: 'pending', reviewer_role: null, publication_warning: null, display_precision: 'unmapped', latitude: null, longitude: null, first_observed_at: null, last_observed_at: null, observation_count: null, lifecycle_status: 'status_unknown', provenance_source_id: 's1', provenance_source: null, provenance_source_name: 'Test source', provenance_source_url: 'https://example.test/source', provenance_retrieved_at: '2026-01-01T00:00:00Z', source_rights_status: 'unknown', release_id: 'test-release', release_ruleset_version: 'rules-1' }; const body = { data: [row], meta: { api_version: 'dev-test-v1', environment: 'test-only', test_only: true, private_preview: true, release_status: 'candidate', release_id: 'test-release', profile: 'official', coverage_scope: 'test_release_public_shaped_rows', count_semantics: 'Rows only', preview_label: TEST_RELEASE_LABEL, result_count: 1, next_cursor: null } }; const candidateVariant = { ...row, canonical_name: null, privacy_screening_status: 'pending', project_approval: 'not-approved', release_ruleset_version: null }; expect(testReleaseLocationSchema.safeParse(candidateVariant).success).toBe(true); diff --git a/frontend/tests/unit/localDetailRepository.test.ts b/frontend/tests/unit/localDetailRepository.test.ts index 0ea4f29..ef102f2 100644 --- a/frontend/tests/unit/localDetailRepository.test.ts +++ b/frontend/tests/unit/localDetailRepository.test.ts @@ -1,5 +1,5 @@ import{describe,expect,it,vi}from'vitest';import{LocalLocationRepository}from'../../src/api/LocalLocationRepository'; -const row={facility_id:'550e8400-e29b-41d4-a716-446655440000',canonical_name:'Detail Local Fixture',city:'North Coast',country_code:'DK',category:'dairy',source_type:'official',publication_profile:'official',factual_review_status:'reviewed',privacy_screening_status:'passed',project_approval:'approved',reviewer_role:null,publication_warning:null,display_precision:'city',latitude:55,longitude:10,first_observed_at:null,last_observed_at:'2026-01-01T00:00:00Z',observation_count:1,lifecycle_status:'active_observed',provenance_source_id:'source-1',provenance_source_name:'Synthetic local source',provenance_source_url:'https://example.test/source',provenance_retrieved_at:'2026-01-01T00:00:00Z',release_id:'rel-1',release_ruleset_version:'rules-1'}; +const row={facility_id:'550e8400-e29b-41d4-a716-446655440000',canonical_name:'Detail Local Fixture',city:'North Coast',country_code:'DK',category:'dairy',source_type:'official',publication_profile:'official',factual_review_status:'reviewed',privacy_screening_status:'passed',project_approval:'approved',reviewer_role:null,publication_warning:null,display_precision:'city',latitude:55,longitude:10,first_observed_at:null,last_observed_at:'2026-01-01T00:00:00Z',observation_count:1,lifecycle_status:'active_observed',provenance_source_id:'source-1',provenance_source:null,provenance_source_name:'Synthetic local source',provenance_source_url:'https://example.test/source',provenance_retrieved_at:'2026-01-01T00:00:00Z',source_rights_status:'cleared',release_id:'rel-1',release_ruleset_version:'rules-1'}; const body=(data=row,meta={release_id:'rel-1',ruleset_version:'rules-1',release_created_at:'2026-01-01T00:00:00Z',profile:'official'})=>({data,api_version:'v2',meta}); describe('LocalLocationRepository detail',()=>{it('maps a valid detail envelope',async()=>{const fetcher=vi.fn().mockResolvedValue(new Response(JSON.stringify(body()),{status:200}));const result=await new LocalLocationRepository(fetcher).detail(row.facility_id);expect(result).toMatchObject({releaseId:'rel-1',profile:'official',location:{id:row.facility_id,name:'Detail Local Fixture'}});expect(fetcher).toHaveBeenCalledWith(`/api/v2/locations/${row.facility_id}?profile=official`,expect.any(Object));});it('rejects wrong profile and aborts',async()=>{const wrong=vi.fn().mockResolvedValue(new Response(JSON.stringify(body(row,{...body().meta,profile:'community'}))));await expect(new LocalLocationRepository(wrong).detail(row.facility_id)).rejects.toMatchObject({kind:'invalid-contract'});const controller=new AbortController();const fetcher=vi.fn().mockRejectedValue(new DOMException('aborted','AbortError'));await expect(new LocalLocationRepository(fetcher).detail(row.facility_id,'official',controller.signal)).rejects.toMatchObject({kind:'aborted'});});}); diff --git a/frontend/tests/unit/localLocationRepository.test.ts b/frontend/tests/unit/localLocationRepository.test.ts index 9f99c16..bcfc3c2 100644 --- a/frontend/tests/unit/localLocationRepository.test.ts +++ b/frontend/tests/unit/localLocationRepository.test.ts @@ -1,5 +1,5 @@ import{describe,expect,it,vi}from'vitest';import{LocalLocationRepository}from'../../src/api/LocalLocationRepository'; -const row={facility_id:'550e8400-e29b-41d4-a716-446655440000',canonical_name:'Local V2 Fixture',city:'North Coast',country_code:'DK',category:'dairy',source_type:'official',publication_profile:'official',factual_review_status:'reviewed',privacy_screening_status:'passed',project_approval:'approved',reviewer_role:null,publication_warning:null,display_precision:'city',latitude:55,longitude:10,first_observed_at:null,last_observed_at:'2026-01-01T00:00:00Z',observation_count:1,lifecycle_status:'active_observed',provenance_source_id:'source-1',provenance_source_name:'Synthetic local source',provenance_source_url:'https://example.test/source',provenance_retrieved_at:'2026-01-01T00:00:00Z',release_id:'rel-1',release_ruleset_version:'rules-1'}; +const row={facility_id:'550e8400-e29b-41d4-a716-446655440000',canonical_name:'Local V2 Fixture',city:'North Coast',country_code:'DK',category:'dairy',source_type:'official',publication_profile:'official',factual_review_status:'reviewed',privacy_screening_status:'passed',project_approval:'approved',reviewer_role:null,publication_warning:null,display_precision:'city',latitude:55,longitude:10,first_observed_at:null,last_observed_at:'2026-01-01T00:00:00Z',observation_count:1,lifecycle_status:'active_observed',provenance_source_id:'source-1',provenance_source:null,provenance_source_name:'Synthetic local source',provenance_source_url:'https://example.test/source',provenance_retrieved_at:'2026-01-01T00:00:00Z',source_rights_status:'cleared',release_id:'rel-1',release_ruleset_version:'rules-1'}; const response=(body:unknown,status=200)=>new Response(JSON.stringify(body),{status,headers:{'content-type':'application/json'}});const envelope=(data=[row],meta={release_id:'rel-1',ruleset_version:'rules-1',profile:'official',next_cursor:null,coverage_note:'Local promoted release.'})=>({data,api_version:'v2',meta}); describe('LocalLocationRepository',()=>{it('maps a valid Rust-shaped envelope',async()=>{const result=await new LocalLocationRepository(vi.fn().mockResolvedValue(response(envelope()))).list();expect(result.locations[0]).toMatchObject({id:row.facility_id,name:'Local V2 Fixture',lat:55});});it('fails closed when no release is promoted',async()=>{await expect(new LocalLocationRepository(vi.fn().mockResolvedValue(response(envelope([],{release_id:null,profile:'official',coverage_note:'No promoted release.'})))).list()).rejects.toMatchObject({kind:'no-release'});});it('classifies server failures as unavailable',async()=>{await expect(new LocalLocationRepository(vi.fn().mockResolvedValue(response({},503))).list()).rejects.toMatchObject({kind:'unavailable',status:503});});it('rejects malformed or restricted payloads',async()=>{await expect(new LocalLocationRepository(vi.fn().mockResolvedValue(response({...envelope(),api_version:'v1'}))).list()).rejects.toMatchObject({kind:'invalid-contract'});await expect(new LocalLocationRepository(vi.fn().mockResolvedValue(response(envelope([{...row,privacy_screening_status:'failed'}])))).list()).rejects.toMatchObject({kind:'invalid-contract'});});}); describe('LocalLocationRepository query contract', () => { @@ -52,4 +52,8 @@ describe('community list safety', () => { it('rejects a non-web source URL before it can become a detail link', async () => { await expect(new LocalLocationRepository(vi.fn().mockResolvedValue(response(envelope([{ ...row, provenance_source_url: 'javascript:alert(1)' }])))).list()).rejects.toMatchObject({ kind: 'invalid-contract' }); }); + it('retains source rights context and rejects unknown rights states', async () => { + await expect(new LocalLocationRepository(vi.fn().mockResolvedValue(response(envelope()))).list()).resolves.toMatchObject({ locations: [{ evidence: { sourceRightsStatus: 'cleared', provenanceSource: null } }] }); + await expect(new LocalLocationRepository(vi.fn().mockResolvedValue(response(envelope([{ ...row, source_rights_status: 'restricted' }])))).list()).rejects.toMatchObject({ kind: 'invalid-contract' }); + }); }); diff --git a/frontend/tests/unit/mapProjection.test.ts b/frontend/tests/unit/mapProjection.test.ts index 2be4117..700b37e 100644 --- a/frontend/tests/unit/mapProjection.test.ts +++ b/frontend/tests/unit/mapProjection.test.ts @@ -1 +1,14 @@ -import{describe,expect,it}from'vitest';import{locations}from'../../src/fixtures/locations';import{projectLocations}from'../../src/map/mapProjection';describe('map projection',()=>it('excludes unmapped fixtures',()=>expect(projectLocations(locations)).toHaveLength(2))); +import{describe,expect,it}from'vitest';import{locations}from'../../src/fixtures/locations';import{clusterFeatures,projectLocations}from'../../src/map/mapProjection'; +describe('map projection',()=>{ + it('excludes unmapped fixtures',()=>expect(projectLocations(locations)).toHaveLength(2)); + it('clusters only the current release-filtered page and preserves member IDs',()=>{ + const features=projectLocations([{...locations[0],id:'one',lat:55,lon:10},{...locations[1],id:'two',lat:55.1,lon:10.1}]); + const display=clusterFeatures(features); + expect(display).toHaveLength(1); + expect(display[0]).toMatchObject({count:2,memberIds:['one','two']}); + }); + it('keeps distant records as individual display points',()=>{ + const features=projectLocations([{...locations[0],id:'one',lat:55,lon:10},{...locations[1],id:'two',lat:56,lon:12}]); + expect(clusterFeatures(features)).toHaveLength(2); + }); +}); From 570892a0e70a4347a3ec50c37602dba0b11e861e Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Thu, 17 Sep 2026 09:15:13 -0700 Subject: [PATCH 221/311] audit public profile and suppression boundaries --- .../src/api/DevCandidatePreviewRepository.ts | 2 +- frontend/src/api/LocalCsvExportRepository.ts | 7 +-- .../tests/unit/devPreviewContract.test.ts | 2 + .../unit/localCsvExportRepository.test.ts | 3 ++ pipeline/common/data_product.py | 27 +++++++++-- pipeline/common/test_data_product.py | 37 +++++++++++++++ src/lib.rs | 42 +++++++++++++++++ static/ethics.html | 2 +- static/modules/__tests__/ethicsPage.test.js | 1 + static/modules/__tests__/staticSafety.test.js | 7 +++ static/sw.js | 46 ++----------------- 11 files changed, 125 insertions(+), 51 deletions(-) diff --git a/frontend/src/api/DevCandidatePreviewRepository.ts b/frontend/src/api/DevCandidatePreviewRepository.ts index dbcb3b6..a8b681b 100644 --- a/frontend/src/api/DevCandidatePreviewRepository.ts +++ b/frontend/src/api/DevCandidatePreviewRepository.ts @@ -7,7 +7,7 @@ const rowSchema = z.object({ candidate_id: z.string().min(1), source_record_id: z.string().min(1), facility_id: z.string().min(1), canonical_name: z.string().min(1), country_code: z.string().min(1), city: z.string().nullable(), category: z.string().min(1), display_precision: z.enum(['exact', 'city', 'unmapped']), latitude: z.number().finite().nullable(), longitude: z.number().finite().nullable(), source_type: z.enum(['official', 'secondary', 'user_submitted']), - provenance_source_id: z.string().min(1), provenance_source_name: z.string().min(1), provenance_source_url: z.string().url(), provenance_retrieved_at: z.string().min(1), + provenance_source_id: z.string().min(1), provenance_source_name: z.string().min(1), provenance_source_url: z.string().url().refine(value => { try { return ['http:', 'https:'].includes(new URL(value).protocol); } catch { return false; } }, 'source URL must use HTTP or HTTPS'), provenance_retrieved_at: z.string().min(1), factual_review_status: z.enum(['unreviewed', 'reviewed', 'rejected']), privacy_screening_status: z.literal('passed'), project_approval: z.literal(false), release_id: z.string().nullable(), release_status: z.literal('candidate'), preview_label: z.string().min(1), }).superRefine((row, ctx) => { if ((row.latitude === null) !== (row.longitude === null)) ctx.addIssue({ code: z.ZodIssueCode.custom, message: 'coordinate pair must be complete' }); }); diff --git a/frontend/src/api/LocalCsvExportRepository.ts b/frontend/src/api/LocalCsvExportRepository.ts index 3d26be1..647cbf8 100644 --- a/frontend/src/api/LocalCsvExportRepository.ts +++ b/frontend/src/api/LocalCsvExportRepository.ts @@ -1,4 +1,4 @@ -import type { FetchLike, LocalProfile } from './LocalLocationRepository'; +import { localOrigin, type FetchLike, type LocalProfile } from './LocalLocationRepository'; import type { ApiError } from './errors'; export type CsvExport = Readonly<{ @@ -9,11 +9,12 @@ export type CsvExport = Readonly<{ }>; export class LocalCsvExportRepository { - constructor(private readonly fetcher: FetchLike = globalThis.fetch, private readonly baseUrl = '') {} + readonly #base: string | undefined; + constructor(private readonly fetcher: FetchLike = globalThis.fetch, baseUrl = '') { this.#base = baseUrl ? localOrigin(baseUrl) : undefined; } async download(profile: LocalProfile = 'official', signal?: AbortSignal): Promise { const init: RequestInit = { cache: 'no-store' }; if (signal) init.signal = signal; - const response = await this.fetcher.call(globalThis, `${this.baseUrl}/api/v2/locations.csv?profile=${profile}`, init); + const response = await this.fetcher.call(globalThis, `${this.#base ?? ''}/api/v2/locations.csv?profile=${profile}`, init); if (!response.ok) { let message = `Local V2 export request failed with status ${response.status}.`; let code: string | undefined; diff --git a/frontend/tests/unit/devPreviewContract.test.ts b/frontend/tests/unit/devPreviewContract.test.ts index c2e96a1..6a3828b 100644 --- a/frontend/tests/unit/devPreviewContract.test.ts +++ b/frontend/tests/unit/devPreviewContract.test.ts @@ -36,6 +36,8 @@ describe('dev preview boundary', () => { const fetcher = async () => new Response(JSON.stringify({ api_version: 'dev-preview-v1', data: [{ candidate_id: 'candidate-1', source_record_id: 'source-row-1', facility_id: 'facility-1', canonical_name: 'Candidate facility', country_code: 'DK', city: null, category: 'dairy', display_precision: 'unmapped', latitude: null, longitude: null, source_type: 'official', provenance_source_id: 'source-1', provenance_source_name: 'Private source', provenance_source_url: 'https://example.test/source', provenance_retrieved_at: '2026-01-01T00:00:00Z', factual_review_status: 'unreviewed', privacy_screening_status: 'passed', project_approval: false, release_id: 'candidate-release', release_status: 'candidate', preview_label: DEV_PREVIEW_LABEL }], meta: { test_only: true, private_preview: true, profile: null, coverage_scope: 'candidate_release_only', next_cursor: null } })); const result = await new DevCandidatePreviewRepository(fetcher).list('operator-token'); expect(result[0]?.evidence).toMatchObject({ projectApproval: false, publicationProfile: null, publicationWarning: DEV_PREVIEW_LABEL }); + const unsafeBody = JSON.parse(JSON.stringify({ api_version: 'dev-preview-v1', data: [{ candidate_id: 'candidate-1', source_record_id: 'source-row-1', facility_id: 'facility-1', canonical_name: 'Candidate facility', country_code: 'DK', city: null, category: 'dairy', display_precision: 'unmapped', latitude: null, longitude: null, source_type: 'official', provenance_source_id: 'source-1', provenance_source_name: 'Private source', provenance_source_url: 'javascript:alert(1)', provenance_retrieved_at: '2026-01-01T00:00:00Z', factual_review_status: 'unreviewed', privacy_screening_status: 'passed', project_approval: false, release_id: 'candidate-release', release_status: 'candidate', preview_label: DEV_PREVIEW_LABEL }], meta: { test_only: true, private_preview: true, profile: null, coverage_scope: 'candidate_release_only', next_cursor: null } })); + await expect(new DevCandidatePreviewRepository(vi.fn().mockResolvedValue(new Response(JSON.stringify(unsafeBody)))).list('operator-token')).rejects.toThrow('rejected safely'); }); it('keeps local review notes non-approval and non-persistent in the model', () => { expect(nextLocalReviewState('unreviewed', 'inspect')).toBe('inspected'); diff --git a/frontend/tests/unit/localCsvExportRepository.test.ts b/frontend/tests/unit/localCsvExportRepository.test.ts index 5287b69..b88117c 100644 --- a/frontend/tests/unit/localCsvExportRepository.test.ts +++ b/frontend/tests/unit/localCsvExportRepository.test.ts @@ -20,4 +20,7 @@ describe('LocalCsvExportRepository', () => { expect(String(fetcher.mock.calls[0]?.[0])).toContain('profile=secondary'); await expect(new LocalCsvExportRepository(vi.fn().mockResolvedValue(new Response('{}', { status: 200, headers: { 'x-uec-release-id': 'release-1', 'content-type': 'application/json' } }))).download('official')).rejects.toMatchObject({ kind: 'invalid-contract' }); }); + it('rejects a non-loopback API origin before sending an export request', () => { + expect(() => new LocalCsvExportRepository(vi.fn(), 'https://external.example')).toThrow('Local API origin must be loopback HTTP.'); + }); }); diff --git a/pipeline/common/data_product.py b/pipeline/common/data_product.py index d3dfcf1..0a59d26 100644 --- a/pipeline/common/data_product.py +++ b/pipeline/common/data_product.py @@ -23,8 +23,11 @@ SCHEMA_VERSION = "uec-location-projection-v1" MANIFEST_VERSION = "uec-release-manifest-v2" SUPPORTED_PROFILES = frozenset(("official", "secondary", "community")) +ALLOWED_SOURCE_TYPES = frozenset(("official", "secondary", "user_submitted")) +ALLOWED_FACTUAL_REVIEW_STATUSES = frozenset(("unreviewed", "reviewed")) ALLOWED_RIGHTS = frozenset(("cleared", "attribution_required")) FORMULA_PREFIXES = ("=", "+", "-", "@") +UNREVIEWED_COMMUNITY_WARNING = "Unreviewed community claim — not verified by Until Every Cage" CSV_FIELDS = ( "facility_id", @@ -206,17 +209,31 @@ def validate_public_rows(rows: Iterable[Mapping[str, Any]], metadata: Mapping[st raise DataProductError(f"row {index} does not match the selected release/profile") if row.get("publication_eligible") is not True or row.get("privacy_screening_status") != "passed": raise DataProductError(f"row {index} is not publication-eligible and privacy-screened") - if selected_profile != "community" and row.get("project_approval") != "approved": + if row.get("source_type") not in ALLOWED_SOURCE_TYPES: + raise DataProductError(f"row {index} has an unsupported source type") + factual_review_status = row.get("factual_review_status") + if factual_review_status not in ALLOWED_FACTUAL_REVIEW_STATUSES: + raise DataProductError(f"row {index} has an unsupported factual review status") + project_approval = row.get("project_approval") + community_unreviewed = ( + selected_profile == "community" + and row.get("source_type") == "user_submitted" + and factual_review_status == "unreviewed" + and project_approval == "pending" + ) + if project_approval != "approved" and not community_unreviewed: raise DataProductError(f"row {index} is not project-approved") - if row.get("source_rights_status") not in ALLOWED_RIGHTS: - raise DataProductError(f"row {index} has unclear or restricted source reuse rights") if row.get("source_type") == "user_submitted" and selected_profile != "community": raise DataProductError(f"row {index} user-submitted claim is outside the community profile") + if row.get("source_rights_status") not in ALLOWED_RIGHTS: + raise DataProductError(f"row {index} has unclear or restricted source reuse rights") + if community_unreviewed and row.get("publication_warning") not in (None, UNREVIEWED_COMMUNITY_WARNING): + raise DataProductError(f"row {index} is missing the required community warning") projection = {field: row.get(field) for field in CSV_FIELDS} projection["release_ruleset_version"] = row.get("release_ruleset_version", release["ruleset_version"]) projection["publication_warning"] = row.get("publication_warning") or ( - "Unreviewed community claim — not verified by Until Every Cage" - if selected_profile == "community" and row.get("factual_review_status") == "unreviewed" + UNREVIEWED_COMMUNITY_WARNING + if community_unreviewed else None ) validated.append(projection) diff --git a/pipeline/common/test_data_product.py b/pipeline/common/test_data_product.py index 24ec9d0..dc324b8 100644 --- a/pipeline/common/test_data_product.py +++ b/pipeline/common/test_data_product.py @@ -122,6 +122,43 @@ def test_rows_cannot_bypass_suppression_or_rights(self): with self.assertRaisesRegex(DataProductError, message): write_package(Path(tempfile.mkdtemp()), metadata(), [row(**changes)]) + def test_community_profile_requires_approval_or_screened_unreviewed_submission(self): + community_metadata = metadata("community") + with self.assertRaisesRegex(DataProductError, "project-approved"): + write_package( + Path(tempfile.mkdtemp()), + community_metadata, + [row("community", project_approval="pending")], + ) + unreviewed = row( + "community", + source_type="user_submitted", + factual_review_status="unreviewed", + project_approval="pending", + ) + with tempfile.TemporaryDirectory() as directory: + write_package(Path(directory), community_metadata, [unreviewed]) + self.assertIn("Unreviewed community claim", (Path(directory) / "locations.csv").read_text()) + + def test_rejected_or_mislabelled_public_rows_fail_closed(self): + for changes, message in [ + ({"factual_review_status": "rejected"}, "factual review status"), + ({"source_type": "unknown"}, "source type"), + ( + { + "source_type": "user_submitted", + "factual_review_status": "unreviewed", + "project_approval": "pending", + "publication_profile": "community", + "publication_warning": "Claim is verified", + }, + "community warning", + ), + ]: + candidate = row("community", **changes) + with self.assertRaisesRegex(DataProductError, message): + write_package(Path(tempfile.mkdtemp()), metadata("community"), [candidate]) + def test_malformed_metadata_and_tampering_fail_verification(self): for field in ("release_id", "generated_at", "source_coverage", "row_counts", "checksums"): invalid = metadata() diff --git a/src/lib.rs b/src/lib.rs index ae2e9c2..f8e50ac 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1140,6 +1140,17 @@ pub async fn get_v2_locations_handler( State(state): State, Query(params): Query, ) -> impl IntoResponse { + if params + .profile + .as_deref() + .is_some_and(|v| !V2_PROFILES.contains(&v)) + { + return v2_error( + StatusCode::BAD_REQUEST, + "invalid_profile", + "profile is unsupported", + ); + } if params .display_precision .as_deref() @@ -1750,6 +1761,37 @@ mod v2_api_tests { } } + #[tokio::test] + async fn list_rejects_an_unsupported_profile_before_database_access() { + let state = ApiState { + database: None, + dev_preview_token: None, + dev_test_release_id: None, + dev_test_release_token: None, + }; + let response = Router::new() + .route( + "/api/v2/locations", + axum::routing::get(get_v2_locations_handler), + ) + .with_state(state) + .oneshot( + Request::builder() + .uri("/api/v2/locations?profile=untrusted") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + let body = axum::body::to_bytes(response.into_body(), usize::MAX) + .await + .unwrap(); + let json: serde_json::Value = serde_json::from_slice(&body).unwrap(); + assert_eq!(json["api_version"], "v2"); + assert_eq!(json["error"]["code"], "invalid_profile"); + } + #[tokio::test] async fn v2_response_is_json_when_database_is_configured() { let url = std::env::var("UEC_DATABASE_URL").unwrap_or_else(|_| { diff --git a/static/ethics.html b/static/ethics.html index 6bee130..b4acf1c 100644 --- a/static/ethics.html +++ b/static/ethics.html @@ -27,7 +27,7 @@

Locations and uncertainty

Data boundaries

Candidate, staged, quarantined, and raw evidence are not public data. Public responses are intended to use eligible records from a promoted release. Internal artifacts and source evidence may have different access and retention rules.

-

Corrections and privacy/location requests

+

Corrections and privacy/location requests

To report an incorrect location, residential/private exposure, harmful identifying detail, or other data concern, email untileverycageproject@protonmail.com. For a location-exposure concern, use the subject “Privacy/location removal” and do not post sensitive evidence publicly.

Requests are assessed under the project policy, including prompt suppression of credible exposure concerns, correction or removal review, and propagation through project-controlled public outputs where applicable. The project cannot guarantee recall of independent copies.

diff --git a/static/modules/__tests__/ethicsPage.test.js b/static/modules/__tests__/ethicsPage.test.js index c41d6b7..adf888a 100644 --- a/static/modules/__tests__/ethicsPage.test.js +++ b/static/modules/__tests__/ethicsPage.test.js @@ -5,6 +5,7 @@ test('public ethics page contains safety disclosures and reporting link', () => const html = readFileSync(resolve(process.cwd(), 'static/ethics.html'), 'utf8'); expect(html).toContain('Candidate, staged, quarantined, and raw evidence are not public data.'); expect(html).toContain('Privacy%2Flocation%20removal'); + expect(html).toContain('id="reporting"'); expect(html).toContain('Data ethics, provenance, and limitations'); expect(html).toContain('does not claim legal immunity'); }); diff --git a/static/modules/__tests__/staticSafety.test.js b/static/modules/__tests__/staticSafety.test.js index 5a350b4..47f873e 100644 --- a/static/modules/__tests__/staticSafety.test.js +++ b/static/modules/__tests__/staticSafety.test.js @@ -12,3 +12,10 @@ test('browser-facing external links opened in new tabs protect the opener', () = const html = readFileSync(resolve(process.cwd(), 'static/howtouse.html'), 'utf8'); expect(html).not.toMatch(/target="_blank"\s+rel="noopener"(?:\s|>)/); }); + +test('service worker never caches API responses that could outlive suppression', () => { + const worker = readFileSync(resolve(process.cwd(), 'static/sw.js'), 'utf8'); + expect(worker).toContain('must not be cached client-side'); + expect(worker).not.toContain('cache.put('); + expect(worker).not.toContain('caches.match('); +}); diff --git a/static/sw.js b/static/sw.js index 911530d..71814a5 100644 --- a/static/sw.js +++ b/static/sw.js @@ -1,12 +1,6 @@ -const CACHE_NAME = 'api-cache-v5'; -const API_URLS = [ - 'https://untileverycage-production.up.railway.app/api/locations', - 'https://untileverycage-production.up.railway.app/api/aphis-reports', - 'https://untileverycage-production.up.railway.app/api/inspection-reports', - 'http://localhost:8000/api/locations', - 'http://localhost:8000/api/aphis-reports', - 'http://localhost:8000/api/inspection-reports', -]; +// Public API responses must not be cached client-side. A cached response can +// outlive a privacy suppression and become an alternate disclosure path. +const CACHE_NAME = 'api-cache-v6'; // @ts-ignore self.addEventListener('install', (event) => { @@ -33,36 +27,6 @@ self.addEventListener('activate', (event) => { }); self.addEventListener('fetch', (event) => { - // @ts-ignore - const { request } = event; - const url = new URL(request.url); - - // @ts-ignore - const isApiRequest = API_URLS.some((apiUrl) => url.href.startsWith(apiUrl)); - - if (!isApiRequest) { - return; - } - - // @ts-ignore - event.respondWith( - caches.match(request).then((cachedResponse) => { - if (cachedResponse) { - return cachedResponse; - } - - return fetch(request).then((response) => { - if (!response || response.status !== 200 || response.type === 'error') { - return response; - } - - const responseToCache = response.clone(); - caches.open(CACHE_NAME).then((cache) => { - cache.put(request, responseToCache); - }); - - return response; - }); - }) - ); + // Deliberately do not intercept API requests. The legacy application and + // V2 routes must always reach the current server-side suppression gates. }); From 8ddc6fcc76bbd312894e4c698616b5c6aa46844c Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Thu, 17 Sep 2026 10:55:51 -0700 Subject: [PATCH 222/311] fix integration diagnostics and local E2E contract --- frontend/tests/e2e/local-backend.spec.ts | 2 +- pipeline/tests/e2e/fixture.py | 2 +- src/main.rs | 9 +++++---- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/frontend/tests/e2e/local-backend.spec.ts b/frontend/tests/e2e/local-backend.spec.ts index 33ac88b..3cd718b 100644 --- a/frontend/tests/e2e/local-backend.spec.ts +++ b/frontend/tests/e2e/local-backend.spec.ts @@ -58,7 +58,7 @@ test('exercises server discovery, cursor state, map, export, and mobile basics', await expect(page.getByText('LOCAL V2 API')).toBeVisible(); const searchTerm = (record.canonical_name ?? record.country_code ?? '').slice(0, 6); await page.getByLabel('Search locations').fill(searchTerm); - await expect(page).toHaveURL(new RegExp(`q=${encodeURIComponent(searchTerm)}`)); + await expect.poll(() => new URL(page.url()).searchParams.get('q')).toBe(searchTerm); await expect(page.getByRole('status').filter({ hasText: 'server' })).toBeVisible(); await page.getByRole('button', { name: 'Show map' }).click(); await expect(page.getByLabel(/Location map showing facility records/)).toBeVisible(); diff --git a/pipeline/tests/e2e/fixture.py b/pipeline/tests/e2e/fixture.py index 359f72f..b536945 100644 --- a/pipeline/tests/e2e/fixture.py +++ b/pipeline/tests/e2e/fixture.py @@ -226,7 +226,7 @@ def seed_official_scenario(self): restricted_record = None with psycopg.connect(self.database_url) as db: with db.transaction(): - db.execute("INSERT INTO uec.sources (source_id,country_code,name,official_url,access_method) VALUES ('e2e.official','DK','Synthetic official source','https://example.invalid/official','fixture')") + db.execute("INSERT INTO uec.sources (source_id,country_code,name,official_url,access_method,attribution) VALUES ('e2e.official','DK','Synthetic official source','https://example.invalid/official','fixture','Synthetic fixture attribution')") release = 'e2e-promoted' db.execute("INSERT INTO uec.releases (release_id,status,ruleset_version,summary) VALUES (%s,'promoted','e2e-v1','{}')", (release,)) db.execute("INSERT INTO uec.release_manifests (release_id,manifest,manifest_sha256) VALUES ('e2e-promoted','{\"eligible_record_count\":3,\"manifest_version\":\"v1\",\"profile\":\"official\",\"release_id\":\"e2e-promoted\",\"ruleset_version\":\"e2e-v1\",\"source_ids\":[\"e2e.official\"]}', 'dcf1cb50c078057cac2527936332e35892c2c13ecdcf2f545176acd17897cde7')") diff --git a/src/main.rs b/src/main.rs index 6d72ecf..7241340 100644 --- a/src/main.rs +++ b/src/main.rs @@ -433,11 +433,12 @@ async fn liveness() -> impl IntoResponse { Json(serde_json::json!({"status": "ok", "service": "uec-api"})) } -async fn diagnostics(Extension(metrics): Extension>) -> impl IntoResponse { +async fn diagnostics( + State(state): State, + Extension(metrics): Extension>, +) -> impl IntoResponse { let mode = std::env::var("UEC_RUNTIME_MODE").unwrap_or_else(|_| "development".into()); - let database_configured = std::env::var("UEC_DATABASE_URL") - .ok() - .is_some_and(|url| !url.trim().is_empty()); + let database_configured = state.database.is_some(); let proxy_trust = match std::env::var("UEC_TRUST_PROXY").as_deref() { Ok("true") => "enabled_with_configured_boundary", Ok("false") | Err(_) => "disabled", From 47dbdaa478de7da3b0c7362bb0d9940a794d6682 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Thu, 17 Sep 2026 12:28:21 -0700 Subject: [PATCH 223/311] fix: isolate and bound E2E startup retries --- docker-compose.e2e.yml | 4 + pipeline/tests/e2e/README.md | 2 +- pipeline/tests/e2e/fixture.py | 326 +++++++++++++++++++---------- pipeline/tests/test_e2e_fixture.py | 68 +++++- 4 files changed, 293 insertions(+), 107 deletions(-) diff --git a/docker-compose.e2e.yml b/docker-compose.e2e.yml index eae896a..24c947a 100644 --- a/docker-compose.e2e.yml +++ b/docker-compose.e2e.yml @@ -18,4 +18,8 @@ services: ports: - "${UEC_E2E_DB_PORT:-55432}:5432" volumes: + - postgres-data:/var/lib/postgresql/data - "./pipeline/tests/e2e/disposable-marker.sql:/docker-entrypoint-initdb.d/uec-disposable-marker.sql:ro" + +volumes: + postgres-data: diff --git a/pipeline/tests/e2e/README.md b/pipeline/tests/e2e/README.md index d390297..0cb5eed 100644 --- a/pipeline/tests/e2e/README.md +++ b/pipeline/tests/e2e/README.md @@ -15,7 +15,7 @@ volume, backend process, and temporary build directory. This requires Docker Desktop, Cargo, and the pinned Python dependencies. The fixture uses isolated random ports and tears down its Compose project even after setup failures. Fast non-Docker checks remain available with `python -m unittest discover -s pipeline/tests -p "test_*.py" -v`. -`fixture.py` owns the environment lifecycle: it selects isolated ports, starts Docker Compose, applies migrations as UTF-8, builds and starts the backend from a per-run temporary Cargo target directory, waits for readiness, and tears everything down. The isolated target prevents E2E builds from contending with a developer's running backend binary. Setup failures also trigger cleanup. Run API modules through `run-suite.ps1` because each module owns a disposable PostGIS environment; running all classes in one discovery process can create avoidable Docker resource/lifecycle contention. +`fixture.py` owns the environment lifecycle: it selects isolated ports, starts Docker Compose, applies migrations as UTF-8, builds and starts the backend from a per-run temporary Cargo target directory, waits for readiness, and tears everything down. The isolated target prevents E2E builds from contending with a developer's running backend binary. A startup retry is bounded to one additional attempt and only recognizes transient Postgres lifecycle messages; it removes the failed project's volume, rotates the Compose project and ports, and emits bounded redacted diagnostics. Deterministic SQL/schema failures are never retried. Setup failures also trigger cleanup. Run API modules through `run-suite.ps1` because each module owns a disposable PostGIS environment; running all classes in one discovery process can create avoidable Docker resource/lifecycle contention. `test_public_api.py` verifies the publication boundary with an empty database: candidate data and filters remain unavailable, and malformed pagination is rejected. diff --git a/pipeline/tests/e2e/fixture.py b/pipeline/tests/e2e/fixture.py index b536945..5af9076 100644 --- a/pipeline/tests/e2e/fixture.py +++ b/pipeline/tests/e2e/fixture.py @@ -6,8 +6,10 @@ import subprocess import tempfile import shutil +import sys import time import uuid +import re from pathlib import Path from datetime import datetime, timezone import psycopg @@ -19,6 +21,42 @@ _READ_MODEL_SPEC = None _READ_MODEL_MODULE = None +MAX_START_ATTEMPTS = 2 +_TRANSIENT_DATABASE_MARKERS = ( + "database system is shutting down", + "database system is starting up", + "could not connect to server", + "connection refused", + "server closed the connection unexpectedly", +) + + +def is_retryable_database_failure(output): + """Return whether output describes a transient Postgres lifecycle failure. + + This deliberately excludes SQL/schema errors. Retrying those would hide a + deterministic migration defect and would only produce a second failure. + """ + text = output if isinstance(output, str) else str(output or "") + lowered = text.lower() + return any(marker in lowered for marker in _TRANSIENT_DATABASE_MARKERS) + + +class _RetryableStartupFailure(RuntimeError): + """A bounded retry may recreate the disposable environment for this error.""" + + +def _process_output(result): + return "\n".join(part for part in (result.stdout, result.stderr) if part) + + +def _sanitize_diagnostics(text): + """Keep Docker diagnostics useful without echoing credentials or URLs.""" + text = text or "" + text = re.sub(r"(?i)postgres(?:ql)?://[^\s]+", "postgresql://[redacted]", text) + text = re.sub(r"(?i)(password|token|secret)=([^\s]+)", r"\1=[redacted]", text) + return text[-12000:] + def _read_model_builder(): global _READ_MODEL_SPEC, _READ_MODEL_MODULE if _READ_MODEL_MODULE is None: @@ -65,120 +103,198 @@ def command(self, *args): def compose_env(self): env = os.environ.copy(); env["UEC_E2E_DB_PORT"] = str(self.db_port); return env - def start(self, migration_files=None, wait_for_ready=True): - self._ensure_build_temp() - try: - print(f"[e2e] starting {self.project}", flush=True) - startup = subprocess.run(self.command("up", "-d", "--wait"), cwd=ROOT, capture_output=True, text=True, env=self.compose_env()) - if startup.returncode: - raise RuntimeError(f"Docker Compose startup failed (exit {startup.returncode})\n{startup.stdout}\n{startup.stderr}") - files = tuple(migration_files if migration_files is not None else sorted((ROOT / "pipeline/migrations").glob("*.sql"))) - print("[e2e] applying migrations", flush=True) - stable_postmaster = None - stable_checks = 0 - for _ in range(120): - # pg_isready only confirms that Postgres accepts connections; - # during container bootstrap it may report ready before the - # POSTGRES_DB database has been created. Query the target DB - # directly so migrations never race initialization in CI. The - # image can still replace its temporary bootstrap server after - # the first successful query, so require the same postmaster - # start time across several checks before attaching migrations. - ready = subprocess.run( - self.command("exec", "-T", "postgres", "psql", "-At", "-U", "uec", "-d", "uec", "-c", "SELECT pg_postmaster_start_time()"), + def _rotate_attempt(self): + """Give a retry a new Compose identity, ports, container, and volume.""" + self.project = f"uec-e2e-{uuid.uuid4().hex[:8]}" + self.db_port = free_port() + self.api_port = free_port() + while self.api_port == self.db_port: + self.api_port = free_port() + self.database_url = f"postgresql://uec:uec-e2e@localhost:{self.db_port}/uec" + + def _container_diagnostics(self): + """Return bounded, redacted diagnostics before failed cleanup removes state.""" + parts = [] + for args in (("ps", "--all"), ("logs", "--no-color", "--tail", "120", "postgres")): + try: + result = subprocess.run( + self.command(*args), cwd=ROOT, capture_output=True, text=True, + check=False, env=self.compose_env(), ) - if ready.returncode == 0 and ready.stdout.strip(): - postmaster = ready.stdout.strip() - if postmaster == stable_postmaster: - stable_checks += 1 - else: - stable_postmaster = postmaster - stable_checks = 1 - if stable_checks >= 3: - break + output = _sanitize_diagnostics(_process_output(result)) + if output: + parts.append(f"$ docker compose {' '.join(args)}\n{output}") + except OSError as exc: + parts.append(f"$ docker compose {' '.join(args)}\n{type(exc).__name__}: unavailable") + return "\n".join(parts) or "(no container diagnostics available)" + + def _database_ready(self): + result = subprocess.run( + self.command( + "exec", "-T", "postgres", "psql", "-At", "-U", "uec", "-d", "uec", + "-c", "SELECT pg_postmaster_start_time()", + ), + cwd=ROOT, + capture_output=True, + text=True, + check=False, + env=self.compose_env(), + ) + if result.returncode == 0 and result.stdout.strip(): + return result + return result + + def _start_once(self, files, wait_for_ready): + self._ensure_build_temp() + print(f"[e2e] starting {self.project}", flush=True) + startup = subprocess.run(self.command("up", "-d", "--wait"), cwd=ROOT, capture_output=True, text=True, env=self.compose_env()) + if startup.returncode: + output = _process_output(startup) + if is_retryable_database_failure(output): + raise _RetryableStartupFailure(f"Docker Compose startup transiently failed (exit {startup.returncode})") + raise RuntimeError(f"Docker Compose startup failed (exit {startup.returncode})\n{_sanitize_diagnostics(output)}") + print("[e2e] applying migrations", flush=True) + stable_postmaster = None + stable_checks = 0 + last_ready = None + for _ in range(120): + # pg_isready only confirms that Postgres accepts connections; + # during container bootstrap it may report ready before the + # POSTGRES_DB database has been created. Query the target DB + # directly so migrations never race initialization in CI. The + # image can still replace its temporary bootstrap server after + # the first successful query, so require the same postmaster + # start time across several checks before attaching migrations. + ready = self._database_ready() + last_ready = ready + if ready.returncode == 0 and (ready.stdout or "").strip(): + postmaster = ready.stdout.strip() + if postmaster == stable_postmaster: + stable_checks += 1 else: - stable_postmaster = None - stable_checks = 0 + stable_postmaster = postmaster + stable_checks = 1 + if stable_checks >= 3: + break + else: + stable_postmaster = None + stable_checks = 0 + time.sleep(.25) + else: + if last_ready and is_retryable_database_failure(_process_output(last_ready)): + raise _RetryableStartupFailure("Postgres did not stabilize before migrations") + raise RuntimeError("PostGIS container did not become ready") + # Apply files one at a time. A transient Postgres restart is retryable; + # all SQL/schema errors remain deterministic and fail immediately. + for migration in files: + print(f"[e2e] applying {migration.name}", flush=True) + result = subprocess.run( + self.command("exec", "-T", "postgres", "psql", "-v", "ON_ERROR_STOP=1", "-U", "uec", "-d", "uec"), + input=migration.read_text(encoding="utf-8"), + cwd=ROOT, + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + check=False, + env=self.compose_env(), + ) + if result.stdout: + print(result.stdout, end="", flush=True) + if result.stderr: + print(result.stderr, end="", file=sys.stderr, flush=True) + if result.returncode: + output = _process_output(result) + if is_retryable_database_failure(output): + raise _RetryableStartupFailure( + f"migration {migration.name} hit a transient Postgres lifecycle failure" + ) + raise subprocess.CalledProcessError( + result.returncode, + result.args, + output=result.stdout, + stderr=result.stderr, + ) + # Revalidate the database after migration application so a container + # that restarted at the boundary cannot launch a backend against an + # unstable database. + final_ready = self._database_ready() + if final_ready.returncode != 0 or not (final_ready.stdout or "").strip(): + output = _process_output(final_ready) + if is_retryable_database_failure(output): + raise _RetryableStartupFailure("Postgres became unavailable after migrations") + raise RuntimeError(f"PostGIS database failed post-migration readiness\n{_sanitize_diagnostics(output)}") + print("[e2e] building backend", flush=True) + build_env = os.environ.copy() + build_env["CARGO_TARGET_DIR"] = str(self.cargo_cache_dir) + subprocess.run(["cargo", "build", "--quiet"], cwd=ROOT, check=True, timeout=180, env=build_env) + env = os.environ.copy(); env.update({"UEC_DATABASE_URL": self.database_url, "PORT": str(self.api_port), "UEC_RUNTIME_MODE": "development", "UEC_BIND_HOST": "127.0.0.1", "UEC_DEV_PREVIEW": "true", "UEC_DEV_PREVIEW_TOKEN": self.dev_preview_token}) + if self.test_release_id: + env.update({"UEC_TEST_RELEASE_ID": self.test_release_id, "UEC_TEST_RELEASE_TOKEN": self.dev_preview_token}) + cached_binary = self.cargo_cache_dir / "debug/uec-api.exe" + if not cached_binary.exists(): + cached_binary = self.cargo_cache_dir / "debug/uec-api" + binary = self.cargo_target_dir / cached_binary.name + shutil.copy2(cached_binary, binary) + self.backend_log = (self.cargo_target_dir / f"e2e-{self.project}.log").open("w", encoding="utf-8") + self.backend = subprocess.Popen([str(binary)], cwd=ROOT, env=env, stdout=self.backend_log, stderr=subprocess.STDOUT, text=True) + print(f"[e2e] waiting for backend on {self.api_port}", flush=True) + if not wait_for_ready: + return self + import urllib.error + import urllib.request + last_error = None + for _ in range(80): + try: + with urllib.request.urlopen(f"http://127.0.0.1:{self.api_port}/health/ready", timeout=1) as response: + payload = json.load(response) + if response.status == 200 and payload.get("schema") == "migrated": + return self + except (urllib.error.HTTPError, urllib.error.URLError, TimeoutError, OSError) as exc: + last_error = repr(exc) + if self.backend.poll() is not None: + break time.sleep(.25) - else: raise RuntimeError("PostGIS container did not become ready") + exit_code = self.backend.poll() if self.backend else None + log_path = self.backend_log.name if self.backend_log else None + if self.backend_log: + self.backend_log.flush() + log_path = self.backend_log.name + self.backend_log.close() + self.backend_log = None + log_text = Path(log_path).read_text(encoding="utf-8") if log_path else "" + raise RuntimeError( + f"backend did not become ready; last_error={last_error}; " + f"exit_code={exit_code}; log_path={log_path}\n{log_text}" + ) + + def start(self, migration_files=None, wait_for_ready=True): + files = tuple(migration_files if migration_files is not None else sorted((ROOT / "pipeline/migrations").glob("*.sql"))) + for attempt in range(MAX_START_ATTEMPTS): + self.start_attempts = attempt + 1 try: - # Apply files one at a time. Streaming the complete migration - # history through a single Windows/Docker exec can terminate - # the disposable Postgres process mid-stream, which leaves a - # misleading partial-schema failure and makes the local load - # harness non-rerunnable. Per-file execution is still - # disposable, ordered, and fail-fast, while keeping the - # migration boundary visible in the log. - for migration in files: - print(f"[e2e] applying {migration.name}", flush=True) - subprocess.run( - self.command("exec", "-T", "postgres", "psql", "-v", "ON_ERROR_STOP=1", "-U", "uec", "-d", "uec"), - input=migration.read_bytes(), - cwd=ROOT, - check=True, - env=self.compose_env(), - ) - except subprocess.CalledProcessError: - # Docker Desktop can restart a freshly initialized PostGIS - # container while the first large SQL stream is attached. - # Recreate the disposable environment once; never retry a - # partially applied migration set in place. - if self.start_attempts < 1: - self.start_attempts += 1 - self.stop() - time.sleep(1) - return self.start() + return self._start_once(files, wait_for_ready) + except _RetryableStartupFailure as exc: + diagnostics = self._container_diagnostics() + self.stop() + if attempt + 1 >= MAX_START_ATTEMPTS: + raise RuntimeError( + f"E2E startup exhausted {MAX_START_ATTEMPTS} isolated attempts: {exc}\n" + f"sanitized container diagnostics:\n{diagnostics}" + ) from exc + print( + f"[e2e] transient startup failure; cleaning {self.project} and rotating the next attempt", + flush=True, + ) + self._rotate_attempt() + time.sleep(1) + except Exception: + self.stop() raise - print("[e2e] building backend", flush=True) - build_env = os.environ.copy() - build_env["CARGO_TARGET_DIR"] = str(self.cargo_cache_dir) - subprocess.run(["cargo", "build", "--quiet"], cwd=ROOT, check=True, timeout=180, env=build_env) - env = os.environ.copy(); env.update({"UEC_DATABASE_URL": self.database_url, "PORT": str(self.api_port), "UEC_RUNTIME_MODE": "development", "UEC_BIND_HOST": "127.0.0.1", "UEC_DEV_PREVIEW": "true", "UEC_DEV_PREVIEW_TOKEN": self.dev_preview_token}) - if self.test_release_id: - env.update({"UEC_TEST_RELEASE_ID": self.test_release_id, "UEC_TEST_RELEASE_TOKEN": self.dev_preview_token}) - cached_binary = self.cargo_cache_dir / "debug/uec-api.exe" - if not cached_binary.exists(): - cached_binary = self.cargo_cache_dir / "debug/uec-api" - binary = self.cargo_target_dir / cached_binary.name - shutil.copy2(cached_binary, binary) - self.backend_log = (self.cargo_target_dir / f"e2e-{self.project}.log").open("w", encoding="utf-8") - self.backend = subprocess.Popen([str(binary)], cwd=ROOT, env=env, stdout=self.backend_log, stderr=subprocess.STDOUT, text=True) - print(f"[e2e] waiting for backend on {self.api_port}", flush=True) - if not wait_for_ready: - return self - import urllib.error - import urllib.request - last_error = None - for _ in range(80): - try: - with urllib.request.urlopen(f"http://127.0.0.1:{self.api_port}/health/ready", timeout=1) as response: - payload = json.load(response) - if response.status == 200 and payload.get("schema") == "migrated": - return self - except (urllib.error.HTTPError, urllib.error.URLError, TimeoutError, OSError) as exc: - last_error = repr(exc) - if self.backend.poll() is not None: - break - time.sleep(.25) - exit_code = self.backend.poll() if self.backend else None - log_path = self.backend_log.name if self.backend_log else None - if self.backend_log: - self.backend_log.flush() - log_path = self.backend_log.name - self.backend_log.close() - self.backend_log = None - log_text = Path(log_path).read_text(encoding="utf-8") if log_path else "" - raise RuntimeError( - f"backend did not become ready; last_error={last_error}; " - f"exit_code={exit_code}; log_path={log_path}\n{log_text}" - ) - except Exception: - self.stop() - raise def wait_for_listening(self, timeout=20): """Wait for the backend socket without requiring schema readiness.""" diff --git a/pipeline/tests/test_e2e_fixture.py b/pipeline/tests/test_e2e_fixture.py index f9520d0..f771f04 100644 --- a/pipeline/tests/test_e2e_fixture.py +++ b/pipeline/tests/test_e2e_fixture.py @@ -1,9 +1,75 @@ import unittest +import subprocess +from unittest.mock import patch -from pipeline.tests.e2e.fixture import E2EEnvironment +from pipeline.tests.e2e.fixture import ( + E2EEnvironment, + MAX_START_ATTEMPTS, + _RetryableStartupFailure, + is_retryable_database_failure, +) class E2EFixtureLifecycleTests(unittest.TestCase): + def test_retry_classification_only_accepts_transient_database_lifecycle_errors(self): + self.assertTrue(is_retryable_database_failure("psql: FATAL: the database system is shutting down")) + self.assertTrue(is_retryable_database_failure("connection refused")) + self.assertFalse(is_retryable_database_failure("psql: ERROR: relation uec.releases does not exist")) + self.assertFalse(is_retryable_database_failure("psql: ERROR: syntax error at or near SELECT")) + + def test_retry_rotates_project_and_ports_after_cleanup(self): + environment = E2EEnvironment() + old_identity = (environment.project, environment.db_port, environment.api_port) + try: + with patch.object(environment, "stop") as cleanup, patch.object( + environment, "_container_diagnostics", return_value="redacted diagnostics" + ): + with patch.object( + environment, + "_start_once", + side_effect=[_RetryableStartupFailure("database system is shutting down"), "started"], + ): + self.assertEqual(environment.start(migration_files=()), "started") + cleanup.assert_called_once() + self.assertNotEqual(old_identity, (environment.project, environment.db_port, environment.api_port)) + self.assertEqual(environment.start_attempts, 2) + finally: + environment.build_temp.cleanup() + environment.build_temp = None + + def test_failed_attempt_cleanup_removes_compose_volume(self): + environment = E2EEnvironment() + try: + with patch("pipeline.tests.e2e.fixture.subprocess.run") as run: + environment.stop() + commands = [call.args[0] for call in run.call_args_list] + self.assertEqual(len(commands), 1) + self.assertIn("down", commands[0]) + self.assertIn("-v", commands[0]) + self.assertIn(environment.project, commands[0]) + finally: + environment.build_temp = None + + def test_retry_is_bounded(self): + self.assertEqual(MAX_START_ATTEMPTS, 2) + + def test_deterministic_start_failure_is_not_retried(self): + environment = E2EEnvironment() + try: + failure = subprocess.CalledProcessError( + 3, ["psql"], output="ERROR: relation uec.releases does not exist", stderr="" + ) + with patch.object(environment, "stop") as cleanup, patch.object( + environment, "_start_once", side_effect=failure + ) as start_once: + with self.assertRaises(subprocess.CalledProcessError): + environment.start(migration_files=()) + start_once.assert_called_once() + cleanup.assert_called_once() + finally: + environment.build_temp.cleanup() + environment.build_temp = None + def test_retry_recreates_build_temp_after_failed_start_cleanup(self): environment = E2EEnvironment() original_target = environment.cargo_target_dir From 5b01b925178400f4216bcd86d97e4cc3ad2de5c4 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Thu, 17 Sep 2026 13:24:18 -0700 Subject: [PATCH 224/311] Add scalable country source platform contracts --- docs/architecture/country-source-platform.md | 57 ++++++ docs/source-status.md | 7 + pipeline/common/review.py | 4 + pipeline/common/review_packet.py | 54 +++++ pipeline/common/source_operations.py | 7 +- pipeline/common/test_review_packet.py | 2 + pipeline/contracts/README.md | 6 + pipeline/contracts/country_contract.py | 133 ++++++++++++ pipeline/contracts/readiness.py | 178 +++++++++++++++++ pipeline/contracts/test_country_contract.py | 57 ++++++ pipeline/contracts/test_readiness.py | 45 +++++ pipeline/platform_registry.json | 14 ++ pipeline/platform_registry.py | 180 +++++++++++++++++ .../rehearse_candidate_private_frontend.py | 189 ++++++++++++++++++ pipeline/tests/test_platform_registry.py | 25 +++ .../tests/test_private_frontend_rehearsal.py | 41 ++++ scripts/dev.py | 10 + scripts/test_dev.py | 6 + 18 files changed, 1014 insertions(+), 1 deletion(-) create mode 100644 docs/architecture/country-source-platform.md create mode 100644 pipeline/contracts/country_contract.py create mode 100644 pipeline/contracts/readiness.py create mode 100644 pipeline/contracts/test_country_contract.py create mode 100644 pipeline/contracts/test_readiness.py create mode 100644 pipeline/platform_registry.json create mode 100644 pipeline/platform_registry.py create mode 100644 pipeline/scripts/maintenance/rehearse_candidate_private_frontend.py create mode 100644 pipeline/tests/test_platform_registry.py create mode 100644 pipeline/tests/test_private_frontend_rehearsal.py diff --git a/docs/architecture/country-source-platform.md b/docs/architecture/country-source-platform.md new file mode 100644 index 0000000..cb69c2e --- /dev/null +++ b/docs/architecture/country-source-platform.md @@ -0,0 +1,57 @@ +# Country/source platform contract + +The country platform is an offline, row-free registry view built from two +existing sources of truth: + +- [`pipeline/source_registry.json`](../../pipeline/source_registry.json) owns + source identity, URL, access method, cadence, expected schema, attribution + notes, and known blockers. +- [`docs/source-status.json`](../source-status.json) owns the conservative + metadata, acquisition, runtime-health, publication-eligibility, evidence, + and next-action status for each source. + +`pipeline/platform_registry.py` joins those files into country contracts and +source records. It currently materializes 234 sources across the registered +country/cross-border prefixes and is designed for the 100+ country / hundreds +of source target without copying every source row into a second hand-maintained +registry. `pipeline/platform_registry.json` records the grouping rule and +scale target. + +## Contracts + +`pipeline/contracts/country_contract.py` validates each country contract. Every +country must state its source IDs, scope, included and excluded populations, +non-closure disappearance semantics, attribution/terms status, owner-review +state, publication state, and readiness state. Completeness is explicitly +`not-claimed` until a separate evidence-backed decision changes it. + +`pipeline/contracts/readiness.py` is the shared state machine. Acquisition or +runtime health never implies approval. A privately acquired lane can reach +`private-validated` and then stops at `awaiting-owner-review`; only an explicit +owner decision can move it to `approved-for-release`. The platform builder +currently records all derived lanes as owner-review pending and publication +blocked. + +## Review packets and private preview + +The existing v1 review packet schemas remain compatible. They now carry a +row-free `platform` context with coverage, attribution, readiness, and the +explicit publication boundary. The packet states that it cannot approve or +promote a release. Address, coordinate, source-row, geocoder, and contact +payloads are rejected from the packet. + +`pipeline/scripts/maintenance/rehearse_candidate_private_frontend.py` checks +an aggregate candidate manifest and, optionally, the authenticated +`/api/dev/preview/candidates` endpoint. It verifies `test_only` and +`private_preview` metadata, treats missing private handoffs as unavailable +rather than zero coverage, and emits only aggregate evidence. The helper is +available through: + +```text +python scripts/dev.py private-frontend +python scripts/dev.py platform-registry +``` + +These commands support private staging and rehearsal only. They do not create, +approve, promote, or publish a release. A human owner remains responsible for +terms, privacy, factual review, project approval, and publication decisions. diff --git a/docs/source-status.md b/docs/source-status.md index 72722c0..ac1cd5a 100644 --- a/docs/source-status.md +++ b/docs/source-status.md @@ -1,5 +1,12 @@ # Source status baseline +The joined country/source platform view is documented in +[`architecture/country-source-platform.md`](architecture/country-source-platform.md) +and validated offline with `python scripts/dev.py platform-registry`. This +baseline remains evidence-backed status, not runtime health or publication +approval; every derived lane stops at `awaiting-owner-review` until an +authorized owner records a decision. + This is the canonical human-readable view of [`source-status.json`](source-status.json). It records repository evidence and reconnaissance state; it is not a live monitor, acquisition log, pipeline health dashboard, release approval, or publication authorization. ## How to read it diff --git a/pipeline/common/review.py b/pipeline/common/review.py index 75875d4..c681af6 100644 --- a/pipeline/common/review.py +++ b/pipeline/common/review.py @@ -6,6 +6,7 @@ from typing import Any, Iterable from pipeline.contracts.source_lifecycle import atomic_json +from .review_packet import _assert_row_free, platform_context def write_operator_review_packet( @@ -40,7 +41,10 @@ def write_operator_review_packet( "geocoding": "disabled", "checks": sorted(set(checks)), "blockers": sorted(set(blockers)), + "platform": platform_context(manifest.get("source_id")), + "publication_boundary": "awaiting-owner-review; this packet is row-free evidence and cannot approve or promote a release", "row_payloads_included": False, } + _assert_row_free(packet) atomic_json(Path(run_dir) / "operator-review-packet.json", packet) return packet diff --git a/pipeline/common/review_packet.py b/pipeline/common/review_packet.py index 92faf55..259940b 100644 --- a/pipeline/common/review_packet.py +++ b/pipeline/common/review_packet.py @@ -16,6 +16,57 @@ REVIEW_PACKET_VERSION = "private-review-packet-v1" +ROW_FREE_REVIEW_PACKET_VERSION = "private-review-packet-v2" + + +def _assert_row_free(value: Any, path: str = "packet") -> None: + """Reject row-shaped or location-bearing payloads in operator artifacts.""" + forbidden = { + "source_values", "raw_fields", "address", "street", "latitude", "longitude", + "coordinates", "geocoder_query", "geocoder_response", "phone", "email", + } + if isinstance(value, dict): + leaked = sorted(forbidden.intersection(value)) + if leaked: + raise ValueError(f"row-free review packet contains prohibited keys at {path}: {leaked}") + for key, child in value.items(): + _assert_row_free(child, f"{path}.{key}") + elif isinstance(value, list): + for index, child in enumerate(value): + _assert_row_free(child, f"{path}[{index}]") + + +def platform_context(source_id: str | None) -> dict[str, Any]: + """Return safe attribution/coverage/readiness metadata for a source.""" + if not source_id: + return { + "registered": False, + "owner_review": "awaiting-owner-review", + "publication_state": "blocked", + "readiness_state": "not-started", + } + try: + from pipeline.platform_registry import source_metadata + + metadata = source_metadata(source_id) + except Exception: + metadata = None + if metadata is None: + return { + "registered": False, + "owner_review": "awaiting-owner-review", + "publication_state": "blocked", + "readiness_state": "not-started", + } + return { + "registered": True, + "country_code": metadata["country_code"], + "coverage": metadata["coverage"], + "attribution": metadata["attribution"], + "readiness": metadata["readiness"], + "owner_review": metadata["owner_review"], + "publication": metadata["publication"], + } def _read_json(path: Path) -> dict[str, Any]: @@ -88,12 +139,15 @@ def build_review_packet( "public_surfaces": status.get("public_surfaces", {surface: False for surface in ("api", "map", "export", "cache", "history")}), "geocoding": manifest.get("geocoding", "disabled"), }, + "platform": platform_context(manifest.get("source_id")), + "publication_boundary": "awaiting-owner-review; this packet is row-free evidence and cannot approve or promote a release", "blockers": blockers or {}, } if not counts["reconciles"] or not counts["qa_matches_manifest"]: packet["blockers"].setdefault("validation", []).append("manifest and QA row counts must reconcile") if packet["gates"]["release_state"] != "not-created" or packet["gates"]["release_promoted"] is not False: packet["blockers"].setdefault("release", []).append("private review requires release_state=not-created and release_promoted=false") + _assert_row_free(packet) return packet diff --git a/pipeline/common/source_operations.py b/pipeline/common/source_operations.py index 434ed45..0aaf2ec 100644 --- a/pipeline/common/source_operations.py +++ b/pipeline/common/source_operations.py @@ -23,6 +23,7 @@ from .acquisition import AcquisitionError from .delta import compare_normalized_paths, compare_runs +from .review_packet import _assert_row_free, platform_context from pipeline.contracts.source_lifecycle import atomic_bytes, atomic_json @@ -395,7 +396,7 @@ def build_review_packet( } qa_counts = {key: qa.get(key) for key in counts} blockers = status.get("review_blockers", {}) - return { + packet = { "schema_version": "private-review-packet-v1", "source_id": source_id, "run_dir_digest": _file_sha256(Path(run_dir) / "manifest.json") or _file_sha256(Path(run_dir) / "run-manifest.json"), @@ -433,6 +434,8 @@ def build_review_packet( "public_surfaces": status.get("public_surfaces", {surface: False for surface in ("api", "map", "export", "cache", "history")}), "geocoding": manifest.get("geocoding", "disabled"), }, + "platform": platform_context(source_id), + "publication_boundary": "awaiting-owner-review; this packet is row-free evidence and cannot approve or promote a release", "blockers": blockers, "prior_eligible_release": prior_eligible_release, "release_promotion_allowed": False, @@ -443,6 +446,8 @@ def build_review_packet( "use a separate authorized release process; this packet cannot promote a release", ], } + _assert_row_free(packet) + return packet def finalize_run_operations( diff --git a/pipeline/common/test_review_packet.py b/pipeline/common/test_review_packet.py index f87843a..329ab01 100644 --- a/pipeline/common/test_review_packet.py +++ b/pipeline/common/test_review_packet.py @@ -27,6 +27,8 @@ def test_packet_is_row_free_and_delta_is_explicitly_not_observed(self): self.assertEqual(packet["release_diff"]["counts"]["not_observed"], 0) self.assertEqual(packet["gates"]["release_state"], "not-created") self.assertFalse(packet["gates"]["release_promoted"]) + self.assertEqual(packet["platform"]["owner_review"]["state"], "awaiting-owner-review") + self.assertEqual(packet["publication_boundary"].split(";", 1)[0], "awaiting-owner-review") self.assertNotIn("source_values", json.dumps(packet)) diff --git a/pipeline/contracts/README.md b/pipeline/contracts/README.md index c252bd5..d6a997e 100644 --- a/pipeline/contracts/README.md +++ b/pipeline/contracts/README.md @@ -49,6 +49,12 @@ It requires recorded URL, UTC retrieval time, hash, and byte size; missing or mismatched provenance fails closed. Database import, geocoding, release approval, and publication remain separately gated. +The country/source platform adds two source-agnostic contracts: +`country_contract.py` validates country scope, attribution, coverage, and +ownership; `readiness.py` prevents acquisition or private validation from being +mistaken for approval. The explicit stopping state for a lane needing a human +decision is `awaiting-owner-review`. + For disposable development teardown, remove only the selected `data/staging/denmark-smiley//` directory after checking retention duties, then recreate the local database through the existing maintenance script with diff --git a/pipeline/contracts/country_contract.py b/pipeline/contracts/country_contract.py new file mode 100644 index 0000000..2b647b5 --- /dev/null +++ b/pipeline/contracts/country_contract.py @@ -0,0 +1,133 @@ +"""Validation contract shared by country lanes. + +The contract describes scope and gates; it does not contain facility rows. A +country may have many source profiles, and each profile remains separately +traceable even when the country summary is materialized for an operator UI. +""" +from __future__ import annotations + +import re +from typing import Any, Iterable, Mapping + +from .readiness import ReadinessError, validate_readiness + + +COUNTRY_CONTRACT_VERSION = "country-contract-v1" +COUNTRY_CODE_RE = re.compile(r"^[A-Z]{2,3}$") +OWNER_REVIEW_STATES = frozenset({"not-requested", "awaiting-owner-review", "approved", "rejected"}) +PUBLICATION_STATES = frozenset({"blocked", "private-only", "approved-for-release", "published", "suppressed"}) + + +class CountryContractError(ValueError): + """Raised when a country contract is incomplete or unsafe.""" + + +def _require_text(value: Mapping[str, Any], key: str, prefix: str) -> str: + result = value.get(key) + if not isinstance(result, str) or not result.strip(): + raise CountryContractError(f"{prefix}.{key} must be a non-empty string") + return result + + +def _require_list(value: Mapping[str, Any], key: str, prefix: str) -> list[str]: + result = value.get(key) + if not isinstance(result, list) or not result or any(not isinstance(item, str) or not item.strip() for item in result): + raise CountryContractError(f"{prefix}.{key} must be a non-empty list of strings") + return result + + +def validate_country_contract(contract: Mapping[str, Any], *, known_source_ids: Iterable[str] | None = None) -> None: + """Validate one source/country contract and all publication boundaries.""" + if not isinstance(contract, Mapping): + raise CountryContractError("country contract must be an object") + prefix = f"country[{contract.get('country_code', '?')}]" + if contract.get("contract_version") != COUNTRY_CONTRACT_VERSION: + raise CountryContractError(f"{prefix}.contract_version must be {COUNTRY_CONTRACT_VERSION}") + code = _require_text(contract, "country_code", prefix) + if not COUNTRY_CODE_RE.fullmatch(code): + raise CountryContractError(f"{prefix}.country_code must be an uppercase ISO-like code") + _require_text(contract, "display_name", prefix) + _require_list(contract, "source_ids", prefix) + source_ids = contract["source_ids"] + if len(set(source_ids)) != len(source_ids): + raise CountryContractError(f"{prefix}.source_ids must be unique") + if known_source_ids is not None: + unknown = sorted(set(source_ids) - set(known_source_ids)) + if unknown: + raise CountryContractError(f"{prefix}.source_ids are not in the source registry: {unknown}") + + coverage = contract.get("coverage") + if not isinstance(coverage, Mapping): + raise CountryContractError(f"{prefix}.coverage must be an object") + _require_text(coverage, "scope_statement", f"{prefix}.coverage") + _require_text(coverage, "completeness", f"{prefix}.coverage") + _require_list(coverage, "included", f"{prefix}.coverage") + _require_list(coverage, "excluded", f"{prefix}.coverage") + if coverage.get("disappearance_semantics") != "not-observed; never inferred as closure": + raise CountryContractError(f"{prefix}.coverage.disappearance_semantics must preserve not-observed semantics") + + attribution = contract.get("attribution") + if not isinstance(attribution, Mapping): + raise CountryContractError(f"{prefix}.attribution must be an object") + _require_text(attribution, "source_origin", f"{prefix}.attribution") + _require_text(attribution, "terms_status", f"{prefix}.attribution") + _require_text(attribution, "notice", f"{prefix}.attribution") + if not isinstance(attribution.get("attribution_required"), bool): + raise CountryContractError(f"{prefix}.attribution.attribution_required must be boolean") + + owner_review = contract.get("owner_review") + if not isinstance(owner_review, Mapping): + raise CountryContractError(f"{prefix}.owner_review must be an object") + owner_state = _require_text(owner_review, "state", f"{prefix}.owner_review") + if owner_state not in OWNER_REVIEW_STATES: + raise CountryContractError(f"{prefix}.owner_review.state is unknown") + if "owner" not in owner_review or owner_review.get("owner") not in (None, "") and not isinstance(owner_review.get("owner"), str): + raise CountryContractError(f"{prefix}.owner_review.owner must be null or a string") + if owner_state == "approved" and not owner_review.get("decision_id"): + raise CountryContractError(f"{prefix}.owner_review approved state requires decision_id") + if owner_state != "approved" and owner_review.get("decision_id") is not None: + raise CountryContractError(f"{prefix}.owner_review decision_id is only valid after approval") + + publication = contract.get("publication") + if not isinstance(publication, Mapping): + raise CountryContractError(f"{prefix}.publication must be an object") + publication_state = _require_text(publication, "state", f"{prefix}.publication") + if publication_state not in PUBLICATION_STATES: + raise CountryContractError(f"{prefix}.publication.state is unknown") + if not isinstance(publication.get("approval_required"), bool) or publication.get("approval_required") is not True: + raise CountryContractError(f"{prefix}.publication.approval_required must remain true") + if publication_state in {"approved-for-release", "published"} and owner_state != "approved": + raise CountryContractError(f"{prefix} cannot be release-ready without owner approval") + if publication_state in {"blocked", "private-only"} and owner_state == "approved": + # An approved owner can still keep a country private, but the contract + # must say why instead of accidentally looking publishable. + if not publication.get("reason"): + raise CountryContractError(f"{prefix}.publication.reason is required for a private approved lane") + + readiness = contract.get("readiness") + if not isinstance(readiness, Mapping): + raise CountryContractError(f"{prefix}.readiness must be an object") + try: + validate_readiness(readiness) + except ReadinessError as error: + raise CountryContractError(f"{prefix}.readiness is invalid: {error}") from error + if readiness.get("owner_review") != owner_state: + raise CountryContractError(f"{prefix}.readiness.owner_review must match owner_review.state") + if owner_state != "approved" and readiness.get("public_release_allowed") is not False: + raise CountryContractError(f"{prefix} cannot allow public release before owner approval") + + +def validate_country_contracts(contracts: Iterable[Mapping[str, Any]], *, known_source_ids: Iterable[str] | None = None) -> None: + """Validate a collection and reject duplicate country/source ownership.""" + seen_countries: set[str] = set() + seen_sources: set[str] = set() + for contract in contracts: + validate_country_contract(contract, known_source_ids=known_source_ids) + code = str(contract["country_code"]) + if code in seen_countries: + raise CountryContractError(f"duplicate country contract: {code}") + seen_countries.add(code) + duplicates = seen_sources.intersection(contract["source_ids"]) + if duplicates: + raise CountryContractError(f"source IDs assigned to multiple country contracts: {sorted(duplicates)}") + seen_sources.update(contract["source_ids"]) diff --git a/pipeline/contracts/readiness.py b/pipeline/contracts/readiness.py new file mode 100644 index 0000000..466e0ee --- /dev/null +++ b/pipeline/contracts/readiness.py @@ -0,0 +1,178 @@ +"""Shared readiness and publication-boundary model for country/source lanes. + +Readiness is deliberately separate from factual review and publication. A +source can be privately validated without being approved, and an approved +record can later be suppressed. This module is a small, dependency-free +contract that adapters, diagnostics, and review tooling can share. +""" +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, Mapping + + +READINESS_SCHEMA_VERSION = "country-source-readiness-v1" + +READINESS_STATES = frozenset( + { + "not-started", + "reconnaissance", + "blocked", + "private-candidate", + "private-validated", + "awaiting-owner-review", + "approved-for-release", + "published", + "suppressed", + } +) +OWNER_REVIEW_STATES = frozenset( + {"not-requested", "awaiting-owner-review", "approved", "rejected"} +) + +_ALLOWED_TRANSITIONS = { + "not-started": {"not-started", "reconnaissance", "blocked"}, + "reconnaissance": {"reconnaissance", "blocked", "private-candidate"}, + "blocked": {"blocked", "reconnaissance", "private-candidate"}, + "private-candidate": { + "private-candidate", + "private-validated", + "blocked", + "awaiting-owner-review", + "suppressed", + }, + "private-validated": { + "private-validated", + "awaiting-owner-review", + "blocked", + "suppressed", + }, + "awaiting-owner-review": { + "awaiting-owner-review", + "approved-for-release", + "blocked", + "suppressed", + }, + "approved-for-release": { + "approved-for-release", + "published", + "blocked", + "suppressed", + }, + "published": {"published", "suppressed", "blocked"}, + "suppressed": {"suppressed", "private-candidate", "blocked"}, +} + + +class ReadinessError(ValueError): + """Raised when a readiness object violates the shared boundary.""" + + +@dataclass(frozen=True) +class Readiness: + """A serializable readiness decision with an explicit owner boundary.""" + + state: str + owner_review: str = "awaiting-owner-review" + private_candidate: bool = False + public_release_allowed: bool = False + reasons: tuple[str, ...] = () + + def __post_init__(self) -> None: + validate_readiness(self.as_mapping()) + + def as_mapping(self) -> dict[str, Any]: + return { + "schema_version": READINESS_SCHEMA_VERSION, + "state": self.state, + "owner_review": self.owner_review, + "private_candidate": self.private_candidate, + "public_release_allowed": self.public_release_allowed, + "reasons": list(self.reasons), + } + + +def validate_readiness(value: Mapping[str, Any]) -> None: + """Validate an externally supplied readiness mapping fail-closed.""" + if value.get("schema_version") != READINESS_SCHEMA_VERSION: + raise ReadinessError(f"schema_version must be {READINESS_SCHEMA_VERSION}") + state = value.get("state") + owner_review = value.get("owner_review") + if state not in READINESS_STATES: + raise ReadinessError(f"unknown readiness state: {state!r}") + if owner_review not in OWNER_REVIEW_STATES: + raise ReadinessError(f"unknown owner review state: {owner_review!r}") + for key in ("private_candidate", "public_release_allowed"): + if not isinstance(value.get(key), bool): + raise ReadinessError(f"{key} must be boolean") + reasons = value.get("reasons") + if not isinstance(reasons, list) or any(not isinstance(item, str) or not item for item in reasons): + raise ReadinessError("reasons must be a list of non-empty strings") + if state in {"approved-for-release", "published"}: + if owner_review != "approved" or value.get("public_release_allowed") is not True: + raise ReadinessError("release-ready states require approved owner review and public_release_allowed=true") + else: + if value.get("public_release_allowed") is not False: + raise ReadinessError("non-release-ready states must not allow public release") + if state == "awaiting-owner-review" and owner_review != "awaiting-owner-review": + raise ReadinessError("awaiting-owner-review must retain the same owner boundary") + + +def readiness_from_status(status: Mapping[str, Any]) -> Readiness: + """Derive a conservative readiness state from the source-status vocabulary. + + Existing status files intentionally do not contain approvals. Therefore a + privately acquired source becomes ``awaiting-owner-review`` at most; it can + never be inferred as release-ready from acquisition or runtime health. + """ + publication = str(status.get("publication_eligibility", "not_assessed")) + acquisition = str(status.get("acquisition", "not_run")) + metadata = str(status.get("metadata", "unknown")) + explicit = status.get("readiness_state") + owner_review = str(status.get("owner_review", "awaiting-owner-review")) + if owner_review not in OWNER_REVIEW_STATES: + owner_review = "awaiting-owner-review" + + if explicit in {"approved-for-release", "published"} and owner_review == "approved": + state = str(explicit) + return Readiness(state, owner_review, True, True, ()) + + reasons: list[str] = [] + if publication in {"blocked", "not_assessed"}: + reasons.append(f"publication:{publication}") + if acquisition in {"blocked", "not_run"}: + reasons.append(f"acquisition:{acquisition}") + if metadata == "unknown": + reasons.append("metadata:unknown") + + if acquisition == "not_run" and metadata in {"verified", "partial"}: + state = "reconnaissance" + elif acquisition == "blocked": + state = "blocked" + elif acquisition in {"artifact_private_only", "verified"}: + state = "awaiting-owner-review" + else: + state = "not-started" + if state == "awaiting-owner-review": + owner_review = "awaiting-owner-review" + return Readiness(state, owner_review, state in {"private-candidate", "private-validated", "awaiting-owner-review"}, False, tuple(sorted(set(reasons)))) + + +def can_transition(current: str, target: str) -> bool: + """Return whether a lane may move between states without skipping gates.""" + return target in _ALLOWED_TRANSITIONS.get(current, set()) + + +def require_transition(current: str, target: str) -> None: + """Raise when a requested state change skips the owner/publication gate.""" + if current not in READINESS_STATES or target not in READINESS_STATES: + raise ReadinessError("readiness transition uses an unknown state") + if not can_transition(current, target): + raise ReadinessError(f"invalid readiness transition: {current} -> {target}") + + +def require_private_boundary(value: Mapping[str, Any]) -> None: + """Assert that a readiness mapping cannot be interpreted as publication.""" + validate_readiness(value) + if value["public_release_allowed"] or value["state"] in {"approved-for-release", "published"}: + raise ReadinessError("private rehearsal cannot contain a release-ready readiness state") diff --git a/pipeline/contracts/test_country_contract.py b/pipeline/contracts/test_country_contract.py new file mode 100644 index 0000000..e888fdc --- /dev/null +++ b/pipeline/contracts/test_country_contract.py @@ -0,0 +1,57 @@ +import unittest + +from pipeline.contracts.country_contract import CountryContractError, validate_country_contract + + +def contract(**overrides): + value = { + "contract_version": "country-contract-v1", + "country_code": "DK", + "display_name": "Denmark", + "source_ids": ["dk.smiley"], + "coverage": { + "scope_statement": "Danish food-business listings", + "completeness": "not-claimed", + "included": ["food-business listings"], + "excluded": ["private records"], + "disappearance_semantics": "not-observed; never inferred as closure", + }, + "attribution": { + "source_origin": "government source", + "terms_status": "pending-human-review", + "attribution_required": True, + "notice": "Attribute source publisher.", + }, + "owner_review": {"state": "awaiting-owner-review", "owner": None, "decision_id": None}, + "publication": {"state": "blocked", "approval_required": True, "reason": "review pending"}, + "readiness": { + "schema_version": "country-source-readiness-v1", + "state": "awaiting-owner-review", + "owner_review": "awaiting-owner-review", + "private_candidate": True, + "public_release_allowed": False, + "reasons": ["publication:blocked"], + }, + } + value.update(overrides) + return value + + +class CountryContractTests(unittest.TestCase): + def test_contract_requires_registered_source_ids(self): + validate_country_contract(contract(), known_source_ids={"dk.smiley"}) + with self.assertRaises(CountryContractError): + validate_country_contract(contract(source_ids=["not-registered"]), known_source_ids={"dk.smiley"}) + + def test_contract_rejects_missing_coverage_semantics_and_publication(self): + coverage = dict(contract()["coverage"]) + coverage.pop("disappearance_semantics") + with self.assertRaises(CountryContractError): + validate_country_contract(contract(coverage=coverage)) + publication = {"state": "approved-for-release", "approval_required": True} + with self.assertRaises(CountryContractError): + validate_country_contract(contract(publication=publication)) + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/contracts/test_readiness.py b/pipeline/contracts/test_readiness.py new file mode 100644 index 0000000..63edeca --- /dev/null +++ b/pipeline/contracts/test_readiness.py @@ -0,0 +1,45 @@ +import unittest + +from pipeline.contracts.readiness import ( + ReadinessError, + can_transition, + readiness_from_status, + require_private_boundary, + require_transition, +) + + +class ReadinessTests(unittest.TestCase): + def test_private_acquisition_stops_at_owner_review(self): + readiness = readiness_from_status({ + "metadata": "verified", + "acquisition": "artifact_private_only", + "publication_eligibility": "blocked", + }) + self.assertEqual(readiness.state, "awaiting-owner-review") + self.assertEqual(readiness.owner_review, "awaiting-owner-review") + self.assertTrue(readiness.private_candidate) + self.assertFalse(readiness.public_release_allowed) + + def test_not_run_source_is_reconnaissance_not_ready(self): + readiness = readiness_from_status({"metadata": "verified", "acquisition": "not_run"}) + self.assertEqual(readiness.state, "reconnaissance") + self.assertFalse(readiness.private_candidate) + + def test_approval_requires_explicit_owner_review(self): + self.assertTrue(can_transition("awaiting-owner-review", "approved-for-release")) + with self.assertRaises(ReadinessError): + require_transition("private-candidate", "approved-for-release") + with self.assertRaises(ReadinessError): + require_private_boundary({ + "schema_version": "country-source-readiness-v1", + "state": "approved-for-release", + "owner_review": "approved", + "private_candidate": True, + "public_release_allowed": True, + "reasons": [], + }) + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/platform_registry.json b/pipeline/platform_registry.json new file mode 100644 index 0000000..b447223 --- /dev/null +++ b/pipeline/platform_registry.json @@ -0,0 +1,14 @@ +{ + "schema_version": "country-source-platform-v1", + "scale_target": { + "countries": 100, + "sources": 300, + "note": "Architecture target only; current registry evidence is not a completeness claim." + }, + "source_of_truth": { + "sources": "pipeline/source_registry.json", + "status": "docs/source-status.json" + }, + "country_grouping": "uppercase source-id prefix; EU is retained as a cross-border scope", + "owner_review_boundary": "awaiting-owner-review" +} diff --git a/pipeline/platform_registry.py b/pipeline/platform_registry.py new file mode 100644 index 0000000..0cd7131 --- /dev/null +++ b/pipeline/platform_registry.py @@ -0,0 +1,180 @@ +"""Materialize the country/source platform from existing checked-in registries. + +The source registry and source-status baseline remain the authoritative inputs. +This module joins them into a typed, country-aware view so new lanes do not +copy 234 source rows into another file. It is intentionally offline and never +fetches a source or grants release approval. +""" +from __future__ import annotations + +import json +from collections import defaultdict +from pathlib import Path +from typing import Any, Mapping + +from pipeline.contracts.country_contract import COUNTRY_CONTRACT_VERSION, validate_country_contracts +from pipeline.contracts.readiness import readiness_from_status +from pipeline.source_registry import load_registry + + +ROOT = Path(__file__).resolve().parents[1] +PLATFORM_CONFIG_PATH = Path(__file__).with_name("platform_registry.json") +STATUS_PATH = ROOT / "docs" / "source-status.json" +PLATFORM_SCHEMA_VERSION = "country-source-platform-v1" + + +class PlatformRegistryError(ValueError): + """Raised when the joined platform registry cannot be trusted.""" + + +def _read_json(path: Path) -> dict[str, Any]: + try: + value = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as error: + raise PlatformRegistryError(f"invalid registry input: {path}") from error + if not isinstance(value, dict): + raise PlatformRegistryError(f"registry input must be an object: {path}") + return value + + +def _country_code(source_id: str) -> str: + prefix = source_id.split(".", 1)[0].strip().upper() + # Cross-border statistical sources are intentionally retained as EU rather + # than being assigned to a country by inference. + return "EU" if prefix == "EU" else prefix + + +def _source_readiness(status: Mapping[str, Any]) -> dict[str, Any]: + return readiness_from_status(status).as_mapping() + + +def _source_record(source: Mapping[str, Any], status: Mapping[str, Any]) -> dict[str, Any]: + source_id = str(source["source_id"]) + readiness = _source_readiness(status) + terms = str(source.get("attribution_licensing_notes") or "unknown") + return { + "source_id": source_id, + "country_code": _country_code(source_id), + "jurisdiction_scope": source["jurisdiction_scope"], + "source_url": source["url"], + "access_method": source["access_method"], + "cadence": source["cadence"], + "coverage": { + "scope_statement": source["jurisdiction_scope"], + "completeness": "not-claimed", + "included": [source["jurisdiction_scope"]], + "excluded": ["unverified or out-of-scope populations", "private or restricted payloads"], + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": list(source.get("blockers") or []), + }, + "attribution": { + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review", + "attribution_required": True, + "notice": terms, + }, + "status": {key: status.get(key) for key in ( + "metadata", "acquisition", "runtime_health", "publication_eligibility", "evidence", "next_action" + )}, + "readiness": readiness, + "owner_review": {"state": "awaiting-owner-review", "owner": None, "decision_id": None}, + "publication": { + "state": "blocked", + "approval_required": True, + "reason": "No owner approval is recorded in the checked-in status baseline.", + }, + } + + +def _country_contract(country_code: str, sources: list[dict[str, Any]]) -> dict[str, Any]: + readiness_states = {str(source["readiness"]["state"]) for source in sources} + if readiness_states and readiness_states.issubset({"awaiting-owner-review", "private-candidate", "private-validated"}): + state = "awaiting-owner-review" + elif "reconnaissance" in readiness_states and readiness_states.issubset({"reconnaissance", "blocked"}): + state = "reconnaissance" + elif "not-started" in readiness_states: + state = "not-started" + else: + state = "blocked" + reasons = sorted({reason for source in sources for reason in source["readiness"]["reasons"]}) + readiness = { + "schema_version": "country-source-readiness-v1", + "state": state, + "owner_review": "awaiting-owner-review", + "private_candidate": state == "awaiting-owner-review", + "public_release_allowed": False, + "reasons": reasons or ["owner review is not recorded"], + } + return { + "contract_version": COUNTRY_CONTRACT_VERSION, + "country_code": country_code, + "display_name": country_code, + "source_ids": [source["source_id"] for source in sources], + "coverage": { + "scope_statement": f"Structured source profiles currently registered for {country_code}; no national completeness claim.", + "completeness": "not-claimed", + "included": sorted({source["jurisdiction_scope"] for source in sources}), + "excluded": ["sources not registered in this checkout", "unverified facility populations", "restricted or removed records"], + "disappearance_semantics": "not-observed; never inferred as closure", + }, + "attribution": { + "source_origin": "source-specific; see each source record", + "terms_status": "pending-human-review", + "attribution_required": True, + "notice": "Source attribution, reuse terms, and publication scope require owner review per source.", + }, + "owner_review": {"state": "awaiting-owner-review", "owner": None, "decision_id": None}, + "publication": { + "state": "blocked", + "approval_required": True, + "reason": "Country summaries are not publication approvals and must not promote source rows.", + }, + "readiness": readiness, + } + + +def build_platform_registry(*, source_path: Path | None = None, status_path: Path | None = None) -> dict[str, Any]: + """Build and validate the joined source/country registry offline.""" + config = _read_json(PLATFORM_CONFIG_PATH) + if config.get("schema_version") != PLATFORM_SCHEMA_VERSION: + raise PlatformRegistryError(f"platform config must use {PLATFORM_SCHEMA_VERSION}") + source_payload = load_registry(source_path or (ROOT / "pipeline" / "source_registry.json")) + status_payload = _read_json(status_path or STATUS_PATH) + statuses = status_payload.get("sources") + if not isinstance(statuses, list): + raise PlatformRegistryError("source status registry must contain a sources list") + status_by_id = {item.get("source_id"): item for item in statuses if isinstance(item, dict)} + source_ids = {item["source_id"] for item in source_payload["sources"]} + if set(status_by_id) != source_ids: + raise PlatformRegistryError("source registry and status registry IDs do not match exactly") + + source_records = [_source_record(source, status_by_id[source["source_id"]]) for source in source_payload["sources"]] + grouped: dict[str, list[dict[str, Any]]] = defaultdict(list) + for source in source_records: + grouped[source["country_code"]].append(source) + countries = [_country_contract(code, sorted(items, key=lambda item: item["source_id"])) for code, items in sorted(grouped.items())] + validate_country_contracts(countries, known_source_ids=source_ids) + return { + "schema_version": PLATFORM_SCHEMA_VERSION, + "contract_versions": {"country": COUNTRY_CONTRACT_VERSION, "readiness": "country-source-readiness-v1"}, + "source_count": len(source_records), + "country_count": len(countries), + "scale_target": config.get("scale_target"), + "sources": source_records, + "countries": countries, + "publication_boundary": "awaiting-owner-review; private staging may continue; no release approval or promotion is implied", + } + + +def source_metadata(source_id: str) -> dict[str, Any] | None: + """Return row-free platform metadata for a source, if it is registered.""" + registry = build_platform_registry() + return next((source for source in registry["sources"] if source["source_id"] == source_id), None) + + +def write_platform_snapshot(path: Path) -> dict[str, Any]: + """Materialize a reviewable row-free snapshot without changing release state.""" + snapshot = build_platform_registry() + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(snapshot, ensure_ascii=False, indent=2, sort_keys=True) + "\n", encoding="utf-8") + return snapshot diff --git a/pipeline/scripts/maintenance/rehearse_candidate_private_frontend.py b/pipeline/scripts/maintenance/rehearse_candidate_private_frontend.py new file mode 100644 index 0000000..bf31803 --- /dev/null +++ b/pipeline/scripts/maintenance/rehearse_candidate_private_frontend.py @@ -0,0 +1,189 @@ +"""Rehearse a private candidate handoff against the frontend preview boundary. + +This tool consumes only an aggregate candidate manifest and optional HTTP +metadata. It never starts a public server, creates a release, or emits source +rows. Missing local handoffs are reported as unavailable private evidence, +not as zero coverage. +""" +from __future__ import annotations + +import argparse +import json +import sys +import urllib.error +import urllib.request +from pathlib import Path +from typing import Any, Callable + + +REPORT_VERSION = "candidate-private-frontend-rehearsal-v1" +FORBIDDEN_KEYS = frozenset({ + "source_values", "raw_fields", "address", "street", "latitude", "longitude", + "coordinates", "records", "rows", "geocoder_query", "geocoder_response", +}) + + +class RehearsalError(ValueError): + """The private preview contract is incomplete or unsafe.""" + + +def _read(path: Path) -> dict[str, Any]: + try: + value = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as error: + raise RehearsalError(f"invalid candidate manifest: {path}") from error + if not isinstance(value, dict): + raise RehearsalError("candidate manifest must be an object") + return value + + +def _assert_safe(value: Any, path: str = "report") -> None: + if isinstance(value, dict): + leaked = sorted(FORBIDDEN_KEYS.intersection(value)) + if leaked: + raise RehearsalError(f"row payload key in {path}: {leaked}") + for key, child in value.items(): + _assert_safe(child, f"{path}.{key}") + elif isinstance(value, list): + for index, child in enumerate(value): + _assert_safe(child, f"{path}[{index}]") + + +def _request(base_url: str, token: str) -> tuple[int, dict[str, Any]]: + request = urllib.request.Request( + base_url.rstrip("/") + "/api/dev/preview/candidates", + headers={"X-UEC-Dev-Preview-Token": token, "Accept": "application/json"}, + ) + try: + with urllib.request.urlopen(request, timeout=5) as response: + body = json.loads(response.read().decode("utf-8")) + return response.status, body if isinstance(body, dict) else {} + except urllib.error.HTTPError as error: + return error.code, {} + except (urllib.error.URLError, TimeoutError, json.JSONDecodeError) as error: + raise RehearsalError(f"private preview request failed: {error}") from error + + +def rehearse_candidate( + manifest_path: str | Path, + *, + root: str | Path = ".", + base_url: str | None = None, + token: str | None = None, + request: Callable[[str, str], tuple[int, dict[str, Any]]] = _request, +) -> dict[str, Any]: + """Validate an aggregate handoff and optionally probe the private preview.""" + manifest_file = Path(manifest_path) + root_path = Path(root) + manifest = _read(manifest_file) + publication = manifest.get("publication") + if not isinstance(publication, dict): + raise RehearsalError("candidate manifest must contain a publication object") + if publication.get("release_created") is not False or publication.get("release_promoted") is not False: + raise RehearsalError("candidate rehearsal requires release_created=false and release_promoted=false") + if publication.get("project_approval") not in {None, "not-approved", "pending"}: + raise RehearsalError("candidate rehearsal cannot contain project approval") + sources = manifest.get("sources") + if not isinstance(sources, list) or not sources: + raise RehearsalError("candidate manifest must contain sources") + + source_reports: list[dict[str, Any]] = [] + for entry in sources: + if not isinstance(entry, dict) or not isinstance(entry.get("source_id"), str): + raise RehearsalError("each candidate source must have a source_id") + state = str(entry.get("status") or entry.get("acquisition") or "unknown") + if "public" in state.lower() and "private" not in state.lower(): + raise RehearsalError(f"source {entry['source_id']} is not private-only") + handoff_value = entry.get("candidate_handoff_manifest") or entry.get("private_manifest") + handoff_state = "not-declared" + normalized_rows = entry.get("normalized_rows") + if isinstance(handoff_value, str): + handoff = Path(handoff_value) + if not handoff.is_absolute(): + handoff = root_path / handoff + if handoff.is_file(): + handoff_data = _read(handoff) + handoff_state = "available-private-handoff" + normalized_rows = handoff_data.get("normalized_rows", normalized_rows) + if handoff_data.get("release_state") not in {None, "not-created"}: + raise RehearsalError(f"source {entry['source_id']} handoff declares a release") + else: + handoff_state = "unavailable-private-handoff" + source_reports.append({ + "source_id": entry["source_id"], + "handoff_state": handoff_state, + "normalized_rows": normalized_rows if isinstance(normalized_rows, int) else None, + "publication_state": "private-only", + "owner_review": "awaiting-owner-review", + }) + + preview = { + "state": "not-run", + "endpoint": "/api/dev/preview/candidates", + "test_only": True, + "private_preview": True, + "raw_fields_absent": None, + } + if base_url is not None: + if not token: + raise RehearsalError("base_url requires a preview token") + status, body = request(base_url, token) + if status != 200: + raise RehearsalError(f"private preview returned HTTP {status}") + meta = body.get("meta") if isinstance(body.get("meta"), dict) else {} + if meta.get("test_only") is not True or meta.get("private_preview") is not True: + raise RehearsalError("private preview response did not carry test_only/private_preview metadata") + encoded = json.dumps(body, ensure_ascii=False) + if any(key in encoded for key in FORBIDDEN_KEYS): + raise RehearsalError("private preview response contains a prohibited raw-field marker") + preview.update({"state": "passed", "http_status": status, "raw_fields_absent": True}) + + report = { + "report_version": REPORT_VERSION, + "status": "passed", + "fail_closed": True, + "candidate_state": "private-only; no publication decision implied", + "manifest": { + "path": manifest_file.name, + "release_created": False, + "release_promoted": False, + "project_approval": "not-approved", + }, + "frontend_preview": preview, + "sources": source_reports, + "limitations": [ + "This rehearsal validates the private preview boundary, not source factual accuracy, privacy eligibility, or release approval.", + "Unavailable private handoffs are not counted as zero coverage.", + ], + "private_payloads_included": False, + "publication_boundary": "awaiting-owner-review", + } + _assert_safe(report) + return report + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--manifest", type=Path, required=True) + parser.add_argument("--root", type=Path, default=Path(".")) + parser.add_argument("--base-url") + parser.add_argument("--token") + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + try: + report = rehearse_candidate(args.manifest, root=args.root, base_url=args.base_url, token=args.token) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(json.dumps({"output": str(args.output), "status": report["status"]}, sort_keys=True)) + return 0 + except Exception as error: + failure = {"report_version": REPORT_VERSION, "status": "blocked", "fail_closed": True, "blocker": str(error), "private_payloads_included": False} + _assert_safe(failure) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(failure, ensure_ascii=False, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(json.dumps({"output": str(args.output), "status": "blocked", "blocker": str(error)}, sort_keys=True), file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/pipeline/tests/test_platform_registry.py b/pipeline/tests/test_platform_registry.py new file mode 100644 index 0000000..812246f --- /dev/null +++ b/pipeline/tests/test_platform_registry.py @@ -0,0 +1,25 @@ +import unittest + +from pipeline.platform_registry import build_platform_registry, source_metadata + + +class PlatformRegistryTests(unittest.TestCase): + def test_materialized_registry_scales_from_existing_source_status_inputs(self): + registry = build_platform_registry() + self.assertEqual(registry["source_count"], 234) + self.assertGreaterEqual(registry["country_count"], 40) + self.assertEqual(registry["publication_boundary"], "awaiting-owner-review; private staging may continue; no release approval or promotion is implied") + self.assertTrue(all(country["publication"]["state"] == "blocked" for country in registry["countries"])) + self.assertTrue(all(source["owner_review"]["state"] == "awaiting-owner-review" for source in registry["sources"])) + + def test_source_context_carries_coverage_and_attribution_without_rows(self): + source = source_metadata("dk.smiley") + self.assertEqual(source["country_code"], "DK") + self.assertEqual(source["coverage"]["completeness"], "not-claimed") + self.assertTrue(source["attribution"]["attribution_required"]) + self.assertEqual(source["readiness"]["state"], "awaiting-owner-review") + self.assertNotIn("source_values", source) + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/tests/test_private_frontend_rehearsal.py b/pipeline/tests/test_private_frontend_rehearsal.py new file mode 100644 index 0000000..41005e2 --- /dev/null +++ b/pipeline/tests/test_private_frontend_rehearsal.py @@ -0,0 +1,41 @@ +import json +import tempfile +import unittest +from pathlib import Path + +from pipeline.scripts.maintenance.rehearse_candidate_private_frontend import rehearse_candidate + + +class PrivateFrontendRehearsalTests(unittest.TestCase): + def test_missing_private_handoff_is_not_counted_as_zero_and_report_is_row_free(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + manifest = root / "candidate.json" + manifest.write_text(json.dumps({ + "publication": {"release_created": False, "release_promoted": False, "project_approval": "not-approved"}, + "sources": [{"source_id": "dk.smiley", "candidate_handoff_manifest": "private/missing.json", "normalized_rows": 12, "status": "candidate-ready; private only"}], + }), encoding="utf-8") + report = rehearse_candidate(manifest, root=root) + self.assertEqual(report["status"], "passed") + self.assertEqual(report["sources"][0]["handoff_state"], "unavailable-private-handoff") + self.assertFalse(report["private_payloads_included"]) + self.assertNotIn("source_values", json.dumps(report)) + + def test_private_preview_probe_requires_test_only_metadata(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + manifest = root / "candidate.json" + manifest.write_text(json.dumps({ + "publication": {"release_created": False, "release_promoted": False, "project_approval": "not-approved"}, + "sources": [{"source_id": "dk.smiley", "status": "private only"}], + }), encoding="utf-8") + + def response(_base_url, _token): + return 200, {"meta": {"test_only": False, "private_preview": True}} + + with self.assertRaises(ValueError): + rehearse_candidate(manifest, root=root, base_url="http://127.0.0.1", token="test", request=response) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/dev.py b/scripts/dev.py index 5dbc2a4..248cde5 100644 --- a/scripts/dev.py +++ b/scripts/dev.py @@ -78,6 +78,9 @@ def main() -> int: sub.add_parser("test", help="root legacy static/Jest tests").add_argument("--full", action="store_true") sub.add_parser("pipeline", help="run Python pipeline tests").add_argument("args", nargs=argparse.REMAINDER) sub.add_parser("contracts", help="run contract tests") + sub.add_parser("platform-registry", help="validate the joined country/source registry") + pf = sub.add_parser("private-frontend", help="rehearse a candidate against the private frontend preview boundary") + pf.add_argument("manifest"); pf.add_argument("output"); pf.add_argument("--root", default=str(ROOT)); pf.add_argument("--base-url"); pf.add_argument("--token") rp = sub.add_parser("review-packet", help="generate a private row-free review packet") rp.add_argument("run_dir"); rp.add_argument("--previous-normalized") args = p.parse_args(); SUMMARY["command"] = args.command @@ -92,6 +95,13 @@ def main() -> int: runner = "pytest" if shutil.which("pytest") else "unittest" targets = ["pipeline/contracts", "pipeline/tests/test_database_contract.py", "pipeline/tests/test_graph_database_contract.py"] if runner == "pytest" else ["discover", "-s", "pipeline/contracts", "-t", str(ROOT)] code = run([sys.executable, "-m", runner, *targets], capture=args.json) + elif args.command == "platform-registry": + code = run([sys.executable, "-c", "import json; from pipeline.platform_registry import build_platform_registry; r=build_platform_registry(); print(json.dumps({'countries':r['country_count'],'sources':r['source_count'],'status':'validated'}))"], capture=args.json) + elif args.command == "private-frontend": + cmd = [sys.executable, str(ROOT / "pipeline/scripts/maintenance/rehearse_candidate_private_frontend.py"), "--manifest", args.manifest, "--root", args.root, "--output", args.output] + if args.base_url: cmd.extend(["--base-url", args.base_url]) + if args.token: cmd.extend(["--token", args.token]) + code = run(cmd, capture=args.json) else: cmd = [sys.executable, "-c", "from pipeline.common.review_packet import write_review_packet; import sys; write_review_packet(sys.argv[1], previous_normalized_path=sys.argv[2] if len(sys.argv)>2 else None)", args.run_dir] if args.previous_normalized: cmd.append(args.previous_normalized) diff --git a/scripts/test_dev.py b/scripts/test_dev.py index a9eccb7..3b2b3fa 100644 --- a/scripts/test_dev.py +++ b/scripts/test_dev.py @@ -9,6 +9,7 @@ def test_help(self): self.assertEqual(result.returncode, 0) self.assertIn("doctor", result.stdout) self.assertIn("review-packet", result.stdout) + self.assertIn("private-frontend", result.stdout) def test_doctor_json_does_not_echo_secret(self): result = subprocess.run([sys.executable, "scripts/dev.py", "--json", "doctor"], cwd=ROOT, env={**os.environ, "UEC_DATABASE_URL": "postgresql://secret.invalid/db"}, capture_output=True, text=True) @@ -20,6 +21,11 @@ def test_contracts_command_uses_package_root_for_relative_imports(self): result = subprocess.run([sys.executable, "scripts/dev.py", "contracts"], cwd=ROOT, capture_output=True, text=True) self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + def test_platform_registry_command_validates_joined_registry(self): + result = subprocess.run([sys.executable, "scripts/dev.py", "platform-registry"], cwd=ROOT, capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertIn("validated", result.stdout) + def test_json_delegated_command_is_machine_readable(self): result = subprocess.run([sys.executable, "scripts/dev.py", "--json", "status"], cwd=ROOT, capture_output=True, text=True) self.assertEqual(result.returncode, 0) From 340acfe65f69f28414783fce5e3cb926c842d842 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Thu, 17 Sep 2026 13:22:09 -0700 Subject: [PATCH 225/311] feat: harden private Denmark and Italy candidate review --- docs/review-packet-denmark.md | 7 + docs/review-packet-italy.md | 10 ++ pipeline/common/graph_candidates.py | 109 ++++++++++++++ pipeline/common/identity.py | 12 +- pipeline/common/orchestrator.py | 7 +- pipeline/common/review_metrics.py | 141 ++++++++++++++++++ pipeline/common/review_packet.py | 21 ++- pipeline/common/test_delta.py | 10 ++ pipeline/common/test_graph_candidates.py | 35 +++++ pipeline/common/test_review_metrics.py | 25 ++++ pipeline/contracts/private_run.py | 11 ++ .../diagnostics/geospatial_readiness_audit.py | 12 +- pipeline/sources/denmark/adapter.py | 25 ++++ pipeline/sources/denmark/pipeline.py | 29 +++- pipeline/sources/italy/it_853_adapter.py | 31 +++- 15 files changed, 479 insertions(+), 6 deletions(-) create mode 100644 pipeline/common/graph_candidates.py create mode 100644 pipeline/common/review_metrics.py create mode 100644 pipeline/common/test_graph_candidates.py create mode 100644 pipeline/common/test_review_metrics.py diff --git a/docs/review-packet-denmark.md b/docs/review-packet-denmark.md index 85e96e0..0badf70 100644 --- a/docs/review-packet-denmark.md +++ b/docs/review-packet-denmark.md @@ -14,3 +14,10 @@ The bounded release lane and current row-free evidence are recorded in [`docs/reviewed-demonstration-release.md`](reviewed-demonstration-release.md) and [`data/manifests/reviewed-demonstration-release-2026-09-17.json`](../data/manifests/reviewed-demonstration-release-2026-09-17.json). That evidence records no public rows and does not represent a release approval. + +The shared private packet now also records aggregate facility-versus- +observation counts, classification review-state counts, coordinate precision +and coordinate-gate counts, source-native graph-candidate counts, and +not-observed diff semantics. These fields are row-free; graph candidate JSONL +remains restricted private evidence. Explicit CVR-to-Find-Smiley-ID operator +edges are emitted only as `review_required` candidates and never as merges. diff --git a/docs/review-packet-italy.md b/docs/review-packet-italy.md index f0ed40b..cbe83a5 100644 --- a/docs/review-packet-italy.md +++ b/docs/review-packet-italy.md @@ -8,4 +8,14 @@ As of 2026-09-15, `it.853-2004` has catalog-linked private acquisition and candi - Classification: recognition number plus activity code is provisional source identity; repeated pairs quarantine. Source establishment/activity categories and codes are emitted as coverage diagnostics without collapsing them. - Coverage/lifecycle: invalid dates, missing geography, unknown status, and coordinate precision remain explicit. Source disappearance is not closure; candidate import/API checks are private and test-only. +The shared row-free packet reports source-row observations separately from +provisional recognition-number facility groups, including repeated-group and +quarantine counts. Repeated recognition/activity rows remain isolated with +occurrence-qualified source keys; they are not merged or treated as separate +canonical facilities. Classification is source-preserved with an explicit +`review_required` decision, while coordinates remain source-precision-unknown +and privacy-gated. Where the source supplies a recognition number plus VAT or +fiscal identifier in the same observation, a private operator graph candidate +is emitted with `review_required`, never a canonical identity or public edge. + Evidence: `pipeline/sources/italy/`, `docs/country-recon-it.md`, and `pipeline/tests/e2e/test_italy_candidate_import.py`. diff --git a/pipeline/common/graph_candidates.py b/pipeline/common/graph_candidates.py new file mode 100644 index 0000000..e6cecd6 --- /dev/null +++ b/pipeline/common/graph_candidates.py @@ -0,0 +1,109 @@ +"""Shared private graph-candidate generation from explicit source identifiers.""" +from __future__ import annotations + +import hashlib +from pathlib import Path +from typing import Any, Iterable + +from pipeline.contracts.graph_candidate_handoff import ( + CONTRACT_VERSION, + canonical_json_bytes, +) +from pipeline.contracts.source_lifecycle import atomic_bytes, atomic_json + + +def _text(value: Any) -> str | None: + return value.strip() if isinstance(value, str) and value.strip() else None + + +def _stable_ref(prefix: str, value: str) -> str: + digest = hashlib.sha256(f"{prefix}|{value}".encode()).hexdigest()[:16] + return f"{prefix}:{digest}" + + +def build_identifier_graph_candidate( + *, + source_id: str, + source_record_key: str, + source_values: dict[str, Any], + facility_identifier: tuple[str, str] | None, + organization_identifier: tuple[str, str] | None, + observed_at: str, + source_row: int = 1, +) -> dict[str, Any]: + """Build a private candidate without fuzzy matching or canonical IDs.""" + if not _text(source_record_key): + raise ValueError("source_record_key is required for graph candidate") + if not facility_identifier and not organization_identifier: + raise ValueError("an explicit facility or organization identifier is required") + facilities: list[dict[str, Any]] = [] + organizations: list[dict[str, Any]] = [] + facility_ref = None + organization_ref = None + if facility_identifier: + identifier_type, value = facility_identifier + facility_ref = _stable_ref("facility", f"{source_id}|{identifier_type}|{value}") + facilities.append({"local_ref": facility_ref, "source_identifier": {"identifier_type": identifier_type, "value": value, "identity_scope": "source_scoped"}}) + if organization_identifier: + identifier_type, value = organization_identifier + organization_ref = _stable_ref("organization", f"{source_id}|{identifier_type}|{value}") + organizations.append({"local_ref": organization_ref, "source_identifier": {"identifier_type": identifier_type, "value": value, "identity_scope": "source_scoped"}}) + relationships = [] + if organization_ref and facility_ref: + relationships.append({ + "relationship_type": "operator", + "from_organization_ref": organization_ref, + "target_facility_ref": facility_ref, + "assertion_status": "asserted", + "observed_at": observed_at, + "valid_from": None, + "valid_to": None, + "confidence": 1.0, + "review_state": "review_required", + "evidence_method": "explicit source-native identifiers in one source observation", + }) + return { + "contract_version": CONTRACT_VERSION, + "source_id": source_id, + "source_record_key": source_record_key, + "source_row": source_row if isinstance(source_row, int) and source_row > 0 else 1, + "source_values": source_values, + "facilities": facilities, + "organizations": organizations, + "relationships": relationships, + "claims": [], + "crosswalks": [], + "publication": { + "storage_state": "private", + "privacy_status": "pending", + "review_state": "review_required", + "publication_status": "not_eligible", + "release_id": None, + }, + } + + +def write_graph_candidates(run_dir: str | Path, candidates: Iterable[dict[str, Any]]) -> dict[str, Any]: + """Write deterministic private JSONL graph candidates and row-free manifest.""" + ordered = sorted((candidate for candidate in candidates), key=lambda value: (value["source_id"], value["source_record_key"])) + payload = b"".join(canonical_json_bytes(candidate) for candidate in ordered) + root = Path(run_dir) + target = root / "graph-candidates.jsonl" + atomic_bytes(target, payload) + relationship_count = sum(len(candidate.get("relationships", [])) for candidate in ordered) + manifest = { + "schema_version": "private-graph-candidate-batch-v1", + "contract_version": CONTRACT_VERSION, + "candidate_count": len(ordered), + "facility_identifier_candidates": sum(bool(candidate.get("facilities")) for candidate in ordered), + "organization_identifier_candidates": sum(bool(candidate.get("organizations")) for candidate in ordered), + "operator_relationship_candidates": relationship_count, + "review_required_count": len(ordered), + "publication_status": "not_eligible", + "storage_state": "private", + "sha256": hashlib.sha256(payload).hexdigest(), + "byte_size": len(payload), + "row_payloads_included": True, + } + atomic_json(root / "graph-candidates-manifest.json", manifest) + return manifest diff --git a/pipeline/common/identity.py b/pipeline/common/identity.py index bb2c581..8a5bc17 100644 --- a/pipeline/common/identity.py +++ b/pipeline/common/identity.py @@ -11,7 +11,7 @@ } -def record_key(record: dict[str, Any]) -> str | tuple[str, str, str]: +def record_key(record: dict[str, Any]) -> str | tuple[str, str] | tuple[str, str, str]: """Keep UK establishment IDs distinct across feeds and nations. Other adapters currently use a record-level source_id. Do not infer a UK @@ -29,4 +29,14 @@ def record_key(record: dict[str, Any]) -> str | tuple[str, str, str]: return source_id, nation.strip(), identifier.strip() if not isinstance(source_id, str) or not source_id: raise ValueError("record lacks stable source_id") + # Source adapters commonly carry many observations under one feed-level + # source_id. Prefer the source-native row key when it is present; using + # only source_id silently collapses Italy activity observations (and any + # future multi-row source) during diffs and suppression checks. + source_record_key = record.get("source_record_key") + if isinstance(source_record_key, str) and source_record_key.strip(): + return source_id, source_record_key.strip() + source_row_id = record.get("source_row_id") + if isinstance(source_row_id, str) and source_row_id.strip(): + return source_id, source_row_id.strip() return source_id diff --git a/pipeline/common/orchestrator.py b/pipeline/common/orchestrator.py index c116d8c..27bd411 100644 --- a/pipeline/common/orchestrator.py +++ b/pipeline/common/orchestrator.py @@ -100,7 +100,12 @@ def run_registered_input(raw_path: str | Path, runs_dir: str | Path, config: dic if "manifest" in status and isinstance(status["manifest"], dict) and status["manifest"].get("source_id"): manifest = status["manifest"] try: - write_private_run_report(run_dir, manifest) + write_private_run_report( + run_dir, + manifest, + normalized_path=run_dir / "normalized" / "records.jsonl", + previous_normalized_path=previous_normalized_path, + ) if manifest.get("publication_state") == "private-candidate": as_of = config.get("health_as_of_utc") or config.get("retrieved_at_utc") if as_of: diff --git a/pipeline/common/review_metrics.py b/pipeline/common/review_metrics.py new file mode 100644 index 0000000..16e3171 --- /dev/null +++ b/pipeline/common/review_metrics.py @@ -0,0 +1,141 @@ +"""Row-free aggregate metrics shared by private source review packets.""" +from __future__ import annotations + +from collections import Counter +from typing import Any, Iterable + + +def _record(value: Any) -> dict[str, Any] | None: + if not isinstance(value, dict): + return None + nested = value.get("record") + return nested if isinstance(nested, dict) else value + + +def _text(value: Any) -> str | None: + return value.strip() if isinstance(value, str) and value.strip() else None + + +def _normalized(row: dict[str, Any]) -> dict[str, Any]: + value = row.get("normalized") + return value if isinstance(value, dict) else row + + +def _facility_key(row: dict[str, Any]) -> str | None: + normalized = _normalized(row) + for field in ("facility_key", "facility_id", "recognition_number", "establishment_id"): + value = _text(normalized.get(field)) + if value: + return value + # A source row key is a safe fallback for sources whose rows are already + # facility observations; this does not assert a cross-source identity. + return _text(row.get("source_record_key")) or _text(row.get("source_row_id")) + + +def _observation_key(row: dict[str, Any]) -> str | None: + return _text(row.get("source_record_key")) or _text(row.get("source_row_id")) + + +def _coordinate_state(row: dict[str, Any]) -> str: + normalized = _normalized(row) + coordinates = normalized.get("coordinates") + if isinstance(coordinates, (list, tuple)) and len(coordinates) == 2: + try: + lon, lat = float(coordinates[0]), float(coordinates[1]) + if -180 <= lon <= 180 and -90 <= lat <= 90 and not (lon == 0 and lat == 0): + return "valid_point" + except (TypeError, ValueError): + pass + return "invalid_point" + state = _text(normalized.get("coordinate_state")) + if state: + return state + coordinate_gate = _text(normalized.get("coordinate_gate")) + if coordinate_gate: + return coordinate_gate + return "missing" + + +def _classification_state(row: dict[str, Any]) -> tuple[str, str]: + normalized = _normalized(row) + classification = normalized.get("classification") + if isinstance(classification, dict): + return ( + _text(classification.get("review_status")) or "unknown", + "present", + ) + return (_text(normalized.get("classification_state")) or "unknown", "absent") + + +def _group_counts(rows: Iterable[dict[str, Any]], key_fn) -> tuple[int, int, int]: + keys = [key_fn(row) for row in rows] + present = [key for key in keys if key is not None] + counts = Counter(present) + return len(set(present)), sum(count > 1 for count in counts.values()), max(counts.values(), default=0) + + +def build_private_review_metrics( + normalized_rows: Iterable[dict[str, Any]], + quarantined_rows: Iterable[dict[str, Any]] = (), +) -> dict[str, Any]: + """Summarize private rows without copying names, identifiers, or values. + + The metrics intentionally describe source-row observations separately from + provisional facility grouping. They are not identity resolution or + publication approval. + """ + accepted = [record for row in normalized_rows if (record := _record(row)) is not None] + quarantined = [record for row in quarantined_rows if (record := _record(row)) is not None] + all_rows = accepted + quarantined + + facility_count, repeated_facilities, max_rows_per_facility = _group_counts(all_rows, _facility_key) + accepted_facilities, _, _ = _group_counts(accepted, _facility_key) + observation_count, repeated_observations, _ = _group_counts(all_rows, _observation_key) + + classifications = Counter() + classification_presence = Counter() + coordinate_states = Counter() + coordinate_precision = Counter() + coordinate_gates = Counter() + status_states = Counter() + for row in all_rows: + normalized = _normalized(row) + classification_state, presence = _classification_state(row) + classifications[classification_state] += 1 + classification_presence[presence] += 1 + coordinate_states[_coordinate_state(row)] += 1 + precision = _text(normalized.get("geography_precision")) or _text(normalized.get("coordinate_precision")) + coordinate_precision[precision or ("point-precision-unknown" if _coordinate_state(row) == "valid_point" else "unresolved")] += 1 + coordinate_gates[_text(normalized.get("coordinate_gate")) or "not-specified"] += 1 + status_states[_text(normalized.get("status_state")) or "not-specified"] += 1 + + return { + "schema_version": "private-review-metrics-v1", + "facility_observation": { + "source_row_unit": "source observation", + "input_observations": len(all_rows), + "accepted_observations": len(accepted), + "quarantined_observations": len(quarantined), + "distinct_provisional_facility_keys": facility_count, + "accepted_distinct_provisional_facility_keys": accepted_facilities, + "distinct_observation_keys": observation_count, + "repeated_provisional_facility_groups": repeated_facilities, + "repeated_observation_keys": repeated_observations, + "max_observations_per_provisional_facility": max_rows_per_facility, + "identity_semantics": "source-scoped provisional grouping only; no canonical merge", + "disappearance_semantics": "not-observed; never inferred as closure", + }, + "classification": { + "rows": len(all_rows), + "review_state_counts": dict(sorted(classifications.items())), + "field_presence": dict(sorted(classification_presence.items())), + "status_state_counts": dict(sorted(status_states.items())), + "interpretation": "source classifications remain separate from project approval", + }, + "geospatial": { + "coordinate_state_counts": dict(sorted(coordinate_states.items())), + "precision_counts": dict(sorted(coordinate_precision.items())), + "coordinate_gate_counts": dict(sorted(coordinate_gates.items())), + "interpretation": "coordinate presence and precision do not establish privacy eligibility or publication approval", + }, + } diff --git a/pipeline/common/review_packet.py b/pipeline/common/review_packet.py index 259940b..b56ee32 100644 --- a/pipeline/common/review_packet.py +++ b/pipeline/common/review_packet.py @@ -107,6 +107,11 @@ def build_review_packet( status = _read_json(root / "run-status.json") counts = _counts(manifest, qa) normalized = root / "normalized" / "records.jsonl" + packet_blockers: dict[str, list[str]] = {key: list(values) for key, values in (blockers or {}).items()} + review_metrics = qa.get("review_metrics", {}) + facility_observation = review_metrics.get("facility_observation", {}) if isinstance(review_metrics, dict) else {} + classification = review_metrics.get("classification", {}) if isinstance(review_metrics, dict) else {} + geospatial = review_metrics.get("geospatial", {}) if isinstance(review_metrics, dict) else {} packet: dict[str, Any] = { "schema_version": REVIEW_PACKET_VERSION, "source_id": manifest.get("source_id"), @@ -123,6 +128,10 @@ def build_review_packet( "schema_status": manifest.get("schema_status", "not-reported"), }, "counts": counts, + "review_metrics": review_metrics, + "facility_observation": facility_observation, + "classification": classification, + "geospatial": geospatial, "quarantine": { "rows": manifest.get("quarantined_rows"), "reasons": manifest.get("anomaly_counts", {}), @@ -141,10 +150,20 @@ def build_review_packet( }, "platform": platform_context(manifest.get("source_id")), "publication_boundary": "awaiting-owner-review; this packet is row-free evidence and cannot approve or promote a release", - "blockers": blockers or {}, + "blockers": packet_blockers, } if not counts["reconciles"] or not counts["qa_matches_manifest"]: packet["blockers"].setdefault("validation", []).append("manifest and QA row counts must reconcile") + if facility_observation.get("repeated_provisional_facility_groups", 0): + packet["blockers"].setdefault("identity", []).append("repeated provisional facility groups require source-scoped identity review; rows were not merged") + if classification.get("review_state_counts", {}).get("review_required", 0) or classification.get("review_state_counts", {}).get("unknown", 0): + packet["blockers"].setdefault("classification", []).append("classification remains source-preserved or unresolved and requires scoped human review") + pending_coordinates = sum( + value for key, value in geospatial.get("coordinate_gate_counts", {}).items() + if key not in {"passed", "not-required"} and isinstance(value, int) + ) + if pending_coordinates: + packet["blockers"].setdefault("geospatial", []).append("coordinate precision/privacy review remains open; coordinate success is not publication approval") if packet["gates"]["release_state"] != "not-created" or packet["gates"]["release_promoted"] is not False: packet["blockers"].setdefault("release", []).append("private review requires release_state=not-created and release_promoted=false") _assert_row_free(packet) diff --git a/pipeline/common/test_delta.py b/pipeline/common/test_delta.py index 8fb1ac7..1f04688 100644 --- a/pipeline/common/test_delta.py +++ b/pipeline/common/test_delta.py @@ -70,6 +70,16 @@ def uk(nation, name): result = compare_runs(old, new, {("fsa_approved_establishments", "Wales", "00017")}) self.assertEqual(result["counts"], {"added": 0, "changed": 1, "not_observed": 0, "suppressed": 1}) + def test_source_native_row_keys_keep_multi_observation_feeds_distinct(self): + def italy(key, name): + return {"source_id": "it.853-2004", "source_record_key": key, "normalized": {"recognition_number": key, "name": name}} + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + old = write_run(root, "old", [italy("REC|A|1", "same"), italy("REC|B|1", "same")]) + new = write_run(root, "new", [italy("REC|A|1", "changed"), italy("REC|B|1", "same")]) + result = compare_runs(old, new) + self.assertEqual(result["counts"], {"added": 0, "changed": 1, "not_observed": 0, "suppressed": 0}) + if __name__ == "__main__": unittest.main() diff --git a/pipeline/common/test_graph_candidates.py b/pipeline/common/test_graph_candidates.py new file mode 100644 index 0000000..0b6ae98 --- /dev/null +++ b/pipeline/common/test_graph_candidates.py @@ -0,0 +1,35 @@ +import json +import tempfile +import unittest +from pathlib import Path + +from pipeline.common.graph_candidates import build_identifier_graph_candidate, write_graph_candidates +from pipeline.contracts.graph_candidate_handoff import validate_graph_candidate + + +class GraphCandidateGenerationTests(unittest.TestCase): + def test_explicit_facility_operator_identifiers_emit_review_required_edge(self): + candidate = build_identifier_graph_candidate( + source_id="it.853-2004", + source_record_key="REC|ACT|1", + source_values={"recognition": "REC", "vat": "VAT"}, + facility_identifier=("eu_recognition_number", "REC"), + organization_identifier=("italian_vat", "VAT"), + observed_at="2026-09-17T00:00:00Z", + source_row=2, + ) + validate_graph_candidate(candidate) + self.assertEqual(len(candidate["relationships"]), 1) + self.assertEqual(candidate["relationships"][0]["relationship_type"], "operator") + self.assertEqual(candidate["relationships"][0]["review_state"], "review_required") + self.assertEqual(candidate["publication"]["publication_status"], "not_eligible") + with tempfile.TemporaryDirectory() as directory: + manifest = write_graph_candidates(directory, [candidate]) + self.assertEqual(manifest["operator_relationship_candidates"], 1) + payload = Path(directory, "graph-candidates.jsonl").read_text(encoding="utf-8") + self.assertEqual(len(payload.splitlines()), 1) + self.assertNotIn("global_id", payload) + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/common/test_review_metrics.py b/pipeline/common/test_review_metrics.py new file mode 100644 index 0000000..d470083 --- /dev/null +++ b/pipeline/common/test_review_metrics.py @@ -0,0 +1,25 @@ +import unittest + +from .review_metrics import build_private_review_metrics + + +class ReviewMetricsTests(unittest.TestCase): + def test_facility_observation_and_precision_metrics_are_aggregate_only(self): + metrics = build_private_review_metrics( + [ + {"source_record_key": "a", "normalized": {"recognition_number": "R", "classification_state": "source-category-preserved", "coordinate_state": "source-value-present-pending-review", "coordinate_precision": "source-precision-unknown", "coordinate_gate": "review_required"}}, + {"source_record_key": "b", "normalized": {"recognition_number": "R", "classification_state": "unknown", "coordinate_state": "unknown", "coordinate_gate": "review_required"}}, + ], + [{"record": {"source_record_key": "c", "normalized": {"recognition_number": "R"}}, "reasons": ["ambiguous"]}], + ) + facility = metrics["facility_observation"] + self.assertEqual(facility["input_observations"], 3) + self.assertEqual(facility["distinct_provisional_facility_keys"], 1) + self.assertEqual(facility["repeated_provisional_facility_groups"], 1) + self.assertEqual(metrics["geospatial"]["precision_counts"]["source-precision-unknown"], 1) + self.assertNotIn("R", str(metrics)) + self.assertNotIn("source_record_key", str(metrics)) + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/contracts/private_run.py b/pipeline/contracts/private_run.py index 0ee8c46..7049b35 100644 --- a/pipeline/contracts/private_run.py +++ b/pipeline/contracts/private_run.py @@ -6,6 +6,7 @@ from typing import Any, Iterable from .adapter_contract import SourceAdapter, SourceArtifact +from pipeline.common.review_metrics import build_private_review_metrics from .source_lifecycle import atomic_json, validate_private_manifest @@ -95,12 +96,22 @@ def write_private_run_report( previous_normalized_path: str | Path | None = None, drift_alarms: Iterable[str] = (), ) -> dict[str, Any]: + run_root = Path(run_dir) + normalized_path = normalized_path or run_root / "normalized" / "records.jsonl" + quarantine_path = run_root / "quarantined" / "records.jsonl" + normalized_rows = [] + quarantined_rows = [] + if Path(normalized_path).is_file(): + normalized_rows = [json.loads(line) for line in Path(normalized_path).read_text(encoding="utf-8").splitlines() if line.strip()] + if quarantine_path.is_file(): + quarantined_rows = [json.loads(line) for line in quarantine_path.read_text(encoding="utf-8").splitlines() if line.strip()] report = summarize_private_run( manifest, normalized_path=normalized_path, previous_normalized_path=previous_normalized_path, drift_alarms=drift_alarms, ) + report["review_metrics"] = build_private_review_metrics(normalized_rows, quarantined_rows) atomic_json(Path(run_dir) / "qa.json", report) return report diff --git a/pipeline/scripts/diagnostics/geospatial_readiness_audit.py b/pipeline/scripts/diagnostics/geospatial_readiness_audit.py index 2f5004c..0296f47 100644 --- a/pipeline/scripts/diagnostics/geospatial_readiness_audit.py +++ b/pipeline/scripts/diagnostics/geospatial_readiness_audit.py @@ -21,6 +21,7 @@ def valid_point(lat, lon): def _audit_rows(files, sample_size, as_of, corpus): funnel = Counter(); countries = Counter(); strata = Counter(); sample_buckets = {} + precision_counts = Counter(); coordinate_review_states = Counter() file_manifest = [] for path in files: digest = hashlib.sha256(path.read_bytes()).hexdigest() @@ -41,6 +42,15 @@ def _audit_rows(files, sample_size, as_of, corpus): lon, lat = num(coordinates[0]), num(coordinates[1]) coord = "source_coordinate_valid" if valid_point(lat, lon) else "source_coordinate_invalid_or_missing" funnel[coord] += 1 + supplied_state = normalized.get("coordinate_state") or normalized.get("coordinate_review_state") + if valid_point(lat, lon): + precision = normalized.get("geography_precision") or normalized.get("coordinate_precision") or "point-precision-unknown" + elif supplied_state: + precision = normalized.get("geography_precision") or normalized.get("coordinate_precision") or "source-precision-unknown" + else: + precision = "unresolved" + precision_counts[str(precision)] += 1 + coordinate_review_states[str(normalized.get("coordinate_gate") or supplied_state or "not-specified")] += 1 address_values = (normalized.get("street", row.get("street")), normalized.get("zip", row.get("zip")), normalized.get("city", row.get("city")), normalized.get("state", row.get("state"))) address = " ".join(filter(None, address_values)) quality = "address_complete" if all(address_values) else ("city_only" if address_values[2] else "address_missing") @@ -60,7 +70,7 @@ def _audit_rows(files, sample_size, as_of, corpus): samples = [v for _, v in sorted(samples)[:sample_size * max(1, len(files))]] composition = Counter("|".join((x["country"], x["address_quality"], x["coordinate_state"])) for x in samples) funnel.setdefault("total", 0) - return {"corpus": corpus, "schema_version": "geospatial-readiness-audit/v1", "as_of": as_of or datetime.now(timezone.utc).isoformat(), "method": "offline deterministic audit; no geocoder calls", "funnel": dict(sorted(funnel.items())), "country_counts": dict(sorted(countries.items())), "strata_counts": {"|".join(k): v for k,v in sorted(strata.items())}, "sample": {"size": len(samples), "composition": dict(composition)}, "sample_rows": samples, "source_files": file_manifest, "provenance_fields_required_for_any_geocode": ["provider", "query_hash", "queried_at", "precision", "review_state"], "publication_rule": "map-ready requires current privacy eligibility and release approval; geocoding success alone never grants publication permission"} + return {"corpus": corpus, "schema_version": "geospatial-readiness-audit/v1", "as_of": as_of or datetime.now(timezone.utc).isoformat(), "method": "offline deterministic audit; no geocoder calls", "funnel": dict(sorted(funnel.items())), "country_counts": dict(sorted(countries.items())), "strata_counts": {"|".join(k): v for k,v in sorted(strata.items())}, "precision_counts": dict(sorted(precision_counts.items())), "coordinate_review_state_counts": dict(sorted(coordinate_review_states.items())), "sample": {"size": len(samples), "composition": dict(composition)}, "sample_rows": samples, "source_files": file_manifest, "provenance_fields_required_for_any_geocode": ["provider", "query_hash", "queried_at", "precision", "review_state"], "publication_rule": "map-ready requires current privacy eligibility and release approval; geocoding success alone never grants publication permission", "precision_rule": "a valid point with unknown precision remains precision-unknown until source or geocoder precision is reviewed"} def _v2_files(root: Path): manifests = sorted(root.rglob("manifest.json")) if root.exists() else [] diff --git a/pipeline/sources/denmark/adapter.py b/pipeline/sources/denmark/adapter.py index d37dfbb..2d0b3fd 100644 --- a/pipeline/sources/denmark/adapter.py +++ b/pipeline/sources/denmark/adapter.py @@ -11,6 +11,8 @@ from pipeline.contracts.adapter_contract import SourceArtifact from pipeline.contracts.candidate_handoff import write_handoff from pipeline.contracts.source_lifecycle import atomic_json, atomic_jsonl, private_manifest +from pipeline.common.graph_candidates import build_identifier_graph_candidate, write_graph_candidates +from pipeline.common.review_metrics import build_private_review_metrics SOURCE_ID = "dk.smiley" ADAPTER_VERSION = "denmark-smiley-contract-v1" @@ -103,6 +105,23 @@ def run(self, raw_path: str | Path, run_dir: str | Path, artifact: SourceArtifac _, parsed_hash, _ = atomic_jsonl(root / "parsed" / "records.jsonl", parsed_rows) _, normalized_hash, _ = atomic_jsonl(root / "normalized" / "records.jsonl", rows) atomic_jsonl(root / "quarantined" / "records.jsonl", quarantined) + graph_candidates = [] + for record in parsed_rows: + fields = record.get("source_fields", {}) + key = record.get("source_record_key") + if not isinstance(key, str) or not key: + continue + cvr = fields.get("CVR_nummer") or fields.get("cvrnr") + graph_candidates.append(build_identifier_graph_candidate( + source_id=SOURCE_ID, + source_record_key=key, + source_values=fields, + facility_identifier=("findsmiley_id", key), + organization_identifier=("cvr", str(cvr).strip()) if cvr and str(cvr).strip() else None, + observed_at=artifact.retrieved_at_utc, + source_row=record.get("source_row", 1), + )) + graph_manifest = write_graph_candidates(root / "graph", graph_candidates) # This state is deliberately private: validation cannot authorize release. manifest = private_manifest( source_id=SOURCE_ID, @@ -117,4 +136,10 @@ def run(self, raw_path: str | Path, run_dir: str | Path, artifact: SourceArtifac anomaly_counts={"missing_source_key": len(quarantined)} if quarantined else {}, ) atomic_json(root / "manifest.json", manifest) + manifest["review_metrics"] = build_private_review_metrics(rows, quarantined) + manifest["graph_candidate_summary"] = {key: graph_manifest[key] for key in ( + "candidate_count", "facility_identifier_candidates", "organization_identifier_candidates", + "operator_relationship_candidates", "review_required_count", "publication_status", + )} + atomic_json(root / "manifest.json", manifest) return manifest diff --git a/pipeline/sources/denmark/pipeline.py b/pipeline/sources/denmark/pipeline.py index ac4fb07..7ef958f 100644 --- a/pipeline/sources/denmark/pipeline.py +++ b/pipeline/sources/denmark/pipeline.py @@ -18,6 +18,7 @@ from pipeline.contracts.adapter_contract import SourceArtifact from pipeline.common.review import write_operator_review_packet from pipeline.common.review_packet import write_review_packet +from pipeline.common.graph_candidates import build_identifier_graph_candidate, write_graph_candidates from .adapter import DenmarkSmileyAdapter @@ -119,7 +120,11 @@ def _canonical_evidence(run_dir: Path, input_path: Path, metadata: dict, } validate_private_manifest(manifest) atomic_json(run_dir / "manifest.json", manifest) - report = write_private_run_report(run_dir, manifest) + report = write_private_run_report( + run_dir, + manifest, + normalized_path=run_dir / "03-classify" / "classified-records.jsonl", + ) # Using the recorded observation time as the default makes local reruns # byte-identical. Callers needing wall-clock freshness may override it. if retrieved: @@ -137,6 +142,28 @@ def _canonical_evidence(run_dir: Path, input_path: Path, metadata: dict, ) classified_rows = [json.loads(line) for line in classified.read_text(encoding="utf-8").splitlines() if line] DenmarkSmileyAdapter().write_candidate_handoff(run_dir / "candidate-handoff", artifact, classified_rows) + graph_candidates = [] + for row in classified_rows: + key = row.get("source_record_key") + fields = row.get("source_fields", {}) + if not isinstance(key, str) or not key or not isinstance(fields, dict): + continue + cvr = fields.get("CVR_nummer") or fields.get("cvrnr") + graph_candidates.append(build_identifier_graph_candidate( + source_id="dk.smiley", + source_record_key=key, + source_values=fields, + facility_identifier=("findsmiley_id", key), + organization_identifier=("cvr", str(cvr).strip()) if cvr and str(cvr).strip() else None, + observed_at=str(retrieved), + source_row=row.get("source_row", 1), + )) + graph_summary = write_graph_candidates(run_dir / "graph", graph_candidates) + manifest["graph_candidate_summary"] = {key: graph_summary[key] for key in ( + "candidate_count", "facility_identifier_candidates", "organization_identifier_candidates", + "operator_relationship_candidates", "review_required_count", "publication_status", + )} + atomic_json(run_dir / "manifest.json", manifest) write_operator_review_packet( run_dir, manifest, diff --git a/pipeline/sources/italy/it_853_adapter.py b/pipeline/sources/italy/it_853_adapter.py index c6d0c42..ad44aa4 100644 --- a/pipeline/sources/italy/it_853_adapter.py +++ b/pipeline/sources/italy/it_853_adapter.py @@ -18,6 +18,8 @@ from pipeline.contracts.adapter_contract import SourceArtifact from pipeline.contracts.candidate_handoff import write_handoff from pipeline.contracts.source_lifecycle import atomic_json, atomic_jsonl, private_manifest +from pipeline.common.graph_candidates import build_identifier_graph_candidate, write_graph_candidates +from pipeline.common.review_metrics import build_private_review_metrics REQUIRED = tuple( @@ -133,6 +135,8 @@ def parse_bytes(self, content: bytes) -> dict[str, Any]: "recognition_number": rec, "source_activity_code": activity, "facility_grouping": "provisional-recognition-number", + "facility_identity_state": "provisional-source-recognition-number", + "observation_identity_state": "source-row-with-occurrence", "name": clean(row.get("ragione_sociale")), "trading_name": clean(row.get("ragione_sociale")), "address": None, @@ -146,6 +150,8 @@ def parse_bytes(self, content: bytes) -> dict[str, Any]: "geography_state": "source-municipality-code" if geography_precision != "unknown" else "unknown", "classification": clean(row.get("classificazione_stabilimento")), "classification_state": "source-category-preserved" if clean(row.get("classificazione_stabilimento")) else "unknown", + "classification_decision": "review_required", + "activity_state": "source-code-preserved-unmapped", "activity_code": activity, "activity_description": clean(row.get("descrizione_impianto_attivita")), "products": clean(row.get("prodotti_abilitati")), @@ -155,6 +161,7 @@ def parse_bytes(self, content: bytes) -> dict[str, Any]: "date_state": {field: _date_state(clean(row.get(field))) for field in DATE_FIELDS}, "coordinates": None, "coordinate_state": "source-value-present-pending-review" if clean(row.get("longitudine")) or clean(row.get("latitudine")) else "unknown", + "coordinate_precision": "source-precision-unknown" if clean(row.get("longitudine")) or clean(row.get("latitudine")) else "unresolved", "privacy_gate": "pending-review", "coordinate_gate": "review_required", "publication_gate": "blocked", @@ -189,6 +196,28 @@ def run(self, raw_path: str | Path, run_dir: str | Path, artifact: SourceArtifac _, parsed_sha256, _ = atomic_jsonl(root / "parsed" / "records.jsonl", parsed) _, normalized_sha256, _ = atomic_jsonl(root / "normalized" / "records.jsonl", accepted) atomic_jsonl(root / "quarantined" / "records.jsonl", quarantined) + graph_candidates = [] + for item in parsed: + record = item.get("record") if isinstance(item, dict) and isinstance(item.get("record"), dict) else item + if not isinstance(record, dict): + continue + source_key = record.get("source_record_key") + source_values = record.get("source_values") + if not isinstance(source_key, str) or not source_key or not isinstance(source_values, dict): + continue + recognition = clean(source_values.get("num_identificativo_produzione_commercializzazione")) + piva = clean(source_values.get("p_iva")) + fiscal = clean(source_values.get("cod_fiscale")) + graph_candidates.append(build_identifier_graph_candidate( + source_id=self.source_id, + source_record_key=source_key, + source_values=source_values, + facility_identifier=("eu_recognition_number", recognition) if recognition else None, + organization_identifier=(("italian_vat", piva) if piva else ("italian_fiscal_code", fiscal) if fiscal else None), + observed_at=artifact.retrieved_at_utc, + source_row=record.get("source_row", 1), + )) + graph_manifest = write_graph_candidates(root / "graph", graph_candidates) anomaly_counts = Counter(reason for item in quarantined for reason in item["reasons"]) manifest = private_manifest( source_id=self.source_id, @@ -202,6 +231,6 @@ def run(self, raw_path: str | Path, run_dir: str | Path, artifact: SourceArtifac parsed_sha256=parsed_sha256, anomaly_counts=dict(sorted(anomaly_counts.items())), ) - manifest.update({"coverage": "Italian Ministry 853/2004 CSV; one source row per establishment/activity; 1069/2009 excluded", "geocoding": "disabled", "schema_fingerprint": result["schema_fingerprint"], "source_category_counts": result["source_category_counts"], "source_activity_counts": result["source_activity_counts"]}) + manifest.update({"coverage": "Italian Ministry 853/2004 CSV; one source row per establishment/activity; 1069/2009 excluded", "geocoding": "disabled", "schema_fingerprint": result["schema_fingerprint"], "source_category_counts": result["source_category_counts"], "source_activity_counts": result["source_activity_counts"], "review_metrics": build_private_review_metrics(accepted, quarantined), "graph_candidate_summary": {key: graph_manifest[key] for key in ("candidate_count", "facility_identifier_candidates", "organization_identifier_candidates", "operator_relationship_candidates", "review_required_count", "publication_status")}}) atomic_json(root / "manifest.json", manifest) return manifest From c133cfc0d2a28d1fdccc27630212de61b6a7535d Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Thu, 17 Sep 2026 13:23:25 -0700 Subject: [PATCH 226/311] Advance France and UK private candidate handoffs --- docs/country-recon-fr.md | 28 ++- docs/country-recon-uk.md | 18 ++ pipeline/common/graph_candidate.py | 167 ++++++++++++++++++ pipeline/common/identity.py | 13 +- pipeline/common/privacy.py | 19 ++ pipeline/common/source_operations.py | 18 ++ pipeline/contracts/GRAPH-CANDIDATE-HANDOFF.md | 7 + pipeline/contracts/candidate_handoff.py | 13 +- pipeline/contracts/private_run.py | 30 ++-- .../contracts/test_graph_candidate_handoff.py | 19 ++ pipeline/sources/france/adapter.py | 13 +- pipeline/sources/france/refresh.py | 13 +- pipeline/sources/france/test_adapter.py | 9 + pipeline/sources/france/test_refresh.py | 14 ++ pipeline/sources/uk/fsa_approved/adapter.py | 13 +- pipeline/sources/uk/fsa_approved/refresh.py | 34 ++-- pipeline/sources/uk/fss_approved/adapter.py | 12 +- pipeline/sources/uk/fss_approved/refresh.py | 9 +- 18 files changed, 393 insertions(+), 56 deletions(-) create mode 100644 pipeline/common/graph_candidate.py create mode 100644 pipeline/common/privacy.py diff --git a/docs/country-recon-fr.md b/docs/country-recon-fr.md index af61372..2419ee9 100644 --- a/docs/country-recon-fr.md +++ b/docs/country-recon-fr.md @@ -1,6 +1,6 @@ # France source reconnaissance -Status: reconnaissance only. No adapter, release, publication, or row-level fixture was created. No row-level records, personal names, addresses, contacts, coordinates, or private artifacts are retained here. +Status: private candidate implementation and source assessment. No release, publication, or row-level fixture is created in Git; real artifacts remain restricted and ignored. Last checked: 2026-09-14 UTC under `docs/ETHICS.md`, policy version 1.0, last reviewed 2026-09-12. This document is source-status evidence, not publication approval or a healthy-pipeline claim. @@ -8,7 +8,7 @@ Last checked: 2026-09-14 UTC under `docs/ETHICS.md`, policy version 1.0, last re | Source | Discovered / verified | Acquisition | Adapter / validation | Terms / privacy | Blocker / next action | |---|---|---|---|---|---| -| DGAL approved CE lists | Official Ministry page and Section I/II TXT routes verified | HTTP 200 bounded retrieval; bytes discarded | Not started | Etalab attribution appears on Ministry page; confirm file-specific terms; screen names, addresses, and precise geocodes | Freeze category dictionary and daily snapshot/provenance handling, then human review | +| DGAL approved CE lists | Official Ministry page and Section I/II TXT routes verified | HTTP 200 bounded retrieval with recorded artifact provenance | Shared private lifecycle, source-local handoff, category/quarantine QA, row-free review packet, and graph-candidate rehearsal | Etalab attribution appears on Ministry page; confirm file-specific terms; screen names, addresses, and precise geocodes | Human terms, privacy, category, duplicate, and release review | | Alim’confiance | Official DGAL Opendatasoft dataset/API/CSV routes verified | Bounded API query HTTP 200; bytes discarded | Not started | Licence Ouverte 2.0 in data.gouv metadata; screen address/coordinate and farm/person fields | Define qualifying activity/agreement labels; do not ingest all food establishments | | INSEE SIRENE | Official open-data page, bulk route, and API terms verified | Not attempted; API account/subscription and multi-GB bulk | Not started | Licence Ouverte 2.0; diffusion-partielle and personal-data rules are material | Authorized access, partitioned import, NAF mapping, and privacy rules | | HVE directory | Official Ministry dataset/current CSV verified | HTTP 200 bounded retrieval; bytes discarded | Not started | Licence Ouverte 2.0; voluntary opt-in, head-office address, possible individual farm names | Treat only as labeled HVE subset, never exhaustive farm source | @@ -77,3 +77,27 @@ All listed requests were bounded and read-only; response bytes were discarded. N 4. Use SIRENE as a cross-source backbone only after authorized access and diffusion/privacy handling. 5. Treat Géorisques as a later regulatory complement after schema/licence/token/rubric review. 6. Obtain authorized project approval before any release; acquisition success and government origin are not publication authorization. + +## Private candidate implementation status (2026-09-17) + +Section I and Section II now run through the shared typed lifecycle and +candidate-handoff contract. Source values remain restricted; normalized rows +keep approval, SIRET, category, activity, species, section, and explicit +uncertainty states. Category codes are tokenized rather than substring-matched; +unknown categories and duplicate source observations stay quarantined. Address +and coordinate fields remain suppressed, geocoding is disabled, and the +non-public graph handoff contains only source-supported claims with +`review_required`, source-scoped identity, and `publication_status: not_eligible`. + +Refresh accepts `--previous-normalized`; the shared delta reports additions, +changes, and `not-observed` rows without inferring closure. Review packets are +aggregate-only and retain the prior-run linkage. France sections remain +separate sources and no candidate is owner-approved or public. + +The current checked-in evidence is aggregate only. The 2026-09-16 private +reacquisition manifest records 1,448 Section I rows and 1,068 Section II rows, +with zero quarantines in that snapshot; those counts are not a release decision +and the underlying artifacts are not present in Git. Remaining maintainer +decisions are file-specific rights/attribution, address/privacy disposition, +category codebook confirmation, duplicate identity handling, and any +release-specific project approval. diff --git a/docs/country-recon-uk.md b/docs/country-recon-uk.md index aa4c977..ff1f4a7 100644 --- a/docs/country-recon-uk.md +++ b/docs/country-recon-uk.md @@ -231,3 +231,21 @@ restricted at `data/restricted/country-recon/uk/runs/2026-09-14-refresh-check/re Prior-run comparisons report disappeared identifiers as `not-observed`; they never infer closure. The command keeps Scotland and Northern Ireland outside this source profile and does not create a release. + +### Shared candidate/graph handoff updates (2026-09-17) + +FSA England/Wales and FSS Scotland accepted rows now carry a stable, +nation-qualified `source_record_key`. The shared delta path therefore keeps +same-number records in different UK feeds or nations distinct; legacy prior-run +files without nation metadata retain a conservative compatibility alias. France +uses the same source-scoped identity rule for one-to-many approval/activity +observations. + +All candidate handoffs emit a private, deterministic graph-candidate set. It +contains only source-supported claims, retains `review_required` and +`publication_status: not_eligible`, performs no auto-merge, and emits no graph +candidate for quarantined rows. Quarantine reasons remain in the row-free QA and +review packet. Source coordinates remain null/suppressed and no geocoding is +enabled. Northern Ireland remains quarantined/out of scope in the monthly FSA +profile until its separate catalogue resource, terms, schema, and privacy +handling are independently supported. diff --git a/pipeline/common/graph_candidate.py b/pipeline/common/graph_candidate.py new file mode 100644 index 0000000..531391c --- /dev/null +++ b/pipeline/common/graph_candidate.py @@ -0,0 +1,167 @@ +"""Shared private graph-candidate generation for source adapters. + +The builder only uses identifiers and claims already present in one source +record. It never performs fuzzy matching, proximity joins, or universal-ID +assignment. Output is private review evidence and cannot authorize import or +publication. +""" +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +from typing import Any, Iterable + +from pipeline.contracts.graph_candidate_handoff import validate_graph_candidate + + +PUBLICATION = { + "storage_state": "private", + "privacy_status": "pending", + "review_state": "review_required", + "publication_status": "not_eligible", + "release_id": None, +} + + +def _text(value: Any) -> str | None: + return value.strip() if isinstance(value, str) and value.strip() else None + + +def _source_key(record: dict[str, Any]) -> str: + for key in ("source_record_key", "source_row_id"): + value = _text(record.get(key)) + if value: + return value + normalized = record.get("normalized") or {} + identifier = _text(normalized.get("establishment_id")) or _text(normalized.get("approval_number")) + if identifier: + return f"{record.get('source_id')}|{identifier}" + return f"{record.get('source_id')}|row-{record.get('source_row')}" + + +def _local_ref(prefix: str, source_id: str, source_key: str) -> str: + digest = hashlib.sha256(f"{source_id}|{source_key}".encode("utf-8")).hexdigest()[:20] + return f"{prefix}:{source_id}:{digest}" + + +def build_graph_candidate( + record: dict[str, Any], + *, + artifact_sha256: str | None = None, + observed_at: str | None = None, +) -> dict[str, Any]: + """Build one source-local, review-required graph candidate.""" + source_id = _text(record.get("source_id")) + source_key = _source_key(record) + if not source_id: + raise ValueError("graph candidate requires source_id") + normalized = record.get("normalized") + if not isinstance(normalized, dict): + raise ValueError("graph candidate requires normalized source evidence") + identifier = _text(normalized.get("establishment_id")) or _text(normalized.get("approval_number")) + if not identifier: + raise ValueError("graph candidate requires a source-native establishment identifier") + observed = _text(observed_at) or _text(normalized.get("observed_at")) or "unknown-observation-date" + facility_ref = _local_ref("facility", source_id, source_key) + facilities = [{ + "local_ref": facility_ref, + "source_identifier": { + "identifier_type": "source_establishment_id", + "value": identifier, + "identity_scope": "source_scoped", + }, + }] + + support: list[dict[str, str]] = [{"source_record_key": source_key}] + if artifact_sha256: + support.append({"artifact_sha256": artifact_sha256}) + claims: list[dict[str, Any]] = [{ + "claim_domain": "identity", + "claim_kind": "source_establishment", + "facility_ref": facility_ref, + "value_state": "known", + "value": identifier, + "observed_at": observed, + "confidence": None, + "review_state": "review_required", + "support": support, + }] + categories = normalized.get("activity_categories") or () + if categories: + claims.append({ + "claim_domain": "operation", + "claim_kind": "source_activity_categories", + "facility_ref": facility_ref, + "value_state": "known", + "value": list(categories), + "observed_at": observed, + "confidence": None, + "review_state": "review_required", + "support": support, + }) + source_label = _text(normalized.get("name")) or _text(normalized.get("trading_name")) + if source_label: + claims.append({ + "claim_domain": "identity", + "claim_kind": "source_label", + "facility_ref": facility_ref, + "value_state": "known", + "value": source_label, + "observed_at": observed, + "confidence": None, + "review_state": "review_required", + "support": support, + }) + + candidate = { + "contract_version": "graph-candidate-handoff-v1", + "source_id": source_id, + "source_record_key": source_key, + "source_row": record.get("source_row"), + "source_values": record.get("source_values", {}), + "facilities": facilities, + "organizations": [], + "relationships": [], + "claims": claims, + "crosswalks": [], + "contradiction_state": "none-observed", + "review_state": "review_required", + "publication": PUBLICATION, + } + validate_graph_candidate(candidate) + return candidate + + +def write_graph_candidates( + run_dir: str | Path, + records: Iterable[dict[str, Any]], + *, + artifact_sha256: str | None = None, + observed_at: str | None = None, + quarantined_rows: int = 0, +) -> dict[str, Any]: + """Write deterministic private graph candidates and a row-free manifest.""" + root = Path(run_dir) + root.mkdir(parents=True, exist_ok=True) + candidates = [build_graph_candidate(record, artifact_sha256=artifact_sha256, observed_at=observed_at) for record in records] + candidates.sort(key=lambda candidate: (candidate["source_id"], candidate["source_record_key"], candidate["source_row"])) + payload = b"".join((json.dumps(candidate, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + "\n").encode("utf-8") for candidate in candidates) + records_path = root / "records.jsonl" + records_path.write_bytes(payload) + manifest = { + "schema_version": "private-graph-candidate-set-v1", + "contract_version": "graph-candidate-handoff-v1", + "candidate_rows": len(candidates), + "quarantined_source_rows": quarantined_rows, + "records_sha256": hashlib.sha256(payload).hexdigest(), + "storage_state": "private", + "privacy_status": "pending", + "review_state": "review_required", + "publication_status": "not_eligible", + "release_id": None, + "auto_merge": False, + "contradictions": "source-local contradiction/review states preserved; no cross-source merge performed", + } + (root / "manifest.json").write_text(json.dumps(manifest, ensure_ascii=False, sort_keys=True, indent=2) + "\n", encoding="utf-8") + return manifest diff --git a/pipeline/common/identity.py b/pipeline/common/identity.py index 8a5bc17..d076a7d 100644 --- a/pipeline/common/identity.py +++ b/pipeline/common/identity.py @@ -29,14 +29,11 @@ def record_key(record: dict[str, Any]) -> str | tuple[str, str] | tuple[str, str return source_id, nation.strip(), identifier.strip() if not isinstance(source_id, str) or not source_id: raise ValueError("record lacks stable source_id") - # Source adapters commonly carry many observations under one feed-level - # source_id. Prefer the source-native row key when it is present; using - # only source_id silently collapses Italy activity observations (and any - # future multi-row source) during diffs and suppression checks. - source_record_key = record.get("source_record_key") + # Source adapters must be able to retain one-to-many observations (for + # example France approval/activity rows) without collapsing a whole feed + # to one source-level key. The key is still source-scoped and is never a + # universal identity assertion. + source_record_key = record.get("source_record_key") or record.get("source_row_id") if isinstance(source_record_key, str) and source_record_key.strip(): return source_id, source_record_key.strip() - source_row_id = record.get("source_row_id") - if isinstance(source_row_id, str) and source_row_id.strip(): - return source_id, source_row_id.strip() return source_id diff --git a/pipeline/common/privacy.py b/pipeline/common/privacy.py new file mode 100644 index 0000000..50f33ab --- /dev/null +++ b/pipeline/common/privacy.py @@ -0,0 +1,19 @@ +"""Shared conservative privacy signals for restricted source adapters. + +These are screening signals, not privacy clearance. A match blocks the +candidate row; a non-match leaves the row behind the explicit privacy gate. +""" +from __future__ import annotations + +import re + + +# Do not classify ordinary facility names such as ``House Farm`` as private on +# their own. Stronger residential/intermediary indicators still require a +# human decision before an address or point can be exposed. +ADDRESS_RISK = re.compile(r"\b(flat|apartment|residential|c/o|care\s+of|caravan)\b", re.I) + + +def address_privacy_risk(*values: object) -> bool: + """Return whether concatenated source address values need quarantine.""" + return bool(ADDRESS_RISK.search(" ".join(str(value).strip() for value in values if value))) diff --git a/pipeline/common/source_operations.py b/pipeline/common/source_operations.py index 0aaf2ec..0c8d132 100644 --- a/pipeline/common/source_operations.py +++ b/pipeline/common/source_operations.py @@ -396,6 +396,20 @@ def build_review_packet( } qa_counts = {key: qa.get(key) for key in counts} blockers = status.get("review_blockers", {}) + graph_manifest = None + for graph_path in ( + Path(run_dir) / "graph-candidates" / "manifest.json", + Path(run_dir) / "candidate-handoff" / "graph-candidates" / "manifest.json", + ): + if graph_path.is_file(): + graph_value = json.loads(graph_path.read_text(encoding="utf-8")) + if isinstance(graph_value, dict): + graph_manifest = {key: graph_value.get(key) for key in ( + "schema_version", "contract_version", "candidate_rows", + "quarantined_source_rows", "records_sha256", "storage_state", + "privacy_status", "review_state", "publication_status", "release_id", "auto_merge", + )} + break packet = { "schema_version": "private-review-packet-v1", "source_id": source_id, @@ -427,6 +441,10 @@ def build_review_packet( "drift_alarms": sorted(set(qa.get("drift_alarms", []))) if isinstance(qa.get("drift_alarms", []), list) else [], }, "release_diff": diff, + "graph_candidates": graph_manifest or { + "status": "not-emitted", "storage_state": "private", + "review_state": "review_required", "publication_status": "not_eligible", + }, "gates": { "release_state": manifest.get("release_state", "not-created"), "publication_state": status.get("publication_state"), diff --git a/pipeline/contracts/GRAPH-CANDIDATE-HANDOFF.md b/pipeline/contracts/GRAPH-CANDIDATE-HANDOFF.md index 444c4ac..a3e63d5 100644 --- a/pipeline/contracts/GRAPH-CANDIDATE-HANDOFF.md +++ b/pipeline/contracts/GRAPH-CANDIDATE-HANDOFF.md @@ -6,6 +6,13 @@ source-native identifiers for facilities and organizations, local references for relationships, optional claims with at least one supporting artifact/record, and optional source-scoped crosswalks. +The shared `candidate_handoff.write_handoff` bridge emits a deterministic +`graph-candidates/records.jsonl` plus a row-free manifest for every accepted +candidate row. Quarantined source rows are not turned into graph edges; their +reason counts remain in source QA and review packets. This keeps ambiguous +category, duplicate, remarks, nation-scope, and privacy cases isolated while +retaining their original evidence for operator review. + The handoff is always `storage_state: private`, `privacy_status: pending`, `review_state: review_required`, `publication_status: not_eligible`, and `release_id: null`. It is not a database import, identity decision, review, or diff --git a/pipeline/contracts/candidate_handoff.py b/pipeline/contracts/candidate_handoff.py index 1e45f39..51ecd31 100644 --- a/pipeline/contracts/candidate_handoff.py +++ b/pipeline/contracts/candidate_handoff.py @@ -4,6 +4,7 @@ from pathlib import Path from typing import Any from .adapter_contract import SourceArtifact +from pipeline.common.graph_candidate import write_graph_candidates CONTRACT_VERSION = "candidate-handoff-v1" @@ -13,7 +14,8 @@ def _atomic(path: Path, payload: bytes) -> None: tmp.write_bytes(payload); os.replace(tmp, path) def write_handoff(run_dir: str | Path, rows: list[dict[str, Any]], artifact: SourceArtifact, - *, source_id: str, profile: str = "default") -> dict[str, Any]: + *, source_id: str, profile: str = "default", + emit_graph_candidates: bool = True) -> dict[str, Any]: """Write importer-compatible JSONL/manifest, rejecting guessed identities.""" for row in rows: normalized = row.get("normalized") @@ -35,4 +37,13 @@ def write_handoff(run_dir: str | Path, rows: list[dict[str, Any]], artifact: Sou "coordinate_gate": "review_required"} # The importer consumes the conventional manifest.json name. _atomic(root / "manifest.json", (json.dumps(manifest, ensure_ascii=False, sort_keys=True, indent=2) + "\n").encode()) + if emit_graph_candidates: + graph_manifest = write_graph_candidates( + root / "graph-candidates", rows, + artifact_sha256=artifact.sha256, + observed_at=artifact.effective_date or artifact.retrieved_at_utc, + ) + manifest["graph_candidate_manifest"] = "graph-candidates/manifest.json" + manifest["graph_candidate_rows"] = graph_manifest["candidate_rows"] + _atomic(root / "manifest.json", (json.dumps(manifest, ensure_ascii=False, sort_keys=True, indent=2) + "\n").encode()) return manifest diff --git a/pipeline/contracts/private_run.py b/pipeline/contracts/private_run.py index 7049b35..ed0129e 100644 --- a/pipeline/contracts/private_run.py +++ b/pipeline/contracts/private_run.py @@ -7,6 +7,7 @@ from .adapter_contract import SourceAdapter, SourceArtifact from pipeline.common.review_metrics import build_private_review_metrics +from pipeline.common.identity import record_key from .source_lifecycle import atomic_json, validate_private_manifest @@ -23,24 +24,29 @@ def _manifest_value(manifest: dict[str, Any], key: str) -> Any: return None -def _record_ids(path: Path) -> set[str]: +def _record_ids(path: Path) -> set[str | tuple[str, str, str]]: if not path.exists(): return set() - ids: set[str] = set() + ids: set[str | tuple[str, str, str]] = set() for line in path.read_text(encoding="utf-8").splitlines(): if not line: continue row = json.loads(line) - normalized = row.get("normalized", {}) - candidates = ( - normalized.get("establishment_id"), - normalized.get("recognition_number"), - row.get("source_record_key"), - row.get("source_row_id"), - ) - value = next((candidate for candidate in candidates if isinstance(candidate, str) and candidate), None) - if value: - ids.add(value) + if row.get("source_id"): + ids.add(record_key(row)) + normalized = row.get("normalized", {}) + legacy_value = normalized.get("establishment_id") or normalized.get("recognition_number") + if isinstance(legacy_value, str) and legacy_value.strip(): + # Alias only for comparison with pre-envelope snapshots. + ids.add(legacy_value.strip()) + else: + # Accept older normalized snapshots that predate the shared + # source-envelope. They still contribute a conservative delta + # signal, but cannot be mistaken for a source-qualified key. + normalized = row.get("normalized", {}) + value = normalized.get("establishment_id") or normalized.get("recognition_number") + if isinstance(value, str) and value.strip(): + ids.add(value.strip()) return ids diff --git a/pipeline/contracts/test_graph_candidate_handoff.py b/pipeline/contracts/test_graph_candidate_handoff.py index eb8bfb6..593619a 100644 --- a/pipeline/contracts/test_graph_candidate_handoff.py +++ b/pipeline/contracts/test_graph_candidate_handoff.py @@ -9,6 +9,7 @@ validate_graph_candidate, write_graph_candidate, ) +from pipeline.common.graph_candidate import build_graph_candidate, write_graph_candidates def candidate(): @@ -73,6 +74,24 @@ def test_requires_explicit_unknown_reason(self): with self.assertRaisesRegex(ValueError, "unknown_reason"): validate_graph_candidate(value) + def test_shared_builder_preserves_source_identity_and_blocks_publication(self): + row = { + "source_id": "fr.dgal.section-i", "source_row": 4, + "source_record_key": "FR-1|SH|BOVINS|1", + "source_values": {"N° d'agrément": "FR-1", "Catégorie": "SH"}, + "normalized": {"establishment_id": "FR-1", "name": "Synthetic", "activity_categories": ("slaughter",)}, + } + candidate = build_graph_candidate(row, artifact_sha256="a" * 64, observed_at="2026-09-16T00:00:00Z") + self.assertEqual(candidate["source_record_key"], row["source_record_key"]) + self.assertEqual(candidate["publication"]["publication_status"], "not_eligible") + self.assertEqual(candidate["review_state"], "review_required") + self.assertNotIn("canonical_id", candidate) + with tempfile.TemporaryDirectory() as directory: + manifest = write_graph_candidates(directory, [row], artifact_sha256="a" * 64, quarantined_rows=2) + self.assertEqual(manifest["candidate_rows"], 1) + self.assertEqual(manifest["quarantined_source_rows"], 2) + self.assertEqual(len(Path(directory, "records.jsonl").read_text().splitlines()), 1) + if __name__ == "__main__": unittest.main() diff --git a/pipeline/sources/france/adapter.py b/pipeline/sources/france/adapter.py index d3b8532..db29768 100644 --- a/pipeline/sources/france/adapter.py +++ b/pipeline/sources/france/adapter.py @@ -4,6 +4,7 @@ import hashlib import json +import re from collections import Counter from pathlib import Path from typing import Any @@ -37,13 +38,16 @@ def _clean(raw: str | None) -> str | None: def _categories(category: str | None, activities: str | None) -> tuple[tuple[str, ...], bool]: text = " ".join(item for item in (category, activities) if item).upper() categories: list[str] = [] - if any(token in text for token in ("SH", "ABAT", "SLAUGHT", "ABATTAGE")): + # Match source codes as tokens; substring matching would turn an + # unrelated value such as ``FRESH`` into a slaughter classification. + codes = set(re.findall(r"(? dict[str, Any]: "establishment_id": approval, "recognition_number": approval, "facility_grouping": "provisional-dgal-approval-number", "identity_review": "required-before-merge", "name": _clean(value(row, mapping, "legal_name")), "trading_name": _clean(value(row, mapping, "legal_name")), + "siret": _clean(value(row, mapping, "siret")), "address": None, "address_state": "source-value-present-pending-review" if _clean(value(row, mapping, "address")) else "unknown", "postal_code": _clean(value(row, mapping, "postal_code")), "municipality": _clean(value(row, mapping, "commune")), "city": _clean(value(row, mapping, "commune")), "department_number": _clean(value(row, mapping, "department_number")), diff --git a/pipeline/sources/france/refresh.py b/pipeline/sources/france/refresh.py index dce2ab4..2a456de 100644 --- a/pipeline/sources/france/refresh.py +++ b/pipeline/sources/france/refresh.py @@ -30,7 +30,7 @@ def _local_metadata(path: Path, retrieved: str) -> dict[str, Any]: return {"acquisition_method": "assisted_local_capture", "source_id": "", "artifact_path": str(path), "sha256": digest, "byte_size": len(raw), "retrieved_at_utc": retrieved} -def refresh(*, section: str, run_dir: str | Path, raw_path: str | Path | None = None, fetch: bool = False, terms_review_path: str | Path | None = None, output_root: str | Path = "data/raw", run_id: str | None = None, retrieved_at_utc: str | None = None, timeout_seconds: float = 60.0, max_bytes: int = 32 * 1024 * 1024) -> dict[str, Any]: +def refresh(*, section: str, run_dir: str | Path, raw_path: str | Path | None = None, fetch: bool = False, terms_review_path: str | Path | None = None, output_root: str | Path = "data/raw", run_id: str | None = None, retrieved_at_utc: str | None = None, timeout_seconds: float = 60.0, max_bytes: int = 32 * 1024 * 1024, previous_normalized: str | Path | None = None) -> dict[str, Any]: if fetch == (raw_path is not None): raise ValueError("specify exactly one of --fetch or --raw") adapter = ADAPTERS[section](); retrieved = retrieved_at_utc or utc_now() if fetch: @@ -45,7 +45,12 @@ def refresh(*, section: str, run_dir: str | Path, raw_path: str | Path | None = metadata.update({"source_id": adapter.source_id, "requested_url": metadata.get("requested_url") or adapter.source_url, "final_url": metadata.get("final_url") or adapter.source_url}) raw = source.read_bytes(); artifact = source_artifact_from_acquisition(metadata, adapter_version=adapter.adapter_version, config_version=adapter.schema_version, source_url=adapter.source_url, coverage=f"France DGAL Regulation (EC) 853/2004 Section {section}; source rows only", rights_caveat="assisted capture; file-specific terms remain pending", privacy_caveat="private staging; privacy review pending") root = Path(run_dir); atomic_json(root / "acquisition-metadata.json", metadata) - lifecycle = run_private_lifecycle(source, root / "lifecycle", artifact, adapter, health_as_of_utc=retrieved) + lifecycle = run_private_lifecycle(source, root / "lifecycle", artifact, adapter, health_as_of_utc=retrieved, previous_normalized_path=previous_normalized, review_blockers={ + "terms": ["DGAL file-specific terms and attribution remain a human gate."], + "privacy": ["Names, addresses, SIRET values, and any coordinates require privacy screening; geocoding is disabled."], + "classification": ["DGAL category/activity labels are preserved; unknown categories and duplicate source observations remain isolated."], + "coverage": ["Section I and Section II are separate source scopes; disappearance is not closure."], + }) if lifecycle.get("status") == "candidate-ready": run_root = Path(lifecycle["run_dir"]); rows = [json.loads(line) for line in (run_root / "normalized" / "records.jsonl").read_text(encoding="utf-8").splitlines() if line] write_handoff(run_root / "candidate-handoff", rows, artifact, source_id=adapter.source_id) @@ -57,9 +62,9 @@ def refresh(*, section: str, run_dir: str | Path, raw_path: str | Path | None = def main() -> int: parser = argparse.ArgumentParser(description=__doc__); parser.add_argument("--section", choices=sorted(ADAPTERS), required=True) source = parser.add_mutually_exclusive_group(required=True); source.add_argument("--fetch", action="store_true"); source.add_argument("--raw", type=Path) - parser.add_argument("--run-dir", type=Path, required=True); parser.add_argument("--terms-review", type=Path); parser.add_argument("--output-root", type=Path, default=Path("data/raw")); parser.add_argument("--run-id"); parser.add_argument("--retrieved-at-utc"); parser.add_argument("--timeout-seconds", type=float, default=60.0); parser.add_argument("--max-bytes", type=int, default=32 * 1024 * 1024) + parser.add_argument("--run-dir", type=Path, required=True); parser.add_argument("--terms-review", type=Path); parser.add_argument("--output-root", type=Path, default=Path("data/raw")); parser.add_argument("--run-id"); parser.add_argument("--retrieved-at-utc"); parser.add_argument("--timeout-seconds", type=float, default=60.0); parser.add_argument("--max-bytes", type=int, default=32 * 1024 * 1024); parser.add_argument("--previous-normalized", type=Path) args = parser.parse_args() - try: result = refresh(section=args.section, run_dir=args.run_dir, raw_path=args.raw, fetch=args.fetch, terms_review_path=args.terms_review, output_root=args.output_root, run_id=args.run_id, retrieved_at_utc=args.retrieved_at_utc, timeout_seconds=args.timeout_seconds, max_bytes=args.max_bytes) + try: result = refresh(section=args.section, run_dir=args.run_dir, raw_path=args.raw, fetch=args.fetch, terms_review_path=args.terms_review, output_root=args.output_root, run_id=args.run_id, retrieved_at_utc=args.retrieved_at_utc, timeout_seconds=args.timeout_seconds, max_bytes=args.max_bytes, previous_normalized=args.previous_normalized) except (OSError, ValueError) as error: print(json.dumps({"status": "failed", "error": str(error)}, sort_keys=True)); return 2 print(json.dumps(result["report"], sort_keys=True)); return 0 if result["report"]["lifecycle_status"] == "candidate-ready" else 1 diff --git a/pipeline/sources/france/test_adapter.py b/pipeline/sources/france/test_adapter.py index e3a6369..96d6020 100644 --- a/pipeline/sources/france/test_adapter.py +++ b/pipeline/sources/france/test_adapter.py @@ -35,6 +35,15 @@ def test_section_ii_missing_identity_is_quarantined(self): result = FranceDgalSectionIIAdapter().parse_file(FIXTURES / "section_ii.csv") self.assertEqual(len(result["accepted"]), 1); self.assertIn("missing_approval_number", result["quarantined"][0]["reasons"]) + def test_category_codes_are_tokenized_and_source_rows_have_distinct_graph_keys(self): + content = ("approval_number;legal_name;commune;category;associated activities\n" + "FR-1;Fresh Foods;Town;FRESH;\n" + "FR-2;Cut Foods;Town;CP;Découpe\n").encode() + result = FranceDgalSectionIAdapter().parse_bytes(content) + self.assertEqual(len(result["accepted"]), 1) + self.assertIn("unknown_category_code", result["quarantined"][0]["reasons"]) + self.assertIn("source_record_key", result["accepted"][0]) + def test_schema_drift_and_lifecycle_are_closed(self): adapter = FranceDgalSectionIAdapter(); raw = (FIXTURES / "section_i.csv").read_bytes(); artifact = SourceArtifact(adapter.source_url, "2026-09-15T00:00:00Z", hashlib.sha256(raw).hexdigest(), len(raw), code_version=adapter.adapter_version, config_version=adapter.schema_version) with self.assertRaises(ValueError): adapter.parse_bytes(raw.replace("N° d'agrément".encode(), b"wrong")) diff --git a/pipeline/sources/france/test_refresh.py b/pipeline/sources/france/test_refresh.py index 02e9609..f809076 100644 --- a/pipeline/sources/france/test_refresh.py +++ b/pipeline/sources/france/test_refresh.py @@ -14,5 +14,19 @@ def test_assisted_refresh_emits_candidate_handoff_and_review_packet(self): run = Path(result["report"]["run_dir"]); self.assertTrue((run / "candidate-handoff/manifest.json").exists()); self.assertTrue((run / "operator-review-packet.json").exists()) packet = json.loads((run / "operator-review-packet.json").read_text()); self.assertFalse(packet["row_payloads_included"]) + def test_prior_run_is_linked_to_standard_delta_and_graph_handoff(self): + with tempfile.TemporaryDirectory() as d: + root = Path(d); source = Path(__file__).parent / "fixtures/section_i.csv" + first = refresh(section="I", raw_path=source, run_dir=root / "first", retrieved_at_utc="2026-09-15T00:00:00Z") + prior = Path(first["report"]["run_dir"]) / "normalized" / "records.jsonl" + second = refresh(section="I", raw_path=source, run_dir=root / "second", retrieved_at_utc="2026-09-16T00:00:00Z", previous_normalized=prior) + run = Path(second["report"]["run_dir"]) + packet = json.loads((run / "review-packet.json").read_text(encoding="utf-8")) + self.assertEqual(packet["release_diff"]["status"], "delta-ready") + self.assertEqual(packet["release_diff"]["counts"]["not_observed"], 0) + graph = json.loads((run / "candidate-handoff/graph-candidates/manifest.json").read_text(encoding="utf-8")) + self.assertEqual(graph["candidate_rows"], 2) + self.assertFalse(graph["publication_status"] == "eligible") + if __name__ == "__main__": unittest.main() diff --git a/pipeline/sources/uk/fsa_approved/adapter.py b/pipeline/sources/uk/fsa_approved/adapter.py index 5869bc6..5e00691 100644 --- a/pipeline/sources/uk/fsa_approved/adapter.py +++ b/pipeline/sources/uk/fsa_approved/adapter.py @@ -1,11 +1,12 @@ """FSA approved-establishments adapter for synthetic and monthly source profiles.""" from __future__ import annotations -import csv, hashlib, json, os, re, tempfile +import csv, hashlib, json, os, tempfile from dataclasses import asdict, dataclass from pathlib import Path from typing import Any from pipeline.contracts.adapter_contract import SourceArtifact from pipeline.common.activity import classify_activities +from pipeline.common.privacy import address_privacy_risk ROOT=Path(__file__).parent CONFIG=json.loads((ROOT/"config.json").read_text(encoding="utf-8")) @@ -16,7 +17,6 @@ # Generic facility-building names such as "house", "home", and "lodge" are # common in legitimate establishment addresses. Keep only terms that are # stronger indicators of a residential, private, or intermediary address. -ADDRESS_RISK=re.compile(r"\b(flat|apartment|residential|c/o|care of|caravan)\b",re.I) MONTHLY_REQUIRED=frozenset({"AppNo","TradingName","Country","CompetentAuthority","X","Y","AddressWithheld","All_Activities"}) MONTHLY_COUNTRIES=frozenset({"England","Wales"}) @@ -61,7 +61,7 @@ def _csv(content): except csv.Error as exc:raise FsaContractError("malformed CSV") from exc raise FsaContractError("unsupported CSV encoding") def _synthetic_record(row,line): - nation=_clean(row.get("nation"));acts=_split(row.get("activities"));return {"source_id":CONFIG["source_id"],"source_row":line,"source_values":dict(row),"normalized":{"establishment_id":_clean(row.get("establishment_id")),"trading_name":_clean(row.get("trading_name")),"address_lines":tuple(_clean(row.get(f"address_line_{n}")) for n in range(1,4)),"postcode":_clean(row.get("postcode")),"activities":acts,"activity_categories":classify_activities(acts),"species":_clean(row.get("species")),"competent_authority":_clean(row.get("competent_authority")),"nation":nation,"authority_nation_key":nation,"status":_clean(row.get("status")),"remarks":_clean(row.get("remarks")),"published_date":_clean(row.get("published_date")),"coordinates":None}} + nation=_clean(row.get("nation"));ident=_clean(row.get("establishment_id"));acts=_split(row.get("activities"));return {"source_id":CONFIG["source_id"],"source_row":line,"source_record_key":f"{nation or 'unknown'}|{ident or 'unknown'}","source_values":dict(row),"normalized":{"establishment_id":ident,"trading_name":_clean(row.get("trading_name")),"address_lines":tuple(_clean(row.get(f"address_line_{n}")) for n in range(1,4)),"postcode":_clean(row.get("postcode")),"activities":acts,"activity_categories":classify_activities(acts),"species":_clean(row.get("species")),"competent_authority":_clean(row.get("competent_authority")),"nation":nation,"authority_nation_key":nation,"status":_clean(row.get("status")),"remarks":_clean(row.get("remarks")),"published_date":_clean(row.get("published_date")),"coordinates":None}} def _coords(row): try:x,y=float(row.get("X","").strip()),float(row.get("Y","").strip()) except ValueError:return None,None,"unresolved-nonnumeric" @@ -73,7 +73,7 @@ def _monthly_record(row,line): acts=tuple(x for x in (_clean(row.get("All_Activities")),_clean(row.get("Part_A__All_sections_")),_clean(row.get("Part B All sections "))) if x) privacy_gate="restricted-withheld-address" if withheld else "privacy-review-required" coordinate_gate="restricted-withheld-address" if withheld else "privacy-review-required" - return {"source_id":CONFIG["source_id"],"source_row":line,"source_values":dict(row),"normalized":{"establishment_id":_clean(row.get("AppNo")),"trading_name":_clean(row.get("TradingName")),"address_lines":None if withheld else tuple(_clean(row.get(k)) for k in ("Address1","Address2","Address3")),"postcode":_clean(row.get("Postcode")),"activities":acts,"activity_categories":classify_activities(acts),"species":_clean(row.get("Species")),"competent_authority":_clean(row.get("CompetentAuthority")),"nation":_clean(row.get("Country")),"authority_nation_key":_clean(row.get("Country")),"status":None,"remarks":_clean(row.get("Remarks")),"published_date":None,"coordinates":None,"coordinate_state":status,"coordinate_precision":"withheld" if withheld else "source-precision-unspecified","coordinate_gate":coordinate_gate,"privacy_gate":privacy_gate,"publication_gate":"blocked"}} + ident=_clean(row.get("AppNo"));nation=_clean(row.get("Country"));return {"source_id":CONFIG["source_id"],"source_row":line,"source_record_key":f"{nation or 'unknown'}|{ident or 'unknown'}","source_values":dict(row),"normalized":{"establishment_id":ident,"trading_name":_clean(row.get("TradingName")),"address_lines":None if withheld else tuple(_clean(row.get(k)) for k in ("Address1","Address2","Address3")),"postcode":_clean(row.get("Postcode")),"activities":acts,"activity_categories":classify_activities(acts),"species":_clean(row.get("Species")),"competent_authority":_clean(row.get("CompetentAuthority")),"nation":nation,"authority_nation_key":nation,"status":None,"remarks":_clean(row.get("Remarks")),"published_date":None,"coordinates":None,"coordinate_state":status,"coordinate_precision":"withheld" if withheld else "source-precision-unspecified","coordinate_gate":coordinate_gate,"privacy_gate":privacy_gate,"publication_gate":"blocked"}} class FsaApprovedEstablishmentsAdapter: source_id=CONFIG["source_id"];schema_version=CONFIG["contract_version"];adapter_version=CONFIG["adapter_version"] @@ -100,7 +100,7 @@ def _synthetic(self,headers,rows,digest,fp): status=_clean(v.get("status")) if status and status.lower() not in ALLOWED_STATUSES:reasons.append("unknown_status") if _clean(v.get("remarks")):reasons.append("remarks_present") - if ADDRESS_RISK.search(" ".join(_clean(v.get(f"address_line_{n}")) or "" for n in range(1,4))):reasons.append("address_privacy_risk") + if address_privacy_risk(*(_clean(v.get(f"address_line_{n}")) for n in range(1,4))):reasons.append("address_privacy_risk") coverage[nation or ""] = coverage.get(nation or "", 0) + 1 for reason in dict.fromkeys(reasons): anomalies[reason] = anomalies.get(reason, 0) + 1 record=_synthetic_record(v,line);(quarantined if reasons else accepted).append({"reasons":tuple(dict.fromkeys(reasons)),"record":record} if reasons else record) @@ -118,9 +118,10 @@ def _monthly(self,headers,rows,digest,fp): if (country,ident) in duplicates:reasons.append("duplicate_id_within_nation") if country not in MONTHLY_COUNTRIES:reasons.append("unknown_nation") if not any(_clean(v.get(k)) for k in ("All_Activities","Part_A__All_sections_","Part B All sections ")):reasons.append("missing_activity") + elif not classify_activities(_split(_clean(v.get("All_Activities")) or _clean(v.get("Part_A__All_sections_")) or _clean(v.get("Part B All sections ")))):reasons.append("no_relevant_activity") if _clean(v.get("Remarks")):reasons.append("remarks_present") withheld=(_clean(v.get("AddressWithheld")) or "").lower()=="yes" - if not withheld and ADDRESS_RISK.search(" ".join(_clean(v.get(k)) or "" for k in ("Address1","Address2","Address3","Town","Postcode"))):reasons.append("address_privacy_risk") + if not withheld and address_privacy_risk(*(v.get(k) for k in ("Address1","Address2","Address3","Town","Postcode"))):reasons.append("address_privacy_risk") record=_monthly_record(v,line) if reasons: for reason in reasons:anomalies[reason]=anomalies.get(reason,0)+1 diff --git a/pipeline/sources/uk/fsa_approved/refresh.py b/pipeline/sources/uk/fsa_approved/refresh.py index 0e6396d..c1e9ba0 100644 --- a/pipeline/sources/uk/fsa_approved/refresh.py +++ b/pipeline/sources/uk/fsa_approved/refresh.py @@ -33,6 +33,23 @@ def _header_fingerprint(raw: bytes) -> tuple[str, int]: return hashlib.sha256(json.dumps(headers, ensure_ascii=False, separators=(",", ":")).encode()).hexdigest(), len(headers) +def _id_keys(normalized: dict[str, Any]) -> set[str]: + identifier = normalized.get("establishment_id") + nation = normalized.get("nation") + if not isinstance(identifier, str) or not identifier.strip(): + return set() + identifier = identifier.strip() + if not isinstance(nation, str) or not nation.strip(): + # Legacy prior-run files may predate nation-qualified IDs. Preserve + # their disappearance signal conservatively rather than treating the + # missing nation as a match to an arbitrary current nation. + return {identifier} + # Keep the bare alias for old prior-run files that predate nation + # qualification, while the qualified key remains authoritative for new + # comparisons. + return {identifier, f"{nation.strip()}|{identifier}"} + + def _read_ids(path: Path) -> set[str]: if not path.exists(): return set() @@ -41,9 +58,7 @@ def _read_ids(path: Path) -> set[str]: if not line: continue normalized = json.loads(line).get("normalized", {}) - value = normalized.get("establishment_id") - if isinstance(value, str) and value: - ids.add(value) + ids.update(_id_keys(normalized)) return ids @@ -135,14 +150,11 @@ def refresh_monthly( if not bounded_sample and abs(len(result.accepted) + len(result.quarantined) - baseline) > max(100, baseline // 10): alarms.append("input_row_count_changed") previous_ids = _read_ids(Path(previous_normalized)) if previous_normalized else set() - current_ids = { - r["normalized"].get("establishment_id") - for r in result.accepted - } | { - item["record"]["normalized"].get("establishment_id") - for item in result.quarantined - } - current_ids.discard(None) + current_ids: set[str] = set() + for record in result.accepted: + current_ids.update(_id_keys(record["normalized"])) + for item in result.quarantined: + current_ids.update(_id_keys(item["record"]["normalized"])) disappeared = len(previous_ids - current_ids) if previous_ids else 0 artifact = SourceArtifact( source_url=source_url, diff --git a/pipeline/sources/uk/fss_approved/adapter.py b/pipeline/sources/uk/fss_approved/adapter.py index b843b02..55c983b 100644 --- a/pipeline/sources/uk/fss_approved/adapter.py +++ b/pipeline/sources/uk/fss_approved/adapter.py @@ -8,19 +8,18 @@ import hashlib import json import os -import re from dataclasses import asdict, dataclass from pathlib import Path from typing import Any from pipeline.contracts.adapter_contract import SourceArtifact from pipeline.common.activity import classify_activities +from pipeline.common.privacy import address_privacy_risk ROOT = Path(__file__).parent CONFIG = json.loads((ROOT / "config.json").read_text(encoding="utf-8")) REQUIRED_COLUMNS = tuple(CONFIG["required_columns"]) ALLOWED_ACTIVITIES = frozenset(CONFIG["allowed_activities"]) ALLOWED_STATUSES = frozenset(CONFIG["allowed_statuses"]) -ADDRESS_RISK = re.compile(r"\b(flat|apartment|house|home|residential|c/o|care of|caravan|lodge)\b", re.I) class FssContractError(ValueError): @@ -53,9 +52,12 @@ def _split(value: str | None) -> tuple[str, ...]: def _record(row: dict[str, str], line: int) -> dict[str, Any]: + approval = _clean(row.get("approval_number")) + nation = _clean(row.get("nation")) return {"source_id": CONFIG["source_id"], "source_row": line, + "source_record_key": f"{nation or 'unknown'}|{approval or 'unknown'}", "source_values": dict(row), "normalized": { - "approval_number": _clean(row.get("approval_number")), + "approval_number": approval, # Keep the source-native identifier and expose the shared # importer identity explicitly; neither value is inferred or # used to merge records across authorities. @@ -66,7 +68,7 @@ def _record(row: dict[str, str], line: int) -> dict[str, Any]: "activity_categories": classify_activities(_split(row.get("activities"))), "species": _clean(row.get("species")), "competent_authority": _clean(row.get("competent_authority")), - "nation": _clean(row.get("nation")), "status": _clean(row.get("status")), + "nation": nation, "status": _clean(row.get("status")), "remarks": _clean(row.get("remarks")), "published_date": _clean(row.get("published_date")), "coordinates": None}} @@ -162,7 +164,7 @@ def parse_bytes(self, content: bytes) -> ValidationResult: if _clean(row.get("remarks")): reasons.append("remarks_present") address = " ".join(_clean(record_row.get(f"address_line_{n}")) or "" for n in range(1, 5 if live else 4)) - if ADDRESS_RISK.search(address): + if address_privacy_risk(address): reasons.append("address_privacy_risk") nation = "Scotland" if live else (_clean(row.get("nation")) or "") coverage[nation] = coverage.get(nation, 0) + 1 diff --git a/pipeline/sources/uk/fss_approved/refresh.py b/pipeline/sources/uk/fss_approved/refresh.py index 5e75749..048e434 100644 --- a/pipeline/sources/uk/fss_approved/refresh.py +++ b/pipeline/sources/uk/fss_approved/refresh.py @@ -70,8 +70,9 @@ def _read_previous_ids(path: Path | None) -> set[str]: if line: normalized = json.loads(line).get("normalized", {}) value = normalized.get("establishment_id") or normalized.get("approval_number") + nation = normalized.get("nation") or "Scotland" if isinstance(value, str) and value: - values.add(value) + values.add(f"{nation}|{value}") return values @@ -128,11 +129,13 @@ def refresh_scotland( adapter = FssApprovedEstablishmentsAdapter() result = adapter.parse_bytes(raw) current_ids = { - record["normalized"].get("approval_number") + f"Scotland|{record['normalized'].get('approval_number')}" for record in result.accepted + if record["normalized"].get("approval_number") } | { - item["record"]["normalized"].get("approval_number") + f"Scotland|{item['record']['normalized'].get('approval_number')}" for item in result.quarantined + if item["record"]["normalized"].get("approval_number") } current_ids.discard(None) previous_ids = _read_previous_ids(Path(previous_normalized) if previous_normalized else None) From 8f3a9e942950957ddc8db48d44a99faf0a43f5b4 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Thu, 17 Sep 2026 13:26:22 -0700 Subject: [PATCH 227/311] Prepare private Canada source graph candidates --- .../current-canada-private-2026-09-17.json | 88 ++++++++++++ .../current-reacquisition-2026-09-16.json | 2 +- ...anada-geospatial-readiness-2026-09-17.json | 136 ++++++++++++++++++ docs/countries/canada/meat-plants-pipeline.md | 8 ++ docs/country-recon-ca.md | 6 +- docs/current-reacquisition.md | 13 +- pipeline/contracts/GRAPH-CANDIDATE-HANDOFF.md | 3 +- pipeline/contracts/graph_candidate_handoff.py | 2 +- pipeline/sources/canada/adapter.py | 74 +++++++++- pipeline/sources/canada/test_adapter.py | 29 ++++ 10 files changed, 352 insertions(+), 9 deletions(-) create mode 100644 data/manifests/current-canada-private-2026-09-17.json create mode 100644 data/reports/current-canada-geospatial-readiness-2026-09-17.json diff --git a/data/manifests/current-canada-private-2026-09-17.json b/data/manifests/current-canada-private-2026-09-17.json new file mode 100644 index 0000000..8b8a485 --- /dev/null +++ b/data/manifests/current-canada-private-2026-09-17.json @@ -0,0 +1,88 @@ +{ + "manifest_version": "current-canada-private-v1", + "as_of_utc": "2026-09-17T20:20:33Z", + "evidence_scope": "private row-free Canada source and graph-candidate rehearsal; no publication decision", + "publication": { + "release_created": false, + "release_promoted": false, + "public_api_rows": 0, + "project_approval": "not-approved" + }, + "sources": [ + { + "source_id": "ca.ontario.meat-plants", + "country_code": "CA", + "jurisdiction_level": "provincial", + "jurisdiction": "Ontario", + "coverage": "Ontario provincially licensed meat plants only; no national completeness claim", + "source_url": "https://data.ontario.ca/dataset/a763088c-018d-48b7-bf47-3027a8c725b8/resource/ee6d559a-78de-40e6-b2ba-ad3c4a674b96/download/1._all_meat_plants.csv", + "retrieved_at_utc": "2026-09-17T20:20:33Z", + "effective_date": "Thu, 06 Aug 2026 19:04:50 GMT", + "raw_artifact": "data/raw/ca.ontario.meat-plants/20260917T000000Z-on/source.csv", + "raw_bytes": 130252, + "raw_sha256": "c4edfdd415812f6a914f5fb29a9f67cf2907ea6fbfe4e09ab96eae1468002adf", + "private_manifest": "data/staging/reacquisition/ca.ontario.meat-plants/20260917T002000Z-rerun/lifecycle/c4edfdd415812f6a-b_z_w5rh/manifest.json", + "candidate_handoff_manifest": "data/staging/reacquisition/ca.ontario.meat-plants/20260917T002000Z-rerun/lifecycle/c4edfdd415812f6a-b_z_w5rh/candidate-handoff/manifest.json", + "graph_candidate_manifest": "data/staging/reacquisition/ca.ontario.meat-plants/20260917T002000Z-rerun/lifecycle/c4edfdd415812f6a-b_z_w5rh/graph-candidates/manifest.json", + "input_rows": 460, + "normalized_rows": 460, + "quarantined_rows": 0, + "graph_candidates": 460, + "graph_relationships": {}, + "status": "candidate-ready; private only; human review required" + }, + { + "source_id": "ca.cfia.federal-meat", + "country_code": "CA", + "jurisdiction_level": "federal", + "jurisdiction": "Canada", + "coverage": "CFIA federally registered meat establishments; federal registry only; provincial establishments excluded", + "source_url": "https://active.inspection.gc.ca/scripts/meavia/reglist/download.asp?lang=e", + "retrieved_at_utc": "2026-09-17T20:20:06Z", + "effective_date": "unknown", + "raw_artifact": "data/raw/ca.cfia.federal-meat/20260917T000000Z-cfia/source.xls", + "raw_bytes": 572928, + "raw_sha256": "d2f042a43e0dc72460c892c67b60e8d0cacf9a91bd85032862664a6deae0cfef", + "response_content_type": "application/octet-stream", + "private_manifest": "data/staging/reacquisition/ca.cfia.federal-meat/20260917T002000Z-rerun/lifecycle/d2f042a43e0dc724-grqcwtl2/manifest.json", + "candidate_handoff_manifest": "data/staging/reacquisition/ca.cfia.federal-meat/20260917T002000Z-rerun/lifecycle/d2f042a43e0dc724-grqcwtl2/candidate-handoff/manifest.json", + "graph_candidate_manifest": "data/staging/reacquisition/ca.cfia.federal-meat/20260917T002000Z-rerun/lifecycle/d2f042a43e0dc724-grqcwtl2/graph-candidates/manifest.json", + "input_rows": 874, + "normalized_rows": 0, + "quarantined_rows": 874, + "anomaly_counts": {"unknown_function_code": 874}, + "graph_candidates": 874, + "graph_relationships": {}, + "status": "candidate-ready; private only; all rows quarantined pending reviewed function-code and operator-field mapping" + } + ], + "reconciliation": { + "input_rows": 1334, + "normalized_rows": 460, + "quarantined_rows": 874, + "identity_safe_graph_candidates": 1334, + "asserted_operator_relationships": 0, + "asserted_regulator_relationships": 0, + "formula": "input = normalized + quarantined" + }, + "private_api_frontend_rehearsal": { + "status": "passed-in-shared-current-candidate-lane", + "evidence_manifest": "data/manifests/current-candidate-rehearsal-2026-09-16.json", + "public_v2_rows": 0, + "test_release_list_detail_facets_cursor": "passed", + "bounded_export_guard": "passed", + "raw_fields_absent": true, + "suppression": "append-only public_access_revoked removed the selected candidate from detail and list", + "frontend_boundary": "loopback development preview requires explicit opt-in and operator token; candidate rows visibly labeled private test data" + }, + "gates": { + "geocoding": "disabled", + "privacy_and_coordinate_review": "required", + "terms_and_attribution_review": "required", + "federal_function_code_mapping": "required", + "federal_operator_field_semantics": "required", + "cross_source_merge": "disabled", + "publication": "blocked" + }, + "private_payloads_included": false +} diff --git a/data/manifests/current-reacquisition-2026-09-16.json b/data/manifests/current-reacquisition-2026-09-16.json index 7529588..841bf0d 100644 --- a/data/manifests/current-reacquisition-2026-09-16.json +++ b/data/manifests/current-reacquisition-2026-09-16.json @@ -156,7 +156,7 @@ "input_rows_known": 116056, "normalized_rows": 108475, "quarantined_rows": 7581, - "raw_only_profiles": 0, + "raw_only_profiles": 1, "requested_min_normalized_rows": 100000, "target_met": true }, diff --git a/data/reports/current-canada-geospatial-readiness-2026-09-17.json b/data/reports/current-canada-geospatial-readiness-2026-09-17.json new file mode 100644 index 0000000..5531312 --- /dev/null +++ b/data/reports/current-canada-geospatial-readiness-2026-09-17.json @@ -0,0 +1,136 @@ +{ + "as_of_utc": "2026-09-17T20:20:33Z", + "candidate_state": "private-only; no publication decision implied", + "manifest_sha256": "b7424efde8539888a499fcedca0c608d0541b988216d20d2296978fd79b4675d", + "privacy_boundary": "aggregate-only; normalized rows, raw artifacts, addresses, coordinates, identifiers, and geocoder payloads are excluded", + "publication": { + "project_approval": "not-approved", + "public_api_rows": 0, + "release_created": false, + "release_promoted": false + }, + "report_version": "current-geospatial-readiness-v1", + "semantics": { + "coordinate_review_queue": "Rows needing coordinate/privacy review or explicitly unresolved coordinate handling; geocoding success alone never grants release eligibility.", + "display_states": "Mutually exclusive current candidate display states; restricted takes precedence. Exact means a valid source point or accepted exact geocode, city means a declared city/coarse signal without an exact point, and unmapped means no displayable location evidence.", + "evidence_states": "Overlapping evidence counters. A row may have source coordinates and a geocoder result; these counters intentionally preserve both facts.", + "privacy_review_queue": "Conservative human-review indicators, not a residential determination or publication approval.", + "unavailable_sources": "A missing private handoff is not counted as zero and does not establish no coverage." + }, + "sources": [ + { + "acquisition_state": "candidate-ready; private only; all rows quarantined pending reviewed function-code and operator-field mapping", + "country_code": "CA", + "declared_input_rows": 874, + "declared_normalized_rows": 0, + "declared_quarantined_rows": 874, + "metrics": { + "audit_status": "complete", + "available": true, + "coordinate_review_queue": { + "by_reason": {}, + "rows": 0 + }, + "display_states": { + "city": 0, + "exact": 0, + "restricted": 0, + "unmapped": 0 + }, + "evidence_states": { + "accepted_geocode_coarse": 0, + "accepted_geocode_exact": 0, + "coarse_city_location": 0, + "geocode_unresolved": 0, + "source_coordinate_invalid": 0, + "source_coordinate_pending_review": 0, + "source_coordinate_valid": 0, + "unresolved": 0 + }, + "normalized_bytes": 0, + "normalized_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "parse_errors": 0, + "privacy_review_queue": { + "by_reason": {}, + "rows": 0 + }, + "records": 0, + "source_local_identifier_rows": 0, + "suppressed_rows": 0 + }, + "publication_state": "private-candidate", + "source_id": "ca.cfia.federal-meat" + }, + { + "acquisition_state": "candidate-ready; private only; human review required", + "country_code": "CA", + "declared_input_rows": 460, + "declared_normalized_rows": 460, + "declared_quarantined_rows": 0, + "metrics": { + "audit_status": "complete", + "available": true, + "coordinate_review_queue": { + "by_reason": { + "review_required": 460 + }, + "rows": 460 + }, + "display_states": { + "city": 460, + "exact": 0, + "restricted": 0, + "unmapped": 0 + }, + "evidence_states": { + "accepted_geocode_coarse": 0, + "accepted_geocode_exact": 0, + "coarse_city_location": 460, + "geocode_unresolved": 460, + "source_coordinate_invalid": 0, + "source_coordinate_pending_review": 0, + "source_coordinate_valid": 0, + "unresolved": 0 + }, + "normalized_bytes": 1093430, + "normalized_sha256": "ffef83bf8f8751dcf0a7344c5feb9637ba5690dd9eb3856a39d1c211e92a536f", + "parse_errors": 0, + "privacy_review_queue": { + "by_reason": { + "pending-review": 460 + }, + "rows": 460 + }, + "records": 460, + "source_local_identifier_rows": 460, + "suppressed_rows": 0 + }, + "publication_state": "private-candidate", + "source_id": "ca.ontario.meat-plants" + } + ], + "sources_available": 2, + "sources_listed": 2, + "sources_unavailable": 0, + "sources_with_parse_errors": 0, + "totals_available_rows_only": { + "coordinate_review_rows": 460, + "display_city": 460, + "display_exact": 0, + "display_restricted": 0, + "display_unmapped": 0, + "evidence_accepted_geocode_coarse": 0, + "evidence_accepted_geocode_exact": 0, + "evidence_coarse_city_location": 460, + "evidence_geocode_unresolved": 460, + "evidence_source_coordinate_invalid": 0, + "evidence_source_coordinate_pending_review": 0, + "evidence_source_coordinate_valid": 0, + "evidence_unresolved": 0, + "parse_errors": 0, + "privacy_review_rows": 460, + "records": 460, + "source_local_identifier_rows": 460, + "suppressed_rows": 0 + } +} diff --git a/docs/countries/canada/meat-plants-pipeline.md b/docs/countries/canada/meat-plants-pipeline.md index 17349fa..a766734 100644 --- a/docs/countries/canada/meat-plants-pipeline.md +++ b/docs/countries/canada/meat-plants-pipeline.md @@ -27,4 +27,12 @@ handoff`. Reruns are deterministic. Missing observations are not closure. No geocoding is performed. Current licence, attribution, redistribution, privacy, function-code semantics, freshness, and project approval remain human gates. +The adapter also emits row-level graph candidates into the private run. Every +accepted row gets a source-scoped facility node keyed by its source +establishment number and a supported operation claim. CFIA rows with an +explicit operator field additionally get operator and federal-registry +regulator edges; Ontario's plant-name field is never guessed to be an operator. +These graph candidates remain `review_required`, private, and not eligible for +publication, with no universal identity or cross-source merge assertion. + Run with `python -m pipeline.sources.canada.refresh --source ontario --raw --run-dir ` or `--source cfia --fetch --terms-review `. Keep federal and provincial candidate releases separate. diff --git a/docs/country-recon-ca.md b/docs/country-recon-ca.md index 540f53d..499dc58 100644 --- a/docs/country-recon-ca.md +++ b/docs/country-recon-ca.md @@ -1,6 +1,6 @@ # Canada source reconnaissance -Status: sanitized metadata handoff; no facility rows, names, addresses, coordinates, or downloaded artifacts are retained here. Last checked 2026-09-14 UTC. This is not publication approval or a healthy-pipeline claim. +Status: sanitized metadata handoff; no facility rows, names, addresses, coordinates, or downloaded artifacts are retained here. Last checked 2026-09-17 UTC. This is not publication approval or a healthy-pipeline claim. ## Comparison @@ -30,8 +30,8 @@ AAFC livestock/slaughter context: dict[str, Any]: if occurrences[key] > 1: reasons.append("duplicate_source_row") normalized = { "establishment_id": plant_number, "recognition_number": plant_number, "facility_grouping": f"provisional-{self.jurisdiction_level}-plant-number", "identity_review": "required-before-merge", - "name": name, "trading_name": _clean(value(row, mapping, "doing_business_as")) or name, "address": None, + "name": name, "operator_name": _clean(value(row, mapping, "operator_name")), "trading_name": _clean(value(row, mapping, "doing_business_as")) or name, "address": None, "address_state": "source-value-present-pending-review" if _clean(value(row, mapping, "address")) else "unknown", "city": _clean(value(row, mapping, "city")), "postal_code": _clean(value(row, mapping, "postal_code")), "province": _clean(value(row, mapping, "province")), "country_code": "CA", "nation": "Canada", "jurisdiction_level": self.jurisdiction_level, "jurisdiction": self.jurisdiction, "source_plant_type": plant_type, "source_function_codes": functions, "animal_class": animal_class, "activity_categories": categories, @@ -201,6 +203,73 @@ def parse_bytes(self, content: bytes) -> dict[str, Any]: def parse_file(self, path: str | Path) -> dict[str, Any]: return self.parse_bytes(Path(path).read_bytes()) + def _graph_candidate(self, record: dict[str, Any], artifact: SourceArtifact) -> dict[str, Any]: + """Build a source-scoped graph candidate from one accepted source row. + + A facility node is always justified by the source establishment number. + Operator and regulator edges are emitted only for the federal adapter: + its workbook has an explicit operator column and its registry scope is + federal. Ontario's plant-name field is deliberately not treated as an + operator identity. + """ + normalized = record["normalized"] + facility_ref = f"facility:{record['source_record_key']}" + support = [{"source_record_key": record["source_record_key"]}, {"artifact_sha256": artifact.sha256}] + candidate: dict[str, Any] = { + "contract_version": "graph-candidate-handoff-v1", + "source_id": self.source_id, + "source_row": record["source_row"], + "source_record_key": record["source_record_key"], + "source_values": record["source_values"], + "publication": {"storage_state": "private", "privacy_status": "pending", "review_state": "review_required", "publication_status": "not_eligible", "release_id": None}, + "facilities": [{"local_ref": facility_ref, "source_identifier": {"identifier_type": "establishment-number", "value": str(normalized["establishment_id"])}}], + "organizations": [], + "relationships": [], + "claims": [{"claim_domain": "operation", "claim_kind": "listed-meat-establishment", "facility_ref": facility_ref, "value_state": "known", "observed_at": artifact.retrieved_at_utc, "review_state": "review_required", "support": support}], + "crosswalks": [], + } + if self.jurisdiction_level == "federal" and normalized.get("operator_name"): + operator_ref = f"organization:operator:{record['source_record_key']}" + regulator_ref = "organization:regulator:cfia" + candidate["organizations"] = [ + {"local_ref": operator_ref, "source_identifier": {"identifier_type": "operator-name", "value": normalized["operator_name"]}}, + {"local_ref": regulator_ref, "source_identifier": {"identifier_type": "authority-source", "value": "CFIA"}}, + ] + candidate["relationships"] = [ + {"relationship_type": "operator", "from_organization_ref": operator_ref, "target_facility_ref": facility_ref, "assertion_status": "asserted", "observed_at": artifact.retrieved_at_utc, "review_state": "review_required", "support": support, "assertion_basis": "explicit CFIA operator field"}, + {"relationship_type": "regulator", "from_organization_ref": regulator_ref, "target_facility_ref": facility_ref, "assertion_status": "asserted", "observed_at": artifact.retrieved_at_utc, "review_state": "review_required", "support": support, "assertion_basis": "CFIA federal registry scope"}, + ] + return candidate + + def _write_graph_candidates(self, root: Path, records: list[dict[str, Any]], artifact: SourceArtifact) -> dict[str, Any]: + """Persist identity-safe graph candidates and a row-free manifest. + + A row quarantined only for an unmapped activity code can still support + a private facility/operator/regulator review edge. Rows with missing + identity fields or duplicate source keys are excluded from graph + staging because their identity is not sufficiently supported. + """ + graph_root = root / "graph-candidates" + relationship_counts: Counter[str] = Counter() + for record in records: + candidate = self._graph_candidate(record, artifact) + candidate_dir = graph_root / hashlib.sha256(record["source_record_key"].encode("utf-8")).hexdigest()[:24] + write_graph_candidate(candidate_dir, candidate) + relationship_counts.update(item["relationship_type"] for item in candidate["relationships"]) + summary = { + "contract_version": "graph-candidate-handoff-v1", + "source_id": self.source_id, + "candidate_count": len(records), + "relationship_counts": dict(sorted(relationship_counts.items())), + "storage_state": "private", + "review_state": "review_required", + "publication_status": "not_eligible", + "universal_identity_assertions": 0, + "row_payloads_included": False, + } + atomic_json(graph_root / "manifest.json", summary) + return summary + def run(self, raw_path: str | Path, run_dir: str | Path, artifact: SourceArtifact) -> dict[str, Any]: raw = Path(raw_path).read_bytes() if artifact.sha256 != hashlib.sha256(raw).hexdigest() or artifact.byte_size != len(raw): raise ValueError("artifact provenance mismatch") @@ -209,6 +278,9 @@ def run(self, raw_path: str | Path, run_dir: str | Path, artifact: SourceArtifac anomaly_counts = Counter(reason for item in quarantined for reason in item["reasons"]) manifest = private_manifest(source_id=self.source_id, adapter_version=self.adapter_version, schema_version=self.schema_version, artifact=artifact, input_rows=result["input_rows"], normalized_rows=len(accepted), quarantined_rows=len(quarantined), normalized_sha256=normalized_sha256, parsed_sha256=parsed_sha256, anomaly_counts=dict(sorted(anomaly_counts.items()))) manifest.update({"country_code": "CA", "jurisdiction_level": self.jurisdiction_level, "jurisdiction": self.jurisdiction, "delimiter": result["delimiter"], "schema_fingerprint": result["schema_fingerprint"], "coverage": self.coverage, "geocoding": "disabled"}) + graph_records = accepted + [item["record"] for item in quarantined if set(item["reasons"]).issubset({"unknown_function_code"})] + manifest["graph_candidates"] = self._write_graph_candidates(root, graph_records, artifact) + manifest["graph_candidates"]["quarantined_identity_safe_rows"] = len(graph_records) - len(accepted) atomic_json(root / "manifest.json", manifest) write_operator_review_packet(root, manifest, source_scope=self.coverage, checks=("keep federal and provincial identities separate", "review address, phone, and coordinate privacy", "review duplicate plant-number rows without silent merge", "confirm function-code or plant-type mapping", "approve any project release separately"), blockers=("no national completeness claim", "publication and privacy approval pending", "source disappearance means not observed, not closure")) return manifest diff --git a/pipeline/sources/canada/test_adapter.py b/pipeline/sources/canada/test_adapter.py index cb026e1..1fd36e7 100644 --- a/pipeline/sources/canada/test_adapter.py +++ b/pipeline/sources/canada/test_adapter.py @@ -1,5 +1,6 @@ import hashlib import io +import json import zipfile import tempfile import unittest @@ -7,6 +8,7 @@ from pipeline.common.orchestrator import run_private_lifecycle from pipeline.contracts.adapter_contract import SourceArtifact +from pipeline.contracts.graph_candidate_handoff import validate_graph_candidate from .adapter import CfiaFederalMeatAdapter, OntarioMeatPlantsAdapter FIXTURES = Path(__file__).parent / "fixtures" @@ -56,5 +58,32 @@ def test_federal_and_provincial_lifecycles_are_separate(self): status = run_private_lifecycle(FIXTURES / fixture, Path(d) / adapter.source_id, artifact, adapter) self.assertEqual(status["status"], "candidate-ready"); self.assertEqual(status["manifest"]["jurisdiction_level"], adapter.jurisdiction_level); self.assertTrue((Path(status["run_dir"]) / "release-candidate" / "records.jsonl").exists()) + def test_graph_candidates_are_source_scoped_and_only_explicit_federal_edges_are_emitted(self): + with tempfile.TemporaryDirectory() as d: + for adapter, fixture, expected_graph_candidates, expected_relationships in ((OntarioMeatPlantsAdapter(), "ontario.csv", 2, {}), (CfiaFederalMeatAdapter(), "cfia.csv", 3, {"operator": 3, "regulator": 3})): + raw = (FIXTURES / fixture).read_bytes(); artifact = SourceArtifact(adapter.source_url, "2026-09-15T00:00:00Z", hashlib.sha256(raw).hexdigest(), len(raw), code_version=adapter.adapter_version, config_version=adapter.schema_version) + status = run_private_lifecycle(FIXTURES / fixture, Path(d) / adapter.source_id, artifact, adapter) + run = Path(status["run_dir"]); graph_root = run / "graph-candidates" + summary = json.loads((graph_root / "manifest.json").read_text(encoding="utf-8")) + self.assertEqual(summary["candidate_count"], expected_graph_candidates) + self.assertEqual(summary["relationship_counts"], expected_relationships) + candidates = [path / "graph-candidate.json" for path in graph_root.iterdir() if path.is_dir()] + self.assertEqual(len(candidates), expected_graph_candidates) + candidate = json.loads(candidates[0].read_text(encoding="utf-8")) + validate_graph_candidate(candidate) + self.assertEqual(candidate["publication"]["publication_status"], "not_eligible") + if adapter.jurisdiction_level == "provincial": + self.assertEqual(candidate["relationships"], []) + + def test_graph_candidate_rerun_is_byte_deterministic(self): + raw = (FIXTURES / "cfia.csv").read_bytes(); adapter = CfiaFederalMeatAdapter() + artifact = SourceArtifact(adapter.source_url, "2026-09-15T00:00:00Z", hashlib.sha256(raw).hexdigest(), len(raw), code_version=adapter.adapter_version, config_version=adapter.schema_version) + with tempfile.TemporaryDirectory() as d: + first = run_private_lifecycle(FIXTURES / "cfia.csv", Path(d) / "first", artifact, adapter) + second = run_private_lifecycle(FIXTURES / "cfia.csv", Path(d) / "second", artifact, adapter) + def payloads(status): + return sorted(path.read_bytes() for path in (Path(status["run_dir"]) / "graph-candidates").glob("*/graph-candidate.json")) + self.assertEqual(payloads(first), payloads(second)) + if __name__ == "__main__": unittest.main() From c7c08f2f58c61494a9823d3116354c8fa6f2689b Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Thu, 17 Sep 2026 13:29:56 -0700 Subject: [PATCH 228/311] Build private US real-data graph rehearsal --- .../us-fsis-current-route-2026-09-17.json | 21 ++ ...eal-legacy-graph-rehearsal-2026-09-17.json | 144 +++++++++++++ docs/architecture/adr-graph-foundation.md | 4 +- docs/architecture/graph-data-dictionary.md | 1 + docs/countries/us/README.md | 8 + docs/country-recon-us.md | 2 +- ...raph_regulatory_authority_relationship.sql | 10 + .../scripts/maintenance/rehearse_us_real.py | 99 +++++++++ pipeline/sources/us/accountability/adapter.py | 1 + .../sources/us/accountability/config.json | 2 +- pipeline/sources/us/real_rehearsal.py | 204 ++++++++++++++++++ pipeline/sources/us/test_real_rehearsal.py | 50 +++++ pipeline/tests/test_graph_migrations.py | 4 +- 13 files changed, 545 insertions(+), 5 deletions(-) create mode 100644 data/manifests/us-fsis-current-route-2026-09-17.json create mode 100644 data/manifests/us-real-legacy-graph-rehearsal-2026-09-17.json create mode 100644 pipeline/migrations/038_graph_regulatory_authority_relationship.sql create mode 100644 pipeline/scripts/maintenance/rehearse_us_real.py create mode 100644 pipeline/sources/us/real_rehearsal.py create mode 100644 pipeline/sources/us/test_real_rehearsal.py diff --git a/data/manifests/us-fsis-current-route-2026-09-17.json b/data/manifests/us-fsis-current-route-2026-09-17.json new file mode 100644 index 0000000..74b8013 --- /dev/null +++ b/data/manifests/us-fsis-current-route-2026-09-17.json @@ -0,0 +1,21 @@ +{ + "manifest_version": "us-fsis-current-route-v1", + "source_id": "us.fsis", + "authority": "USDA Food Safety and Inspection Service", + "page_url": "https://www.fsis.usda.gov/inspection/establishments/meat-poultry-and-egg-product-inspection-directory", + "page_observed_in_normal_browser": true, + "page_last_updated_observed": "2026-09-14", + "current_file_routes": [ + "https://www.fsis.usda.gov/sites/default/files/media_file/documents/MPI_Directory_by_Establishment_Name.csv", + "https://www.fsis.usda.gov/sites/default/files/media_file/documents/MPI_Directory_by_Establishment_Number.csv", + "https://www.fsis.usda.gov/sites/default/files/media_file/documents/Dataset_Establishment_Demographic_Data.csv" + ], + "current_raw_artifacts": { + "status": "not_captured", + "reason": "The exact CSV routes returned HTTP 403 to the bounded direct acquisition attempt; the in-app browser displayed the official page and links but did not expose a local downloaded artifact path.", + "bypass_attempted": false, + "next_action": "Authorized operator-assisted browser capture, then record retrieval timestamp, final URL, byte size, hash, edition/effective date, and terms review before V2 handoff." + }, + "publication_state": "blocked", + "row_payloads_included": false +} diff --git a/data/manifests/us-real-legacy-graph-rehearsal-2026-09-17.json b/data/manifests/us-real-legacy-graph-rehearsal-2026-09-17.json new file mode 100644 index 0000000..50858b8 --- /dev/null +++ b/data/manifests/us-real-legacy-graph-rehearsal-2026-09-17.json @@ -0,0 +1,144 @@ +{ + "corpus_state": "private-regression-only", + "generated_at_utc": "2026-09-17T00:00:00Z", + "graph": { + "accepted_relationship_counts": { + "inspection_observes": 1332, + "regulatory_authority_for": 1332 + }, + "accepted_relationships": 2664, + "all_candidates": { + "auto_merge": false, + "geocoding": "disabled", + "publication_gate": "blocked", + "review_state": "review_required" + }, + "candidate_rule": "exact source-native IDs in the same legacy source row only", + "cross_source_identity_joins_attempted": 0, + "dated_relationship_counts": { + "legacy-aphis-annual-reports": 2026, + "legacy-aphis-inspections": 9014, + "legacy-fsis-locations": 14198 + }, + "graph_manifest_sha256": "6fd68d2caad56a3b671dc455723b25520a690a8728aff8fa1659c77e53a40458", + "ledger_input_rows": 25238, + "name_address_phone_coordinate_joins_attempted": 0, + "profile_counts": { + "legacy-aphis-annual-reports": 2026, + "legacy-aphis-inspections": 9014, + "legacy-fsis-locations": 14198 + }, + "quarantine_reason_counts": { + "retrieval_precedes_observation": 5868, + "stale_evidence": 16706 + }, + "quarantined_relationships": 22574, + "relationship_counts": { + "aggregate_describes": 1013, + "establishment_approval_for": 7099, + "inspection_observes": 4507, + "regulatory_authority_for": 12619 + } + }, + "input_kind": "existing-v1-derived-snapshot; not current raw acquisition", + "lifecycle": { + "aphis-annual-reports": { + "candidate_created": true, + "coordinate_gate": "review_required", + "input_rows": 1013, + "normalized_rows": 1013, + "privacy_gate": "pending", + "publication_state": "private-candidate", + "quarantined_rows": 0, + "release_state": "not-created", + "review_state": "review_required", + "schema_fingerprint": "a1fd908ab662cadd330310aeddba586c34d170c12a0f75b0dfb367b3f978ccc5", + "sha256": "c243e64ba804ad55cc470644cc9c41678bb542622a7dbba75c938fff9a789ff1", + "source_id": "us.aphis", + "status": "candidate-ready" + }, + "aphis-inspections": { + "candidate_created": true, + "coordinate_gate": "review_required", + "input_rows": 4507, + "normalized_rows": 4507, + "privacy_gate": "pending", + "publication_state": "private-candidate", + "quarantined_rows": 0, + "release_state": "not-created", + "review_state": "review_required", + "schema_fingerprint": "8a7c46f3068955b88275377da2418a2cb232260eb85327924460a4791f1f1971", + "sha256": "edbd3cd963b097cd0d72049af9cdc661429e849d3f29b9cb31827d228c1cd1a7", + "source_id": "us.aphis", + "status": "candidate-ready" + }, + "fsis-locations": { + "candidate_created": true, + "coordinate_gate": "review_required", + "input_rows": 7101, + "normalized_rows": 7101, + "privacy_gate": "pending", + "publication_state": "private-candidate", + "quarantined_rows": 0, + "release_state": "not-created", + "review_state": "review_required", + "schema_fingerprint": "9d68e76e6adb5be3a5033a5e99a2b8954d898a39d8eba51ed1cb804f0ae939d9", + "sha256": "2dca259076a16a324ad9565d2d4057aa0f5e6e51bbc25a8dc1c80a4e4fbdf5c7", + "source_id": "us.fsis", + "status": "candidate-ready" + } + }, + "limitations": [ + "The legacy V1 snapshots are not current raw FSIS or APHIS captures and are not represented as current evidence.", + "The current FSIS page was observed in a browser with a Sep 14, 2026 update, but exact CSV downloads returned 403 outside that session; no current raw artifact is claimed.", + "Graph yield is a candidate count, not accuracy, ownership truth, facility operation, or publication permission.", + "A source disappearance is not closure; state inspection programs remain outside this federal-only rehearsal." + ], + "privacy_and_provenance": { + "coordinates": "not used for identity; geocoding disabled", + "raw_values": "private adapter output only; not included in report", + "source_hashes": { + "aphis_annual_reports": "c243e64ba804ad55cc470644cc9c41678bb542622a7dbba75c938fff9a789ff1", + "aphis_inspections": "edbd3cd963b097cd0d72049af9cdc661429e849d3f29b9cb31827d228c1cd1a7", + "fsis_locations": "2dca259076a16a324ad9565d2d4057aa0f5e6e51bbc25a8dc1c80a4e4fbdf5c7" + } + }, + "publication_eligibility": "blocked", + "quality": { + "accuracy": "not measured; no adjudicated real labels available", + "input_rows_reconciled_to_source_local_graph_or_skip": true, + "skipped_or_quarantined_before_ledger": { + "fsis_missing_observation_date": 2 + } + }, + "schema_version": "us-real-legacy-graph-rehearsal-v1", + "source_boundaries": { + "aphis_annual_reports": { + "evidence_family": "annual aggregate observations; may be amended", + "input_rows": 1013, + "jurisdiction": "federal", + "source_url": "https://www.aphis.usda.gov/animal-care/awa-services/usda-animal-care-public-search-tool" + }, + "aphis_inspections": { + "evidence_family": "inspection observations; no FSIS facility merge", + "input_rows": 4507, + "jurisdiction": "federal", + "source_url": "https://www.aphis.usda.gov/animal-care/awa-services/usda-animal-care-public-search-tool" + }, + "fsis": { + "input_rows": 7101, + "jurisdiction": "federal", + "source_url": "https://www.fsis.usda.gov/sites/default/files/media_file/documents/MPI_Directory_by_Establishment_Number.csv", + "state_inspection_programs": "excluded" + }, + "state_programs": { + "included": false, + "reason": "no state inspection source was supplied or inferred" + } + }, + "strata": { + "aphis_annual_reports": 1013, + "aphis_inspections": 4507, + "fsis_locations": 7101 + } +} diff --git a/docs/architecture/adr-graph-foundation.md b/docs/architecture/adr-graph-foundation.md index 08fe783..04971e2 100644 --- a/docs/architecture/adr-graph-foundation.md +++ b/docs/architecture/adr-graph-foundation.md @@ -12,8 +12,8 @@ source-scoped decision (`candidate`, `accepted`, `disputed`, or `rejected`), but never merges rows or creates a universal identity. Relationship rows are dated observations, not mutable edges. They support -operator, owner, parent, brand, supplier, and customer assertions, including -explicit unknown observations. Conflicting observations remain queryable and +operator, owner, parent, brand, supplier, customer, and regulatory-authority +assertions, including explicit unknown observations. Conflicting observations remain queryable and are not resolved by a latest-write overwrite. `uec.organization_relationship_current` is only a convenience projection over retained observations. diff --git a/docs/architecture/graph-data-dictionary.md b/docs/architecture/graph-data-dictionary.md index ce38de9..fd6f9a7 100644 --- a/docs/architecture/graph-data-dictionary.md +++ b/docs/architecture/graph-data-dictionary.md @@ -6,6 +6,7 @@ | `source_entity_identifiers` | Native identifier observed in one source record | Source-qualified; exactly one facility or organization target; append-only. | | `source_entity_crosswalks` | A reviewed or candidate mapping between two native identifiers | `identity_scope = source_scoped`; does not merge or assert universal identity. | | `organization_relationship_observations` | Dated organization-to-facility or organization-to-organization assertion | Validity dates, observation time, source, confidence, review state, and explicit unknowns are distinct. | +| `regulatory_authority_for` relationship | Source-backed regulatory-scope assertion from a legal-entity authority to a facility, operator, or inspection | Authority role is a relationship, not a new canonical entity type; it does not establish project approval, operation, or factual accuracy. | | `organization_relationship_current` | Latest observation per scoped endpoint/type | Projection only; different targets remain visible for contradictions. | | `claims` | Source-backed typed value/unknown for one facility or organization | Multiple values can coexist; unknown requires a reason. | | `claim_support` | Link from a claim to a source record/artifact | Role is explicit: primary, corroborating, contradicting, or context. | diff --git a/docs/countries/us/README.md b/docs/countries/us/README.md index b94c037..bd99fc2 100644 --- a/docs/countries/us/README.md +++ b/docs/countries/us/README.md @@ -35,6 +35,14 @@ stale, conflicting, overlapping-ownership, and suppressed relationships are quarantined. The checked-in fixture is synthetic/sanitized, private/test-only, and does not add a graph migration or public release. +## Legacy real-data V2 and graph rehearsal + +Run `python -m pipeline.scripts.maintenance.rehearse_us_real --root . --output data/manifests/us-real-legacy-graph-rehearsal-2026-09-17.json --private-dir data/graph-rehearsal/us-real-20260917` to replay the checked-in V1-derived US snapshots through the typed FSIS and APHIS private lifecycle contracts and build a private graph ledger. The rehearsal keeps FSIS federal facility/establishment-approval evidence separate from APHIS inspection and annual-report evidence, emits regulator edges only from source scope, and never joins across FSIS and APHIS by name, address, phone, or coordinates. All output rows remain ignored private staging; the checked-in manifest is aggregate-only. + +The 2026-09-17 rehearsal measured 7,101 FSIS rows, 4,507 APHIS inspection rows, and 1,013 APHIS annual-report rows. It produced 25,238 explicit source-local ledger assertions; 2,664 survived the stale/retrieval safety checks and 22,574 were quarantined for review. These are candidate and queue counts, not accuracy, ownership, operating-status, approval, or publication claims. State inspection programs remain excluded. + +The current FSIS page was observed in a normal browser with a September 14, 2026 update and three CSV routes, but the exact file routes returned HTTP 403 to bounded direct acquisition. See the row-free [current-route manifest](../../../data/manifests/us-fsis-current-route-2026-09-17.json). + ## Review checklist - authority, edition/effective date, URL, terms/attribution, and retention are recorded; diff --git a/docs/country-recon-us.md b/docs/country-recon-us.md index d13195c..a8e7a98 100644 --- a/docs/country-recon-us.md +++ b/docs/country-recon-us.md @@ -10,7 +10,7 @@ V1’s “USDA” layer is the FSIS Meat, Poultry and Egg Product Inspection (MP | V1 component | Primary route | Evidence / readiness | Caveats and blocker | |---|---|---|---| -| FSIS establishments/demographics | [FSIS MPI Directory](https://www.fsis.usda.gov/inspection/establishments) and [inspected establishments](https://www.fsis.usda.gov/inspection/fsis-inspected-establishments) | Official page exposes downloadable CSV directory/demographic files and documentation; not privately fetched and no hash/bytes claimed | Weekly replacement; no API/rate contract verified; FSIS coverage is not all slaughter/processing sites and state programs are separate | +| FSIS establishments/demographics | [FSIS MPI Directory](https://www.fsis.usda.gov/inspection/establishments) and [inspected establishments](https://www.fsis.usda.gov/inspection/fsis-inspected-establishments) | Official page observed in a normal browser on 2026-09-17; current page update observed as 2026-09-14 and three CSV routes exposed, but exact CSV routes returned HTTP 403 to bounded direct acquisition; no current raw hash/bytes claimed | Weekly replacement; FSIS coverage is not all slaughter/processing sites and state programs are separate; use operator-assisted capture contract | | APHIS research annual use | [Annual Usage Summary](https://www.aphis.usda.gov/awa/research-facility-report/annual-summary), [Public Search Tool](https://direct.aphis.usda.gov/animal-care/awa-services/usda-animal-care-public-search-tool), [annual reports](https://efile.aphis.usda.gov/PublicSearchTool/s/annual-reports) | Official fiscal-year/search routes identified; not privately fetched | Interactive/UI-mediated, no documented bulk API/rate contract; amended annual reports may differ; use sanctioned route only | | APHIS inspections/registrants | [AWA inspections and annual reports](https://www.aphis.usda.gov/awa/annual-inspection-reports) | Separate public-search/inspection population identified; not reproduced | Redactions/changes and FOIA boundary; absence/presence does not prove operation or violation | diff --git a/pipeline/migrations/038_graph_regulatory_authority_relationship.sql b/pipeline/migrations/038_graph_regulatory_authority_relationship.sql new file mode 100644 index 0000000..ba2ef81 --- /dev/null +++ b/pipeline/migrations/038_graph_regulatory_authority_relationship.sql @@ -0,0 +1,10 @@ +-- Add the source-backed authority-role edge used by private graph candidates. +-- Authorities remain organizations; this does not create a new canonical entity +-- type and does not grant approval or publication. + +ALTER TABLE uec.organization_relationship_observations + DROP CONSTRAINT organization_relationship_observations_relationship_type_check; + +ALTER TABLE uec.organization_relationship_observations + ADD CONSTRAINT organization_relationship_observations_relationship_type_check + CHECK (relationship_type IN ('operator', 'owner', 'parent', 'brand', 'supplier', 'customer', 'regulatory_authority_for')); diff --git a/pipeline/scripts/maintenance/rehearse_us_real.py b/pipeline/scripts/maintenance/rehearse_us_real.py new file mode 100644 index 0000000..3cf354a --- /dev/null +++ b/pipeline/scripts/maintenance/rehearse_us_real.py @@ -0,0 +1,99 @@ +"""Run the private US legacy V2 and accountability-graph rehearsal.""" +from __future__ import annotations + +import argparse +import hashlib +import json +import tempfile +from datetime import datetime, timezone +from pathlib import Path + +from pipeline.contracts.adapter_contract import SourceArtifact +from pipeline.common.orchestrator import run_private_lifecycle +from pipeline.sources.us.accountability.adapter import UsAccountabilityAdapter +from pipeline.sources.us.real_rehearsal import ( + APHIS_ROUTE, + FSIS_ROUTE, + build_ledger_rows, + row_free_metrics, + write_ledger, +) + + +def _artifact(path: Path, url: str, code: str, schema: str) -> SourceArtifact: + raw = path.read_bytes() + return SourceArtifact( + source_url=url, + retrieved_at_utc="2026-09-17T00:00:00Z", + sha256=hashlib.sha256(raw).hexdigest(), + byte_size=len(raw), + effective_date="unknown", + code_version=code, + config_version=schema, + rights_caveat="Legacy snapshot; source terms and field-level publication review remain open", + privacy_caveat="Private staging; addresses, coordinates, contacts, and names require review", + coverage="Legacy V1 snapshot only; not a current-source claim", + ) + + +def run(*, root: Path, output: Path, private_dir: Path) -> dict: + fsis = root / "static_data/us/locations.csv" + inspections = root / "static_data/us/inspection_reports.csv" + annual = root / "static_data/us/aphis_data_final.csv" + for path in (fsis, inspections, annual): + if not path.is_file(): + raise ValueError(f"missing US legacy input: {path}") + + from pipeline.sources.us.fsis.adapter import FsisMpiAdapter + from pipeline.sources.us.aphis.adapter import AphisPublicSearchAdapter + + lifecycle = {} + for name, path, adapter, url in ( + ("fsis-locations", fsis, FsisMpiAdapter(), FSIS_ROUTE), + ("aphis-inspections", inspections, AphisPublicSearchAdapter(), APHIS_ROUTE), + ("aphis-annual-reports", annual, AphisPublicSearchAdapter(), APHIS_ROUTE), + ): + run_dir = private_dir / "v2" / name + lifecycle[name] = run_private_lifecycle(path, run_dir, _artifact(path, url, adapter.adapter_version, adapter.schema_version), adapter, health_as_of_utc="2026-09-17T00:00:00Z") + + ledger_rows, skipped = build_ledger_rows( + __import__("csv").DictReader(fsis.open(newline="", encoding="utf-8-sig")), + __import__("csv").DictReader(inspections.open(newline="", encoding="utf-8-sig")), + __import__("csv").DictReader(annual.open(newline="", encoding="utf-8-sig")), + ) + ledger = private_dir / "graph" / "link-ledger.csv" + write_ledger(ledger, ledger_rows) + graph = UsAccountabilityAdapter(reference_date=datetime(2026, 9, 15, tzinfo=timezone.utc).date()).run( + ledger, private_dir / "graph" / "candidate", _artifact(ledger, "https://www.fsis.usda.gov/inspection/establishments/meat-poultry-and-egg-product-inspection-directory", "us-accountability-real-rehearsal-v1", "us-accountability-real-rehearsal-v1") + ) + report = row_free_metrics(fsis_path=fsis, inspection_path=inspections, annual_path=annual, ledger_rows=ledger_rows, skipped=skipped) + report["lifecycle"] = { + name: {key: value for key, value in status.get("manifest", {}).items() if key in {"source_id", "input_rows", "normalized_rows", "quarantined_rows", "sha256", "schema_fingerprint", "release_state", "publication_state", "review_state", "privacy_gate", "coordinate_gate"}} + | {"status": status.get("status"), "candidate_created": status.get("candidate_created", False)} + for name, status in lifecycle.items() + } + report["graph"]["accepted_relationships"] = graph["relationship_rows"] + report["graph"]["quarantined_relationships"] = graph["quarantined_relationship_rows"] + report["graph"]["accepted_relationship_counts"] = graph["relationship_types"] + report["graph"]["quarantine_reason_counts"] = graph["anomaly_counts"] + report["graph"]["graph_manifest_sha256"] = hashlib.sha256(json.dumps(graph, sort_keys=True, default=list).encode()).hexdigest() + report["generated_at_utc"] = "2026-09-17T00:00:00Z" + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True) + "\n", encoding="utf-8") + return report + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--root", type=Path, default=Path(".")) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--private-dir", type=Path) + args = parser.parse_args() + private_dir = args.private_dir or Path(tempfile.mkdtemp(prefix="uec-us-real-graph-")) + report = run(root=args.root, output=args.output, private_dir=private_dir) + print(json.dumps({"input_rows": sum(report["strata"].values()), "ledger_rows": report["graph"]["ledger_input_rows"], "accepted_relationships": report["graph"]["accepted_relationships"], "quarantined_relationships": report["graph"]["quarantined_relationships"], "publication_eligibility": report["publication_eligibility"]}, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/pipeline/sources/us/accountability/adapter.py b/pipeline/sources/us/accountability/adapter.py index 3cdb1a6..57214a4 100644 --- a/pipeline/sources/us/accountability/adapter.py +++ b/pipeline/sources/us/accountability/adapter.py @@ -37,6 +37,7 @@ "enforcement_for": ("enforcement", frozenset({"violation"})), "laboratory_supports": ("laboratory", frozenset({"inspection", "aggregate_observation"})), "aggregate_describes": ("aggregate_observation", frozenset({"facility", "operator"})), + "regulatory_authority_for": ("legal_entity", frozenset({"facility", "operator", "inspection"})), } CONFIDENCE = frozenset({"high", "medium", "low"}) REVIEW_STATES = frozenset({"evidence_verified", "review_required", "quarantined"}) diff --git a/pipeline/sources/us/accountability/config.json b/pipeline/sources/us/accountability/config.json index 3e7f782..1b1bd64 100644 --- a/pipeline/sources/us/accountability/config.json +++ b/pipeline/sources/us/accountability/config.json @@ -5,7 +5,7 @@ "source_url": "https://www.fsis.usda.gov/inspection/establishments/meat-poultry-and-egg-product-inspection-directory", "acquisition": "assisted-link-ledger-after-source-specific-review", "release_allowed_by_default": false, - "entity_policy": "facilities, establishment approvals, operators, legal entities, parents, brands, inspections, violations, enforcements, laboratories, and aggregate observations remain distinct", + "entity_policy": "facilities, establishment approvals, operators, legal entities (including authorities in a regulator role), parents, brands, inspections, violations, enforcements, laboratories, and aggregate observations remain distinct", "match_policy": "exact source identifiers or an explicit reviewed link event; never name, address, phone, geocoder, or fuzzy matching", "stale_after_days": 365 } diff --git a/pipeline/sources/us/real_rehearsal.py b/pipeline/sources/us/real_rehearsal.py new file mode 100644 index 0000000..bef13a6 --- /dev/null +++ b/pipeline/sources/us/real_rehearsal.py @@ -0,0 +1,204 @@ +"""Private, row-free rehearsal of the real legacy US USDA snapshots. + +The checked-in files are V1-derived legacy snapshots, not current raw source +captures. This module uses them only to exercise the V2 adapters and the +accountability graph contract. It creates a private link ledger from explicit +source-local identifiers and dated source fields; it never joins FSIS to APHIS +by name, address, phone, coordinates, or proximity. +""" +from __future__ import annotations + +import csv +import hashlib +import json +from collections import Counter +from datetime import datetime +from pathlib import Path +from typing import Any, Iterable + +from pipeline.sources.us.accountability.adapter import REQUIRED_HEADERS + + +FSIS_ROUTE = "https://www.fsis.usda.gov/sites/default/files/media_file/documents/MPI_Directory_by_Establishment_Number.csv" +APHIS_ROUTE = "https://www.aphis.usda.gov/animal-care/awa-services/usda-animal-care-public-search-tool" +CURRENT_PAGE = "https://www.fsis.usda.gov/inspection/establishments/meat-poultry-and-egg-product-inspection-directory" +LEGACY_AS_OF = "2026-09-17T00:00:00Z" + + +def _clean(value: Any) -> str: + return str(value or "").strip() + + +def _date(value: str, *, year_end: bool = False) -> str: + text = _clean(value) + for fmt in ("%m/%d/%Y", "%Y-%m-%d", "%m/%d/%y"): + try: + return datetime.strptime(text, fmt).date().isoformat() + except ValueError: + pass + if year_end and len(text) == 4 and text.isdigit(): + return f"{text}-12-31" + return "" + + +def _read(path: Path) -> list[dict[str, str]]: + with path.open(newline="", encoding="utf-8-sig") as handle: + return list(csv.DictReader(handle)) + + +def _row_fingerprint(namespace: str, row: dict[str, str]) -> str: + material = "\0".join((namespace, *(f"{key}={row.get(key, '')}" for key in sorted(row)))) + return hashlib.sha256(material.encode("utf-8", "surrogateescape")).hexdigest() + + +def _base(*, subject_type: str, subject_id: str, subject_name: str, + object_type: str, object_id: str, object_name: str, + relationship_type: str, evidence_source_id: str, + evidence_native_id: str, observation_date: str, + evidence_url: str, excerpt: str, profile: str) -> dict[str, str]: + return { + "subject_type": subject_type, + "subject_source_id": evidence_source_id, + "subject_source_native_id": subject_id, + "subject_name": subject_name, + "object_type": object_type, + "object_source_id": evidence_source_id, + "object_source_native_id": object_id, + "object_name": object_name, + "relationship_type": relationship_type, + "evidence_source_id": evidence_source_id, + "evidence_source_native_id": evidence_native_id, + "observation_date": observation_date, + "retrieved_at_utc": LEGACY_AS_OF, + "valid_from": "", + "valid_to": "", + "confidence": "high" if relationship_type in {"establishment_approval_for", "regulatory_authority_for"} else "medium", + "review_state": "review_required", + "match_method": "exact_source_id", + "evidence_url": evidence_url, + "evidence_excerpt": excerpt, + "suppression_state": "eligible", + "evidence_profile": profile, + } + + +def build_ledger_rows(fsis_rows: Iterable[dict[str, str]], inspection_rows: Iterable[dict[str, str]], annual_rows: Iterable[dict[str, str]]) -> tuple[list[dict[str, str]], dict[str, int]]: + """Build explicit, source-local graph assertions from legacy rows.""" + rows: list[dict[str, str]] = [] + skipped = Counter() + + for row in fsis_rows: + facility_id = _clean(row.get("establishment_id")) + approval_id = _clean(row.get("establishment_number")) + observed = _date(row.get("grant_date", "")) + name = _clean(row.get("establishment_name")) + if not facility_id or not approval_id or not name: + skipped["fsis_missing_explicit_identity"] += 1 + continue + if not observed: + skipped["fsis_missing_observation_date"] += 1 + continue + evidence_id = f"legacy-row:{facility_id}" + excerpt = "FSIS legacy V1 snapshot row retains an establishment ID and establishment number; currentness and publication remain unverified." + rows.append(_base(subject_type="establishment_approval", subject_id=f"approval:{approval_id}", subject_name=f"FSIS establishment approval {approval_id}", object_type="facility", object_id=facility_id, object_name=name, relationship_type="establishment_approval_for", evidence_source_id="us.fsis", evidence_native_id=evidence_id, observation_date=observed, evidence_url=FSIS_ROUTE, excerpt=excerpt, profile="legacy-fsis-locations")) + rows.append(_base(subject_type="legal_entity", subject_id="authority:fsis", subject_name="USDA Food Safety and Inspection Service", object_type="facility", object_id=facility_id, object_name=name, relationship_type="regulatory_authority_for", evidence_source_id="us.fsis", evidence_native_id=evidence_id, observation_date=observed, evidence_url=CURRENT_PAGE, excerpt="The FSIS MPI Directory describes this source population as FSIS-regulated establishments; this is a source-scope assertion, not project approval.", profile="legacy-fsis-locations")) + + for row in inspection_rows: + customer = _clean(row.get("Customer Number")) + certificate = _clean(row.get("Certificate Number")) + name = _clean(row.get("Account Name")) + observed = _date(row.get("Status Date", "")) + if not customer or not certificate or not name: + skipped["aphis_inspections_missing_explicit_identity"] += 1 + continue + if not observed: + skipped["aphis_inspections_missing_observation_date"] += 1 + continue + operator_id = f"operator:customer:{customer}" + inspection_id = f"inspection:certificate:{certificate}:{observed}" + evidence_id = f"legacy-row:{certificate}:{customer}" + rows.append(_base(subject_type="inspection", subject_id=inspection_id, subject_name=f"APHIS inspection {certificate}", object_type="operator", object_id=operator_id, object_name=name, relationship_type="inspection_observes", evidence_source_id="us.aphis", evidence_native_id=evidence_id, observation_date=observed, evidence_url=APHIS_ROUTE, excerpt="APHIS legacy inspection snapshot retains certificate and customer identifiers in the same source row; it is not an FSIS facility assertion.", profile="legacy-aphis-inspections")) + rows.append(_base(subject_type="legal_entity", subject_id="authority:aphis-animal-care", subject_name="USDA APHIS Animal Care", object_type="operator", object_id=operator_id, object_name=name, relationship_type="regulatory_authority_for", evidence_source_id="us.aphis", evidence_native_id=evidence_id, observation_date=observed, evidence_url=APHIS_ROUTE, excerpt="APHIS Animal Care public-search evidence is retained as a separate regulatory observation; no facility merge is inferred.", profile="legacy-aphis-inspections")) + + for row in annual_rows: + customer = _clean(row.get("Customer Number_y")) or _clean(row.get("Customer Number_x")) + certificate = _clean(row.get("Certificate Number")) + name = _clean(row.get("Account Name")) + observed = _date(row.get("Year", ""), year_end=True) + if not customer or not certificate or not name: + skipped["aphis_annual_reports_missing_explicit_identity"] += 1 + continue + if not observed: + skipped["aphis_annual_reports_missing_observation_date"] += 1 + continue + operator_id = f"operator:customer:{customer}" + aggregate_id = f"annual-report:{certificate}:{observed[:4]}" + evidence_id = f"legacy-row:{certificate}:{customer}:{observed[:4]}" + rows.append(_base(subject_type="aggregate_observation", subject_id=aggregate_id, subject_name=f"APHIS annual report {certificate} {observed[:4]}", object_type="operator", object_id=operator_id, object_name=name, relationship_type="aggregate_describes", evidence_source_id="us.aphis", evidence_native_id=evidence_id, observation_date=observed, evidence_url=APHIS_ROUTE, excerpt="APHIS annual-report snapshot is retained as an aggregate observation and does not establish an FSIS facility or laboratory identity.", profile="legacy-aphis-annual-reports")) + rows.append(_base(subject_type="legal_entity", subject_id="authority:aphis-animal-care", subject_name="USDA APHIS Animal Care", object_type="operator", object_id=operator_id, object_name=name, relationship_type="regulatory_authority_for", evidence_source_id="us.aphis", evidence_native_id=evidence_id, observation_date=observed, evidence_url=APHIS_ROUTE, excerpt="APHIS annual-report evidence remains a separate source family under the Animal Care authority; amended reports and currentness require review.", profile="legacy-aphis-annual-reports")) + + return rows, dict(sorted(skipped.items())) + + +def write_ledger(path: Path, rows: list[dict[str, str]]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", newline="", encoding="utf-8") as handle: + writer = csv.DictWriter(handle, fieldnames=REQUIRED_HEADERS, lineterminator="\n") + writer.writeheader() + writer.writerows([{key: row.get(key, "") for key in REQUIRED_HEADERS} for row in rows]) + + +def row_free_metrics(*, fsis_path: Path, inspection_path: Path, annual_path: Path, ledger_rows: list[dict[str, str]], skipped: dict[str, int]) -> dict[str, Any]: + source_rows = {"fsis_locations": _read(fsis_path), "aphis_inspections": _read(inspection_path), "aphis_annual_reports": _read(annual_path)} + relationships = Counter(row["relationship_type"] for row in ledger_rows) + profiles = Counter(row["evidence_profile"] for row in ledger_rows) + dated = Counter(row["evidence_profile"] for row in ledger_rows if row.get("observation_date")) + return { + "schema_version": "us-real-legacy-graph-rehearsal-v1", + "corpus_state": "private-regression-only", + "input_kind": "existing-v1-derived-snapshot; not current raw acquisition", + "publication_eligibility": "blocked", + "source_boundaries": { + "fsis": {"jurisdiction": "federal", "input_rows": len(source_rows["fsis_locations"]), "source_url": FSIS_ROUTE, "state_inspection_programs": "excluded"}, + "aphis_inspections": {"jurisdiction": "federal", "input_rows": len(source_rows["aphis_inspections"]), "source_url": APHIS_ROUTE, "evidence_family": "inspection observations; no FSIS facility merge"}, + "aphis_annual_reports": {"jurisdiction": "federal", "input_rows": len(source_rows["aphis_annual_reports"]), "source_url": APHIS_ROUTE, "evidence_family": "annual aggregate observations; may be amended"}, + "state_programs": {"included": False, "reason": "no state inspection source was supplied or inferred"}, + }, + "strata": {name: len(rows) for name, rows in source_rows.items()}, + "graph": { + "ledger_input_rows": len(ledger_rows), + "relationship_counts": dict(sorted(relationships.items())), + "profile_counts": dict(sorted(profiles.items())), + "dated_relationship_counts": dict(sorted(dated.items())), + "cross_source_identity_joins_attempted": 0, + "name_address_phone_coordinate_joins_attempted": 0, + "candidate_rule": "exact source-native IDs in the same legacy source row only", + "all_candidates": {"review_state": "review_required", "publication_gate": "blocked", "auto_merge": False, "geocoding": "disabled"}, + }, + "quality": { + "skipped_or_quarantined_before_ledger": skipped, + "input_rows_reconciled_to_source_local_graph_or_skip": ( + sum(1 for row in ledger_rows if row.get("evidence_profile") == "legacy-fsis-locations") // 2 + + skipped.get("fsis_missing_explicit_identity", 0) + + skipped.get("fsis_missing_observation_date", 0) == len(source_rows["fsis_locations"]) + and sum(1 for row in ledger_rows if row.get("evidence_profile") == "legacy-aphis-inspections") // 2 + + skipped.get("aphis_inspections_missing_explicit_identity", 0) + + skipped.get("aphis_inspections_missing_observation_date", 0) == len(source_rows["aphis_inspections"]) + and sum(1 for row in ledger_rows if row.get("evidence_profile") == "legacy-aphis-annual-reports") // 2 + + skipped.get("aphis_annual_reports_missing_explicit_identity", 0) + + skipped.get("aphis_annual_reports_missing_observation_date", 0) == len(source_rows["aphis_annual_reports"]) + ), + "accuracy": "not measured; no adjudicated real labels available", + }, + "privacy_and_provenance": { + "raw_values": "private adapter output only; not included in report", + "coordinates": "not used for identity; geocoding disabled", + "source_hashes": {key: hashlib.sha256(path.read_bytes()).hexdigest() for key, path in (("fsis_locations", fsis_path), ("aphis_inspections", inspection_path), ("aphis_annual_reports", annual_path))}, + }, + "limitations": [ + "The legacy V1 snapshots are not current raw FSIS or APHIS captures and are not represented as current evidence.", + "The current FSIS page was observed in a browser with a Sep 14, 2026 update, but exact CSV downloads returned 403 outside that session; no current raw artifact is claimed.", + "Graph yield is a candidate count, not accuracy, ownership truth, facility operation, or publication permission.", + "A source disappearance is not closure; state inspection programs remain outside this federal-only rehearsal.", + ], + } diff --git a/pipeline/sources/us/test_real_rehearsal.py b/pipeline/sources/us/test_real_rehearsal.py new file mode 100644 index 0000000..f87db36 --- /dev/null +++ b/pipeline/sources/us/test_real_rehearsal.py @@ -0,0 +1,50 @@ +import csv +import unittest +from pathlib import Path + +from .real_rehearsal import build_ledger_rows, row_free_metrics + + +ROOT = Path(__file__).resolve().parents[3] + + +def _rows(path: Path): + with path.open(newline="", encoding="utf-8-sig") as handle: + return list(csv.DictReader(handle)) + + +class UsRealRehearsalTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.fsis = ROOT / "static_data/us/locations.csv" + cls.inspections = ROOT / "static_data/us/inspection_reports.csv" + cls.annual = ROOT / "static_data/us/aphis_data_final.csv" + cls.ledger, cls.skipped = build_ledger_rows(_rows(cls.fsis), _rows(cls.inspections), _rows(cls.annual)) + cls.report = row_free_metrics(fsis_path=cls.fsis, inspection_path=cls.inspections, annual_path=cls.annual, ledger_rows=cls.ledger, skipped=cls.skipped) + + def test_real_strata_reconcile_and_keep_federal_state_boundary(self): + self.assertEqual(self.report["strata"], {"fsis_locations": 7101, "aphis_inspections": 4507, "aphis_annual_reports": 1013}) + self.assertTrue(self.report["quality"]["input_rows_reconciled_to_source_local_graph_or_skip"]) + self.assertFalse(self.report["source_boundaries"]["state_programs"]["included"]) + self.assertEqual(self.report["source_boundaries"]["fsis"]["jurisdiction"], "federal") + + def test_graph_exercises_source_local_edges_without_cross_source_matching(self): + counts = self.report["graph"]["relationship_counts"] + self.assertGreater(counts["establishment_approval_for"], 7000) + self.assertGreater(counts["inspection_observes"], 4000) + self.assertGreater(counts["aggregate_describes"], 1000) + self.assertGreater(counts["regulatory_authority_for"], 12000) + self.assertEqual(self.report["graph"]["cross_source_identity_joins_attempted"], 0) + self.assertEqual(self.report["graph"]["name_address_phone_coordinate_joins_attempted"], 0) + + def test_candidate_gates_and_row_free_report(self): + self.assertEqual(self.report["publication_eligibility"], "blocked") + self.assertFalse(self.report["graph"]["all_candidates"]["auto_merge"]) + self.assertEqual(self.report["graph"]["all_candidates"]["publication_gate"], "blocked") + serialized = str(self.report) + for forbidden in ("Godshall", "Auburn University", "street", "latitude", "longitude", "source_values", "display_name"): + self.assertNotIn(forbidden, serialized) + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/tests/test_graph_migrations.py b/pipeline/tests/test_graph_migrations.py index 8796e27..bbc30b6 100644 --- a/pipeline/tests/test_graph_migrations.py +++ b/pipeline/tests/test_graph_migrations.py @@ -20,7 +20,7 @@ def test_reserved_migrations_are_present_and_ordered(self): positions = [migrations.index(name) for name in graph_migrations] self.assertEqual(positions, sorted(positions)) self.assertEqual([migrations[position] for position in positions], graph_migrations) - self.assertEqual(migrations[-12:], [ + self.assertEqual(migrations[-13:], [ "026_graph_entities_crosswalks.sql", "027_graph_relationship_observations.sql", "028_graph_claims_support.sql", @@ -33,6 +33,7 @@ def test_reserved_migrations_are_present_and_ordered(self): "035_public_discovery_planner_indexes.sql", "036_public_facility_discovery_view.sql", "037_public_discovery_read_model.sql", + "038_graph_regulatory_authority_relationship.sql", ]) def test_entities_are_distinct_and_crosswalk_is_scoped(self): @@ -54,6 +55,7 @@ def test_relationships_and_claims_preserve_uncertainty_and_state_separation(self self.assertIn("unknown_reason", relationship) self.assertIn("support_role", claims) self.assertIn("contradicting", claims) + self.assertIn("regulatory_authority_for", (ROOT / "migrations" / "038_graph_regulatory_authority_relationship.sql").read_text(encoding="utf-8")) def test_public_projections_are_release_and_suppression_aware(self): sql = self.read("029_graph_publication_projections.sql") From 157d3cd47f6a8a8460131d0f68a08bc9701a7500 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Thu, 17 Sep 2026 13:20:50 -0700 Subject: [PATCH 229/311] Document Portugal source reconnaissance --- docs/country-recon-pt.md | 66 +++++++++++++++++++ docs/source-status.json | 8 ++- docs/source-status.md | 4 ++ pipeline/source_registry.json | 8 ++- .../tests/test_portugal_recon_metadata.py | 34 ++++++++++ pipeline/tests/test_source_registry.py | 4 +- 6 files changed, 120 insertions(+), 4 deletions(-) create mode 100644 docs/country-recon-pt.md create mode 100644 pipeline/tests/test_portugal_recon_metadata.py diff --git a/docs/country-recon-pt.md b/docs/country-recon-pt.md new file mode 100644 index 0000000..664a309 --- /dev/null +++ b/docs/country-recon-pt.md @@ -0,0 +1,66 @@ +# Portugal source reconnaissance + +Status: row-free reconnaissance only. No facility rows, raw exports, coordinates, or personal data were retained. Official routes were checked 2026-09-17. This is not publication approval, legal clearance, or a healthy-pipeline claim. + +## Decision + +Portugal is a strong medium/high-effort candidate for the animal-products lane. The Direção-Geral de Alimentação e Veterinária (DGAV) publishes a national establishment-list family through SIPACE/+SIPACE, with food-origin approval, animal-by-product, feed, derogation, activity, species/product, municipality, region, and approval/registration identifiers. The source is valuable because it can support facility/activity observations and relationship edges without pretending that every row is a unique facility. + +The route is transitional rather than a verified bulk API. DGAV's current guidance links +SIPACE and says the older SIPACE remains available for consultation during transition. The public legacy list is an HTML/filter/pagination view; the +SIPACE application route resolved as a public URL during reconnaissance, but the text fetch lost its connection and no export contract was verified. The +SIPACE operator API is explicitly credentialed and requires a DGAV VPN, so it is not a public acquisition path. Do not bypass login, VPN, rate limits, or application controls. + +The first implementation target should be a private, assisted-capture adapter for the public approved-food list, followed by separate ABP and feed list scopes. Keep SNIRA animal/holding data restricted, TUA permits as an evidence overlay, and INE as aggregate context. Publication remains blocked pending source terms, privacy review, schema/count validation, coverage, identity reconciliation, and maintainer approval. + +Any production integration must be fully automated from scheduled acquisition through hashing, validation, normalization, quarantine, provenance, review gating, and guarded ingestion. Browser/manual capture is an assisted reconnaissance fallback until a stable source contract is verified. + +## Source inventory + +| Source | Verified official route and ordinary-access result | Format, identifiers, cadence, and coverage | Privacy, graph, and acquisition assessment | +|---|---|---|---| +| DGAV approved food establishments | [DGAV approved/operator page](https://www.dgav.pt/alimentos/conteudo/generos-alimenticios/iniciar-uma-empresa-alimentar/registo-e-aprovacao-de-estabelecimentos-e-operadores/lista-de-operadores-estabelecimentos-aprovados-e-operadores-registados/) links [public +SIPACE listings](https://maissipace.dgav.pt/Listagens). The legacy [SIPACE public list](https://sipace.dgav.pt/Estabelecimentos/PublicacaoNCV) is indexed as an HTML table and remains consultation-only during the transition. | Server-rendered HTML list/filter/pagination observed; no stable CSV/XLS/API export verified. Fields exposed by the public view include NCV/approval or registration number, establishment/operator name, address/locality, municipality, regional service, category, activity, associated activity, and species/products. DGAV guidance also documents NII and separate NCV status. Cadence is not stated; retrieval timestamp is required. National scope is intended, but completeness/effective-date semantics are unresolved. | Names and full addresses may include sole traders, primary producers, or mixed residential/business sites; retain privately and suppress precise output until screened. Model one NCV as an establishment identity candidate and each activity/species/category row as a dated observation. Medium-high effort because a robust browser-assisted capture and schema fingerprint are needed. | +| DGAV animal by-products (ABP) | [DGAV ABP approval/registration guidance](https://www.dgav.pt/alimentos/conteudo/subprodutos-animais/iniciar-uma-empresa-de-subprodutos-animais/registo-e-aprovacao-dos-estabelecimentos/) links public lists in +SIPACE; [ABP activity page](https://www.dgav.pt/alimentos/conteudo/subprodutos-animais/iniciar-uma-empresa-de-subprodutos-animais/atividades-do-setor/) documents the separate approval/registration scope. | Same listing family, but do not union with food establishments. Expected observations include NCV or registration/authorization number, operator/site, address/locality, category 1/2/3 or derived product, activity, and status/remarks. Public HTML route is the only verified format; cadence and export contract remain unknown. | ABP processing, incineration, storage, composting, biogas, and special-use records can expose operationally sensitive locations. Link an ABP activity observation to a reviewed establishment/NCV only; do not infer facility type from a name or from a permit. Medium-high effort after the food-list capture contract is understood. | +| DGAV feed establishments | [DGAV feed registration/approval guidance](https://www.dgav.pt/alimentos/conteudo/alimentos-para-animais/iniciar-uma-empresa-de-alimentos-para-animais/registo-e-aprovacao-de-estabelecimentos/) states that DGAV assigns an individual identification number to feed-sector establishments and directs users to the official list family via the food/operator pages. | Expected public list fields include NII/individual identifier, operator/site, address, feed activity and approval/registration state. The identifier is scope-specific and must not be treated as the food NCV. No stable machine-readable export, cadence, or codebook was verified. | Feed mills, storage, transport and primary feed operators may include private farms or sole traders. Keep feed as a separate source/product layer; link to a food/ABP NCV only on explicit evidence. Reuse the approved-food adapter only after confirming section-specific headers and codes. | +| APA Título Único Ambiental (TUA) | [Portuguese Environment Agency LUA page](https://apambiente.pt/avaliacao-e-gestao-ambiental/licenciamento-unico-ambiental) and [TUA explanation](https://apambiente.pt/avaliacao-e-gestao-ambiental/titulo-unico-ambiental) are current official pages. They describe an electronic environmental title combining decisions for an establishment, activity, or project, including REAP-linked activity where applicable. | Permit/title documents and decisions are expected to carry a TUA/process or document identifier, holder, activity/project, authority, decision date, validity, regime, and conditions. A national bulk/API route and exact public document search contract were not verified. Cadence is decision-specific. | Use as a separate permit/evidence overlay, not a facility master. Environmental documents may contain personal, residential, precise-geometry, or sensitive operational details; retain and publish only reviewed fields. Identity joins require explicit holder/site evidence and must preserve unresolved matches. High effort until a stable public search/export route is captured. | +| IFAP/DGAV SNIRA animal and holding register | [IFAP restricted-area description](https://www.ifap.pt/portal/en/registo-area-reservada), [SNIRA network description](https://www.ifap.pt/portal/en/rede-snira), and [credentialed web-service notice](https://www.ifap.pt/portal/en/temprss/-/asset_publisher/8NqcGH0YvyEu/content/snira-novo-webservice-informacao-sobre-os-animais-na-exploracao) confirm that holding/animal queries are for registered or credentialed users. | Operational service/webservice keyed by animal identification, NIF/holder and Marca de Exploração; exact current contract is not public. Historical public spreadsheets are stale aggregate/statistical artifacts, not a current facility register. | Treat holding, farmer, animal, NIF, and parcel information as restricted. No public acquisition or geocoding is authorized. A reviewed slaughterhouse-to-holding relationship may later be an event edge, but never expose source holding coordinates or infer a farm from an NCV. High difficulty/access-controlled. | +| INE animal production and slaughter statistics | [Statistics Portugal metadata for meat production](https://ine.pt/bddXplorer/htdocs/minfo.jsp?lingua=EN&var_cd=0000916&var_cd=0000917&var_cd=0000918) and [INE slaughter survey description](https://webinq.ine.pt/public/pages/queryInfo.aspx?id=IMAAC) are current official routes. | PxWeb/metadata-backed aggregate time series by period, geography, species/meat type and measure; annual meat-production metadata was current through 2025 when checked. The poultry/rabbit slaughter survey is monthly and confidential/mandatory. Table/API identifiers for a reproducible pull still need pinning. | Aggregate context only. Do not link confidential survey observations to named establishments, use totals to infer facility completeness, or double-count with DGAV approval rows. Low/medium effort once table IDs and terms are pinned. | + +## Reusable adapter and graph shape + +The DGAV family should reuse the existing source-local lifecycle pattern used by the France, Italy, Norway, and Poland reconnaissance lanes: capture the exact response, timestamp it, hash it, record content type/bytes and URL, fingerprint headers/section labels, preserve raw Portuguese values, and quarantine row-length/schema drift. The adapter must be section-aware and must not collapse repeated activity/species/product rows into duplicate facilities. + +Recommended graph entities and observations: + +- `establishment_candidate`: source NCV/registration identity, source name/address, municipality, region, and source status. +- `activity_observation`: section, category, activity code/label, associated activity, species/product, and observation/list date. +- `operator_identity_link`: operator name and any explicit NII/NIF or corporate identifier, kept separate from site identity and screened for personal data. +- `permit_observation`: TUA/process/document and decision/validity fields, linked only after reviewed evidence. +- `slaughter_or_movement_event`: future +SIPACE/SNIRA event evidence, never substituted for an approval record. + +Potential relationships are `operator operates establishment`, `establishment has approved activity`, `establishment handles species/product`, and `establishment has permit`. A shared NCV is evidence of a source identity, not proof that every activity row is a separate facility or that the site is currently operating. + +## Rights, privacy, and publication gates + +DGAV and APA government origin is verified; a source-specific reuse licence or attribution rule was not verified in this pass. Record the page/file terms actually displayed at acquisition time and obtain maintainer/legal review before redistribution. Public availability is not permission to expose a full address, precise point, contact, NIF, sole-trader identity, worker, resident, or farm/holding information. + +Required gates before any real capture is integrated: + +1. Obtain an authorized bounded capture of each DGAV section and confirm whether the +SIPACE list is public, authenticated, or export-limited for the intended use. +2. Fingerprint section-specific headers, code lists, pagination, maximum rows, status/effective-date meaning, and national coverage; fail closed on drift. +3. Keep NCV, NII, operator, permit, feed, ABP, and animal-register identifiers distinct and retain source values alongside normalized interpretations. +4. Apply residential/mixed-site and sole-trader screening before any coordinates or address text can enter public projections. No geocoding is authorized during reconnaissance. +5. Reconcile repeated observations and source disappearance as `not_observed`, not closure. Create reviewable identity links rather than fuzzy merges. +6. Keep publication blocked until terms, privacy, validation, coverage, source review, and named release approval are complete. + +## Recommended next targets + +1. `pt.dgav.approved-food`: assisted public-list capture and a section-aware adapter, using synthetic Portuguese fixtures first. +2. `pt.dgav.abp` and `pt.dgav.feed`: reuse the capture framework only after confirming separate identifiers/codebooks and scope boundaries. +3. `pt.apambiente.tua`: metadata-only route capture to determine whether a safe permit index/export exists. +4. Keep `pt.ifap.snira` restricted and `pt.ine.animal-production` aggregate until explicit access and table contracts are obtained. + +## Evidence checked + +- DGAV [approved establishments page](https://www.dgav.pt/alimentos/conteudo/generos-alimenticios/iniciar-uma-empresa-alimentar/registo-e-aprovacao-de-estabelecimentos-e-operadores/lista-de-operadores-estabelecimentos-aprovados-e-operadores-registados/), [SIPACE/+SIPACE transition page](https://www.dgav.pt/alimentos/conteudo/sipace/), [ABP approval guidance](https://www.dgav.pt/alimentos/conteudo/subprodutos-animais/iniciar-uma-empresa-de-subprodutos-animais/registo-e-aprovacao-dos-estabelecimentos/), and [feed approval guidance](https://www.dgav.pt/alimentos/conteudo/alimentos-para-animais/iniciar-uma-empresa-de-alimentos-para-animais/registo-e-aprovacao-de-estabelecimentos/). +- Public legacy [SIPACE list route](https://sipace.dgav.pt/Estabelecimentos/PublicacaoNCV) and [derogations view](https://sipace.dgav.pt/Estabelecimentos/PublicacaoDerrogacoes) were observed through current indexed pages; direct automated export and stable bulk format were not established. +- APA [LUA](https://apambiente.pt/avaliacao-e-gestao-ambiental/licenciamento-unico-ambiental) and [TUA](https://apambiente.pt/avaliacao-e-gestao-ambiental/titulo-unico-ambiental). +- IFAP [restricted animal register](https://www.ifap.pt/portal/en/registo-area-reservada), [SNIRA network](https://www.ifap.pt/portal/en/rede-snira), and [credentialed webservice notice](https://www.ifap.pt/portal/en/temprss/-/asset_publisher/8NqcGH0YvyEu/content/snira-novo-webservice-informacao-sobre-os-animais-na-exploracao). +- INE [animal production metadata](https://ine.pt/bddXplorer/htdocs/minfo.jsp?lingua=EN&var_cd=0000916&var_cd=0000917&var_cd=0000918) and [poultry/rabbit slaughter survey](https://webinq.ine.pt/public/pages/queryInfo.aspx?id=IMAAC). diff --git a/docs/source-status.json b/docs/source-status.json index 6c5c054..d3b1e6d 100644 --- a/docs/source-status.json +++ b/docs/source-status.json @@ -242,6 +242,12 @@ {"source_id":"lb.justice.companies","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-lb.md","pipeline/source_registry.json"],"next_action":"Confirm authorized access, coverage, identifiers, fees, terms, and personal-address policy."}, {"source_id":"lb.industry.food-guide","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-lb.md","pipeline/source_registry.json"],"next_action":"Confirm current list schema, licensing, cadence, and safe address boundary."}, {"source_id":"lb.cas.livestock-statistics","metadata":"partial","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-lb.md","pipeline/source_registry.json"],"next_action":"Pin aggregate table/API identifiers, cadence, revisions, terms, and suppression rules."}, - {"source_id":"ch.blv.approved-food","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ch.md","pipeline/source_registry.json"],"next_action":"Obtain an authorized bounded export or capture; fingerprint multilingual schema/list version, preserve approval/activity observations separately, and complete coverage, privacy, terms, and project-approval review."} + {"source_id":"ch.blv.approved-food","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ch.md","pipeline/source_registry.json"],"next_action":"Obtain an authorized bounded export or capture; fingerprint multilingual schema/list version, preserve approval/activity observations separately, and complete coverage, privacy, terms, and project-approval review."}, + {"source_id":"pt.dgav.approved-food","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-pt.md","pipeline/source_registry.json"],"next_action":"Obtain an authorized bounded public-list capture; fingerprint section headers, pagination, NCV/NII status semantics, codes, coverage, terms, privacy, and project approval before an adapter."}, + {"source_id":"pt.dgav.abp","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-pt.md","pipeline/source_registry.json"],"next_action":"Capture ABP sections separately from food; resolve approval/registration IDs, category/activity codes, cadence, coverage, terms, privacy, and project approval."}, + {"source_id":"pt.dgav.feed","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-pt.md","pipeline/source_registry.json"],"next_action":"Confirm the public feed section and NII semantics with an authorized capture; keep feed separate from NCV food/ABP rows and review private/primary-producer exposure."}, + {"source_id":"pt.apambiente.tua","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-pt.md","pipeline/source_registry.json"],"next_action":"Locate a current public TUA search/export contract and safe permit fields; keep environmental decisions as a separate reviewed evidence overlay."}, + {"source_id":"pt.ifap.snira","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-pt.md","pipeline/source_registry.json"],"next_action":"Do not acquire without explicit authorization; keep holder, animal, farm, parcel, and precise-location data restricted and assess any future relationship study separately."}, + {"source_id":"pt.ine.animal-production","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-pt.md","pipeline/source_registry.json"],"next_action":"Pin reproducible INE table/API identifiers and revision/confidentiality terms; keep statistics aggregate and separate from named-facility evidence."} ] } diff --git a/docs/source-status.md b/docs/source-status.md index ac1cd5a..c572966 100644 --- a/docs/source-status.md +++ b/docs/source-status.md @@ -71,3 +71,7 @@ The machine-readable file is the source of truth for these statuses. Legacy `.lo ## Poland additions (2026-09-16) Poland reconnaissance added GIW approved-food, registered-food, ABP and RRW sources; GUS slaughter statistics; GIOŚ/WIOŚ integrated permits; GDOŚ EIA; Geoportal Urban Register; ARiMR processing support; KRS and REGON identity routes; and the EU TRACES mirror. The GIW HTML views were verified with current displayed counts, but the XLS body was not acquired because shell HTTP was refused and the browser export timed out. The run therefore records a private metadata-only artifact, not a source-data capture. All Poland entries remain `publication_eligibility=blocked`, with runtime health `not_run`; GDOŚ and REGON acquisition are additionally blocked pending the source’s access/authorization conditions. See [`docs/country-recon-pl.md`](country-recon-pl.md), [`docs/countries/pl/source-crosswalk.json`](countries/pl/source-crosswalk.json), and [`data/manifests/pl-source-artifacts.json`](../data/manifests/pl-source-artifacts.json). + +## Portugal reconnaissance (2026-09-17) + +Portugal adds six source-local scopes: DGAV approved food, DGAV animal by-products, DGAV feed, APA TUA permits, restricted IFAP/DGAV SNIRA animal/holding data, and INE aggregate animal-production statistics. DGAV’s current pages link the +SIPACE listing family and document NCV/NII concepts, while the legacy SIPACE HTML list remains available for consultation during transition. No stable bulk/API export was verified, no row-level artifact was retained, and all six entries remain `publication_eligibility=blocked` with runtime health `not_run`. The recommended next implementation is a private, section-aware assisted capture for `pt.dgav.approved-food`, followed by separately validated ABP/feed sections; SNIRA remains access-controlled and INE remains aggregate-only. See [`docs/country-recon-pt.md`](country-recon-pt.md). diff --git a/pipeline/source_registry.json b/pipeline/source_registry.json index c175f12..286db0c 100644 --- a/pipeline/source_registry.json +++ b/pipeline/source_registry.json @@ -858,7 +858,13 @@ {"source_id":"lb.justice.companies","jurisdiction_scope":"Lebanon; Ministry of Justice commercial register","legacy_paths":[],"url":"https://cr.justice.gov.lb/index.aspx","access_method":"official interactive search; do not bypass controls","cadence":"unknown","attribution_licensing_notes":"Coverage, fees, terms, and personal-address policy require review","adapter_status":"reference_only","expected_artifact_schema":"Organization metadata; suppress personal addresses","blockers":["Machine route and nationwide coverage not verified."]}, {"source_id":"lb.industry.food-guide","jurisdiction_scope":"Lebanon; Ministry of Industry licensed food factories","legacy_paths":[],"url":"https://www.industry.gov.lb/IndustrialStatistics/IndustrialGuide","access_method":"official guide/list; authorized bounded download only","cadence":"2022 visible; current unknown","attribution_licensing_notes":"Terms, schema, IDs, and safe address boundary require review","adapter_status":"reference_only","expected_artifact_schema":"Metadata-only; no rows or coordinates","blockers":["Current export and reuse terms not pinned."]}, {"source_id":"lb.cas.livestock-statistics","jurisdiction_scope":"Lebanon; Central Administration of Statistics aggregate livestock indicators","legacy_paths":[],"url":"https://www.cas.gov.lb/","access_method":"official tables/publications; authorized query/download to be pinned","cadence":"table-specific; unknown","attribution_licensing_notes":"Revisions, licensing, and suppression rules require review","adapter_status":"reference_only","expected_artifact_schema":"Aggregate time-series metadata; no establishment rows","blockers":["Current table/API identifiers and terms not pinned."]}, - {"source_id":"ch.blv.approved-food","jurisdiction_scope":"Switzerland; FSVO list of approved food businesses, including animal-origin establishments and slaughterhouses","legacy_paths":[],"url":"https://www.blv.admin.ch/de/listen-bewilligter-schweizer-betriebe","access_method":"official multilingual FSVO search/list route; authorized bounded export or browser capture only","cadence":"source-specific; current page and list route observed 2026-09-16","attribution_licensing_notes":"Swiss government source; public visibility does not settle reuse, attribution, personal-address, coordinate, or publication rights","adapter_status":"reference_only","expected_artifact_schema":"Metadata-first contract for multilingual list records: approval number, approval date/status, operator/name, contact/address, establishment type, activities/species, source language, list/version date, and source URL; preserve raw labels and avoid row publication","blockers":["No stable bulk/API contract or complete national export verified; federal list may be assembled from cantonal authorities; language normalization, effective/status semantics, privacy, terms, coverage, and project approval remain unresolved."]} ] + {"source_id":"ch.blv.approved-food","jurisdiction_scope":"Switzerland; FSVO list of approved food businesses, including animal-origin establishments and slaughterhouses","legacy_paths":[],"url":"https://www.blv.admin.ch/de/listen-bewilligter-schweizer-betriebe","access_method":"official multilingual FSVO search/list route; authorized bounded export or browser capture only","cadence":"source-specific; current page and list route observed 2026-09-16","attribution_licensing_notes":"Swiss government source; public visibility does not settle reuse, attribution, personal-address, coordinate, or publication rights","adapter_status":"reference_only","expected_artifact_schema":"Metadata-first contract for multilingual list records: approval number, approval date/status, operator/name, contact/address, establishment type, activities/species, source language, list/version date, and source URL; preserve raw labels and avoid row publication","blockers":["No stable bulk/API contract or complete national export verified; federal list may be assembled from cantonal authorities; language normalization, effective/status semantics, privacy, terms, coverage, and project approval remain unresolved."]}, + {"source_id":"pt.dgav.approved-food","jurisdiction_scope":"Portugal; DGAV approved and registered food establishments, including animal-origin establishments under Regulation (EC) 853/2004","legacy_paths":[],"url":"https://maissipace.dgav.pt/Listagens","access_method":"official DGAV public +SIPACE/legacy SIPACE list family; browser-assisted bounded capture only until a stable export is verified","cadence":"not stated; timestamp every retrieval and preserve list/section context","attribution_licensing_notes":"Portuguese government source; source-specific reuse, attribution, personal-address, and redistribution terms were not verified","adapter_status":"reference_only","expected_artifact_schema":"Sectioned HTML/filter/pagination observations with NCV or registration number, NII where present, operator/establishment name, address/locality, municipality, regional service, category, activity, associated activity, species/products, and source status","blockers":["The new +SIPACE route is transitional and no stable bulk/API contract was verified; confirm section coverage, pagination, status/effective-date semantics, codes, privacy, terms, and project approval before acquisition."]}, + {"source_id":"pt.dgav.abp","jurisdiction_scope":"Portugal; DGAV animal-by-product establishments, installations, and operators approved, registered, or authorized under Regulations (EC) 1069/2009 and 142/2011","legacy_paths":[],"url":"https://maissipace.dgav.pt/Listagens","access_method":"official DGAV public +SIPACE/legacy SIPACE list family; separate ABP section capture required","cadence":"not stated; timestamp every retrieval","attribution_licensing_notes":"Portuguese government source; ABP-specific reuse, attribution, privacy, and redistribution terms were not verified","adapter_status":"reference_only","expected_artifact_schema":"ABP observations with NCV or registration/authorization number, operator/site, address/locality, category 1/2/3 or derived product, activity, status, and remarks; preserve section identity","blockers":["Do not union with food or feed lists; exact ABP section routes, codebook, export, cadence, privacy, terms, and coverage remain unresolved."]}, + {"source_id":"pt.dgav.feed","jurisdiction_scope":"Portugal; DGAV feed-sector establishments and operators registered or approved under Regulation (EC) 183/2005","legacy_paths":[],"url":"https://maissipace.dgav.pt/Listagens","access_method":"official DGAV public +SIPACE/legacy SIPACE list family; separate feed section capture required","cadence":"not stated; timestamp every retrieval","attribution_licensing_notes":"Portuguese government source; feed-list reuse, attribution, privacy, and redistribution terms were not verified","adapter_status":"reference_only","expected_artifact_schema":"Feed observations with NII/individual identifier, operator/site, address/locality, feed activity, and registration/approval state; preserve raw section and codes","blockers":["The feed identifier is not interchangeable with an NCV; confirm current public section, codebook, pagination, export, cadence, private/primary-producer boundary, terms, and coverage."]}, + {"source_id":"pt.apambiente.tua","jurisdiction_scope":"Portugal; Agência Portuguesa do Ambiente Título Único Ambiental and linked environmental licensing decisions","legacy_paths":[],"url":"https://apambiente.pt/avaliacao-e-gestao-ambiental/titulo-unico-ambiental","access_method":"official guidance and electronic-title/document route; no stable national public export verified","cadence":"decision/document-specific","attribution_licensing_notes":"Portuguese environmental authority source; document reuse, geometry, personal-address, and permit-condition publication terms require review","adapter_status":"reference_only","expected_artifact_schema":"Permit/title/process/document observations with holder, establishment/activity/project, regime, authority, decision date, validity, status, and reviewed document links; geometry unknown","blockers":["Locate and verify a current public search/export contract; keep TUA separate from DGAV approval and do not infer facility identity from holder or permit text."]}, + {"source_id":"pt.ifap.snira","jurisdiction_scope":"Portugal; IFAP/DGAV SNIRA animal-identification, holding, movement, and herd information","legacy_paths":[],"url":"https://www.ifap.pt/portal/en/registo-area-reservada","access_method":"restricted IFAP area and credentialed webservice; no public acquisition authorized","cadence":"operational system; provider-specific","attribution_licensing_notes":"Restricted animal/holder data; purpose limitation, access control, retention, and privacy review are mandatory","adapter_status":"reference_only","expected_artifact_schema":"Credentialed animal/holding observations keyed by animal identification, NIF/holder, Marca de Exploração, species, and time period; not a public facility master","blockers":["Do not scrape or retain rows; obtain explicit authorization only if a narrowly scoped relationship study is approved, and suppress holder, farm, parcel, and precise-location details."]}, + {"source_id":"pt.ine.animal-production","jurisdiction_scope":"Portugal; Statistics Portugal aggregate animal-production, meat, and slaughter statistics","legacy_paths":[],"url":"https://ine.pt/bddXplorer/htdocs/minfo.jsp?lingua=EN&var_cd=0000916&var_cd=0000917&var_cd=0000918","access_method":"official INE metadata/PxWeb statistics route; table-specific API or download to be pinned","cadence":"annual/semestral/monthly by table; metadata checked through 2025/2026","attribution_licensing_notes":"Official statistics; preserve table metadata, revisions, confidentiality flags, and source terms","adapter_status":"reference_only","expected_artifact_schema":"Aggregate time-series observations by reference period, geography, species/meat type and measure; no named-facility rows","blockers":["Pin reproducible table IDs, API contract, revisions, terms, and confidentiality rules; never infer facility coverage or join confidential slaughter surveys to named establishments."]} ] } diff --git a/pipeline/tests/test_portugal_recon_metadata.py b/pipeline/tests/test_portugal_recon_metadata.py new file mode 100644 index 0000000..1973ddb --- /dev/null +++ b/pipeline/tests/test_portugal_recon_metadata.py @@ -0,0 +1,34 @@ +import json +import unittest +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[2] +IDS = { + "pt.dgav.approved-food", + "pt.dgav.abp", + "pt.dgav.feed", + "pt.apambiente.tua", + "pt.ifap.snira", + "pt.ine.animal-production", +} + + +class PortugalReconMetadataTests(unittest.TestCase): + def test_row_free_recon_and_registry_entries(self): + registry = json.loads((ROOT / "pipeline" / "source_registry.json").read_text(encoding="utf-8")) + self.assertTrue(IDS.issubset({source["source_id"] for source in registry["sources"]})) + document = (ROOT / "docs" / "country-recon-pt.md").read_text(encoding="utf-8") + self.assertIn("row-free", document) + self.assertIn("fully automated", document) + self.assertNotIn("NCV |", document) + + def test_publication_remains_blocked(self): + status = json.loads((ROOT / "docs" / "source-status.json").read_text(encoding="utf-8")) + by_id = {source["source_id"]: source for source in status["sources"]} + for source_id in IDS: + self.assertEqual(by_id[source_id]["publication_eligibility"], "blocked") + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/tests/test_source_registry.py b/pipeline/tests/test_source_registry.py index 29c3ec2..ad41dcc 100644 --- a/pipeline/tests/test_source_registry.py +++ b/pipeline/tests/test_source_registry.py @@ -9,8 +9,8 @@ class SourceRegistryTests(unittest.TestCase): def test_repository_registry_loads_and_references_existing_legacy_paths(self): registry = load_registry() - self.assertEqual(len(registry["sources"]), 234) - self.assertEqual(len({source["source_id"] for source in registry["sources"]}), 234) + self.assertEqual(len(registry["sources"]), 240) + self.assertEqual(len({source["source_id"] for source in registry["sources"]}), 240) def test_unknowns_are_explicit(self): registry = json.loads(REGISTRY_PATH.read_text(encoding="utf-8")) From 9359842005928c2083acf2f58d446ec4d2b689e2 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Thu, 17 Sep 2026 13:45:35 -0700 Subject: [PATCH 230/311] Acquire private Germany and Belgium current candidates --- .../de-be-private-candidates-2026-09-17.json | 82 +++++++++++++++++++ docs/country-recon-be.md | 22 ++--- docs/germany-source-assessment.md | 19 +++-- docs/review-packet-belgium.md | 6 +- docs/review-packet-germany.md | 6 +- docs/source-status.json | 5 +- docs/source-status.md | 6 +- pipeline/germany/source-registry.json | 17 ++-- pipeline/source_registry.json | 8 +- pipeline/sources/belgium/README.md | 7 +- pipeline/sources/belgium/adapter.py | 54 ++++++------ pipeline/sources/belgium/config.json | 4 +- pipeline/sources/belgium/refresh.py | 37 +++++++-- pipeline/sources/belgium/test_refresh.py | 18 ++++ pipeline/sources/germany/README.md | 3 + pipeline/sources/germany/adapter.py | 45 ++++++++-- pipeline/sources/germany/config.json | 4 +- pipeline/sources/germany/refresh.py | 29 ++++++- pipeline/sources/germany/test_refresh.py | 13 +++ 19 files changed, 293 insertions(+), 92 deletions(-) create mode 100644 data/manifests/de-be-private-candidates-2026-09-17.json diff --git a/data/manifests/de-be-private-candidates-2026-09-17.json b/data/manifests/de-be-private-candidates-2026-09-17.json new file mode 100644 index 0000000..61e8466 --- /dev/null +++ b/data/manifests/de-be-private-candidates-2026-09-17.json @@ -0,0 +1,82 @@ +{ + "manifest_version": "1.0", + "as_of_utc": "2026-09-17T20:26:41Z", + "purpose": "Row-free evidence index for current Germany and Belgium private candidate rehearsals.", + "evidence_scope": "private_candidate_only", + "private_payloads_included": false, + "public_release_created": false, + "public_api_rows": 0, + "public_map_rows": 0, + "public_export_rows": 0, + "geocoding": "disabled", + "graph_candidates": { + "count": 0, + "reason": "No cross-source edge was generated without an explicit source-native identifier and reviewable evidence." + }, + "sources": [ + { + "source_id": "be.locations", + "stable_url": "https://www.static.favv.be/bo-documents/inter_actieve_actoren_EN.csv", + "catalog_url": "https://data.gov.be/en/datasets/favv-afsca-operators", + "companion_artifact": "https://www.static.favv.be/bo-documents/inter_PAP_omschrijving_EN.csv", + "retrieved_at_utc": "2026-09-17T20:20:48Z", + "effective_date": "2026-08-05", + "operator_bytes": 87903986, + "operator_sha256": "9df1a9626cfbe30229f509e3004abbc291bd5b2aac3cd7437d96cceb124ffbb4", + "activity_code_bytes": 101987, + "activity_code_sha256": "c50d5ff8db705664a56bef73aec88c94159ff7f810cf6125e978e0e16c5a0812", + "operator_schema_fingerprint": "e225b6f7e11d3cf0d441dd96d1da61e57cd817d3d1e8fd2dad066073d3582a1e", + "activity_code_schema_fingerprint": "4b48513b1169bb0c64d51f7c1cfad5074b1326c1db971b4f4b1273ad7c729087", + "input_rows": 310660, + "normalized_rows": 310640, + "quarantined_rows": 20, + "anomalies": { + "duplicate_source_row": 5, + "unresolved_activity_code": 15 + }, + "source_name_status": "not-supplied-by-source", + "status": "candidate-ready", + "release_state": "not-created", + "blockers": [ + "CC BY attribution and non-misleading-use requirements remain a human terms gate.", + "Postal, municipality, enterprise and identifier fields remain restricted pending privacy review.", + "Activity classification, project approval and release review remain open." + ] + }, + { + "source_id": "de.locations", + "stable_url": "https://www.bvl.bund.de/bltu", + "portal_url": "https://gis.bvl.bund.de/datenportal/", + "retrieved_at_utc": "2026-09-17T20:26:41Z", + "effective_date": null, + "bytes": 2068022, + "sha256": "65e71e324a2d905aff75e145c8e85823172f0e5825205313bdf38cf7dc1a63c0", + "schema_fingerprint": "52fd9ed6ecd543cff2f96b99c8bce5a84cea51cc65d000c00793ff0689e37f78", + "schema_status": "matched", + "input_rows": 15788, + "normalized_rows": 2691, + "quarantined_rows": 13097, + "anomalies": { + "malformed_csv_row": 8, + "missing_activity_code": 5, + "missing_current_approval_id": 4, + "missing_establishment_name": 4, + "physical_column_count_mismatch": 11, + "unmapped_activity_code": 13084 + }, + "status": "candidate-ready", + "release_state": "not-created", + "blockers": [ + "The export route is session-bound; recurring acquisition and dataset-specific reuse terms require human confirmation.", + "The export effective/publication date is unknown.", + "Facility names, addresses and coordinates remain restricted; geocoding is disabled.", + "Project approval, coverage interpretation and release review remain open." + ] + } + ], + "integration_notes": { + "adapter_refresh": "Both current captures completed deterministic parse/normalize/quarantine and shared private lifecycle handoff with no drift alarms.", + "api_frontend": "No current private rows were imported into a public API or frontend surface; existing API/frontend checks remain synthetic/test-only.", + "replay": "Raw artifacts and lifecycle payloads remain in ignored private storage; checked-in evidence is limited to this row-free manifest and documentation." + } +} diff --git a/docs/country-recon-be.md b/docs/country-recon-be.md index 18ad455..51e926c 100644 --- a/docs/country-recon-be.md +++ b/docs/country-recon-be.md @@ -1,8 +1,8 @@ # Belgium source reconnaissance -Status: reconnaissance plus private/test-only adapter implementation. No real facility rows, names, addresses, contacts, coordinates, release, or publication are retained here. Synthetic fixtures contain no real operators. +Status: reconnaissance plus private/test-only adapter implementation. A current official operator/codebook pair was captured on 2026-09-17 into ignored private storage; no real facility rows, names, addresses, contacts, coordinates, release, or publication are retained in Git. Synthetic fixtures contain no real operators. -Last checked: 2026-09-15 UTC under `docs/ETHICS.md`, policy version 1.0, last reviewed 2026-09-12. This is source-status evidence, not publication approval or a runtime-health claim. +Last checked: 2026-09-17 UTC under `docs/ETHICS.md`, policy version 1.0, last reviewed 2026-09-12. This is source-status evidence, not publication approval or a runtime-health claim. The implementation is in [`pipeline/sources/belgium/`](../pipeline/sources/belgium/). It requires two independently preserved official artifacts: the operator CSV at `https://www.static.favv.be/bo-documents/inter_actieve_actoren_EN.csv` and the LAP/PAP codebook at `https://www.static.favv.be/bo-documents/inter_PAP_omschrijving_EN.csv`. The bounded assisted command is repeatable when a browser or authorized operator supplies both files: @@ -10,14 +10,14 @@ The implementation is in [`pipeline/sources/belgium/`](../pipeline/sources/belgi python -m pipeline.sources.belgium.refresh --operators --activity-codes --run-dir --retrieved-at-utc 2026-09-15T00:00:00Z ``` -The codebook join is exact and deterministic; unresolved or ambiguous codes quarantine. The adapter preserves source activity text and distinguishes slaughter, cutting, processing, storage, animal-by-products, and export domains. It never geocodes and keeps source address/coordinate/enterprise values out of normalized/API-shaped rows. The live operator header was not obtainable in this environment, so the checked-in fixture is a schema contract and the first real capture must be reviewed for schema drift. +The codebook join is exact and deterministic; unresolved or ambiguous codes quarantine. The adapter preserves source activity text and distinguishes slaughter, cutting, processing, storage, animal-by-products, and export domains. It never geocodes and keeps source address/coordinate/enterprise values out of normalized/API-shaped rows. The current live operator header is recorded as a cp1252, 20-column identifier/activity/location schema in the private run manifest; the checked-in fixture remains synthetic and no raw rows are committed. ## Readiness | Candidate | Evidence | Acquisition | Terms / privacy | Readiness / next action | |---|---|---|---|---| -| FASFC operator list | Official [data.gov.be dataset](https://data.gov.be/en/datasets/favv-afsca-operators) and published [English CSV](https://www.static.favv.be/bo-documents/inter_actieve_actoren_EN.csv) verified | Published CSV route; bounded header retrieval timed out from the static host in this environment | CC Attribution 4.0; FASFC says attribute source and last-update date, do not imply FASFC affiliation/approval, and do not mislead | Candidate for a private adapter after header/schema capture, delimiter/encoding test, and category mapping | -| FASFC activity-code list | Official [data.gov.be dataset](https://data.gov.be/en/datasets/fasfc-activity-codes) and [English CSV](https://www.static.favv.be/bo-documents/inter_PAP_omschrijving_EN.csv) verified | In-memory bounded retrieval succeeded; bytes discarded | CC Attribution 4.0; weekly metadata on data.gov.be | Companion codebook; not a facility list and not sufficient by itself | +| FASFC operator list | Official [data.gov.be dataset](https://data.gov.be/en/datasets/favv-afsca-operators) and published [English CSV](https://www.static.favv.be/bo-documents/inter_actieve_actoren_EN.csv) verified | 2026-09-17 normal HTTPS capture; cp1252, 20 columns, 87,903,986 bytes | CC Attribution 4.0; FASFC says attribute source and last-update date, do not imply FASFC affiliation/approval, and do not mislead | Private adapter validated; human terms/privacy/classification gates remain open | +| FASFC activity-code list | Official [data.gov.be dataset](https://data.gov.be/en/datasets/fasfc-activity-codes) and [English CSV](https://www.static.favv.be/bo-documents/inter_PAP_omschrijving_EN.csv) verified | 2026-09-17 normal HTTPS capture; cp1252, 13 columns, 101,987 bytes | CC Attribution 4.0; weekly metadata on data.gov.be | Companion codebook; not a facility list and not sufficient by itself | | EU-approved food-establishment PDF lists | Official [FASFC approved-establishments page](https://www.foodweb.favv-afsca.be/professionals/foodstuffs/establishments/default.asp) verified | PDF links are a separate publication surface | Scope and currentness differ from open-data CSV; PDF schema/version handling required | Useful cross-check only; do not merge with operator rows without explicit identity/version evidence | | Animal by-products lists | Official [FASFC animal-by-products page](https://www.foodweb.favv-afsca.be/professionals/animalbyproducts/approvedoperators/) verified | Sectioned PDF lists | Separate legal scope under Regulation (EC) 1069/2009 | Keep separate from food slaughter/processing coverage | @@ -31,23 +31,23 @@ The operator page does not establish that every row is a slaughterhouse. It cove ## Identity, geography, and privacy risks -The dataset description confirms enterprise/establishment scope and approval identifiers, but the operator CSV header was not captured in this reconnaissance. Address, postal-code, municipality, establishment-versus-enterprise identifiers, coordinates, effective dates, and status fields therefore remain unverified. Treat any address as a facility claim requiring source-field preservation and privacy screening; a registered office or mixed residential/business address is not automatically an operating site. Do not geocode until provider/query/time/precision/review metadata and the ETHICS.md residential/private-location rules are implemented. +The captured operator CSV uses cp1252 CSV with 20 columns and 310,660 data rows. It supplies operator/location identifiers, PAP activity fields, postal code, municipality, province, approval number, and dates, but no establishment-name or street-address column. The adapter records `name_state=not-supplied-by-source` and never invents a name. Treat any postal or municipality value as a facility claim requiring source-field preservation and privacy screening; a registered office or mixed residential/business address is not automatically an operating site. Do not geocode until provider/query/time/precision/review metadata and the ETHICS.md residential/private-location rules are implemented. Foodweb is an interactive lookup and inspection-results surface, not a bulk inspection dataset. Its published FAQ says inspection-result publication is limited to B2C operators and cannot produce a complete list by municipality or activity. It must not be used as a substitute for the operator master or treated as slaughterhouse evidence. Coverage is Belgium-wide according to data.gov.be, but completeness is bounded by FASFC registrations/approvals/authorizations currently represented in that feed. It is not evidence of operating status beyond the publisher's stated current eligibility, nor a census of all animal-agriculture facilities. -## Acquisition provenance (private, no raw artifact retained) +## Acquisition provenance (private, no raw artifact retained in Git) -The activity-code request was performed read-only in memory and response bytes were discarded. The operator request was attempted but did not yield bytes; no operator rows were retained. +The current pair was fetched over normal HTTPS from the official static host into ignored private storage. The row-free sidecars preserve response headers, catalog update date, byte size, SHA-256, and schema fingerprints; no raw rows are checked in. | Artifact | Retrieval UTC | HTTP | Content type | Bytes | SHA-256 | Supplied update/effective date | |---|---|---:|---|---:|---|---| | `inter_PAP_omschrijving_EN.csv` | 2026-09-15T17:05:26.4902745Z | 200 | `text/csv` | 101,987 | `c50d5ff8db705664a56bef73aec88c94159ff7f810cf6125e978e0e16c5a0812` | data.gov.be page: 2026-08-05; no artifact-level effective date observed | -| `inter_actieve_actoren_EN.csv` | 2026-09-15; no response body | unavailable | unavailable | unavailable | unavailable | data.gov.be page: 2026-08-05 | +| `inter_actieve_actoren_EN.csv` | 2026-09-17T20:20:48Z | 200 | `text/csv` | 87,903,986 | `9df1a9626cfbe30229f509e3004abbc291bd5b2aac3cd7437d96cceb124ffbb4` | data.gov.be page: 2026-08-05; source last-modified 2026-09-14 | ## Adapter readiness and recommended next step -Readiness: medium difficulty, not ready for implementation. The authoritative source pair and reuse terms are clear, and the activity codebook is machine-readable. The main work is schema capture for the operator CSV, deterministic delimiter/encoding handling, codebook versioning, establishment/enterprise identity semantics, multilingual labels, status/effective-date interpretation, and explicit mappings for slaughterhouse versus cutting/processing/storage and other PAP categories. Expect one operator row per activity or repeated establishment identifiers; this must be verified rather than assumed. +Readiness: private adapter validated against the current pair, not ready for publication. The authoritative source pair and reuse terms are clear, and the activity codebook is machine-readable. The live operator schema is now recorded as cp1252 CSV with 20 columns; repeated location/activity identifiers are retained as distinct source observations, while five exact duplicate rows and 15 unresolved activity codes are quarantined. The private run produced 310,660 input rows, 310,640 normalized rows, and 20 quarantined rows. Human terms/attribution, privacy, classification, and project approval gates remain open. -Next step: obtain an authorized, bounded operator-CSV retrieval; record its redirect chain, headers, byte size, SHA-256, supplied date, and sanitized schema; then build synthetic fixtures for repeated activities, missing identifiers, multilingual text, category ambiguity, and mixed residential/business addresses. Keep acquisition private and publication blocked pending human privacy and release review. +Next step: obtain human confirmation of attribution/privacy/reuse and review the current candidate classifications. Keep the private artifacts restricted and publication blocked; build additional synthetic regression fixtures for repeated activities, missing identifiers, multilingual text, category ambiguity, and mixed residential/business addresses. diff --git a/docs/germany-source-assessment.md b/docs/germany-source-assessment.md index df7ebf7..22fb49b 100644 --- a/docs/germany-source-assessment.md +++ b/docs/germany-source-assessment.md @@ -37,12 +37,15 @@ export. Therefore acquisition is **blocked pending human confirmation** of: 4. required attribution, notices, update/deletion obligations, and any restrictions on address, approval-number, or activity/species fields. -This is a terms uncertainty, not a claim that the source prohibits use. A single -real BLtU export was downloaded earlier at the user's direction as a restricted -local research artifact and staged privately (15,797 input rows; 6,346 normalized; -9,451 quarantined). It is not a repository fixture, release candidate, API source, -map layer, export, or publication. That retrieval does not establish permission for -recurring acquisition, retention, or redistribution; those decisions remain open. +This is a terms uncertainty, not a claim that the source prohibits use. A current +real BLtU general-list export was captured on 2026-09-17 through the ordinary +public browser flow as a restricted local research artifact and staged privately +(15,788 input rows; 2,691 normalized; 13,097 quarantined). The row-free capture +sidecar records the stable landing route, session navigation steps, export route, +response headers, byte size, and SHA-256. It is not a repository fixture, release +candidate, API source, map layer, export, or publication. That retrieval does not +establish permission for recurring acquisition, retention, or redistribution; those +decisions remain open. Privacy/safety review is separately required because facility addresses can overlap with residences or identify individuals; source origin does not resolve that risk. @@ -59,10 +62,10 @@ by BLtU without dataset-specific evidence. [`pipeline/sources/germany/`](../pipeline/sources/germany/) now provides a typed BLtU adapter and assisted refresh. Use the stable landing page to select the current CSV export, save it in private ignored storage, and run: ```text -python -m pipeline.sources.germany.refresh --raw --run-dir --retrieved-at-utc 2026-09-15T00:00:00Z +python -m pipeline.sources.germany.refresh --raw --acquisition-metadata --run-dir --retrieved-at-utc 2026-09-17T20:26:41Z ``` -The adapter preserves repeated activity columns and current approval numbers, quarantines schema/identity/unmapped-code anomalies, emits shared QA and health evidence, and produces only a private candidate handoff. Address and coordinate values remain private and geocoding is disabled. Candidate import, if used, must target the disposable database guard and remains test-only; no public release is created. +The adapter preserves repeated activity columns, current/legacy approval-number provenance, and malformed source rows, quarantines schema/identity/unmapped-code anomalies, emits shared QA and health evidence, and produces only a private candidate handoff. The 2026-09-17 run matched the live 50-column cp1252 schema; eight malformed CSV rows and 13,084 unmapped activity rows were quarantined. Address and coordinate values remain private and geocoding is disabled. Candidate import, if used, must target the disposable database guard and remains test-only; no public release is created. ## Planned recurring acquisition after approval diff --git a/docs/review-packet-belgium.md b/docs/review-packet-belgium.md index 52f40cf..872c1bb 100644 --- a/docs/review-packet-belgium.md +++ b/docs/review-packet-belgium.md @@ -1,11 +1,11 @@ # Belgium private review packet -As of 2026-09-15, Belgium has a private two-artifact FASFC operator/codebook lifecycle with synthetic schema fixtures. No real operator rows are retained in Git and publication is blocked. +As of 2026-09-17, Belgium has a private two-artifact FASFC operator/codebook lifecycle validated against a current official pair. No real operator rows are retained in Git and publication is blocked. - Terms/licensing: CC BY 4.0 is indicated by data.gov.be; FASFC attribution, last-update labeling, non-misleading use, and project publication review remain required. - Privacy: address, postal, enterprise, and coordinate values stay in restricted source evidence; normalized rows suppress them and geocoding is disabled. - Completeness: the operator feed represents current FASFC registrations/approvals/authorizations, not every animal-agriculture facility or every slaughterhouse. - Classification: the LAP/PAP codebook join is exact; slaughter, cutting, processing, storage, animal-by-products, and export remain distinct source categories. Ambiguous/repeated codes quarantine. -- Coverage/lifecycle: operator and activity-code artifacts must be captured together; the live operator header was not available in this environment. Missing rows are `not-observed`, never closure. +- Coverage/lifecycle: operator and activity-code artifacts were captured together. The live operator feed is cp1252 CSV with an identifier/activity-only schema and repeated source observations; exact duplicates quarantine, while distinct rows sharing an establishment/activity key remain separate. Missing rows are `not-observed`, never closure. -Evidence: `pipeline/sources/belgium/`, `docs/country-recon-be.md`, and `pipeline/tests/e2e/test_germany_belgium_candidate_import.py`. +Current row-free run evidence: `data/manifests/de-be-private-candidates-2026-09-17.json` and the private lifecycle sidecars under ignored `data/staging/be.fasfc/20260917T000000Z-v3/`. Adapter/sidecar tests are in `pipeline/sources/belgium/`; the existing candidate-import E2E remains synthetic/test-only. diff --git a/docs/review-packet-germany.md b/docs/review-packet-germany.md index e6f8d05..a26fcef 100644 --- a/docs/review-packet-germany.md +++ b/docs/review-packet-germany.md @@ -1,11 +1,11 @@ # Germany private review packet -As of 2026-09-15, Germany uses the typed BLtU adapter and assisted/private refresh. No release, geocode, or public API exposure is allowed. +As of 2026-09-17, Germany uses the typed BLtU adapter and assisted/private refresh against a current public general-list CSV export. No release, geocode, or public API exposure is allowed. - Terms/licensing: BVL documents public access/export, but dataset-specific reuse and redistribution terms require named human confirmation. - Privacy: facility addresses may overlap residences or identify people; addresses and coordinates stay private and geocoding is disabled. - Completeness: BLtU covers the approved 853/2004 list, not all animal-agriculture facilities. Export effective date and currentness are run-specific. - Classification: only pinned SH/CP mappings are accepted; unmapped activities, missing identity, duplicate approval IDs, and physical schema drift quarantine. -- Coverage/lifecycle: the portal export URL is session/request-specific and must be recorded per run. Missing rows are `not-observed`, never closure. +- Coverage/lifecycle: the ordinary public form flow and session/request-specific export URL are recorded per run. The current run reconciles 15,788 input rows into 2,691 normalized and 13,097 quarantined rows. Missing rows are `not-observed`, never closure. -Evidence: `pipeline/sources/germany/`, `pipeline/germany/bltu_adapter.py`, and `docs/germany-source-assessment.md`. +Current row-free run evidence: `data/manifests/de-be-private-candidates-2026-09-17.json` and the private lifecycle sidecars under ignored `data/staging/de.bltu/20260917T000000Z-v3/`. Adapter/sidecar tests are in `pipeline/sources/germany/`; the existing candidate-import E2E remains synthetic/test-only. diff --git a/docs/source-status.json b/docs/source-status.json index d3b1e6d..7f32aa0 100644 --- a/docs/source-status.json +++ b/docs/source-status.json @@ -2,6 +2,7 @@ "schema_version": "1.0", "purpose": "Evidence-backed source readiness baseline; not a runtime health monitor or publication approval register.", "latest_private_rehearsal": "docs/country-rehearsal-2026-09-15.json", + "latest_country_capture": "data/manifests/de-be-private-candidates-2026-09-17.json", "status_vocabulary": { "metadata": ["verified", "partial", "unknown"], "acquisition": ["not_run", "blocked", "artifact_private_only", "verified"], @@ -13,7 +14,7 @@ {"source_id":"br.sif.export","metadata":"verified","acquisition":"verified","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-br.md","docs/countries/br/v1-field-crosswalk.json","pipeline/source_registry.json"],"next_action":"Model country/product authorizations as separate dated observations keyed to SIF; confirm validity/suspension semantics, terms, privacy, and no-double-counting rules before integration."}, {"source_id":"br.sisbi.public","metadata":"verified","acquisition":"artifact_private_only","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-br.md","docs/countries/br/v1-field-crosswalk.json","pipeline/source_registry.json"],"next_action":"Use bounded GET samples only while an authorized operator confirms pagination, code lists, lifecycle/status, address linkage, cadence, terms, privacy, and project approval."}, {"source_id":"br.trase.facilities","metadata":"verified","acquisition":"artifact_private_only","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-br.md","docs/countries/br/v1-field-crosswalk.json","pipeline/source_registry.json"],"next_action":"Keep Trase as separately labeled secondary evidence; review source lineage, geocoding, constructed IDs, raw-data terms, privacy, coverage, and any exact-ID reconciliation before use."}, - {"source_id":"be.locations","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-be.md","pipeline/sources/belgium/adapter.py","pipeline/sources/belgium/refresh.py","pipeline/sources/belgium/fixtures/synthetic_operators.csv","pipeline/sources/belgium/test_adapter.py","pipeline/common/review_packet.py","docs/review-packet-belgium.md","pipeline/source_registry.json"],"next_action":"Use the assisted two-file refresh with an authorized operator capture and official activity-code CSV; compare live schema and physical row lengths to the synthetic contract. Keep category/privacy/terms gates and publication blocked."}, + {"source_id":"be.locations","metadata":"verified","acquisition":"artifact_private_only","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-be.md","data/manifests/de-be-private-candidates-2026-09-17.json","pipeline/sources/belgium/adapter.py","pipeline/sources/belgium/refresh.py","docs/review-packet-belgium.md","pipeline/source_registry.json"],"next_action":"Review the private current pair: 310,660 input, 310,640 normalized, 20 quarantined; names are not supplied by this snapshot. Keep privacy, attribution, classification, terms, and project approval gates closed."}, {"source_id":"fr.dgal.section-i","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-fr.md","docs/countries/france/dgal-853-pipeline.md","pipeline/sources/france/adapter.py","pipeline/sources/france/acquire.py","pipeline/sources/france/refresh.py","pipeline/sources/france/fixtures/section_i.csv","pipeline/common/review_packet.py","pipeline/source_registry.json"],"next_action":"Run the bounded private Section I refresh with an approved terms record or authorized capture; review category semantics, address privacy, schema drift, and release approval."}, {"source_id":"fr.dgal.section-ii","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-fr.md","docs/countries/france/dgal-853-pipeline.md","pipeline/sources/france/adapter.py","pipeline/sources/france/acquire.py","pipeline/sources/france/refresh.py","pipeline/sources/france/fixtures/section_ii.csv","pipeline/common/review_packet.py","pipeline/source_registry.json"],"next_action":"Run the bounded private Section II refresh separately from Section I; review category/species semantics, address privacy, schema drift, and release approval."}, {"source_id":"it.853-2004","metadata":"verified","acquisition":"artifact_private_only","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-it.md","pipeline/source_registry.json","pipeline/sources/italy/acquire.py","pipeline/sources/italy/it_853_adapter.py","pipeline/sources/italy/README.md","pipeline/common/review_packet.py","docs/review-packet-italy.md","pipeline/tests/e2e/test_italy_candidate_import.py"],"next_action":"Keep catalog acquisition, lifecycle, candidate import, and guarded API evidence private; use source-category/activity diagnostics to resolve repeated identity, coordinate/address privacy, coverage, and project approval before release review."}, @@ -22,7 +23,7 @@ {"source_id":"nz.locations","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-nz.md","pipeline/source_registry.json"],"next_action":"Confirm MPI permitted acquisition route, list-specific terms, and schema before private staging."}, {"source_id":"uk.locations","metadata":"partial","acquisition":"artifact_private_only","runtime_health":"unknown","publication_eligibility":"blocked","evidence":["docs/country-recon-uk.md","docs/countries/uk/fss-approved-establishments-source-assessment.md","pipeline/sources/uk/fsa_approved/README.md","pipeline/sources/uk/fsa_approved/handoff.py","pipeline/sources/uk/fsa_approved/refresh.py","pipeline/sources/uk/fss_approved/refresh.py","pipeline/common/review_packet.py","docs/review-packet-united-kingdom.md","docs/architecture/disposable-candidate-import.md","pipeline/scripts/maintenance/import-candidate.py","pipeline/source_registry.json"],"next_action":"Repeatable source-local refresh dry-run passed on the retained snapshot (5,342 input, 4,300 normalized, 1,042 quarantined, expected schema fingerprint, no drift alarms); use refresh.py dry-run/handoff only in restricted staging. This does not establish production/runtime health or publication eligibility. Complete source-rights, privacy/coordinate, duplicate, coverage, and release review while keeping NI and Scotland separate."}, {"source_id":"dk.smiley","metadata":"verified","acquisition":"verified","runtime_health":"unknown","publication_eligibility":"blocked","evidence":["pipeline/sources/denmark/README.md","pipeline/contracts/README.md","pipeline/common/review_packet.py","docs/review-packet-denmark.md","pipeline/source_registry.json"],"next_action":"Keep the full privately staged artifact and candidate import validation restricted; run a bounded guarded API check with explicit test-only labeling, capture rerun exit output, and resolve coverage/effective-date/category semantics before any release review. This is not a production-health or publication claim."}, - {"source_id":"de.locations","metadata":"partial","acquisition":"artifact_private_only","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/germany-source-assessment.md","pipeline/sources/germany/adapter.py","pipeline/sources/germany/refresh.py","pipeline/common/review_packet.py","docs/review-packet-germany.md","pipeline/source_registry.json"],"next_action":"Use the stable BVL landing to select an export, then run the assisted/private refresh with duplicate-approval and coordinate-precision checks. The export URL, reuse terms, privacy, and project approval remain unresolved; no release or public API exposure is allowed."}, + {"source_id":"de.locations","metadata":"partial","acquisition":"artifact_private_only","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/germany-source-assessment.md","data/manifests/de-be-private-candidates-2026-09-17.json","pipeline/sources/germany/adapter.py","pipeline/sources/germany/refresh.py","pipeline/common/review_packet.py","docs/review-packet-germany.md","pipeline/source_registry.json"],"next_action":"Review the private current export: 15,788 input, 2,691 normalized, 13,097 quarantined, with 50-column schema matched and no release created. The session-bound export route, unknown effective date, terms, privacy, coverage, and project approval remain unresolved."}, {"source_id":"ca.ontario.meat-plants","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ca.md","docs/countries/canada/meat-plants-pipeline.md","pipeline/sources/canada/adapter.py","pipeline/sources/canada/acquire.py","pipeline/sources/canada/refresh.py","pipeline/sources/canada/fixtures/ontario.csv","pipeline/common/review_packet.py","pipeline/source_registry.json"],"next_action":"Run the bounded private Ontario refresh; keep plant/contact/coordinate fields restricted pending privacy and licence review, and do not generalize Ontario coverage nationally."}, {"source_id":"ca.cfia.federal-meat","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-ca.md","docs/countries/canada/meat-plants-pipeline.md","pipeline/sources/canada/adapter.py","pipeline/sources/canada/acquire.py","pipeline/sources/canada/refresh.py","pipeline/sources/canada/fixtures/cfia.csv","pipeline/common/review_packet.py","pipeline/source_registry.json"],"next_action":"Run the bounded private CFIA registry refresh; validate the live export schema/function codes, keep federal scope separate from provincial scope, and retain publication gates."}, {"source_id":"au.daff.export-establishments","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-au.md","docs/countries/australia/artifact-metadata.json","docs/countries/australia/source-crosswalk.json","pipeline/source_registry.json"],"next_action":"Obtain an authorised bounded current DAFF establishment report/export, preserve commodity/list type, record response metadata/hash/bytes, and keep export-only scope and privacy/terms/release gates explicit."}, diff --git a/docs/source-status.md b/docs/source-status.md index c572966..bdc1611 100644 --- a/docs/source-status.md +++ b/docs/source-status.md @@ -20,7 +20,7 @@ No last-success timestamp is invented. Private artifacts are not proof of a publ ## Latest private rehearsal -The owner-authorized 2026-09-15 live-country rehearsal completed private candidate integration for Denmark, France Sections I/II, Italy 853/2004, Germany, Belgium, Canada Ontario/CFIA, and UK FSA/FSS. It created no public surface and kept publication blocked. See the detailed [country rehearsal report](country-rehearsal-2026-09-15.md) and machine-readable [rehearsal status](country-rehearsal-2026-09-15.json). +The owner-authorized 2026-09-15 live-country rehearsal remains the historical synthetic/private baseline. On 2026-09-17, current official Germany and Belgium artifacts were captured privately, parsed through the live-schema adapters, and kept publication-blocked. See the row-free [Germany/Belgium candidate manifest](../data/manifests/de-be-private-candidates-2026-09-17.json), plus the detailed country packets. ## Ireland reconnaissance update @@ -34,7 +34,7 @@ The 2026-09-16 Ireland reconnaissance verified the current FSAI/DAFM/HSE/SFPA so | `br.sif.export` | verified | verified | not_run | blocked | Current MAPA SIF export-authorized CSV captured privately; model country/product authorizations as dated observations, not facility rows, and complete terms/privacy/reconciliation review; see `docs/country-recon-br.md` | | `br.sisbi.public` | verified | artifact_private_only | not_run | blocked | Public e-SISBI JSON/GIS routes returned bounded samples; pagination, code lists, ID lifecycle, status/effective-date semantics, terms, privacy, and cadence remain unresolved; see `docs/country-recon-br.md` | | `br.trase.facilities` | verified | artifact_private_only | not_run | blocked | Current Trase GeoJSON/methodology captured privately as secondary evidence; keep separate from MAPA and review source lineage, geocoding, constructed IDs, terms, privacy, and coverage; see `docs/country-recon-br.md` | -| `be.locations` | verified | blocked | not_run | blocked | Shared private adapter, row-length/quarantine checks, and assisted two-file refresh are covered by synthetic fixtures; obtain an authorized operator capture and compare live schema before any real run; see `docs/review-packet-belgium.md` | +| `be.locations` | verified | artifact_private_only | not_run | blocked | Current official operator/codebook pair parsed privately: 310,660 input, 310,640 normalized, 20 quarantined; names are not supplied by this snapshot and privacy/terms/classification/project approval remain open; see `docs/review-packet-belgium.md` | | `fr.dgal.section-i` | verified | not_run | not_run | blocked | DGAL Section I private adapter/refresh is implemented; run only with approved terms or authorized capture, then review category semantics, address privacy, schema drift, and release approval | | `fr.dgal.section-ii` | verified | not_run | not_run | blocked | DGAL Section II remains a separate private adapter/refresh scope; review species/category semantics, address privacy, schema drift, and release approval | | `it.853-2004` | verified | artifact_private_only | not_run | blocked | Catalog acquisition, shared lifecycle, source-category/activity diagnostics, private candidate import, and guarded API checks remain review-gated; repeated activity identity, coordinate/address privacy, coverage, and project approval remain open; see `docs/review-packet-italy.md` | @@ -43,7 +43,7 @@ The 2026-09-16 Ireland reconnaissance verified the current FSAI/DAFM/HSE/SFPA so | `nz.locations` | verified | blocked | not_run | blocked | MPI/Stats NZ reconnaissance; resolve 403/access and aggregate-vs-facility boundaries | | `uk.locations` | partial | artifact_private_only | unknown | blocked | FSA and FSS private V2 lifecycle paths, nation-qualified identity, coordinate precision states, and synthetic handoff tests pass; no real UK candidate has been imported or previewed; privacy/coordinate, source-rights, duplicate, coverage, and release review remain open; NI/Scotland stay separate; see `docs/review-packet-united-kingdom.md` | | `dk.smiley` | verified | verified | unknown | blocked | Shared private lifecycle and registered adapter are validated on synthetic/retained evidence with explicit not-observed semantics; coverage/effective-date uncertainty and terms/privacy/release review remain open; see `docs/review-packet-denmark.md` | -| `de.locations` | partial | artifact_private_only | not_run | blocked | Stable BVL `/bltu` landing and portal route are verified; typed private adapter and assisted export refresh now include duplicate-approval and coordinate-precision checks, while export-specific terms/privacy/release review remain unresolved; see `docs/review-packet-germany.md` | +| `de.locations` | partial | artifact_private_only | not_run | blocked | Current public general-list export parsed privately: 15,788 input, 2,691 normalized, 13,097 quarantined; session-bound export route, unknown effective date, terms/privacy/coverage and project approval remain unresolved; see `docs/review-packet-germany.md` | | `ca.ontario.meat-plants` | verified | not_run | not_run | blocked | Ontario private adapter/refresh is implemented; keep plant/contact/coordinate fields restricted pending privacy and licence review, and do not generalize Ontario coverage nationally | | `ca.cfia.federal-meat` | verified | not_run | not_run | blocked | CFIA federal private adapter/refresh is implemented; validate live export schema/function codes, keep federal scope separate from provincial scope, and retain publication gates | | `es.locations` | partial | blocked | not_run | blocked | AESAN RGSEAA and MAPA sector routes are documented, but direct acquisition was refused; rights, export/schema, effective dates, sector coverage, privacy, and legacy/source boundaries remain unresolved | diff --git a/pipeline/germany/source-registry.json b/pipeline/germany/source-registry.json index 67cc83b..6aa0515 100644 --- a/pipeline/germany/source-registry.json +++ b/pipeline/germany/source-registry.json @@ -1,15 +1,18 @@ { "country_code": "de", - "adapter_version": "de-v2-foundation-1", - "schema_version": "location-v2-foundation-1", + "adapter_version": "de-bltu-private-v2", + "schema_version": "de-bltu-csv-v2", "mapping_version": "de-bltu-activity-map-1", - "source_url": "https://gis.bvl.bund.de/datenportal/", - "retrieval_timestamp": "NOT-ACQUIRED", + "source_url": "https://www.bvl.bund.de/bltu", + "portal_url": "https://gis.bvl.bund.de/datenportal/", + "retrieval_timestamp": "2026-09-17T20:26:41Z", "source_publication_date": "UNKNOWN-UNTIL-EXPORT", - "checksum_sha256": "COMPUTED-FROM-RAW-ARTIFACT", - "byte_size": "COMPUTED-FROM-RAW-ARTIFACT", + "checksum_sha256": "65e71e324a2d905aff75e145c8e85823172f0e5825205313bdf38cf7dc1a63c0", + "byte_size": 2068022, + "schema_fingerprint": "52fd9ed6ecd543cff2f96b99c8bce5a84cea51cc65d000c00793ff0689e37f78", + "capture_evidence": "data/manifests/de-be-private-candidates-2026-09-17.json", "terms_review": "BLOCKED-PENDING-HUMAN-CONFIRMATION", - "acquisition_status": "restricted_pending_terms", + "acquisition_status": "private_candidate_captured", "privacy_review": "REQUIRED-HUMAN-GATE", "publication_approval": "REQUIRED-HUMAN-GATE", "geocoding": "disabled", diff --git a/pipeline/source_registry.json b/pipeline/source_registry.json index 286db0c..8c65a9b 100644 --- a/pipeline/source_registry.json +++ b/pipeline/source_registry.json @@ -60,8 +60,8 @@ "cadence": "weekly", "attribution_licensing_notes": "CC Attribution 4.0; attribute FASFC and the last update date, do not imply FASFC affiliation/approval, and do not mislead", "adapter_status": "implemented_partial", - "expected_artifact_schema": "UTF-8/CP1252 CSV operator rows joined by LAP/PAP code to the separate FASFC activity-code CSV; live operator header remains to be captured", - "blockers": ["The official operator body was inaccessible from the current execution environment; obtain an authorized bounded capture and compare its header/field semantics with the synthetic contract before any real run. Privacy, source-terms interpretation, coverage, and project publication approval remain human gates."] + "expected_artifact_schema": "CP1252 CSV with 20 live identifier/activity/location fields joined by LAP/PAP code to the separate FASFC activity-code CSV; establishment name is not supplied by this snapshot", + "blockers": ["A current official pair was captured privately and validated through the deterministic adapter. Five exact duplicates and 15 unresolved activity codes remain quarantined; privacy, source-terms interpretation, classification, coverage, and project publication approval remain human gates."] }, { "source_id": "ca.ontario.meat-plants", @@ -96,8 +96,8 @@ "cadence": "unknown", "attribution_licensing_notes": "unknown; verify BVL reuse and attribution terms", "adapter_status": "implemented_partial", - "expected_artifact_schema": "BLtU general-list semicolon CSV with repeated activity-code columns, current approval number, establishment name, state, address, and activity flags", - "blockers": ["The stable BVL landing and portal export route are verified, but the selected export URL is session/request-specific and dataset reuse terms remain pending human confirmation. Keep raw data private and publication blocked."] + "expected_artifact_schema": "BLtU general-list cp1252 CSV with 50 live columns, repeated activity-code columns, current approval number, establishment name, state, address, and activity flags", + "blockers": ["A current public general-list export was captured privately: 15,788 input, 2,691 normalized, and 13,097 quarantined. The export URL is session/request-specific, effective date is unknown, and dataset reuse terms, privacy, coverage, and project approval remain pending human confirmation. Keep raw data private and publication blocked."] }, { "source_id": "dk.smiley", diff --git a/pipeline/sources/belgium/README.md b/pipeline/sources/belgium/README.md index b4427f7..971c91f 100644 --- a/pipeline/sources/belgium/README.md +++ b/pipeline/sources/belgium/README.md @@ -10,6 +10,7 @@ review, but the current operator URL may require an assisted browser download. python -m pipeline.sources.belgium.refresh --operators --activity-codes --run-dir --retrieved-at-utc 2026-09-15T00:00:00Z ``` -The checked-in fixtures are synthetic only. A successful run is a private -candidate with publication blocked; it is not FASFC approval, project review, -or a public release. +The checked-in fixtures are synthetic only. A current official pair may be +provided through private ignored storage with `--acquisition-metadata`; a +successful run is still only a private candidate with publication blocked. It +is not FASFC approval, project review, or a public release. diff --git a/pipeline/sources/belgium/adapter.py b/pipeline/sources/belgium/adapter.py index cb626a6..581b85d 100644 --- a/pipeline/sources/belgium/adapter.py +++ b/pipeline/sources/belgium/adapter.py @@ -29,30 +29,30 @@ class BelgiumSchemaError(ValueError): _ALIASES = { - "operator_id": ("operator_id", "operator id", "operator number", "enterprise number", "enterprise id", "nummer operator", "numero operateur"), - "establishment_id": ("establishment_id", "establishment id", "establishment number", "establishment nr", "n establishment", "nummer vestiging", "numero etablissement", "n etablissement"), + "operator_id": ("operator_id", "operator id", "operator number", "enterprise number", "enterprise id", "nummer operator", "numero operateur", "op n unique id", "op unique id"), + "establishment_id": ("establishment_id", "establishment id", "establishment number", "establishment nr", "n establishment", "nummer vestiging", "numero etablissement", "n etablissement", "lno n unique", "lno unique id"), "name": ("name", "operator name", "establishment name", "name operator", "naam", "nom", "name establishment"), "address": ("address", "address line", "street", "street address", "street and number", "adres", "adresse"), - "postcode": ("postcode", "postal code", "zip", "code postal"), - "municipality": ("municipality", "city", "town", "gemeente", "commune", "municipality name"), - "region": ("region", "province", "provincie", "province"), + "postcode": ("postcode", "postal code", "zip", "code postal", "pc code postal"), + "municipality": ("municipality", "city", "town", "gemeente", "commune", "municipality name", "gem nom"), + "region": ("region", "province", "provincie", "province", "pr nom"), "latitude": ("latitude", "lat", "latitude y"), "longitude": ("longitude", "lon", "lng", "longitude x"), "activity_code": ("activity_code", "activity code", "activity codes", "lap code", "lap id", "pap code", "pap id", "code lap", "code pap", "activiteiten code", "code activite"), - "activity_description": ("activity_description", "activity description", "activity", "description", "omschrijving activiteit", "description activite"), - "approval_number": ("approval_number", "approval number", "approval nr", "agrément", "erkenningsnummer", "numero agrement"), + "activity_description": ("activity_description", "activity description", "activity", "description", "omschrijving activiteit", "description activite", "pap description"), + "approval_number": ("approval_number", "approval number", "approval nr", "agrément", "erkenningsnummer", "numero agrement", "erk nummer", "erk numero"), "authorization_number": ("authorization_number", "authorization number", "authorization nr", "authorisation number", "autorisatienummer", "numero autorisation"), "status": ("status", "current status", "state", "statuut", "statut"), - "effective_date": ("effective_date", "effective date", "valid from", "start date", "geldigheid vanaf", "date debut"), + "effective_date": ("effective_date", "effective date", "valid from", "start date", "geldigheid vanaf", "date debut", "erk date debut"), } _CODE_ALIASES = { "lap_code": ("lap_code", "lap code", "lap id", "pap_code", "pap code", "pap id", "activity_code", "activity code", "code lap", "code pap"), "place_code": ("place_code", "place code", "pl code", "location code", "code lieu", "plaats code"), "place_description": ("place_description", "place description", "location description", "lieu", "plaats"), - "activity_description": ("activity_description", "activity description", "activity", "activiteit", "activite"), - "product_description": ("product_description", "product description", "product", "produit", "productomschrijving"), - "approval_code": ("approval_code", "approval code", "approval form", "code agrement", "erkenningscode"), - "approval_description": ("approval_description", "approval description", "approval", "agrement", "erkenning"), + "activity_description": ("activity_description", "activity description", "activity", "activiteit", "activite", "activiteit omschrijving"), + "product_description": ("product_description", "product description", "product", "produit", "productomschrijving", "product omschrijving"), + "approval_code": ("approval_code", "approval code", "approval form", "code agrement", "erkenningscode", "erkenning code"), + "approval_description": ("approval_description", "approval description", "approval", "agrement", "erkenning", "erkenning omschrijving"), } _CATEGORY_RULES: tuple[tuple[str, tuple[str, ...]], ...] = ( ("slaughter", ("slaughter", "abattoir", "slachthuis", "killing", "abattage")), @@ -174,36 +174,39 @@ def _codebook(self) -> tuple[dict[str, dict[str, str | None]], dict[str, Any]]: def parse_bytes(self, content: bytes) -> dict[str, Any]: codebook, codebook_meta = self._codebook() headers, rows, encoding, delimiter = _csv(content) - fields = _header_map(headers, _ALIASES, {"establishment_id", "name", "activity_code"}) + fields = _header_map(headers, _ALIASES, {"establishment_id", "activity_code"}) + name_header = fields.get("name") accepted: list[dict[str, Any]] = [] quarantined: list[dict[str, Any]] = [] anomalies: Counter[str] = Counter() - seen: Counter[tuple[str | None, str | None]] = Counter() - prepared: list[tuple[int, dict[str, str | None], tuple[str, ...], int]] = [] + exact_counts: Counter[tuple[str, ...]] = Counter() + first_lines: dict[tuple[str, ...], int] = {} + prepared: list[tuple[int, dict[str, str | None], tuple[str, ...], int, tuple[str, ...]]] = [] row_lengths: Counter[str] = Counter() for line, values in enumerate(rows, start=2): raw = _row_values(headers, values) row_lengths[str(len(values))] += 1 codes = _split_codes(_clean(raw.get(fields["activity_code"]))) - prepared.append((line, raw, codes, len(values))) - for code in codes: - seen[(_clean(raw.get(fields["establishment_id"])), code)] += 1 - for line, raw, codes, value_count in prepared: + row_key = tuple(raw.get(header) or "" for header in headers) + prepared.append((line, raw, codes, len(values), row_key)) + exact_counts[row_key] += 1 + first_lines.setdefault(row_key, line) + for line, raw, codes, value_count, row_key in prepared: establishment_id = _clean(raw.get(fields["establishment_id"])) - name = _clean(raw.get(fields["name"])) + name = _clean(raw.get(name_header)) if name_header else None reasons: list[str] = [] if value_count != len(headers) or any(value is None for value in raw.values()): reasons.append("malformed_row") if not establishment_id: reasons.append("missing_establishment_id") - if not name: + if name_header and not name: reasons.append("missing_name") if not codes: reasons.append("missing_activity_code") if any(code not in codebook for code in codes): reasons.append("unresolved_activity_code") - if any(seen[(establishment_id, code)] > 1 for code in codes): - reasons.append("ambiguous_repeated_establishment_activity") + if exact_counts[row_key] > 1 and first_lines[row_key] != line: + reasons.append("duplicate_source_row") address = _clean(raw.get(fields.get("address", ""))) if "address" in fields else None if address and _RISK.search(address): reasons.append("address_privacy_risk") @@ -220,6 +223,7 @@ def parse_bytes(self, content: bytes) -> dict[str, Any]: "establishment_id": establishment_id, "operator_id": _clean(raw.get(fields.get("operator_id", ""))) if "operator_id" in fields else None, "name": name, + "name_state": "source-supplied" if name_header else "not-supplied-by-source", "trading_name": name, "country_code": "BE", "nation": "Belgium", @@ -251,7 +255,7 @@ def parse_bytes(self, content: bytes) -> dict[str, Any]: quarantined.append({"reasons": unique, "record": record}) else: accepted.append(record) - return {"accepted": accepted, "quarantined": quarantined, "input_rows": len(rows), "source_sha256": hashlib.sha256(content).hexdigest(), "operator_schema_fingerprint": _fingerprint(headers), "operator_column_count": len(headers), "operator_encoding": encoding, "operator_delimiter": delimiter, "row_length_counts": dict(sorted(row_lengths.items())), "codebook": codebook_meta, "coverage_counts": dict(Counter(category for item in accepted for category in item["normalized"]["source_activity_categories"])), "anomaly_counts": dict(sorted(anomalies.items()))} + return {"accepted": accepted, "quarantined": quarantined, "input_rows": len(rows), "source_sha256": hashlib.sha256(content).hexdigest(), "operator_schema_fingerprint": _fingerprint(headers), "operator_column_count": len(headers), "operator_encoding": encoding, "operator_delimiter": delimiter, "row_length_counts": dict(sorted(row_lengths.items())), "codebook": codebook_meta, "name_header": name_header, "repeated_establishment_activity_keys": sum(1 for key, count in exact_counts.items() if count > 1), "coverage_counts": dict(Counter(category for item in accepted for category in item["normalized"]["source_activity_categories"])), "anomaly_counts": dict(sorted(anomalies.items()))} def run(self, raw_path: str | Path, run_dir: str | Path, artifact: SourceArtifact) -> dict[str, Any]: raw = Path(raw_path).read_bytes() @@ -265,7 +269,7 @@ def run(self, raw_path: str | Path, run_dir: str | Path, artifact: SourceArtifac _, normalized_sha, _ = atomic_jsonl(root / "normalized" / "records.jsonl", result["accepted"]) atomic_jsonl(root / "quarantined" / "records.jsonl", result["quarantined"]) manifest = private_manifest(source_id=self.source_id, adapter_version=self.adapter_version, schema_version=self.schema_version, artifact=artifact, input_rows=result["input_rows"], normalized_rows=len(result["accepted"]), quarantined_rows=len(result["quarantined"]), normalized_sha256=normalized_sha, parsed_sha256=parsed_sha, anomaly_counts=result["anomaly_counts"]) - manifest.update({"country_code": "BE", "coverage": CONFIG["coverage"], "geocoding": "disabled", "operator_schema_fingerprint": result["operator_schema_fingerprint"], "operator_column_count": result["operator_column_count"], "operator_encoding": result["operator_encoding"], "operator_delimiter": result["operator_delimiter"], "row_length_counts": result["row_length_counts"], "coverage_counts": result["coverage_counts"], "activity_codebook": result["codebook"], "codebook_source_url": self.activity_artifact.source_url if self.activity_artifact else "unrecorded-companion-artifact", "codebook_retrieved_at_utc": self.activity_artifact.retrieved_at_utc if self.activity_artifact else None, "codebook_sha256": result["codebook"]["sha256"], "codebook_byte_size": result["codebook"]["byte_size"]}) + manifest.update({"country_code": "BE", "coverage": CONFIG["coverage"], "geocoding": "disabled", "operator_schema_fingerprint": result["operator_schema_fingerprint"], "operator_column_count": result["operator_column_count"], "operator_encoding": result["operator_encoding"], "operator_delimiter": result["operator_delimiter"], "row_length_counts": result["row_length_counts"], "repeated_establishment_activity_keys": result["repeated_establishment_activity_keys"], "coverage_counts": result["coverage_counts"], "activity_codebook": result["codebook"], "codebook_source_url": self.activity_artifact.source_url if self.activity_artifact else "unrecorded-companion-artifact", "codebook_retrieved_at_utc": self.activity_artifact.retrieved_at_utc if self.activity_artifact else None, "codebook_sha256": result["codebook"]["sha256"], "codebook_byte_size": result["codebook"]["byte_size"], "source_name_field": result["name_header"], "source_name_status": "available" if result["name_header"] else "not-supplied"}) atomic_json(root / "manifest.json", manifest) return manifest diff --git a/pipeline/sources/belgium/config.json b/pipeline/sources/belgium/config.json index e862dcf..6142af6 100644 --- a/pipeline/sources/belgium/config.json +++ b/pipeline/sources/belgium/config.json @@ -4,8 +4,8 @@ "activity_code_url": "https://www.static.favv.be/bo-documents/inter_PAP_omschrijving_EN.csv", "catalog_url": "https://data.gov.be/en/datasets/favv-afsca-operators", "activity_catalog_url": "https://data.gov.be/en/datasets/fasfc-activity-codes", - "adapter_version": "be-fasfc-private-v1", - "schema_version": "be-fasfc-csv-v1", + "adapter_version": "be-fasfc-private-v2", + "schema_version": "be-fasfc-csv-v2", "coverage": "Belgium FASFC operators with a current registration, approval, or authorization; activity codebook joined separately", "terms": "CC Attribution 4.0 is indicated by the data.gov.be dataset pages; attribution and project publication review remain required", "geocoding": "disabled" diff --git a/pipeline/sources/belgium/refresh.py b/pipeline/sources/belgium/refresh.py index 3d0a999..b8b0b3b 100644 --- a/pipeline/sources/belgium/refresh.py +++ b/pipeline/sources/belgium/refresh.py @@ -32,12 +32,35 @@ class RefreshError(ValueError): "privacy": ["Address, postal, enterprise, and coordinate fields require field-level privacy review; normalized rows intentionally suppress them and geocoding is disabled."], "completeness": ["The feed covers current FASFC registrations/approvals/authorizations, not every animal-agriculture facility; live operator schema and currentness semantics still require capture review."], "classification": ["LAP/PAP codebook joins are explicit, but slaughter, cutting, processing, storage, animal-by-products, and export remain separate source categories."], - "coverage": ["Operator CSV and activity-code CSV must be captured together; the live operator header was not available in this environment."], + "coverage": ["Operator CSV and activity-code CSV were captured together; the live operator feed is broader than slaughterhouses and its identifier/activity-only schema must remain source-labeled."], } -def _local_metadata(path: Path, *, source_id: str, source_url: str, retrieved_at: str, coverage: str) -> dict[str, Any]: +def _local_metadata(path: Path, *, source_id: str, source_url: str, retrieved_at: str, coverage: str, metadata_path: str | Path | None = None) -> dict[str, Any]: raw = path.read_bytes() + if metadata_path is not None: + metadata = json.loads(Path(metadata_path).read_text(encoding="utf-8")) + if not isinstance(metadata, dict): + raise RefreshError("acquisition metadata must be an object") + expected_hash = str(metadata.get("sha256") or "").lower() + expected_size = metadata.get("byte_size") + if expected_hash != hashlib.sha256(raw).hexdigest() or int(expected_size or -1) != len(raw): + raise RefreshError(f"acquisition metadata does not match {path.name}") + metadata.setdefault("acquisition_method", "assisted_local_capture") + metadata.setdefault("artifact_path", str(path.resolve())) + metadata.setdefault("requested_at_utc", retrieved_at) + metadata.setdefault("retrieved_at_utc", retrieved_at) + metadata.setdefault("effective_date", "unknown") + metadata.setdefault("publication_date", None) + metadata.setdefault("redirects", []) + metadata.setdefault("response_headers", {}) + metadata.setdefault("adapter_version", CONFIG["adapter_version"]) + metadata.setdefault("code_version", CONFIG["adapter_version"]) + metadata.setdefault("config_version", CONFIG["schema_version"]) + metadata.setdefault("coverage", coverage) + metadata.setdefault("rights_caveat", CONFIG["terms"]) + metadata.setdefault("privacy_caveat", "private staging; address and coordinate review pending") + return metadata return {"acquisition_method": "assisted_local_capture", "source_id": source_id, "artifact": path.name, "artifact_path": str(path.resolve()), "requested_url": source_url, "final_url": source_url, "redirects": [], "response_headers": {}, "requested_at_utc": retrieved_at, "retrieved_at_utc": retrieved_at, "effective_date": "unknown", "publication_date": None, "sha256": hashlib.sha256(raw).hexdigest(), "byte_size": len(raw), "adapter_version": CONFIG["adapter_version"], "code_version": CONFIG["adapter_version"], "config_version": CONFIG["schema_version"], "coverage": coverage, "rights_caveat": CONFIG["terms"], "privacy_caveat": "private staging; address and coordinate review pending", "terms_review": "operator-assisted capture; source terms review remains a separate gate"} @@ -45,7 +68,7 @@ def _artifact(metadata: dict[str, Any], *, default_url: str, default_coverage: s return SourceArtifact(source_url=str(metadata.get("final_url") or metadata.get("requested_url") or default_url), retrieved_at_utc=str(metadata.get("retrieved_at_utc") or ""), sha256=str(metadata["sha256"]), byte_size=int(metadata["byte_size"]), publication_date=metadata.get("publication_date"), effective_date=metadata.get("effective_date"), code_version=str(metadata.get("code_version") or CONFIG["adapter_version"]), config_version=str(metadata.get("config_version") or CONFIG["schema_version"]), rights_caveat=metadata.get("rights_caveat") or CONFIG["terms"], privacy_caveat=metadata.get("privacy_caveat") or "private staging; privacy review pending", coverage=metadata.get("coverage") or default_coverage, redirects=tuple(metadata.get("redirects") or ())) -def refresh(*, run_dir: str | Path, operators_path: str | Path | None = None, activity_codes_path: str | Path | None = None, fetch_pair: bool = False, output_root: str | Path = "data/raw", run_id: str | None = None, terms_review_path: str | Path | None = None, retrieved_at_utc: str | None = None, timeout_seconds: float = 60, max_bytes: int = 128 * 1024 * 1024, previous_normalized: str | Path | None = None) -> dict[str, Any]: +def refresh(*, run_dir: str | Path, operators_path: str | Path | None = None, activity_codes_path: str | Path | None = None, fetch_pair: bool = False, output_root: str | Path = "data/raw", run_id: str | None = None, terms_review_path: str | Path | None = None, retrieved_at_utc: str | None = None, timeout_seconds: float = 60, max_bytes: int = 128 * 1024 * 1024, previous_normalized: str | Path | None = None, acquisition_metadata_path: str | Path | None = None) -> dict[str, Any]: if fetch_pair == (operators_path is not None or activity_codes_path is not None): raise RefreshError("specify --fetch or both --operators and --activity-codes") root = Path(run_dir) @@ -65,8 +88,9 @@ def refresh(*, run_dir: str | Path, operators_path: str | Path | None = None, ac operator_path, code_path = Path(operators_path).resolve(), Path(activity_codes_path).resolve() if not operator_path.is_file() or not code_path.is_file(): raise RefreshError("operator and activity-code artifacts must exist") - operator_meta = _local_metadata(operator_path, source_id=CONFIG["source_id"], source_url=CONFIG["operator_url"], retrieved_at=retrieved, coverage=CONFIG["coverage"]) - code_meta = _local_metadata(code_path, source_id="be.activity-codes", source_url=CONFIG["activity_code_url"], retrieved_at=retrieved, coverage="FASFC LAP/PAP codebook; not a facility list") + metadata_root = json.loads(Path(acquisition_metadata_path).read_text(encoding="utf-8")) if acquisition_metadata_path else {} + operator_meta = _local_metadata(operator_path, source_id=CONFIG["source_id"], source_url=CONFIG["operator_url"], retrieved_at=retrieved, coverage=CONFIG["coverage"], metadata_path=metadata_root.get("operator_path") if isinstance(metadata_root, dict) and metadata_root.get("operator_path") else None) + code_meta = _local_metadata(code_path, source_id="be.activity-codes", source_url=CONFIG["activity_code_url"], retrieved_at=retrieved, coverage="FASFC LAP/PAP codebook; not a facility list", metadata_path=metadata_root.get("activity_codes_path") if isinstance(metadata_root, dict) and metadata_root.get("activity_codes_path") else None) atomic_json(root / "acquisition-metadata.json", {"operator": operator_meta, "activity_codes": code_meta}) operator_artifact = _artifact(operator_meta, default_url=CONFIG["operator_url"], default_coverage=CONFIG["coverage"]) code_artifact = _artifact(code_meta, default_url=CONFIG["activity_code_url"], default_coverage="FASFC LAP/PAP codebook; not a facility list") @@ -103,9 +127,10 @@ def main() -> int: parser.add_argument("--timeout-seconds", type=float, default=60) parser.add_argument("--max-bytes", type=int, default=128 * 1024 * 1024) parser.add_argument("--previous-normalized", type=Path) + parser.add_argument("--acquisition-metadata", type=Path, help="row-free JSON mapping with operator_path and activity_codes_path sidecars") args = parser.parse_args() try: - result = refresh(run_dir=args.run_dir, operators_path=args.operators, activity_codes_path=args.activity_codes, fetch_pair=args.fetch, output_root=args.output_root, run_id=args.run_id, terms_review_path=args.terms_review, retrieved_at_utc=args.retrieved_at_utc, timeout_seconds=args.timeout_seconds, max_bytes=args.max_bytes, previous_normalized=args.previous_normalized) + result = refresh(run_dir=args.run_dir, operators_path=args.operators, activity_codes_path=args.activity_codes, fetch_pair=args.fetch, output_root=args.output_root, run_id=args.run_id, terms_review_path=args.terms_review, retrieved_at_utc=args.retrieved_at_utc, timeout_seconds=args.timeout_seconds, max_bytes=args.max_bytes, previous_normalized=args.previous_normalized, acquisition_metadata_path=args.acquisition_metadata) except (OSError, RefreshError) as error: print(json.dumps({"status": "failed", "error": str(error)}, sort_keys=True)) return 2 diff --git a/pipeline/sources/belgium/test_refresh.py b/pipeline/sources/belgium/test_refresh.py index 1fc2caf..ca28f24 100644 --- a/pipeline/sources/belgium/test_refresh.py +++ b/pipeline/sources/belgium/test_refresh.py @@ -1,3 +1,5 @@ +import hashlib +import json import tempfile import unittest from pathlib import Path @@ -9,6 +11,22 @@ class BelgiumRefreshTests(unittest.TestCase): + def test_row_free_acquisition_sidecars_are_verified_and_preserved(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + operators = ROOT / "fixtures" / "synthetic_operators.csv" + codes = ROOT / "fixtures" / "synthetic_activity_codes.csv" + operator_meta = root / "operator-metadata.json" + code_meta = root / "code-metadata.json" + operator_meta.write_text(json.dumps({"sha256": hashlib.sha256(operators.read_bytes()).hexdigest(), "byte_size": operators.stat().st_size, "final_url": "https://example.invalid/current-operators.csv", "effective_date": "2026-09-14"}), encoding="utf-8") + code_meta.write_text(json.dumps({"sha256": hashlib.sha256(codes.read_bytes()).hexdigest(), "byte_size": codes.stat().st_size, "final_url": "https://example.invalid/current-codes.csv", "effective_date": "2026-09-14"}), encoding="utf-8") + metadata = root / "pair.json" + metadata.write_text(json.dumps({"operator_path": str(operator_meta), "activity_codes_path": str(code_meta)}), encoding="utf-8") + result = refresh(run_dir=root / "run", operators_path=operators, activity_codes_path=codes, acquisition_metadata_path=metadata, retrieved_at_utc="2026-09-14T00:00:00Z") + self.assertEqual(result["report"]["publication_eligibility"], "blocked") + saved = json.loads((root / "run" / "acquisition-metadata.json").read_text(encoding="utf-8")) + self.assertEqual(saved["operator"]["final_url"], "https://example.invalid/current-operators.csv") + def test_assisted_pair_is_repeatable_and_keeps_artifacts_private(self): with tempfile.TemporaryDirectory() as directory: root = Path(directory) diff --git a/pipeline/sources/germany/README.md b/pipeline/sources/germany/README.md index b2666ca..be5af68 100644 --- a/pipeline/sources/germany/README.md +++ b/pipeline/sources/germany/README.md @@ -8,6 +8,9 @@ and run the assisted refresh: python -m pipeline.sources.germany.refresh --raw --run-dir --retrieved-at-utc 2026-09-15T00:00:00Z ``` +For a captured export with row-free acquisition metadata, also pass +`--acquisition-metadata `. + The typed adapter preserves repeated activity columns and source evidence, quarantines schema and mapping anomalies, emits shared health evidence, and cannot create a release. Address/coordinate review, BVL reuse terms, and human diff --git a/pipeline/sources/germany/adapter.py b/pipeline/sources/germany/adapter.py index 6b3de76..0e4cd0e 100644 --- a/pipeline/sources/germany/adapter.py +++ b/pipeline/sources/germany/adapter.py @@ -13,6 +13,16 @@ from pipeline.contracts.source_lifecycle import atomic_json, atomic_jsonl, private_manifest from pipeline.germany.bltu_adapter import EXPECTED_HEADERS +CURRENT_HEADERS = tuple(( + "# Bundesland", "Name des Betriebs ", "Straße / Haus-Nr.", "Ort", "Alte Zulassungs-nummern", + "Neue Zulassungsnummer", "Zulassungsnummer Eierpackstellen", "CS", "RW", "WM", "SH", "CP", + "SH", "CP", "SH", "CP", "GHE", "CP", "MM", "MP", "MSM", "PP", "CC", "PP", "CC", + "PP", "CC", "PP", "CC", "PP", "CC", "PP", "EPC", "LEP", "PP", "AH", "FV", "ZV", "FFPP", + "PP", "WM", "PC", "DC", "PP", "Einschränkungen", "Bemerkungen", "Zulassung befristet bis", + "Zulassung ruht seit", "Drittlandzulassungen", "" +)) +SUPPORTED_HEADERS = {tuple(EXPECTED_HEADERS), CURRENT_HEADERS} + CONFIG = json.loads((Path(__file__).parent / "config.json").read_text(encoding="utf-8")) CURRENT_ID_INDEX = 5 NAME_INDEX = 1 @@ -40,22 +50,39 @@ def parse_bytes(self, content: bytes) -> dict[str, Any]: except UnicodeDecodeError: text = content.decode("cp1252") encoding = "cp1252" - rows = list(csv.reader(text.splitlines(), delimiter=";", strict=True)) - headers = rows[0] if rows else [] - matched = headers == list(EXPECTED_HEADERS) - id_counts = Counter(values[CURRENT_ID_INDEX].strip() for values in rows[1:] if len(values) > CURRENT_ID_INDEX and values[CURRENT_ID_INDEX].strip()) + lines = text.splitlines() + try: + headers = next(csv.reader([lines[0]], delimiter=";", strict=True)) if lines else [] + except csv.Error as error: + raise ValueError("BLtU header is malformed") from error + rows: list[tuple[int, list[str] | None, str | None]] = [] + for line_number, raw_line in enumerate(lines[1:], start=2): + try: + values = next(csv.reader([raw_line], delimiter=";", strict=True)) + except csv.Error: + rows.append((line_number, None, raw_line)) + continue + rows.append((line_number, values, None)) + matched = tuple(headers) in SUPPORTED_HEADERS + id_counts = Counter((values[CURRENT_ID_INDEX].strip() or values[4].strip()) for _, values, _ in rows if values is not None and len(values) > CURRENT_ID_INDEX and (values[CURRENT_ID_INDEX].strip() or values[4].strip())) accepted: list[dict[str, Any]] = [] quarantined: list[dict[str, Any]] = [] anomalies: Counter[str] = Counter() categories: Counter[str] = Counter() - for line, values in enumerate(rows[1:], start=2): + for line, values, raw_line in rows: + if values is None: + evidence = {"source_row": line, "source_headers": headers, "raw_line": raw_line, "source_values": None} + quarantined.append({**evidence, "reasons": ("malformed_csv_row",), "record": {"source_id": self.source_id, "source_row": line, "source_record_key": f"unknown|{line}"}}) + anomalies["malformed_csv_row"] += 1 + continue source = {"source_row": line, "source_headers": headers, "source_values": values} reasons: list[str] = [] if not matched: reasons.append("unrecognized_header_schema") - if len(values) != len(EXPECTED_HEADERS): + if len(values) != len(CURRENT_HEADERS): reasons.append("physical_column_count_mismatch") - current_id = values[CURRENT_ID_INDEX].strip() if len(values) > CURRENT_ID_INDEX else "" + has_current_id = len(values) > CURRENT_ID_INDEX and bool(values[CURRENT_ID_INDEX].strip()) + current_id = (values[CURRENT_ID_INDEX].strip() if has_current_id else (values[4].strip() if len(values) > 4 else "")) name = values[NAME_INDEX].strip() if len(values) > NAME_INDEX else "" if not current_id: reasons.append("missing_current_approval_id") @@ -73,7 +100,7 @@ def parse_bytes(self, content: bytes) -> dict[str, Any]: reasons.append("unmapped_activity_code") for category in mapped: categories[category] += 1 - record = {"source_id": self.source_id, "source_row": line, "source_record_key": f"{current_id or 'unknown'}|{line}", "source_values": source, "normalized": {"establishment_id": current_id or None, "approval_number": current_id or None, "name": name or None, "trading_name": name or None, "country_code": "DE", "nation": "Germany", "state": values[STATE_INDEX].strip() if len(values) > STATE_INDEX else None, "city": values[CITY_INDEX].strip() if len(values) > CITY_INDEX else None, "address": None, "address_state": "source-present-pending-privacy-review" if len(values) > STREET_INDEX and values[STREET_INDEX].strip() else "unknown", "activity_codes": tuple(activity_codes), "activity_categories": mapped, "source_activity_categories": mapped, "classification_state": "mapped" if mapped and not any(code not in ACTIVITY_MAP for code in activity_codes) else "unresolved", "coordinates": None, "coordinate_state": "unknown", "coordinate_precision": "not-supplied", "privacy_gate": "pending-review", "coordinate_gate": "review_required", "publication_gate": "blocked"}} + record = {"source_id": self.source_id, "source_row": line, "source_record_key": f"{current_id or 'unknown'}|{line}", "source_values": source, "normalized": {"establishment_id": current_id or None, "approval_number": current_id or None, "approval_number_kind": "current" if has_current_id else "legacy", "name": name or None, "trading_name": name or None, "country_code": "DE", "nation": "Germany", "state": values[STATE_INDEX].strip() if len(values) > STATE_INDEX else None, "city": values[CITY_INDEX].strip() if len(values) > CITY_INDEX else None, "address": None, "address_state": "source-present-pending-privacy-review" if len(values) > STREET_INDEX and values[STREET_INDEX].strip() else "unknown", "activity_codes": tuple(activity_codes), "activity_categories": mapped, "source_activity_categories": mapped, "classification_state": "mapped" if mapped and not any(code not in ACTIVITY_MAP for code in activity_codes) else "unresolved", "coordinates": None, "coordinate_state": "unknown", "coordinate_precision": "not-supplied", "privacy_gate": "pending-review", "coordinate_gate": "review_required", "publication_gate": "blocked"}} if reasons: unique = tuple(dict.fromkeys(reasons)) for reason in unique: @@ -81,7 +108,7 @@ def parse_bytes(self, content: bytes) -> dict[str, Any]: quarantined.append({"reasons": unique, "record": record}) else: accepted.append(record) - return {"accepted": accepted, "quarantined": quarantined, "input_rows": len(rows) - 1 if rows else 0, "source_sha256": hashlib.sha256(content).hexdigest(), "schema_status": "matched" if matched else "unrecognized", "schema_fingerprint": hashlib.sha256(json.dumps(headers, ensure_ascii=False, separators=(",", ":")).encode()).hexdigest(), "encoding": encoding, "row_length_counts": dict(Counter(str(len(row)) for row in rows[1:])), "coverage_counts": dict(categories), "anomaly_counts": dict(sorted(anomalies.items()))} + return {"accepted": accepted, "quarantined": quarantined, "input_rows": len(rows), "source_sha256": hashlib.sha256(content).hexdigest(), "schema_status": "matched" if matched else "unrecognized", "schema_fingerprint": hashlib.sha256(json.dumps(headers, ensure_ascii=False, separators=(",", ":")).encode()).hexdigest(), "encoding": encoding, "row_length_counts": dict(Counter(str(len(values)) for _, values, _ in rows if values is not None)), "coverage_counts": dict(categories), "anomaly_counts": dict(sorted(anomalies.items()))} def run(self, raw_path: str | Path, run_dir: str | Path, artifact: SourceArtifact) -> dict[str, Any]: raw = Path(raw_path).read_bytes() diff --git a/pipeline/sources/germany/config.json b/pipeline/sources/germany/config.json index b96c9ae..f8bea88 100644 --- a/pipeline/sources/germany/config.json +++ b/pipeline/sources/germany/config.json @@ -2,8 +2,8 @@ "source_id": "de.locations", "source_url": "https://www.bvl.bund.de/bltu", "portal_url": "https://gis.bvl.bund.de/datenportal/", - "adapter_version": "de-bltu-private-v1", - "schema_version": "de-bltu-csv-v1", + "adapter_version": "de-bltu-private-v2", + "schema_version": "de-bltu-csv-v2", "coverage": "BVL BLtU list of German establishments approved under Regulation (EC) 853/2004; export selected from the BVL portal", "terms": "Public access and export are documented by BVL, but dataset-specific reuse and redistribution terms remain pending human confirmation", "geocoding": "disabled" diff --git a/pipeline/sources/germany/refresh.py b/pipeline/sources/germany/refresh.py index 97a42c8..0358a47 100644 --- a/pipeline/sources/germany/refresh.py +++ b/pipeline/sources/germany/refresh.py @@ -30,12 +30,32 @@ class RefreshError(ValueError): } -def _local_metadata(path: Path, retrieved: str) -> dict[str, Any]: +def _local_metadata(path: Path, retrieved: str, metadata_path: str | Path | None = None) -> dict[str, Any]: raw = path.read_bytes() + if metadata_path is not None: + metadata = json.loads(Path(metadata_path).read_text(encoding="utf-8")) + if not isinstance(metadata, dict): + raise RefreshError("acquisition metadata must be an object") + if str(metadata.get("sha256") or "").lower() != hashlib.sha256(raw).hexdigest() or int(metadata.get("byte_size") or -1) != len(raw): + raise RefreshError("acquisition metadata does not match the BLtU artifact") + metadata.setdefault("artifact_path", str(path.resolve())) + metadata.setdefault("retrieved_at_utc", retrieved) + metadata.setdefault("requested_at_utc", retrieved) + metadata.setdefault("effective_date", "unknown") + metadata.setdefault("publication_date", None) + metadata.setdefault("redirects", []) + metadata.setdefault("response_headers", {}) + metadata.setdefault("adapter_version", CONFIG["adapter_version"]) + metadata.setdefault("code_version", CONFIG["adapter_version"]) + metadata.setdefault("config_version", CONFIG["schema_version"]) + metadata.setdefault("coverage", CONFIG["coverage"]) + metadata.setdefault("rights_caveat", CONFIG["terms"]) + metadata.setdefault("privacy_caveat", "private staging; address and coordinate review pending") + return metadata return {"acquisition_method": "assisted_bvl_portal_export", "source_id": CONFIG["source_id"], "artifact": path.name, "artifact_path": str(path.resolve()), "requested_url": CONFIG["source_url"], "final_url": CONFIG["source_url"], "redirects": [], "response_headers": {}, "requested_at_utc": retrieved, "retrieved_at_utc": retrieved, "effective_date": "unknown", "publication_date": None, "sha256": hashlib.sha256(raw).hexdigest(), "byte_size": len(raw), "adapter_version": CONFIG["adapter_version"], "code_version": CONFIG["adapter_version"], "config_version": CONFIG["schema_version"], "coverage": CONFIG["coverage"], "rights_caveat": CONFIG["terms"], "privacy_caveat": "private staging; address and coordinate review pending", "terms_review": "assisted capture; recurring acquisition and redistribution remain human-gated"} -def refresh(*, run_dir: str | Path, raw_path: str | Path | None = None, fetch: bool = False, export_url: str | None = None, terms_review_path: str | Path | None = None, output_root: str | Path = "data/raw", run_id: str | None = None, retrieved_at_utc: str | None = None, timeout_seconds: float = 60, max_bytes: int = 128 * 1024 * 1024, previous_normalized: str | Path | None = None) -> dict[str, Any]: +def refresh(*, run_dir: str | Path, raw_path: str | Path | None = None, fetch: bool = False, export_url: str | None = None, terms_review_path: str | Path | None = None, output_root: str | Path = "data/raw", run_id: str | None = None, retrieved_at_utc: str | None = None, timeout_seconds: float = 60, max_bytes: int = 128 * 1024 * 1024, previous_normalized: str | Path | None = None, acquisition_metadata_path: str | Path | None = None) -> dict[str, Any]: if fetch == (raw_path is not None): raise RefreshError("specify exactly one of --raw or --fetch") retrieved = retrieved_at_utc or utc_now() @@ -55,7 +75,7 @@ def refresh(*, run_dir: str | Path, raw_path: str | Path | None = None, fetch: b input_path = Path(raw_path).resolve() # type: ignore[arg-type] if not input_path.is_file(): raise RefreshError("BLtU raw artifact does not exist") - metadata = _local_metadata(input_path, retrieved) + metadata = _local_metadata(input_path, retrieved, acquisition_metadata_path) atomic_json(Path(run_dir) / "acquisition-metadata.json", metadata) adapter = BltuAdapter() raw = input_path.read_bytes() @@ -92,9 +112,10 @@ def main() -> int: parser.add_argument("--timeout-seconds", type=float, default=60) parser.add_argument("--max-bytes", type=int, default=128 * 1024 * 1024) parser.add_argument("--previous-normalized", type=Path) + parser.add_argument("--acquisition-metadata", type=Path, help="row-free JSON sidecar for a browser-assisted export") args = parser.parse_args() try: - result = refresh(run_dir=args.run_dir, raw_path=args.raw, fetch=args.fetch, export_url=args.export_url, terms_review_path=args.terms_review, output_root=args.output_root, run_id=args.run_id, retrieved_at_utc=args.retrieved_at_utc, timeout_seconds=args.timeout_seconds, max_bytes=args.max_bytes, previous_normalized=args.previous_normalized) + result = refresh(run_dir=args.run_dir, raw_path=args.raw, fetch=args.fetch, export_url=args.export_url, terms_review_path=args.terms_review, output_root=args.output_root, run_id=args.run_id, retrieved_at_utc=args.retrieved_at_utc, timeout_seconds=args.timeout_seconds, max_bytes=args.max_bytes, previous_normalized=args.previous_normalized, acquisition_metadata_path=args.acquisition_metadata) except (OSError, RefreshError) as error: print(json.dumps({"status": "failed", "error": str(error)}, sort_keys=True)) return 2 diff --git a/pipeline/sources/germany/test_refresh.py b/pipeline/sources/germany/test_refresh.py index a247e79..670ac9f 100644 --- a/pipeline/sources/germany/test_refresh.py +++ b/pipeline/sources/germany/test_refresh.py @@ -1,3 +1,5 @@ +import hashlib +import json import tempfile import unittest from pathlib import Path @@ -9,6 +11,17 @@ class GermanyRefreshTests(unittest.TestCase): + def test_row_free_acquisition_sidecar_is_verified_and_preserved(self): + raw = ROOT.parent.parent / "germany" / "fixtures" / "synthetic_bltu.csv" + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + metadata = root / "capture.json" + metadata.write_text(json.dumps({"sha256": hashlib.sha256(raw.read_bytes()).hexdigest(), "byte_size": raw.stat().st_size, "final_url": "https://example.invalid/current-bltu.csv", "effective_date": "2026-09-14"}), encoding="utf-8") + result = refresh(run_dir=root / "run", raw_path=raw, acquisition_metadata_path=metadata, retrieved_at_utc="2026-09-14T00:00:00Z") + self.assertEqual(result["report"]["publication_eligibility"], "blocked") + saved = json.loads((root / "run" / "acquisition-metadata.json").read_text(encoding="utf-8")) + self.assertEqual(saved["final_url"], "https://example.invalid/current-bltu.csv") + def test_assisted_refresh_is_private_and_repeatable(self): raw = ROOT.parent.parent / "germany" / "fixtures" / "synthetic_bltu.csv" with tempfile.TemporaryDirectory() as directory: From 34b56b4137ae06a13667850599af50b532ef64c8 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Thu, 17 Sep 2026 14:03:19 -0700 Subject: [PATCH 231/311] fix: update country platform registry count --- pipeline/platform_registry.py | 2 +- pipeline/tests/test_platform_registry.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pipeline/platform_registry.py b/pipeline/platform_registry.py index 0cd7131..d103588 100644 --- a/pipeline/platform_registry.py +++ b/pipeline/platform_registry.py @@ -2,7 +2,7 @@ The source registry and source-status baseline remain the authoritative inputs. This module joins them into a typed, country-aware view so new lanes do not -copy 234 source rows into another file. It is intentionally offline and never +copy source rows into another file. It is intentionally offline and never fetches a source or grants release approval. """ from __future__ import annotations diff --git a/pipeline/tests/test_platform_registry.py b/pipeline/tests/test_platform_registry.py index 812246f..0643be4 100644 --- a/pipeline/tests/test_platform_registry.py +++ b/pipeline/tests/test_platform_registry.py @@ -6,7 +6,7 @@ class PlatformRegistryTests(unittest.TestCase): def test_materialized_registry_scales_from_existing_source_status_inputs(self): registry = build_platform_registry() - self.assertEqual(registry["source_count"], 234) + self.assertEqual(registry["source_count"], 240) self.assertGreaterEqual(registry["country_count"], 40) self.assertEqual(registry["publication_boundary"], "awaiting-owner-review; private staging may continue; no release approval or promotion is implied") self.assertTrue(all(country["publication"]["state"] == "blocked" for country in registry["countries"])) From 1630f4c13f7496dbc648dd7832af0c08fffdb7ac Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Thu, 17 Sep 2026 18:46:27 -0700 Subject: [PATCH 232/311] Add private accountability graph query layer --- docs/api/private-graph-contract.md | 18 ++++++ .../039_private_graph_query_indexes.sql | 13 +++++ src/graph_private.rs | 57 +++++++++++++++++++ src/lib.rs | 4 +- src/main.rs | 3 + 5 files changed, 94 insertions(+), 1 deletion(-) create mode 100644 docs/api/private-graph-contract.md create mode 100644 pipeline/migrations/039_private_graph_query_indexes.sql create mode 100644 src/graph_private.rs diff --git a/docs/api/private-graph-contract.md b/docs/api/private-graph-contract.md new file mode 100644 index 0000000..042c795 --- /dev/null +++ b/docs/api/private-graph-contract.md @@ -0,0 +1,18 @@ +# Private accountability graph query contract + +The authenticated endpoints under `/api/private/graph` are private review +tools. They require `X-UEC-Private-Graph-Token`; missing or invalid +authentication returns `404` to avoid endpoint discovery. The token is never +accepted on public routes. + +`GET /entities` supports bounded name lookup (`q`, max `limit=100`) and a +stable UUID cursor. `GET /entities/{id}/neighborhood` supports one-hop +relationship traversal only, with allowlisted type/source/date filters and a +maximum of 100 rows. Queue endpoints expose only structured review metadata: +`contradictions`, `unresolved-identities`, `quarantine`, and `statistics`. + +All reads use deterministic tie-breakers, explicit limits, and the private +tables. Raw payloads, addresses, geocoder queries, and private notes are not +returned. No endpoint infers ownership, merges identities, computes targeting +scores, or publishes a release. Database/auth failures fail closed with a +versioned error envelope. diff --git a/pipeline/migrations/039_private_graph_query_indexes.sql b/pipeline/migrations/039_private_graph_query_indexes.sql new file mode 100644 index 0000000..b855386 --- /dev/null +++ b/pipeline/migrations/039_private_graph_query_indexes.sql @@ -0,0 +1,13 @@ +-- Private graph reads are bounded and deterministic. These indexes support +-- entity lookup, dated relationship filters, and review/quarantine queues. +CREATE INDEX IF NOT EXISTS organizations_name_lookup_idx + ON uec.organizations (lower(canonical_name), organization_id); +CREATE INDEX IF NOT EXISTS facilities_name_lookup_idx + ON uec.facilities (lower(canonical_name), facility_id); +CREATE INDEX IF NOT EXISTS graph_relationship_private_query_idx + ON uec.organization_relationship_observations + (from_organization_id, observed_at DESC, relationship_observation_id DESC); +CREATE INDEX IF NOT EXISTS graph_crosswalk_queue_idx + ON uec.source_entity_crosswalks (assertion_status, observed_at DESC, crosswalk_id DESC); +CREATE INDEX IF NOT EXISTS graph_claim_review_queue_idx + ON uec.claims (review_state, observed_at DESC, claim_id DESC); diff --git a/src/graph_private.rs b/src/graph_private.rs new file mode 100644 index 0000000..bf9214e --- /dev/null +++ b/src/graph_private.rs @@ -0,0 +1,57 @@ +use axum::{extract::{Path, Query, State}, http::{HeaderMap, StatusCode}, response::IntoResponse, Json}; +use serde::Deserialize; +use serde_json::json; +use uuid::Uuid; +use crate::{v2_error, ApiState}; + +const TOKEN_HEADER: &str = "x-uec-private-graph-token"; +const MAX_LIMIT: i64 = 100; + +#[derive(Debug, Deserialize)] +pub struct GraphQuery { + pub q: Option, pub relationship_type: Option, pub source_id: Option, + pub state: Option, pub min_confidence: Option, pub from: Option, + pub to: Option, pub limit: Option, pub cursor: Option, +} + +fn authorized(headers: &HeaderMap) -> bool { + let Some(expected) = std::env::var("UEC_PRIVATE_GRAPH_TOKEN").ok().filter(|v| !v.is_empty()) else { return false }; + headers.get(TOKEN_HEADER).and_then(|v| v.to_str().ok()).is_some_and(|v| crate::constant_time_token_matches(&expected, v)) +} +fn limit(v: Option) -> Result { let n = v.unwrap_or(50); if (1..=MAX_LIMIT).contains(&n) { Ok(n) } else { Err("limit must be between 1 and 100") } } +fn denied() -> axum::response::Response { v2_error(StatusCode::NOT_FOUND, "private_graph_unavailable", "private graph unavailable") } +fn bad(msg: &'static str) -> axum::response::Response { v2_error(StatusCode::BAD_REQUEST, "invalid_graph_query", msg) } + +pub async fn entities(State(state): State, headers: HeaderMap, Query(p): Query) -> impl IntoResponse { + if !authorized(&headers) { return denied(); } + let Ok(limit) = limit(p.limit) else { return bad("limit must be between 1 and 100") }; + let Some(pool) = state.database else { return v2_error(StatusCode::SERVICE_UNAVAILABLE, "database_not_configured", "private graph database unavailable") }; + let Ok(client) = pool.get().await else { return v2_error(StatusCode::SERVICE_UNAVAILABLE, "database_pool_unavailable", "private graph database unavailable") }; + let q = p.q.unwrap_or_default(); + let rows = match client.query("SELECT entity_id, entity_type, canonical_name, country_code, created_at FROM (SELECT facility_id AS entity_id, 'facility' AS entity_type, canonical_name, country_code, created_at FROM uec.facilities UNION ALL SELECT organization_id, 'organization', canonical_name, country_code, created_at FROM uec.organizations) e WHERE ($1 = '' OR canonical_name ILIKE '%' || $1 || '%') AND ($2::uuid IS NULL OR entity_id > $2) ORDER BY entity_id LIMIT $3", &[&q, &p.cursor, &limit]).await { Ok(v) => v, Err(_) => return v2_error(StatusCode::SERVICE_UNAVAILABLE, "private_graph_query_failed", "private graph query failed") }; + let data: Vec<_> = rows.into_iter().map(|r| json!({"entity_id":r.get::<_,Uuid>(0),"entity_type":r.get::<_,String>(1),"canonical_name":r.get::<_,Option>(2),"country_code":r.get::<_,Option>(3),"created_at":r.get::<_,chrono::DateTime>(4)})).collect(); + Json(json!({"api_version":"private-graph-v1","data":data,"meta":{"private":true,"limit":limit,"next_cursor":data.last().and_then(|v|v["entity_id"].as_str())}})).into_response() +} + +pub async fn neighborhood(State(state): State, headers: HeaderMap, Path(entity_id): Path, Query(p): Query) -> impl IntoResponse { + if !authorized(&headers) { return denied(); } + let Ok(limit) = limit(p.limit) else { return bad("limit must be between 1 and 100") }; + if p.relationship_type.as_deref().is_some_and(|v| !["operator","owner","parent","brand","supplier","customer","regulatory_authority_for"].contains(&v)) { return bad("unsupported relationship type"); } + let Some(pool) = state.database else { return v2_error(StatusCode::SERVICE_UNAVAILABLE, "database_not_configured", "private graph database unavailable") }; + let Ok(client) = pool.get().await else { return v2_error(StatusCode::SERVICE_UNAVAILABLE, "database_pool_unavailable", "private graph database unavailable") }; + let rows = match client.query("SELECT relationship_observation_id, from_organization_id, target_facility_id, target_organization_id, relationship_type, assertion_status, observed_at, confidence, review_state, storage_state, privacy_status, publication_status, source_id, source_record_id FROM uec.organization_relationship_observations WHERE ($1 = from_organization_id OR $1 = target_facility_id OR $1 = target_organization_id) AND ($2::text IS NULL OR relationship_type=$2) AND ($3::text IS NULL OR source_id=$3) AND ($4::timestamptz IS NULL OR observed_at >= $4) AND ($5::timestamptz IS NULL OR observed_at < $5) ORDER BY observed_at DESC, relationship_observation_id DESC LIMIT $6", &[&entity_id, &p.relationship_type, &p.source_id, &p.from.map(|d| d.and_hms_opt(0,0,0).unwrap().and_utc()), &p.to.map(|d| d.and_hms_opt(0,0,0).unwrap().and_utc()), &limit]).await { Ok(v) => v, Err(_) => return v2_error(StatusCode::SERVICE_UNAVAILABLE, "private_graph_query_failed", "private graph query failed") }; + let data: Vec<_> = rows.into_iter().map(|r| json!({"relationship_observation_id":r.get::<_,Uuid>(0),"from_organization_id":r.get::<_,Option>(1),"target_facility_id":r.get::<_,Option>(2),"target_organization_id":r.get::<_,Option>(3),"relationship_type":r.get::<_,Option>(4),"assertion_status":r.get::<_,String>(5),"observed_at":r.get::<_,chrono::DateTime>(6),"confidence":r.get::<_,Option>(7),"review_state":r.get::<_,String>(8),"storage_state":r.get::<_,String>(9),"privacy_status":r.get::<_,String>(10),"publication_status":r.get::<_,String>(11),"source_id":r.get::<_,String>(12),"source_record_id":r.get::<_,Uuid>(13)})).collect(); + Json(json!({"api_version":"private-graph-v1","data":data,"meta":{"private":true,"entity_id":entity_id,"limit":limit}})).into_response() +} + +pub async fn queue(State(state): State, headers: HeaderMap, Path(kind): Path, Query(p): Query) -> impl IntoResponse { + if !authorized(&headers) { return denied(); } + let Ok(limit) = limit(p.limit) else { return bad("limit must be between 1 and 100") }; + if !["contradictions","unresolved-identities","quarantine","statistics"].contains(&kind.as_str()) { return v2_error(StatusCode::NOT_FOUND, "private_graph_unavailable", "private graph unavailable"); } + let Some(pool) = state.database else { return v2_error(StatusCode::SERVICE_UNAVAILABLE, "database_not_configured", "private graph database unavailable") }; + let Ok(client) = pool.get().await else { return v2_error(StatusCode::SERVICE_UNAVAILABLE, "database_pool_unavailable", "private graph database unavailable") }; + let sql = match kind.as_str() { "contradictions" => "SELECT claim_id, facility_id, organization_id, claim_domain, claim_kind, observed_at, confidence FROM uec.claim_current c WHERE c.review_state IN ('disputed','review_required') ORDER BY observed_at DESC, claim_id DESC LIMIT $1", "unresolved-identities" => "SELECT crosswalk_id, left_identifier_id, right_identifier_id, assertion_status, confidence, observed_at FROM uec.source_entity_crosswalks WHERE assertion_status IN ('candidate','review_required','disputed') ORDER BY observed_at DESC, crosswalk_id DESC LIMIT $1", "quarantine" => "SELECT source_record_id, source_id, source_state, received_at FROM uec.source_records WHERE source_state IN ('quarantined','rejected') ORDER BY received_at DESC, source_record_id DESC LIMIT $1", _ => "SELECT 'claims' AS metric, count(*)::bigint AS value FROM uec.claims UNION ALL SELECT 'relationships', count(*) FROM uec.organization_relationship_observations UNION ALL SELECT 'quarantined_records', count(*) FROM uec.source_records WHERE source_state IN ('quarantined','rejected')" }; + let rows = match client.query(sql, &[&limit]).await { Ok(v) => v, Err(_) => return v2_error(StatusCode::SERVICE_UNAVAILABLE, "private_graph_query_failed", "private graph query failed") }; + let data: Vec<_> = rows.into_iter().map(|r| (0..r.len()).map(|i| r.try_get::<_, String>(i).or_else(|_| r.try_get::<_, i64>(i).map(|v| v.to_string())).unwrap_or_default()).collect::>()).collect(); + Json(json!({"api_version":"private-graph-v1","data":data,"meta":{"private":true,"queue":kind,"limit":limit}})).into_response() +} diff --git a/src/lib.rs b/src/lib.rs index f8e50ac..d9c944c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -32,6 +32,8 @@ use std::error::Error; use std::time::{Duration, Instant}; use tokio::sync::Mutex; +pub mod graph_private; + pub fn v2_error( status: StatusCode, code: &'static str, @@ -705,7 +707,7 @@ pub async fn get_dev_test_release_export_handler( .into_response() } -fn constant_time_token_matches(expected: &str, provided: &str) -> bool { +pub(crate) fn constant_time_token_matches(expected: &str, provided: &str) -> bool { let mut difference = expected.len() ^ provided.len(); for (left, right) in expected.bytes().zip(provided.bytes()) { difference |= usize::from(left ^ right); diff --git a/src/main.rs b/src/main.rs index 7241340..36e622c 100644 --- a/src/main.rs +++ b/src/main.rs @@ -68,6 +68,9 @@ pub fn app(state: uec_api::ApiState, proxy: private_environment::ProxyConfig) -> "/api/dev/preview/candidates", get(uec_api::get_dev_candidate_preview_handler), ) + .route("/api/private/graph/entities", get(uec_api::graph_private::entities)) + .route("/api/private/graph/entities/{entity_id}/neighborhood", get(uec_api::graph_private::neighborhood)) + .route("/api/private/graph/queues/{kind}", get(uec_api::graph_private::queue)) .route( "/api/dev/preview/test-release/locations", get(uec_api::get_dev_test_release_locations_handler), From 2bd4b1b8462faeac994b58aa47a7ddb15b53afdf Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Thu, 17 Sep 2026 21:34:16 -0700 Subject: [PATCH 233/311] quarantine duplicate accountability relationships --- pipeline/sources/us/accountability/adapter.py | 9 +++++++++ pipeline/sources/us/accountability/test_adapter.py | 9 +++++++++ 2 files changed, 18 insertions(+) diff --git a/pipeline/sources/us/accountability/adapter.py b/pipeline/sources/us/accountability/adapter.py index 57214a4..fbb5c84 100644 --- a/pipeline/sources/us/accountability/adapter.py +++ b/pipeline/sources/us/accountability/adapter.py @@ -186,6 +186,7 @@ def parse_bytes(self, content: bytes) -> dict[str, Any]: quarantined: list[dict[str, Any]] = [] entities: dict[tuple[str, str, str], dict[str, Any]] = {} identifiers: dict[tuple[str, str], tuple[str, str]] = {} + relationship_keys: set[tuple[str, str, str, str, str]] = set() relationship_rows: list[tuple[dict[str, Any], dict[str, str], int]] = [] for line, row in enumerate(rows, 2): @@ -235,9 +236,17 @@ def parse_bytes(self, content: bytes) -> dict[str, Any]: same_name = [key for key, value in identifiers.items() if value[1] == row.get("object_name") and key[0] == row.get("object_source_id")] if len(same_name) > 1 and (row["object_source_id"], row["object_source_native_id"]) not in same_name: reasons.append("ambiguous_duplicate_name") + relationship_key = ( + row["subject_type"], row["subject_source_id"], + row["subject_source_native_id"], row["object_source_native_id"], + row["relationship_type"], + ) + if relationship_key in relationship_keys: + reasons.append("duplicate_relationship_observation") if not reasons: for identity_key, identity_value in row_identities: identifiers[identity_key] = identity_value + relationship_keys.add(relationship_key) except AccountabilityContractError as exc: reasons.append(str(exc).replace(" ", "_")) observed = None diff --git a/pipeline/sources/us/accountability/test_adapter.py b/pipeline/sources/us/accountability/test_adapter.py index e56aef6..ea16cbf 100644 --- a/pipeline/sources/us/accountability/test_adapter.py +++ b/pipeline/sources/us/accountability/test_adapter.py @@ -122,6 +122,15 @@ def test_schema_drift_fails_closed(self): with self.assertRaises(AccountabilityContractError): UsAccountabilityAdapter().parse_bytes(b"subject_type,object_type\nfacility,operator\n") + def test_exact_duplicate_relationship_is_quarantined(self): + rows = rows_from_fixture() + rows.append(dict(rows[0])) + result = UsAccountabilityAdapter().parse_bytes(content_for(rows)) + self.assertEqual(result["input_rows"], 13) + self.assertEqual(len(result["accepted"]), 12) + self.assertEqual(len(result["quarantined"]), 1) + self.assertEqual(result["quarantined"][0]["reasons"], ("duplicate_relationship_observation",)) + if __name__ == "__main__": unittest.main() From d06a50ba3da4e45e34b5ede23e490be84acde943 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Thu, 17 Sep 2026 21:31:24 -0700 Subject: [PATCH 234/311] preserve declared totals for unavailable handoffs --- .../scripts/maintenance/rehearse_current_reacquisition.py | 8 +++++++- pipeline/tests/test_current_reacquisition_rehearsal.py | 1 + 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/pipeline/scripts/maintenance/rehearse_current_reacquisition.py b/pipeline/scripts/maintenance/rehearse_current_reacquisition.py index f21a83a..d1a935c 100644 --- a/pipeline/scripts/maintenance/rehearse_current_reacquisition.py +++ b/pipeline/scripts/maintenance/rehearse_current_reacquisition.py @@ -145,6 +145,12 @@ def build_report(manifest_path: Path, root: Path, output: Path) -> dict[str, Any results.append(_missing_result(profile, [str(exc)], [], status="validation_error")) complete = [item for item in results if item["status"] == "validated-private-candidate"] totals = {key: sum(int(item[key]) for item in complete) for key in ("input", "normalized", "quarantined")} + # A missing handoff is unavailable evidence, not a zero-row source. Keep + # manifest-declared scope separate from validated totals for auditability. + declared_totals = { + key: sum(int(profile[key]) for profile in selected if profile.get(key) is not None) + for key in ("input_rows", "normalized_rows", "quarantined_rows") + } unavailable = [item["source_id"] for item in results if item["status"] in {"unavailable_private_handoff", "unavailable_raw_only"}] raw_only = [item["source_id"] for item in results if item["status"] == "validated-raw-only"] failed = [item["source_id"] for item in results if item["status"] not in {"validated-private-candidate", "validated-raw-only", "unavailable_private_handoff", "unavailable_raw_only"}] @@ -154,7 +160,7 @@ def build_report(manifest_path: Path, root: Path, output: Path) -> dict[str, Any "release_id": source_manifest["publication"]["candidate_release"], "publication": {"release_created": False, "release_promoted": False, "public_api_rows": 0, "candidate_only": True}, - "sources": results, "totals": totals, + "sources": results, "totals": totals, "declared_totals": declared_totals, "availability": {"expected_profiles": len(EXPECTED), "validated_private_profiles": len(complete), "validated_raw_only_profiles": len(raw_only), "unavailable_profiles": unavailable, "failed_profiles": failed}, diff --git a/pipeline/tests/test_current_reacquisition_rehearsal.py b/pipeline/tests/test_current_reacquisition_rehearsal.py index bb2d0c6..08522d0 100644 --- a/pipeline/tests/test_current_reacquisition_rehearsal.py +++ b/pipeline/tests/test_current_reacquisition_rehearsal.py @@ -63,6 +63,7 @@ def test_missing_private_inputs_are_enumerated_without_becoming_zero(self): report = build_report(manifest, root, root / "out.json") self.assertFalse(report["reconciliation"]["passed"]) self.assertEqual(report["totals"], {"input": 0, "normalized": 0, "quarantined": 0}) + self.assertEqual(report["declared_totals"], {"input_rows": 21, "normalized_rows": 14, "quarantined_rows": 7}) self.assertEqual(len(report["availability"]["unavailable_profiles"]), len(EXPECTED)) self.assertEqual(report["availability"]["failed_profiles"], []) self.assertTrue(all(item["missing_artifacts"] for item in report["sources"])) From 5c31297d1c319ec0f4b9514647ac875bd931bc53 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Thu, 17 Sep 2026 18:42:40 -0700 Subject: [PATCH 235/311] Improve private graph developer workflows --- pipeline/ONBOARDING.md | 18 ++++++++++++++++++ scripts/dev.py | 16 ++++++++++++++-- scripts/test_dev.py | 7 +++++++ 3 files changed, 39 insertions(+), 2 deletions(-) diff --git a/pipeline/ONBOARDING.md b/pipeline/ONBOARDING.md index 639e8dd..04832b8 100644 --- a/pipeline/ONBOARDING.md +++ b/pipeline/ONBOARDING.md @@ -28,6 +28,24 @@ Run the complete Python suite before submitting pipeline changes: python -m unittest discover -s pipeline -p 'test*.py' ``` +The one-command private demo runs the graph-candidate and review-packet +contracts without acquiring, importing, or publishing anything: + +```powershell +python scripts/dev.py demo +python scripts/dev.py preflight +python scripts/dev.py diagnostics data/reports/real-corpus-report.json +python scripts/dev.py review-export data/staging// +``` + +`review-export` is a row-free operator packet. It is not approval. Source +adapters remain the canonical acquire -> parse/normalize -> quarantine -> +candidate handoff path; candidate database import is restricted to the +disposable loopback database. Rebuild candidates by rerunning the source-owned +adapter with a new private run directory; never edit or promote a candidate in +`static_data`. Missing evidence, source drift, ambiguous identities, and +quarantined rows remain blockers and are reported in the packet/health files. + ## 3. Inspect private operational evidence For an existing private run, build a row-free report from its manifest root: diff --git a/scripts/dev.py b/scripts/dev.py index 248cde5..0e08263 100644 --- a/scripts/dev.py +++ b/scripts/dev.py @@ -78,13 +78,19 @@ def main() -> int: sub.add_parser("test", help="root legacy static/Jest tests").add_argument("--full", action="store_true") sub.add_parser("pipeline", help="run Python pipeline tests").add_argument("args", nargs=argparse.REMAINDER) sub.add_parser("contracts", help="run contract tests") + sub.add_parser("demo", help="run the safe synthetic/private tooling demo") + sub.add_parser("preflight", help="alias for doctor: verify local prerequisites before a run") sub.add_parser("platform-registry", help="validate the joined country/source registry") pf = sub.add_parser("private-frontend", help="rehearse a candidate against the private frontend preview boundary") pf.add_argument("manifest"); pf.add_argument("output"); pf.add_argument("--root", default=str(ROOT)); pf.add_argument("--base-url"); pf.add_argument("--token") rp = sub.add_parser("review-packet", help="generate a private row-free review packet") rp.add_argument("run_dir"); rp.add_argument("--previous-normalized") + re = sub.add_parser("review-export", help="export the private row-free review packet") + re.add_argument("run_dir"); re.add_argument("--previous-normalized") + diag = sub.add_parser("diagnostics", help="build a row-free manifest/source diagnostic") + diag.add_argument("output"); diag.add_argument("--manifest-root", default="data/manifests") args = p.parse_args(); SUMMARY["command"] = args.command - if args.command == "doctor": code = doctor(args) + if args.command in ("doctor", "preflight"): code = doctor(args) elif args.command in ("up", "down", "status", "probe"): code = run(["powershell", "-ExecutionPolicy", "Bypass", "-File", str(LOCAL_V2), {"up":"start","down":"stop"}.get(args.command,args.command)], capture=args.json) elif args.command == "logs": code = run(["docker", "compose", "-p", "uec-local-v2", "-f", "docker-compose.pipeline.yml", "logs", "--tail=100"], capture=args.json) elif args.command == "test": code = run(["npm", "test", "--", *( ["--runInBand"] if not args.full else [])], capture=args.json) @@ -95,6 +101,10 @@ def main() -> int: runner = "pytest" if shutil.which("pytest") else "unittest" targets = ["pipeline/contracts", "pipeline/tests/test_database_contract.py", "pipeline/tests/test_graph_database_contract.py"] if runner == "pytest" else ["discover", "-s", "pipeline/contracts", "-t", str(ROOT)] code = run([sys.executable, "-m", runner, *targets], capture=args.json) + elif args.command == "demo": + # The demo is intentionally synthetic and read-only: it exercises the + # contract/review boundary without acquiring, importing, or publishing. + code = run([sys.executable, "-m", "unittest", "pipeline.common.test_graph_candidates", "pipeline.common.test_review_packet"], capture=args.json) elif args.command == "platform-registry": code = run([sys.executable, "-c", "import json; from pipeline.platform_registry import build_platform_registry; r=build_platform_registry(); print(json.dumps({'countries':r['country_count'],'sources':r['source_count'],'status':'validated'}))"], capture=args.json) elif args.command == "private-frontend": @@ -102,7 +112,9 @@ def main() -> int: if args.base_url: cmd.extend(["--base-url", args.base_url]) if args.token: cmd.extend(["--token", args.token]) code = run(cmd, capture=args.json) - else: + elif args.command == "diagnostics": + code = run([sys.executable, str(ROOT / "pipeline/scripts/diagnostics/real_corpus_report.py"), "--manifest-root", args.manifest_root, "--output", args.output], capture=args.json) + elif args.command in ("review-packet", "review-export"): cmd = [sys.executable, "-c", "from pipeline.common.review_packet import write_review_packet; import sys; write_review_packet(sys.argv[1], previous_normalized_path=sys.argv[2] if len(sys.argv)>2 else None)", args.run_dir] if args.previous_normalized: cmd.append(args.previous_normalized) code = run(cmd, capture=args.json) diff --git a/scripts/test_dev.py b/scripts/test_dev.py index 3b2b3fa..4fdd7e9 100644 --- a/scripts/test_dev.py +++ b/scripts/test_dev.py @@ -10,6 +10,13 @@ def test_help(self): self.assertIn("doctor", result.stdout) self.assertIn("review-packet", result.stdout) self.assertIn("private-frontend", result.stdout) + self.assertIn("demo", result.stdout) + self.assertIn("diagnostics", result.stdout) + + def test_preflight_is_doctor_alias(self): + result = subprocess.run([sys.executable, "scripts/dev.py", "--json", "preflight"], cwd=ROOT, capture_output=True, text=True) + self.assertEqual(result.returncode, 0) + self.assertEqual(json.loads(result.stdout)["command"], "preflight") def test_doctor_json_does_not_echo_secret(self): result = subprocess.run([sys.executable, "scripts/dev.py", "--json", "doctor"], cwd=ROOT, env={**os.environ, "UEC_DATABASE_URL": "postgresql://secret.invalid/db"}, capture_output=True, text=True) From 68ad23cd260332ee8dbf446798012d95fb3ab9a9 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Thu, 17 Sep 2026 18:37:53 -0700 Subject: [PATCH 236/311] Add private accountability graph explorer --- docs/private-accountability-graph.md | 18 +++++++++++++ src/lib.rs | 27 +++++++++++++++++++ src/main.rs | 2 ++ static/modules/__tests__/privateGraph.test.js | 11 ++++++++ static/private-graph.css | 1 + static/private-graph.html | 1 + static/private-graph.js | 8 ++++++ 7 files changed, 68 insertions(+) create mode 100644 docs/private-accountability-graph.md create mode 100644 static/modules/__tests__/privateGraph.test.js create mode 100644 static/private-graph.css create mode 100644 static/private-graph.html create mode 100644 static/private-graph.js diff --git a/docs/private-accountability-graph.md b/docs/private-accountability-graph.md new file mode 100644 index 0000000..ae136dd --- /dev/null +++ b/docs/private-accountability-graph.md @@ -0,0 +1,18 @@ +# Private accountability-graph explorer + +`/private-graph.html` is a restricted research surface for source-qualified +graph evidence. It is not linked from the public map and does not call the +`/api/v2` release routes. The private API is bounded to 100 search results and +200 observations per traversal, supports direction and depth controls, and +returns evidence-state fields separately (review, privacy, and publication). + +The explorer preserves contradictory dated observations instead of resolving +them into an ownership or operational-status claim. It shows source IDs and +observation dates, confidence, unknown reasons, and notes when present. It +does not return addresses, coordinates, raw source payloads, risk scores, +targeting recommendations, or a claim that an entity owns or operates a site. + +The database migrations remain the authority for append-only behavior and +source-scoped identity. Deployment must place this surface behind the +project's private environment access control; the route itself is not an +authentication mechanism. diff --git a/src/lib.rs b/src/lib.rs index d9c944c..d813f0a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -46,6 +46,33 @@ pub fn v2_error( .into_response() } +#[derive(Debug, Deserialize)] +pub struct PrivateGraphSearchParams { pub q: Option, pub limit: Option } + +/// Private evidence-only graph search. This is intentionally not a public projection. +pub async fn get_private_graph_search_handler(State(state): State, Query(params): Query) -> impl IntoResponse { + let Some(pool) = state.database else { return v2_error(StatusCode::SERVICE_UNAVAILABLE, "private_graph_unavailable", "private graph database unavailable"); }; + let q = params.q.unwrap_or_default().trim().to_string(); + let limit = params.limit.unwrap_or(25).clamp(1, 100); + let client = match pool.get().await { Ok(c) => c, Err(_) => return v2_error(StatusCode::SERVICE_UNAVAILABLE, "private_graph_unavailable", "private graph database unavailable") }; + let rows = match client.query("SELECT entity_type, entity_id, display_name, source_id, source_identifier, observed_at FROM (SELECT 'facility' AS entity_type, f.facility_id AS entity_id, COALESCE(f.canonical_name, '[unnamed facility]') AS display_name, sei.source_id, sei.source_identifier, sei.observed_at FROM uec.facilities f LEFT JOIN uec.source_entity_identifiers sei ON sei.facility_id=f.facility_id UNION ALL SELECT 'organization', o.organization_id, COALESCE(o.canonical_name, '[unnamed organization]'), sei.source_id, sei.source_identifier, sei.observed_at FROM uec.organizations o LEFT JOIN uec.source_entity_identifiers sei ON sei.organization_id=o.organization_id) entities WHERE ($1='' OR display_name ILIKE '%' || $1 || '%' OR source_identifier ILIKE '%' || $1 || '%') ORDER BY display_name, observed_at DESC NULLS LAST LIMIT $2", &[&q, &limit]).await { Ok(r) => r, Err(_) => return v2_error(StatusCode::SERVICE_UNAVAILABLE, "private_graph_query_failed", "private graph search failed") }; + let data: Vec = rows.into_iter().map(|r| json!({"entity_type":r.get::<_,String>(0),"entity_id":r.get::<_,uuid::Uuid>(1),"display_name":r.get::<_,String>(2),"source_id":r.get::<_,Option>(3),"source_identifier":r.get::<_,Option>(4),"observed_at":r.get::<_,Option>>(5),"review_state":"unknown","privacy_status":"unknown","publication_status":"unknown"})).collect(); + Json(json!({"api_version":"private-graph-v1","data":data,"meta":{"scope":"private_evidence_only","bounded":true,"public_projection":false}})).into_response() +} + +#[derive(Debug, Deserialize)] +pub struct PrivateGraphTraverseParams { pub entity_type: String, pub entity_id: uuid::Uuid, pub direction: Option, pub depth: Option } + +pub async fn get_private_graph_traverse_handler(State(state): State, Query(params): Query) -> impl IntoResponse { + let Some(pool) = state.database else { return v2_error(StatusCode::SERVICE_UNAVAILABLE, "private_graph_unavailable", "private graph database unavailable"); }; + let depth = params.depth.unwrap_or(1).clamp(1, 2); let direction = params.direction.as_deref().unwrap_or("both"); + if !matches!(params.entity_type.as_str(), "organization" | "facility") || !matches!(direction, "in" | "out" | "both") { return v2_error(StatusCode::BAD_REQUEST, "invalid_traversal", "entity type, direction, or depth is invalid"); } + let client = match pool.get().await { Ok(c) => c, Err(_) => return v2_error(StatusCode::SERVICE_UNAVAILABLE, "private_graph_unavailable", "private graph database unavailable") }; + let rows = match client.query("SELECT relationship_observation_id, source_id, source_record_id, from_organization_id, target_facility_id, target_organization_id, relationship_type, assertion_status, unknown_reason, valid_from, valid_to, observed_at, confidence, review_state, storage_state, privacy_status, publication_status, note FROM uec.organization_relationship_observations WHERE (($1='organization' AND ((($3 IN ('out','both')) AND from_organization_id=$2) OR (($3 IN ('in','both')) AND target_organization_id=$2))) OR ($1='facility' AND target_facility_id=$2) ORDER BY observed_at DESC LIMIT 200", &[¶ms.entity_type, ¶ms.entity_id, &direction]).await { Ok(r) => r, Err(_) => return v2_error(StatusCode::SERVICE_UNAVAILABLE, "private_graph_query_failed", "private graph traversal failed") }; + let data: Vec = rows.into_iter().map(|r| json!({"relationship_observation_id":r.get::<_,uuid::Uuid>(0),"source_id":r.get::<_,String>(1),"source_record_id":r.get::<_,uuid::Uuid>(2),"from_organization_id":r.get::<_,Option>(3),"target_facility_id":r.get::<_,Option>(4),"target_organization_id":r.get::<_,Option>(5),"relationship_type":r.get::<_,Option>(6),"assertion_status":r.get::<_,String>(7),"unknown_reason":r.get::<_,Option>(8),"valid_from":r.get::<_,Option>(9),"valid_to":r.get::<_,Option>(10),"observed_at":r.get::<_,chrono::DateTime>(11),"confidence":r.get::<_,Option>(12),"review_state":r.get::<_,String>(13),"storage_state":r.get::<_,String>(14),"privacy_status":r.get::<_,String>(15),"publication_status":r.get::<_,String>(16),"note":r.get::<_,Option>(17)})).collect(); + Json(json!({"api_version":"private-graph-v1","data":data,"meta":{"scope":"private_evidence_only","bounded":true,"depth":depth,"direction":direction,"contradictions_preserved":true,"public_projection":false}})).into_response() +} + fn canonical_json(value: &Value) -> String { match value { Value::Object(map) => { diff --git a/src/main.rs b/src/main.rs index 36e622c..f72e86b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -96,6 +96,8 @@ pub fn app(state: uec_api::ApiState, proxy: private_environment::ProxyConfig) -> get(uec_api::get_inspection_reports_handler), ) .route("/api/aphis-query", get(uec_api::get_aphis_query_handler)) + .route("/api/private/graph/search", get(uec_api::get_private_graph_search_handler)) + .route("/api/private/graph/traverse", get(uec_api::get_private_graph_traverse_handler)) .fallback_service(ServeDir::new("static")) .layer(CompressionLayer::new().br(true)) .layer(axum::middleware::from_fn_with_state( diff --git a/static/modules/__tests__/privateGraph.test.js b/static/modules/__tests__/privateGraph.test.js new file mode 100644 index 0000000..7bce7aa --- /dev/null +++ b/static/modules/__tests__/privateGraph.test.js @@ -0,0 +1,11 @@ +import { jest } from '@jest/globals'; + +test('private graph surface is explicitly bounded and not a public projection', async () => { + const html = await (await import('fs/promises')).readFile(new URL('../../../../static/private-graph.html', import.meta.url), 'utf8'); + expect(html).toContain('Private evidence only'); + expect(html).toContain('does not establish ownership'); + const js = await (await import('fs/promises')).readFile(new URL('../../private-graph.js', import.meta.url), 'utf8'); + expect(js).toContain('/api/private/graph/search'); + expect(js).toContain('/api/private/graph/traverse'); + expect(js).toContain('replace'); +}); diff --git a/static/private-graph.css b/static/private-graph.css new file mode 100644 index 0000000..eb5880e --- /dev/null +++ b/static/private-graph.css @@ -0,0 +1 @@ +:root{font:16px/1.45 system-ui,sans-serif;color:#17251e;background:#eef2ed}*{box-sizing:border-box}body{margin:0}main{max-width:1200px;margin:auto;padding:2rem}header,.search,.workspace{background:#fff;border:1px solid #cbd7ce;border-radius:12px;padding:1.25rem;margin-bottom:1rem}.eyebrow{font-size:.75rem;letter-spacing:.12em;color:#567263}h1,h2{margin:.2rem 0 .7rem}.notice{border-left:4px solid #ba6b25;padding:.7rem 1rem;background:#fff6e9}.search div{display:flex;gap:.5rem}.search input{flex:1;padding:.7rem;border:1px solid #9daf9f;border-radius:6px}.search button{padding:.7rem 1rem;background:#1e5940;color:#fff;border:0;border-radius:6px}.workspace{display:grid;grid-template-columns:290px 1fr;gap:1rem;min-height:480px}aside{border-right:1px solid #d5ded8;padding-right:1rem}ul{list-style:none;padding:0;margin:0}li{padding:.65rem;border-bottom:1px solid #e4ebe5;cursor:pointer}li:hover,li.active{background:#e8f1eb}.toolbar{display:flex;align-items:center;gap:1rem;flex-wrap:wrap}.toolbar h2{margin-right:auto}.toolbar select{margin-left:.3rem;padding:.35rem}.empty{padding:2rem;color:#607267}.edge{border:1px solid #d5ded8;border-radius:8px;padding:1rem;margin:.7rem 0}.edge header{border:0;padding:0;margin:0;background:none}.chips{display:flex;gap:.4rem;flex-wrap:wrap}.chip{font-size:.78rem;padding:.2rem .45rem;border-radius:99px;background:#e8f1eb}.warn{background:#fff0de;color:#874912}@media(max-width:700px){main{padding:.7rem}.workspace{grid-template-columns:1fr}aside{border-right:0;border-bottom:1px solid #d5ded8;padding:0 0 1rem}} diff --git a/static/private-graph.html b/static/private-graph.html new file mode 100644 index 0000000..b0aee7f --- /dev/null +++ b/static/private-graph.html @@ -0,0 +1 @@ +Private accountability graph

RESTRICTED RESEARCH INTERFACE

Accountability graph explorer

Private evidence only. This view is not a public release, does not establish ownership or operational status, and must not be used to target people or locations.

Search requires a configured private graph database.

Select an entity

Relationships, history, contradictions, and quarantine notes appear here.
diff --git a/static/private-graph.js b/static/private-graph.js new file mode 100644 index 0000000..9762432 --- /dev/null +++ b/static/private-graph.js @@ -0,0 +1,8 @@ +const $=s=>document.querySelector(s); const esc=v=>String(v??'unknown').replace(/[&<>"']/g,c=>({'&':'&','<':'<','>':'>','"':'"',"'":'''}[c])); +let entities=[]; let selected=null; +async function api(path,params){const u=new URL(path,location.href);Object.entries(params).forEach(([k,v])=>u.searchParams.set(k,v));const r=await fetch(u,{headers:{Accept:'application/json'}});const b=await r.json();if(!r.ok)throw Error(b.error?.message||`HTTP ${r.status}`);return b} +function renderEntities(){const list=$('#entities');list.innerHTML=entities.map((e,i)=>`
  • ${esc(e.display_name)}
    ${esc(e.entity_type)} · ${esc(e.source_id)} · ${esc(e.source_identifier)}
    • `).join('')||'
  • No entities found.
    • ';list.querySelectorAll('li[data-index]').forEach(x=>x.onclick=()=>select(Number(x.dataset.index)))} +async function select(i){selected=i;renderEntities();const e=entities[i];$('#selected').textContent=e.display_name;$('#graph').innerHTML='

    Loading bounded evidence…

    ';try{const b=await api('/api/private/graph/traverse',{entity_type:e.entity_type,entity_id:e.entity_id,direction:$('#direction').value,depth:$('#depth').value});renderEdges(b.data)}catch(err){$('#graph').innerHTML=`

    ${esc(err.message)}

    `}} +function renderEdges(edges){if(!edges.length){$('#graph').innerHTML='

    No retained relationship observations for this entity.

    ';return}$('#graph').innerHTML=edges.map(e=>`
    ${esc(e.relationship_type||'unknown relationship')} · ${esc(e.assertion_status)}

    ${esc(e.from_organization_id||'unknown')} → ${esc(e.target_facility_id||e.target_organization_id||'unknown')}

    observed ${esc(e.observed_at)}confidence ${esc(e.confidence)}review ${esc(e.review_state)}privacy ${esc(e.privacy_status)}publication ${esc(e.publication_status)}
    ${e.unknown_reason?`

    Unknown reason: ${esc(e.unknown_reason)}

    `:''}${e.note?`

    ${esc(e.note)}

    `:''}Source ID: ${esc(e.source_id)} · Source record: ${esc(e.source_record_id)}
    `).join('')} +async function search(){try{$('#status').textContent='Searching private evidence…';const b=await api('/api/private/graph/search',{q:$('#query').value,limit:50});entities=b.data;selected=null;renderEntities();$('#status').textContent=`${entities.length} private entities returned. Contradictions remain separate observations.`}catch(e){$('#status').textContent=e.message;$('#entities').innerHTML=''}} +$('#search').onclick=search;$('#query').onkeydown=e=>{if(e.key==='Enter')search()};$('#direction').onchange=()=>selected!==null&&select(selected);$('#depth').onchange=()=>selected!==null&&select(selected); From 47b98bbae28ead7123e622f0b1f773b31afd4a2e Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Thu, 17 Sep 2026 21:32:48 -0700 Subject: [PATCH 237/311] Document India accountability source reconnaissance --- docs/country-recon-in.md | 33 +++++++++++++++++++++ docs/source-status.json | 6 +++- pipeline/source_registry.json | 6 +++- pipeline/tests/test_india_recon_metadata.py | 28 +++++++++++++++++ pipeline/tests/test_source_registry.py | 4 +-- 5 files changed, 73 insertions(+), 4 deletions(-) create mode 100644 docs/country-recon-in.md create mode 100644 pipeline/tests/test_india_recon_metadata.py diff --git a/docs/country-recon-in.md b/docs/country-recon-in.md new file mode 100644 index 0000000..0bcab67 --- /dev/null +++ b/docs/country-recon-in.md @@ -0,0 +1,33 @@ +# India accountability-source reconnaissance + +Status: row-free reconnaissance only. No facility rows, raw exports, personal data, or production ingestion artifacts are committed. Checked 2026-09-17 against `AGENTS.md` and `docs/ETHICS.md` (policy v1.0, reviewed 2026-09-12). + +## Why India + +India is not represented by a country adapter or existing `country-recon-in.md`. It is a high-value accountability graph candidate because food-business licensing, corporate identity, environmental compliance, and national livestock statistics are administered by different authorities. The sources can support relationships among a regulated premise, its legal entity, applicable authority, and regional animal-production context without treating an aggregate count as a facility fact. + +## Source assessment + +| Source | Evidence and shape | Linkage value | Difficulty/status | +| --- | --- | --- | --- | +| FSSAI FoSCoS | Public Food Safety Compliance System; FSSAI states that FBOs can be searched/verified and the portal exposes licensing/registration statistics. Expected fields: FSSAI number, premise/business name, kind of business, state, validity/status. [Portal](https://foscos.fssai.gov.in/) · [FSSAI FAQ](https://www.fssai.gov.in/upload/uploadfiles/files/FAQs_Licensing_Registration_26_07_2022.pdf) | Primary regulated-premise identifier; join candidate to company name, state and business category | Medium; blocked pending authorized search/export contract, code lists, terms, privacy and rate limits | +| DAHD livestock statistics | Animal Husbandry Statistics division publishes annual Basic Animal Husbandry Statistics and quinquennial livestock census, including species, use, sex/age and state-level production/population. [Official page](https://dahd.gov.in/schemes/programmes/animal-husbandry-statistics) | Independent regional/species context; detects implausible interpretations but cannot identify a facility | Low–medium; not-run pending stable table/download IDs and revision metadata | +| MCA company master | Ministry of Corporate Affairs describes public company/LLP master information, including legal status, incorporation and registered-office fields. [MCA](https://www.mca.gov.in/) · [MCA compendium](https://www.mca.gov.in/Ministry/pdf/Compendium.pdf) | CIN/LLPIN and legal name can corroborate FSSAI operator identity and continuity | High; public lookup is not a verified bulk/API contract; personal/director fields must be excluded | +| CPCB/environmental compliance | CPCB publishes national environmental monitoring/compliance material and technology-provider policy; environmental control is also implemented by State Pollution Control Boards. [CPCB](https://cpcb.nic.in/) · [data policy](https://cpcb.nic.in/upload/thrust-area/DATA-POLICY.pdf) | Permit/consent/monitoring/enforcement edges can connect regulated premises to environmental authority and time | High; no single national facility export verified; state coverage and identifiers unresolved | + +## Accountability graph and sequencing + +1. Establish a bounded, authorized FoSCoS observation contract. Preserve FSSAI number, raw status, validity dates, state, business category and source timestamp; do not assume a license proves current operation or animal agriculture. +2. Resolve only organization-level MCA fields needed to corroborate the operator. Keep registered-office addresses distinct from operating premises and never publish director or residential information. +3. Map CPCB and State PCB public routes as separate authority families. A consent, inspection, monitoring record or enforcement action is not interchangeable with an FSSAI license; preserve authority, document, date and status separately. +4. Add DAHD aggregates as a context layer keyed by state/district, species, measure and reference period. Never allocate an aggregate to a named company or reverse-engineer small cells. + +## Provenance, privacy and publication gates + +Future acquisition must record URL, retrieval timestamp, HTTP method/status, content type, byte size, SHA-256, supplied publication/effective date, schema fingerprint and adapter/configuration version. Raw, parsed, normalized, quarantined, reviewed and released layers remain separate. Source disappearance is “not observed,” not closure. Schema drift, missing identifiers, CAPTCHA/authentication, suspicious count changes and ambiguous identity matches fail closed into review/quarantine. + +FSSAI premises and MCA registered offices may expose personal or mixed residential/business locations; suppress precise addresses and coordinates while privacy status is unresolved. Government origin is not project approval, factual review, current operation or completeness. Publication remains blocked until terms, privacy eligibility, project review, release approval and authorized maintainer availability are separately recorded. + +## Blockers and recommended effort + +The main blockers are the absence of a documented public FoSCoS bulk contract, MCA access/rate-limit semantics, the fragmented CPCB/SPCB landscape, and unclear reuse/coordinate rules. A first adapter tranche should take approximately 3–5 engineering days after authorized contracts are pinned; environmental/state expansion is a separate 5–10 day effort. No adapter is built in this reconnaissance. diff --git a/docs/source-status.json b/docs/source-status.json index 7f32aa0..6d02620 100644 --- a/docs/source-status.json +++ b/docs/source-status.json @@ -249,6 +249,10 @@ {"source_id":"pt.dgav.feed","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-pt.md","pipeline/source_registry.json"],"next_action":"Confirm the public feed section and NII semantics with an authorized capture; keep feed separate from NCV food/ABP rows and review private/primary-producer exposure."}, {"source_id":"pt.apambiente.tua","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-pt.md","pipeline/source_registry.json"],"next_action":"Locate a current public TUA search/export contract and safe permit fields; keep environmental decisions as a separate reviewed evidence overlay."}, {"source_id":"pt.ifap.snira","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-pt.md","pipeline/source_registry.json"],"next_action":"Do not acquire without explicit authorization; keep holder, animal, farm, parcel, and precise-location data restricted and assess any future relationship study separately."}, - {"source_id":"pt.ine.animal-production","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-pt.md","pipeline/source_registry.json"],"next_action":"Pin reproducible INE table/API identifiers and revision/confidentiality terms; keep statistics aggregate and separate from named-facility evidence."} + {"source_id":"pt.ine.animal-production","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-pt.md","pipeline/source_registry.json"],"next_action":"Pin reproducible INE table/API identifiers and revision/confidentiality terms; keep statistics aggregate and separate from named-facility evidence."}, + {"source_id":"in.fssai.foscos","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-in.md","pipeline/source_registry.json"],"next_action":"Confirm authorized FBO search/export, identifiers, status semantics, terms, privacy, and state coverage."}, + {"source_id":"in.dahd.livestock-statistics","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-in.md","pipeline/source_registry.json"],"next_action":"Pin report/table identifiers and machine-readable downloads; retain aggregate-only livestock context."}, + {"source_id":"in.mca.company-master","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-in.md","pipeline/source_registry.json"],"next_action":"Confirm public lookup contract and safe corporate fields; exclude director/personal exposure."}, + {"source_id":"in.cpcb.environmental-compliance","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-in.md","pipeline/source_registry.json"],"next_action":"Map state/federal routes, identifiers, terms, privacy and status semantics before acquisition."} ] } diff --git a/pipeline/source_registry.json b/pipeline/source_registry.json index 8c65a9b..7ccfb67 100644 --- a/pipeline/source_registry.json +++ b/pipeline/source_registry.json @@ -864,7 +864,11 @@ {"source_id":"pt.dgav.feed","jurisdiction_scope":"Portugal; DGAV feed-sector establishments and operators registered or approved under Regulation (EC) 183/2005","legacy_paths":[],"url":"https://maissipace.dgav.pt/Listagens","access_method":"official DGAV public +SIPACE/legacy SIPACE list family; separate feed section capture required","cadence":"not stated; timestamp every retrieval","attribution_licensing_notes":"Portuguese government source; feed-list reuse, attribution, privacy, and redistribution terms were not verified","adapter_status":"reference_only","expected_artifact_schema":"Feed observations with NII/individual identifier, operator/site, address/locality, feed activity, and registration/approval state; preserve raw section and codes","blockers":["The feed identifier is not interchangeable with an NCV; confirm current public section, codebook, pagination, export, cadence, private/primary-producer boundary, terms, and coverage."]}, {"source_id":"pt.apambiente.tua","jurisdiction_scope":"Portugal; Agência Portuguesa do Ambiente Título Único Ambiental and linked environmental licensing decisions","legacy_paths":[],"url":"https://apambiente.pt/avaliacao-e-gestao-ambiental/titulo-unico-ambiental","access_method":"official guidance and electronic-title/document route; no stable national public export verified","cadence":"decision/document-specific","attribution_licensing_notes":"Portuguese environmental authority source; document reuse, geometry, personal-address, and permit-condition publication terms require review","adapter_status":"reference_only","expected_artifact_schema":"Permit/title/process/document observations with holder, establishment/activity/project, regime, authority, decision date, validity, status, and reviewed document links; geometry unknown","blockers":["Locate and verify a current public search/export contract; keep TUA separate from DGAV approval and do not infer facility identity from holder or permit text."]}, {"source_id":"pt.ifap.snira","jurisdiction_scope":"Portugal; IFAP/DGAV SNIRA animal-identification, holding, movement, and herd information","legacy_paths":[],"url":"https://www.ifap.pt/portal/en/registo-area-reservada","access_method":"restricted IFAP area and credentialed webservice; no public acquisition authorized","cadence":"operational system; provider-specific","attribution_licensing_notes":"Restricted animal/holder data; purpose limitation, access control, retention, and privacy review are mandatory","adapter_status":"reference_only","expected_artifact_schema":"Credentialed animal/holding observations keyed by animal identification, NIF/holder, Marca de Exploração, species, and time period; not a public facility master","blockers":["Do not scrape or retain rows; obtain explicit authorization only if a narrowly scoped relationship study is approved, and suppress holder, farm, parcel, and precise-location details."]}, - {"source_id":"pt.ine.animal-production","jurisdiction_scope":"Portugal; Statistics Portugal aggregate animal-production, meat, and slaughter statistics","legacy_paths":[],"url":"https://ine.pt/bddXplorer/htdocs/minfo.jsp?lingua=EN&var_cd=0000916&var_cd=0000917&var_cd=0000918","access_method":"official INE metadata/PxWeb statistics route; table-specific API or download to be pinned","cadence":"annual/semestral/monthly by table; metadata checked through 2025/2026","attribution_licensing_notes":"Official statistics; preserve table metadata, revisions, confidentiality flags, and source terms","adapter_status":"reference_only","expected_artifact_schema":"Aggregate time-series observations by reference period, geography, species/meat type and measure; no named-facility rows","blockers":["Pin reproducible table IDs, API contract, revisions, terms, and confidentiality rules; never infer facility coverage or join confidential slaughter surveys to named establishments."]} ] + {"source_id":"pt.ine.animal-production","jurisdiction_scope":"Portugal; Statistics Portugal aggregate animal-production, meat, and slaughter statistics","legacy_paths":[],"url":"https://ine.pt/bddXplorer/htdocs/minfo.jsp?lingua=EN&var_cd=0000916&var_cd=0000917&var_cd=0000918","access_method":"official INE metadata/PxWeb statistics route; table-specific API or download to be pinned","cadence":"annual/semestral/monthly by table; metadata checked through 2025/2026","attribution_licensing_notes":"Official statistics; preserve table metadata, revisions, confidentiality flags, and source terms","adapter_status":"reference_only","expected_artifact_schema":"Aggregate time-series observations by reference period, geography, species/meat type and measure; no named-facility rows","blockers":["Pin reproducible table IDs, API contract, revisions, terms, and confidentiality rules; never infer facility coverage or join confidential slaughter surveys to named establishments."]}, + {"source_id":"in.fssai.foscos","jurisdiction_scope":"India; FSSAI Food Safety Compliance System food-business licensing and registration","legacy_paths":[],"url":"https://foscos.fssai.gov.in/","access_method":"official public portal and FSSAI-described FBO search/verification; no automated query performed","cadence":"operational portal; statistics show publisher update date","attribution_licensing_notes":"Government source; portal access does not settle bulk reuse, personal-address exposure, or publication rights","adapter_status":"reference_only","expected_artifact_schema":"License/registration observations with FSSAI number, business name, premises, kind of business, state, validity/status, and inspection/compliance references where public","blockers":["Search/export contract, stable identifiers, rate limits, privacy boundary, state coverage, and terms require authorized validation; do not scrape login or CAPTCHA surfaces."]}, + {"source_id":"in.dahd.livestock-statistics","jurisdiction_scope":"India; Department of Animal Husbandry and Dairying livestock census and Basic Animal Husbandry Statistics","legacy_paths":[],"url":"https://dahd.gov.in/schemes/programmes/animal-husbandry-statistics","access_method":"official reports, spreadsheets, and manuals from DAHD","cadence":"annual BAHS; quinquennial livestock census; revisions and publication lag possible","attribution_licensing_notes":"Official aggregate statistics; retain table/report metadata and suppress re-identification of small cells","adapter_status":"reference_only","expected_artifact_schema":"Aggregate species, sex/age/use, production, state/district, reference year, report/table and revision observations","blockers":["Pin stable report/table download identifiers and machine-readable route; household/holding microdata must remain restricted and is not a facility registry."]}, + {"source_id":"in.mca.company-master","jurisdiction_scope":"India; Ministry of Corporate Affairs company/LLP master data","legacy_paths":[],"url":"https://www.mca.gov.in/","access_method":"official MCA public company-master lookup; no bulk extraction or authenticated route used","cadence":"operational registry; provider-specific","attribution_licensing_notes":"Corporate identity source; registered-office and director-personal fields require minimization and privacy review","adapter_status":"reference_only","expected_artifact_schema":"Company/LLP identity, CIN/LLPIN, legal name, status, incorporation, registered office, capital and industry fields where public","blockers":["Current lookup/API contract, terms, rate limits, historical identity semantics, and safe fields require authorized validation; do not expose director or residential information."]}, + {"source_id":"in.cpcb.environmental-compliance","jurisdiction_scope":"India; Central Pollution Control Board industry monitoring and environmental compliance surfaces","legacy_paths":[],"url":"https://cpcb.nic.in/","access_method":"official portal, published policy, and sector/monitoring systems; no uncontrolled portal extraction","cadence":"system/report-specific; unknown nationally","attribution_licensing_notes":"Government environmental source; permit, monitoring and enforcement records require source-specific terms, safety and privacy review","adapter_status":"reference_only","expected_artifact_schema":"Permit/consent/monitoring/enforcement observations with industry, authority, location, dates, status, pollutant or compliance measure and source document","blockers":["No single national facility export verified; state PCB authority coverage, identifiers, route permissions, coordinate precision and status semantics remain unresolved."]} ] } diff --git a/pipeline/tests/test_india_recon_metadata.py b/pipeline/tests/test_india_recon_metadata.py new file mode 100644 index 0000000..9ba1cdb --- /dev/null +++ b/pipeline/tests/test_india_recon_metadata.py @@ -0,0 +1,28 @@ +import json +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] + + +class IndiaReconMetadataTests(unittest.TestCase): + def test_row_free_recon_and_policy_boundaries_are_present(self): + text = (ROOT / "docs" / "country-recon-in.md").read_text(encoding="utf-8") + self.assertIn("row-free", text) + self.assertIn("Publication remains blocked", text) + self.assertIn("No adapter is built", text) + + def test_india_registry_and_status_are_aligned_and_blocked(self): + registry = json.loads((ROOT / "pipeline" / "source_registry.json").read_text(encoding="utf-8")) + status = json.loads((ROOT / "docs" / "source-status.json").read_text(encoding="utf-8")) + registry_ids = {s["source_id"] for s in registry["sources"] if s["source_id"].startswith("in.")} + status_rows = [s for s in status["sources"] if s["source_id"].startswith("in.")] + self.assertEqual(registry_ids, {s["source_id"] for s in status_rows}) + self.assertEqual(len(registry_ids), 4) + for row in status_rows: + self.assertEqual(row["publication_eligibility"], "blocked") + self.assertNotEqual(row["runtime_health"], "healthy") + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/tests/test_source_registry.py b/pipeline/tests/test_source_registry.py index ad41dcc..193f1a1 100644 --- a/pipeline/tests/test_source_registry.py +++ b/pipeline/tests/test_source_registry.py @@ -9,8 +9,8 @@ class SourceRegistryTests(unittest.TestCase): def test_repository_registry_loads_and_references_existing_legacy_paths(self): registry = load_registry() - self.assertEqual(len(registry["sources"]), 240) - self.assertEqual(len({source["source_id"] for source in registry["sources"]}), 240) + self.assertEqual(len(registry["sources"]), 244) + self.assertEqual(len({source["source_id"] for source in registry["sources"]}), 244) def test_unknowns_are_explicit(self): registry = json.loads(REGISTRY_PATH.read_text(encoding="utf-8")) From 518ae1a41c69ba7648ae4bd87414131d4a40d2e5 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Thu, 17 Sep 2026 21:38:03 -0700 Subject: [PATCH 238/311] Format consolidated private graph handlers --- src/graph_private.rs | 215 +++++++++++++++++++++++++++++++++++++------ src/lib.rs | 73 +++++++++++++-- src/main.rs | 25 ++++- 3 files changed, 268 insertions(+), 45 deletions(-) diff --git a/src/graph_private.rs b/src/graph_private.rs index bf9214e..58bcdea 100644 --- a/src/graph_private.rs +++ b/src/graph_private.rs @@ -1,57 +1,212 @@ -use axum::{extract::{Path, Query, State}, http::{HeaderMap, StatusCode}, response::IntoResponse, Json}; +use crate::{ApiState, v2_error}; +use axum::{ + Json, + extract::{Path, Query, State}, + http::{HeaderMap, StatusCode}, + response::IntoResponse, +}; use serde::Deserialize; use serde_json::json; use uuid::Uuid; -use crate::{v2_error, ApiState}; const TOKEN_HEADER: &str = "x-uec-private-graph-token"; const MAX_LIMIT: i64 = 100; #[derive(Debug, Deserialize)] pub struct GraphQuery { - pub q: Option, pub relationship_type: Option, pub source_id: Option, - pub state: Option, pub min_confidence: Option, pub from: Option, - pub to: Option, pub limit: Option, pub cursor: Option, + pub q: Option, + pub relationship_type: Option, + pub source_id: Option, + pub state: Option, + pub min_confidence: Option, + pub from: Option, + pub to: Option, + pub limit: Option, + pub cursor: Option, } fn authorized(headers: &HeaderMap) -> bool { - let Some(expected) = std::env::var("UEC_PRIVATE_GRAPH_TOKEN").ok().filter(|v| !v.is_empty()) else { return false }; - headers.get(TOKEN_HEADER).and_then(|v| v.to_str().ok()).is_some_and(|v| crate::constant_time_token_matches(&expected, v)) + let Some(expected) = std::env::var("UEC_PRIVATE_GRAPH_TOKEN") + .ok() + .filter(|v| !v.is_empty()) + else { + return false; + }; + headers + .get(TOKEN_HEADER) + .and_then(|v| v.to_str().ok()) + .is_some_and(|v| crate::constant_time_token_matches(&expected, v)) +} +fn limit(v: Option) -> Result { + let n = v.unwrap_or(50); + if (1..=MAX_LIMIT).contains(&n) { + Ok(n) + } else { + Err("limit must be between 1 and 100") + } +} +fn denied() -> axum::response::Response { + v2_error( + StatusCode::NOT_FOUND, + "private_graph_unavailable", + "private graph unavailable", + ) +} +fn bad(msg: &'static str) -> axum::response::Response { + v2_error(StatusCode::BAD_REQUEST, "invalid_graph_query", msg) } -fn limit(v: Option) -> Result { let n = v.unwrap_or(50); if (1..=MAX_LIMIT).contains(&n) { Ok(n) } else { Err("limit must be between 1 and 100") } } -fn denied() -> axum::response::Response { v2_error(StatusCode::NOT_FOUND, "private_graph_unavailable", "private graph unavailable") } -fn bad(msg: &'static str) -> axum::response::Response { v2_error(StatusCode::BAD_REQUEST, "invalid_graph_query", msg) } -pub async fn entities(State(state): State, headers: HeaderMap, Query(p): Query) -> impl IntoResponse { - if !authorized(&headers) { return denied(); } - let Ok(limit) = limit(p.limit) else { return bad("limit must be between 1 and 100") }; - let Some(pool) = state.database else { return v2_error(StatusCode::SERVICE_UNAVAILABLE, "database_not_configured", "private graph database unavailable") }; - let Ok(client) = pool.get().await else { return v2_error(StatusCode::SERVICE_UNAVAILABLE, "database_pool_unavailable", "private graph database unavailable") }; +pub async fn entities( + State(state): State, + headers: HeaderMap, + Query(p): Query, +) -> impl IntoResponse { + if !authorized(&headers) { + return denied(); + } + let Ok(limit) = limit(p.limit) else { + return bad("limit must be between 1 and 100"); + }; + let Some(pool) = state.database else { + return v2_error( + StatusCode::SERVICE_UNAVAILABLE, + "database_not_configured", + "private graph database unavailable", + ); + }; + let Ok(client) = pool.get().await else { + return v2_error( + StatusCode::SERVICE_UNAVAILABLE, + "database_pool_unavailable", + "private graph database unavailable", + ); + }; let q = p.q.unwrap_or_default(); let rows = match client.query("SELECT entity_id, entity_type, canonical_name, country_code, created_at FROM (SELECT facility_id AS entity_id, 'facility' AS entity_type, canonical_name, country_code, created_at FROM uec.facilities UNION ALL SELECT organization_id, 'organization', canonical_name, country_code, created_at FROM uec.organizations) e WHERE ($1 = '' OR canonical_name ILIKE '%' || $1 || '%') AND ($2::uuid IS NULL OR entity_id > $2) ORDER BY entity_id LIMIT $3", &[&q, &p.cursor, &limit]).await { Ok(v) => v, Err(_) => return v2_error(StatusCode::SERVICE_UNAVAILABLE, "private_graph_query_failed", "private graph query failed") }; let data: Vec<_> = rows.into_iter().map(|r| json!({"entity_id":r.get::<_,Uuid>(0),"entity_type":r.get::<_,String>(1),"canonical_name":r.get::<_,Option>(2),"country_code":r.get::<_,Option>(3),"created_at":r.get::<_,chrono::DateTime>(4)})).collect(); Json(json!({"api_version":"private-graph-v1","data":data,"meta":{"private":true,"limit":limit,"next_cursor":data.last().and_then(|v|v["entity_id"].as_str())}})).into_response() } -pub async fn neighborhood(State(state): State, headers: HeaderMap, Path(entity_id): Path, Query(p): Query) -> impl IntoResponse { - if !authorized(&headers) { return denied(); } - let Ok(limit) = limit(p.limit) else { return bad("limit must be between 1 and 100") }; - if p.relationship_type.as_deref().is_some_and(|v| !["operator","owner","parent","brand","supplier","customer","regulatory_authority_for"].contains(&v)) { return bad("unsupported relationship type"); } - let Some(pool) = state.database else { return v2_error(StatusCode::SERVICE_UNAVAILABLE, "database_not_configured", "private graph database unavailable") }; - let Ok(client) = pool.get().await else { return v2_error(StatusCode::SERVICE_UNAVAILABLE, "database_pool_unavailable", "private graph database unavailable") }; +pub async fn neighborhood( + State(state): State, + headers: HeaderMap, + Path(entity_id): Path, + Query(p): Query, +) -> impl IntoResponse { + if !authorized(&headers) { + return denied(); + } + let Ok(limit) = limit(p.limit) else { + return bad("limit must be between 1 and 100"); + }; + if p.relationship_type.as_deref().is_some_and(|v| { + ![ + "operator", + "owner", + "parent", + "brand", + "supplier", + "customer", + "regulatory_authority_for", + ] + .contains(&v) + }) { + return bad("unsupported relationship type"); + } + let Some(pool) = state.database else { + return v2_error( + StatusCode::SERVICE_UNAVAILABLE, + "database_not_configured", + "private graph database unavailable", + ); + }; + let Ok(client) = pool.get().await else { + return v2_error( + StatusCode::SERVICE_UNAVAILABLE, + "database_pool_unavailable", + "private graph database unavailable", + ); + }; let rows = match client.query("SELECT relationship_observation_id, from_organization_id, target_facility_id, target_organization_id, relationship_type, assertion_status, observed_at, confidence, review_state, storage_state, privacy_status, publication_status, source_id, source_record_id FROM uec.organization_relationship_observations WHERE ($1 = from_organization_id OR $1 = target_facility_id OR $1 = target_organization_id) AND ($2::text IS NULL OR relationship_type=$2) AND ($3::text IS NULL OR source_id=$3) AND ($4::timestamptz IS NULL OR observed_at >= $4) AND ($5::timestamptz IS NULL OR observed_at < $5) ORDER BY observed_at DESC, relationship_observation_id DESC LIMIT $6", &[&entity_id, &p.relationship_type, &p.source_id, &p.from.map(|d| d.and_hms_opt(0,0,0).unwrap().and_utc()), &p.to.map(|d| d.and_hms_opt(0,0,0).unwrap().and_utc()), &limit]).await { Ok(v) => v, Err(_) => return v2_error(StatusCode::SERVICE_UNAVAILABLE, "private_graph_query_failed", "private graph query failed") }; let data: Vec<_> = rows.into_iter().map(|r| json!({"relationship_observation_id":r.get::<_,Uuid>(0),"from_organization_id":r.get::<_,Option>(1),"target_facility_id":r.get::<_,Option>(2),"target_organization_id":r.get::<_,Option>(3),"relationship_type":r.get::<_,Option>(4),"assertion_status":r.get::<_,String>(5),"observed_at":r.get::<_,chrono::DateTime>(6),"confidence":r.get::<_,Option>(7),"review_state":r.get::<_,String>(8),"storage_state":r.get::<_,String>(9),"privacy_status":r.get::<_,String>(10),"publication_status":r.get::<_,String>(11),"source_id":r.get::<_,String>(12),"source_record_id":r.get::<_,Uuid>(13)})).collect(); Json(json!({"api_version":"private-graph-v1","data":data,"meta":{"private":true,"entity_id":entity_id,"limit":limit}})).into_response() } -pub async fn queue(State(state): State, headers: HeaderMap, Path(kind): Path, Query(p): Query) -> impl IntoResponse { - if !authorized(&headers) { return denied(); } - let Ok(limit) = limit(p.limit) else { return bad("limit must be between 1 and 100") }; - if !["contradictions","unresolved-identities","quarantine","statistics"].contains(&kind.as_str()) { return v2_error(StatusCode::NOT_FOUND, "private_graph_unavailable", "private graph unavailable"); } - let Some(pool) = state.database else { return v2_error(StatusCode::SERVICE_UNAVAILABLE, "database_not_configured", "private graph database unavailable") }; - let Ok(client) = pool.get().await else { return v2_error(StatusCode::SERVICE_UNAVAILABLE, "database_pool_unavailable", "private graph database unavailable") }; - let sql = match kind.as_str() { "contradictions" => "SELECT claim_id, facility_id, organization_id, claim_domain, claim_kind, observed_at, confidence FROM uec.claim_current c WHERE c.review_state IN ('disputed','review_required') ORDER BY observed_at DESC, claim_id DESC LIMIT $1", "unresolved-identities" => "SELECT crosswalk_id, left_identifier_id, right_identifier_id, assertion_status, confidence, observed_at FROM uec.source_entity_crosswalks WHERE assertion_status IN ('candidate','review_required','disputed') ORDER BY observed_at DESC, crosswalk_id DESC LIMIT $1", "quarantine" => "SELECT source_record_id, source_id, source_state, received_at FROM uec.source_records WHERE source_state IN ('quarantined','rejected') ORDER BY received_at DESC, source_record_id DESC LIMIT $1", _ => "SELECT 'claims' AS metric, count(*)::bigint AS value FROM uec.claims UNION ALL SELECT 'relationships', count(*) FROM uec.organization_relationship_observations UNION ALL SELECT 'quarantined_records', count(*) FROM uec.source_records WHERE source_state IN ('quarantined','rejected')" }; - let rows = match client.query(sql, &[&limit]).await { Ok(v) => v, Err(_) => return v2_error(StatusCode::SERVICE_UNAVAILABLE, "private_graph_query_failed", "private graph query failed") }; - let data: Vec<_> = rows.into_iter().map(|r| (0..r.len()).map(|i| r.try_get::<_, String>(i).or_else(|_| r.try_get::<_, i64>(i).map(|v| v.to_string())).unwrap_or_default()).collect::>()).collect(); +pub async fn queue( + State(state): State, + headers: HeaderMap, + Path(kind): Path, + Query(p): Query, +) -> impl IntoResponse { + if !authorized(&headers) { + return denied(); + } + let Ok(limit) = limit(p.limit) else { + return bad("limit must be between 1 and 100"); + }; + if ![ + "contradictions", + "unresolved-identities", + "quarantine", + "statistics", + ] + .contains(&kind.as_str()) + { + return v2_error( + StatusCode::NOT_FOUND, + "private_graph_unavailable", + "private graph unavailable", + ); + } + let Some(pool) = state.database else { + return v2_error( + StatusCode::SERVICE_UNAVAILABLE, + "database_not_configured", + "private graph database unavailable", + ); + }; + let Ok(client) = pool.get().await else { + return v2_error( + StatusCode::SERVICE_UNAVAILABLE, + "database_pool_unavailable", + "private graph database unavailable", + ); + }; + let sql = match kind.as_str() { + "contradictions" => { + "SELECT claim_id, facility_id, organization_id, claim_domain, claim_kind, observed_at, confidence FROM uec.claim_current c WHERE c.review_state IN ('disputed','review_required') ORDER BY observed_at DESC, claim_id DESC LIMIT $1" + } + "unresolved-identities" => { + "SELECT crosswalk_id, left_identifier_id, right_identifier_id, assertion_status, confidence, observed_at FROM uec.source_entity_crosswalks WHERE assertion_status IN ('candidate','review_required','disputed') ORDER BY observed_at DESC, crosswalk_id DESC LIMIT $1" + } + "quarantine" => { + "SELECT source_record_id, source_id, source_state, received_at FROM uec.source_records WHERE source_state IN ('quarantined','rejected') ORDER BY received_at DESC, source_record_id DESC LIMIT $1" + } + _ => { + "SELECT 'claims' AS metric, count(*)::bigint AS value FROM uec.claims UNION ALL SELECT 'relationships', count(*) FROM uec.organization_relationship_observations UNION ALL SELECT 'quarantined_records', count(*) FROM uec.source_records WHERE source_state IN ('quarantined','rejected')" + } + }; + let rows = match client.query(sql, &[&limit]).await { + Ok(v) => v, + Err(_) => { + return v2_error( + StatusCode::SERVICE_UNAVAILABLE, + "private_graph_query_failed", + "private graph query failed", + ); + } + }; + let data: Vec<_> = rows + .into_iter() + .map(|r| { + (0..r.len()) + .map(|i| { + r.try_get::<_, String>(i) + .or_else(|_| r.try_get::<_, i64>(i).map(|v| v.to_string())) + .unwrap_or_default() + }) + .collect::>() + }) + .collect(); Json(json!({"api_version":"private-graph-v1","data":data,"meta":{"private":true,"queue":kind,"limit":limit}})).into_response() } diff --git a/src/lib.rs b/src/lib.rs index d813f0a..f730885 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -47,27 +47,80 @@ pub fn v2_error( } #[derive(Debug, Deserialize)] -pub struct PrivateGraphSearchParams { pub q: Option, pub limit: Option } +pub struct PrivateGraphSearchParams { + pub q: Option, + pub limit: Option, +} /// Private evidence-only graph search. This is intentionally not a public projection. -pub async fn get_private_graph_search_handler(State(state): State, Query(params): Query) -> impl IntoResponse { - let Some(pool) = state.database else { return v2_error(StatusCode::SERVICE_UNAVAILABLE, "private_graph_unavailable", "private graph database unavailable"); }; +pub async fn get_private_graph_search_handler( + State(state): State, + Query(params): Query, +) -> impl IntoResponse { + let Some(pool) = state.database else { + return v2_error( + StatusCode::SERVICE_UNAVAILABLE, + "private_graph_unavailable", + "private graph database unavailable", + ); + }; let q = params.q.unwrap_or_default().trim().to_string(); let limit = params.limit.unwrap_or(25).clamp(1, 100); - let client = match pool.get().await { Ok(c) => c, Err(_) => return v2_error(StatusCode::SERVICE_UNAVAILABLE, "private_graph_unavailable", "private graph database unavailable") }; + let client = match pool.get().await { + Ok(c) => c, + Err(_) => { + return v2_error( + StatusCode::SERVICE_UNAVAILABLE, + "private_graph_unavailable", + "private graph database unavailable", + ); + } + }; let rows = match client.query("SELECT entity_type, entity_id, display_name, source_id, source_identifier, observed_at FROM (SELECT 'facility' AS entity_type, f.facility_id AS entity_id, COALESCE(f.canonical_name, '[unnamed facility]') AS display_name, sei.source_id, sei.source_identifier, sei.observed_at FROM uec.facilities f LEFT JOIN uec.source_entity_identifiers sei ON sei.facility_id=f.facility_id UNION ALL SELECT 'organization', o.organization_id, COALESCE(o.canonical_name, '[unnamed organization]'), sei.source_id, sei.source_identifier, sei.observed_at FROM uec.organizations o LEFT JOIN uec.source_entity_identifiers sei ON sei.organization_id=o.organization_id) entities WHERE ($1='' OR display_name ILIKE '%' || $1 || '%' OR source_identifier ILIKE '%' || $1 || '%') ORDER BY display_name, observed_at DESC NULLS LAST LIMIT $2", &[&q, &limit]).await { Ok(r) => r, Err(_) => return v2_error(StatusCode::SERVICE_UNAVAILABLE, "private_graph_query_failed", "private graph search failed") }; let data: Vec = rows.into_iter().map(|r| json!({"entity_type":r.get::<_,String>(0),"entity_id":r.get::<_,uuid::Uuid>(1),"display_name":r.get::<_,String>(2),"source_id":r.get::<_,Option>(3),"source_identifier":r.get::<_,Option>(4),"observed_at":r.get::<_,Option>>(5),"review_state":"unknown","privacy_status":"unknown","publication_status":"unknown"})).collect(); Json(json!({"api_version":"private-graph-v1","data":data,"meta":{"scope":"private_evidence_only","bounded":true,"public_projection":false}})).into_response() } #[derive(Debug, Deserialize)] -pub struct PrivateGraphTraverseParams { pub entity_type: String, pub entity_id: uuid::Uuid, pub direction: Option, pub depth: Option } +pub struct PrivateGraphTraverseParams { + pub entity_type: String, + pub entity_id: uuid::Uuid, + pub direction: Option, + pub depth: Option, +} -pub async fn get_private_graph_traverse_handler(State(state): State, Query(params): Query) -> impl IntoResponse { - let Some(pool) = state.database else { return v2_error(StatusCode::SERVICE_UNAVAILABLE, "private_graph_unavailable", "private graph database unavailable"); }; - let depth = params.depth.unwrap_or(1).clamp(1, 2); let direction = params.direction.as_deref().unwrap_or("both"); - if !matches!(params.entity_type.as_str(), "organization" | "facility") || !matches!(direction, "in" | "out" | "both") { return v2_error(StatusCode::BAD_REQUEST, "invalid_traversal", "entity type, direction, or depth is invalid"); } - let client = match pool.get().await { Ok(c) => c, Err(_) => return v2_error(StatusCode::SERVICE_UNAVAILABLE, "private_graph_unavailable", "private graph database unavailable") }; +pub async fn get_private_graph_traverse_handler( + State(state): State, + Query(params): Query, +) -> impl IntoResponse { + let Some(pool) = state.database else { + return v2_error( + StatusCode::SERVICE_UNAVAILABLE, + "private_graph_unavailable", + "private graph database unavailable", + ); + }; + let depth = params.depth.unwrap_or(1).clamp(1, 2); + let direction = params.direction.as_deref().unwrap_or("both"); + if !matches!(params.entity_type.as_str(), "organization" | "facility") + || !matches!(direction, "in" | "out" | "both") + { + return v2_error( + StatusCode::BAD_REQUEST, + "invalid_traversal", + "entity type, direction, or depth is invalid", + ); + } + let client = match pool.get().await { + Ok(c) => c, + Err(_) => { + return v2_error( + StatusCode::SERVICE_UNAVAILABLE, + "private_graph_unavailable", + "private graph database unavailable", + ); + } + }; let rows = match client.query("SELECT relationship_observation_id, source_id, source_record_id, from_organization_id, target_facility_id, target_organization_id, relationship_type, assertion_status, unknown_reason, valid_from, valid_to, observed_at, confidence, review_state, storage_state, privacy_status, publication_status, note FROM uec.organization_relationship_observations WHERE (($1='organization' AND ((($3 IN ('out','both')) AND from_organization_id=$2) OR (($3 IN ('in','both')) AND target_organization_id=$2))) OR ($1='facility' AND target_facility_id=$2) ORDER BY observed_at DESC LIMIT 200", &[¶ms.entity_type, ¶ms.entity_id, &direction]).await { Ok(r) => r, Err(_) => return v2_error(StatusCode::SERVICE_UNAVAILABLE, "private_graph_query_failed", "private graph traversal failed") }; let data: Vec = rows.into_iter().map(|r| json!({"relationship_observation_id":r.get::<_,uuid::Uuid>(0),"source_id":r.get::<_,String>(1),"source_record_id":r.get::<_,uuid::Uuid>(2),"from_organization_id":r.get::<_,Option>(3),"target_facility_id":r.get::<_,Option>(4),"target_organization_id":r.get::<_,Option>(5),"relationship_type":r.get::<_,Option>(6),"assertion_status":r.get::<_,String>(7),"unknown_reason":r.get::<_,Option>(8),"valid_from":r.get::<_,Option>(9),"valid_to":r.get::<_,Option>(10),"observed_at":r.get::<_,chrono::DateTime>(11),"confidence":r.get::<_,Option>(12),"review_state":r.get::<_,String>(13),"storage_state":r.get::<_,String>(14),"privacy_status":r.get::<_,String>(15),"publication_status":r.get::<_,String>(16),"note":r.get::<_,Option>(17)})).collect(); Json(json!({"api_version":"private-graph-v1","data":data,"meta":{"scope":"private_evidence_only","bounded":true,"depth":depth,"direction":direction,"contradictions_preserved":true,"public_projection":false}})).into_response() diff --git a/src/main.rs b/src/main.rs index f72e86b..c1807c4 100644 --- a/src/main.rs +++ b/src/main.rs @@ -68,9 +68,18 @@ pub fn app(state: uec_api::ApiState, proxy: private_environment::ProxyConfig) -> "/api/dev/preview/candidates", get(uec_api::get_dev_candidate_preview_handler), ) - .route("/api/private/graph/entities", get(uec_api::graph_private::entities)) - .route("/api/private/graph/entities/{entity_id}/neighborhood", get(uec_api::graph_private::neighborhood)) - .route("/api/private/graph/queues/{kind}", get(uec_api::graph_private::queue)) + .route( + "/api/private/graph/entities", + get(uec_api::graph_private::entities), + ) + .route( + "/api/private/graph/entities/{entity_id}/neighborhood", + get(uec_api::graph_private::neighborhood), + ) + .route( + "/api/private/graph/queues/{kind}", + get(uec_api::graph_private::queue), + ) .route( "/api/dev/preview/test-release/locations", get(uec_api::get_dev_test_release_locations_handler), @@ -96,8 +105,14 @@ pub fn app(state: uec_api::ApiState, proxy: private_environment::ProxyConfig) -> get(uec_api::get_inspection_reports_handler), ) .route("/api/aphis-query", get(uec_api::get_aphis_query_handler)) - .route("/api/private/graph/search", get(uec_api::get_private_graph_search_handler)) - .route("/api/private/graph/traverse", get(uec_api::get_private_graph_traverse_handler)) + .route( + "/api/private/graph/search", + get(uec_api::get_private_graph_search_handler), + ) + .route( + "/api/private/graph/traverse", + get(uec_api::get_private_graph_traverse_handler), + ) .fallback_service(ServeDir::new("static")) .layer(CompressionLayer::new().br(true)) .layer(axum::middleware::from_fn_with_state( From 9daf535614add1b97199f22050b6f9c1ea37bc87 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Thu, 17 Sep 2026 22:36:04 -0700 Subject: [PATCH 239/311] Update migration and registry contract tests --- pipeline/tests/test_graph_migrations.py | 3 ++- pipeline/tests/test_platform_registry.py | 7 ++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/pipeline/tests/test_graph_migrations.py b/pipeline/tests/test_graph_migrations.py index bbc30b6..a0e1c48 100644 --- a/pipeline/tests/test_graph_migrations.py +++ b/pipeline/tests/test_graph_migrations.py @@ -20,7 +20,7 @@ def test_reserved_migrations_are_present_and_ordered(self): positions = [migrations.index(name) for name in graph_migrations] self.assertEqual(positions, sorted(positions)) self.assertEqual([migrations[position] for position in positions], graph_migrations) - self.assertEqual(migrations[-13:], [ + self.assertEqual(migrations[-14:], [ "026_graph_entities_crosswalks.sql", "027_graph_relationship_observations.sql", "028_graph_claims_support.sql", @@ -34,6 +34,7 @@ def test_reserved_migrations_are_present_and_ordered(self): "036_public_facility_discovery_view.sql", "037_public_discovery_read_model.sql", "038_graph_regulatory_authority_relationship.sql", + "039_private_graph_query_indexes.sql", ]) def test_entities_are_distinct_and_crosswalk_is_scoped(self): diff --git a/pipeline/tests/test_platform_registry.py b/pipeline/tests/test_platform_registry.py index 0643be4..83c227f 100644 --- a/pipeline/tests/test_platform_registry.py +++ b/pipeline/tests/test_platform_registry.py @@ -1,12 +1,17 @@ import unittest from pipeline.platform_registry import build_platform_registry, source_metadata +from pipeline.source_registry import load_registry class PlatformRegistryTests(unittest.TestCase): def test_materialized_registry_scales_from_existing_source_status_inputs(self): registry = build_platform_registry() - self.assertEqual(registry["source_count"], 240) + # The materialized count is derived from the authoritative source + # registry; country reconnaissance may add sources without requiring + # this contract test to be edited again. + source_registry = load_registry() + self.assertEqual(registry["source_count"], len(source_registry["sources"])) self.assertGreaterEqual(registry["country_count"], 40) self.assertEqual(registry["publication_boundary"], "awaiting-owner-review; private staging may continue; no release approval or promotion is implied") self.assertTrue(all(country["publication"]["state"] == "blocked" for country in registry["countries"])) From 4d0dbca0156736317d91fe77f225ce95223acb86 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Thu, 17 Sep 2026 22:39:49 -0700 Subject: [PATCH 240/311] Fix private graph test asset paths --- static/modules/__tests__/privateGraph.test.js | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/static/modules/__tests__/privateGraph.test.js b/static/modules/__tests__/privateGraph.test.js index 7bce7aa..f5ee89a 100644 --- a/static/modules/__tests__/privateGraph.test.js +++ b/static/modules/__tests__/privateGraph.test.js @@ -1,11 +1,14 @@ import { jest } from '@jest/globals'; test('private graph surface is explicitly bounded and not a public projection', async () => { - const html = await (await import('fs/promises')).readFile(new URL('../../../../static/private-graph.html', import.meta.url), 'utf8'); + const fs = await import('fs/promises'); + const html = await fs.readFile(new URL('../../private-graph.html', import.meta.url), 'utf8'); expect(html).toContain('Private evidence only'); expect(html).toContain('does not establish ownership'); - const js = await (await import('fs/promises')).readFile(new URL('../../private-graph.js', import.meta.url), 'utf8'); + const js = await fs.readFile(new URL('../../private-graph.js', import.meta.url), 'utf8'); expect(js).toContain('/api/private/graph/search'); expect(js).toContain('/api/private/graph/traverse'); expect(js).toContain('replace'); + const css = await fs.readFile(new URL('../../private-graph.css', import.meta.url), 'utf8'); + expect(css).toContain('.private-graph'); }); From 6ea4034fb298e679686022c84771fc941288f764 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Thu, 17 Sep 2026 22:58:41 -0700 Subject: [PATCH 241/311] Align private graph stylesheet test with explorer markup --- static/modules/__tests__/privateGraph.test.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/static/modules/__tests__/privateGraph.test.js b/static/modules/__tests__/privateGraph.test.js index f5ee89a..c7e3411 100644 --- a/static/modules/__tests__/privateGraph.test.js +++ b/static/modules/__tests__/privateGraph.test.js @@ -10,5 +10,7 @@ test('private graph surface is explicitly bounded and not a public projection', expect(js).toContain('/api/private/graph/traverse'); expect(js).toContain('replace'); const css = await fs.readFile(new URL('../../private-graph.css', import.meta.url), 'utf8'); - expect(css).toContain('.private-graph'); + expect(css).toContain('.workspace'); + expect(css).toContain('.notice'); + expect(html).toContain('class="workspace"'); }); From c19caeaec9acf667b2e4549835523e1674cd91f8 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Fri, 18 Sep 2026 08:47:31 -0700 Subject: [PATCH 242/311] Consolidate US current-data parity sprint --- docs/countries/us/README.md | 56 +- .../us/fec-corporate-identity-graph-recon.md | 404 ++++++++++++ docs/countries/us/v1-field-crosswalk.json | 19 +- docs/country-recon-us.md | 2 +- pipeline/common/acquisition.py | 55 +- pipeline/common/test_acquisition.py | 2 +- pipeline/source-inventory.csv | 2 +- pipeline/source_registry.json | 2 +- pipeline/sources/us/accountability/README.md | 26 + .../us/accountability/current_identity.py | 590 ++++++++++++++++++ .../fixtures/current_identity.json | 46 ++ .../accountability/test_current_identity.py | 116 ++++ pipeline/sources/us/aphis/acquire.py | 318 ++++++++++ pipeline/sources/us/aphis/adapter.py | 282 +++++++-- pipeline/sources/us/aphis/config.json | 7 +- pipeline/sources/us/aphis/handoff.py | 115 ++++ pipeline/sources/us/aphis/refresh.py | 237 ++++++- pipeline/sources/us/aphis/test_acquire.py | 156 +++++ pipeline/sources/us/aphis/test_adapter.py | 39 ++ pipeline/sources/us/aphis/test_refresh.py | 38 ++ pipeline/sources/us/fsis/README.md | 62 ++ pipeline/sources/us/fsis/adapter.py | 386 ++++++++++-- pipeline/sources/us/fsis/config.json | 7 +- .../sources/us/fsis/fixtures/demographics.csv | 3 + pipeline/sources/us/fsis/refresh.py | 254 +++++++- pipeline/sources/us/fsis/test_adapter.py | 28 +- pipeline/sources/us/fsis/test_refresh.py | 48 ++ 27 files changed, 3116 insertions(+), 184 deletions(-) create mode 100644 docs/countries/us/fec-corporate-identity-graph-recon.md create mode 100644 pipeline/sources/us/accountability/current_identity.py create mode 100644 pipeline/sources/us/accountability/fixtures/current_identity.json create mode 100644 pipeline/sources/us/accountability/test_current_identity.py create mode 100644 pipeline/sources/us/aphis/acquire.py create mode 100644 pipeline/sources/us/aphis/handoff.py create mode 100644 pipeline/sources/us/aphis/test_acquire.py create mode 100644 pipeline/sources/us/aphis/test_refresh.py create mode 100644 pipeline/sources/us/fsis/README.md create mode 100644 pipeline/sources/us/fsis/fixtures/demographics.csv create mode 100644 pipeline/sources/us/fsis/test_refresh.py diff --git a/docs/countries/us/README.md b/docs/countries/us/README.md index bd99fc2..51a2260 100644 --- a/docs/countries/us/README.md +++ b/docs/countries/us/README.md @@ -4,19 +4,59 @@ This packet is private pipeline documentation, not publication approval. ## Source boundaries -The facility-master candidate is USDA FSIS's [Meat, Poultry and Egg Product Inspection Directory](https://www.fsis.usda.gov/inspection/establishments/meat-poultry-and-egg-product-inspection-directory) and its supplemental establishment-demographic CSV. FSIS describes the directory as a listing of FSIS-regulated meat, poultry, and egg establishments, with a weekly replacement edition and generalized activity categories. State meat-and-poultry inspection programs are not silently included. +The facility-master candidate is USDA FSIS's [Meat, Poultry and Egg Product Inspection Directory](https://www.fsis.usda.gov/inspection/establishments/meat-poultry-and-egg-product-inspection-directory) and its supplemental establishment-demographic CSV. FSIS describes the directory as a listing of FSIS-regulated meat, poultry, and egg establishments, with a weekly replacement edition and generalized activity categories. State meat-and-poultry inspection programs are not silently included. The source-local bundle adapter is documented in [`pipeline/sources/us/fsis/README.md`](../../../pipeline/sources/us/fsis/README.md): it joins only exact source-native IDs/numbers and keeps all acquired data private/test-only. -APHIS is a separate evidence family. The [Animal Care Public Search Tool](https://www.aphis.usda.gov/animal-care/awa-services/usda-animal-care-public-search-tool) exposes licensed/registered persons, inspection reports, and research facility annual reports. The [annual usage summary](https://direct.aphis.usda.gov/awa/research-facility-report/annual-summary) notes that annual reports can be amended. The adapters require an explicit `registrations`, `annual_reports`, or `inspections` profile, and never turn an APHIS row into an FSIS facility or laboratory master record. +APHIS is a separate evidence family. The [Animal Care Public Search Tool](https://www.aphis.usda.gov/animal-care/awa-services/usda-animal-care-public-search-tool) exposes licensed/registered persons, inspection reports, and research facility annual reports. The [annual usage summary](https://direct.aphis.usda.gov/awa/research-facility-report/annual-summary) notes that annual reports can be amended. The adapters require an explicit `registrations`, `annual_reports`, or `inspections` profile, preserve certificate and customer-number variants, version explicit amendments, and never turn an APHIS row into an FSIS facility or laboratory master record. ## Acquisition boundary -FSIS direct links returned HTTP 403 during the prior reconnaissance. This sprint does not bypass that control. `pipeline.sources.us.fsis.refresh` provides a reproducible operator-assisted capture contract: an authorized operator saves the current CSV export shown on the official page, records the edition/date and final URL, and runs the private adapter. Direct fetch is available only with a terms-review JSON and the shared bounded acquisition primitive; HTML, login, 403, content-type, and schema failures remain fail-closed. +FSIS direct links returned HTTP 403 during the prior reconnaissance. This sprint does not bypass that control. `pipeline.sources.us.fsis.refresh` provides a reproducible operator-assisted capture contract: an authorized operator saves one current directory CSV plus the demographic CSV shown on the official page, records the edition/date and final URL for each, and runs the private adapter. Direct fetch is available only with a terms-review JSON and the shared bounded acquisition primitive; HTML, login, 403, content-type, and schema failures remain fail-closed. -APHIS is UI-mediated. `pipeline.sources.us.aphis.refresh` records the selected profile, official route, query/export context, retrieval time, hash, byte size, and separate evidence type. No hidden endpoint automation is required. +APHIS is UI-mediated. `pipeline.sources.us.aphis.refresh` records the selected profile, official route, query/export context, retrieval time, hash, byte size, source dates, and separate evidence type. `pipeline.sources.us.aphis.acquire` can fetch only a terms-reviewed URL for a documented download control; it rejects empty, malformed, HTML/challenge, truncated, and invalid-signature responses before committing bytes. No hidden endpoint automation is required. + +### APHIS renewable capture + +The supported operator workflow is: + +1. Open the official APHIS Animal Care search/report page in an authorized browser session. +2. Select exactly one profile: `registrations`, `annual_reports`, or `inspections`. Preserve the selected year/date, amendment indicator, filters, displayed result count, final URL, and any linked report/document identifier. Do not combine profiles or edit the saved export. +3. Save the CSV export, or save a linked document/amendment separately. Documents are source evidence only and are not automatically merged into annual-report or inspection rows. +4. Run a private refresh. For a saved CSV: + +```powershell +python -m pipeline.sources.us.aphis.refresh ` + --raw C:\path\to\authorized-private\aphis.csv ` + --run-dir data/staging/aphis/ ` + --profile annual_reports ` + --source-url "https://the-final-url-used-by-the-operator" ` + --effective-date 2025 ` + --query-context '{"selected_year":"2025","amended_reports_included":true,"displayed_result_count":"recorded-privately"}' +``` + +For a linked PDF/XLSX document or amendment, preserve it without parsing or release promotion: + +```powershell +python -m pipeline.sources.us.aphis.acquire ` + --profile documents --raw C:\path\to\authorized-private\report.pdf ` + --run-dir data/staging/aphis-documents/ ` + --retrieved-at-utc 2026-09-18T00:00:00Z ` + --source-url "https://the-final-document-url" +``` + +`acquire` currently supports the documented-download route; the browser-assisted route is the normal fallback for UI exports. A direct fetch requires an approved terms record and a documented final download URL: + +```powershell +python -m pipeline.sources.us.aphis.acquire ` + --profile annual_reports --terms-review data/restricted/aphis-terms-review.json ` + --source-url "https://the-documented-download-url" --output-root C:\path\to\authorized-private\data\raw ` + --run-id --query-context '{"selected_year":"2025","amended_reports_included":true}' +``` + +The fetch path writes `acquisition-metadata.json` and a row-free `manifest.json`; failures write `acquisition-failure.json` with the failure class, attempts, query context, and no committed artifact. Reuse a new run ID for every observation. A failed or empty capture must not replace or delete the previous validated artifact. All APHIS outputs remain restricted private research evidence, with `release_state=not-created`, `publication_state=private-research-evidence`, and `publication_gate=blocked` until separate human review and approval. ## Validation and handoff -Both adapters preserve source values only in restricted staging and emit parsed, normalized, quarantined, QA, run-status, health, and private candidate-handoff artifacts. They disable geocoding, mark address/coordinate review as pending, quarantine duplicate or missing identities, and set `release_state=not-created`, `publication_state=private-candidate`, and `publication_gate=blocked`. Candidate import remains limited to the disposable loopback database and existing test-only API path; no public promotion is performed by these commands. +Both adapters preserve source values only in restricted staging and emit parsed, normalized, quarantined, QA, run-status, health, and private candidate-handoff artifacts. They disable geocoding, mark address/coordinate review as pending, quarantine duplicate or missing identities, and set `release_state=not-created`, `publication_state=private-candidate`, and `publication_gate=blocked`. APHIS uses a source-specific test-only import packet rather than the facility importer: its `establishment_id` stays null, no graph candidates or edges are emitted, and no database release or public promotion is performed. The APHIS export is not a complete AWA or animal-use census; annual-report absence is not closure or non-use, and amendments/currentness require review. ## V1 reconciliation @@ -24,6 +64,12 @@ Both adapters preserve source values only in restricted staging and emit parsed, ## Accountability pilot +The private [FEC and corporate-identity graph reconnaissance](fec-corporate-identity-graph-recon.md) +documents official FEC, SEC EDGAR, IRS TEOS, SAM.gov, and state-registry +evidence families, their identifiers and access limits, and a conservative +event/matching/test contract. It does not authorize contribution-derived +facility links, automatic parent/subsidiary merges, or publication. + The private accountability pilot in [`pipeline/sources/us/accountability`](../../../pipeline/sources/us/accountability/README.md) adds a deterministic, graph-foundation-compatible link ledger. It starts from diff --git a/docs/countries/us/fec-corporate-identity-graph-recon.md b/docs/countries/us/fec-corporate-identity-graph-recon.md new file mode 100644 index 0000000..de76ef8 --- /dev/null +++ b/docs/countries/us/fec-corporate-identity-graph-recon.md @@ -0,0 +1,404 @@ +# US FEC and corporate-identity graph reconnaissance + +Status/date: private research design, observed 2026-09-18 UTC. This document +does not authorize acquisition, graph import, publication, or a claim that a +facility, organization, committee, candidate, donor, officer, parent, or +subsidiary is related to another entity. All proposed examples and tests are +synthetic/test-only. The governing policy is [`docs/ETHICS.md`](../../ETHICS.md). + +## Executive finding + +There is no single public, authoritative US corporate master that covers all +private companies, public issuers, nonprofits, political committees, and state +registrations. The safe design is a set of source-qualified evidence families: + +| Source family | Strong identifier and useful evidence | Boundary that must remain explicit | +| --- | --- | --- | +| Federal Election Commission (FEC) | `committee_id`/`CMTE_ID`, `candidate_id`/`CAND_ID`, filing/image/file identifiers, candidate-to-committee linkage, Form 1 connected organization/sponsor fields, Schedule A/B/E transactions | FEC records describe filers and reported transactions. A committee name, contributor name, employer string, or donation does not establish a legal parent, facility operator, ownership, control, wrongdoing, or facility connection. | +| Securities and Exchange Commission (SEC) EDGAR | CIK, accession number, filing form/date, company submissions, former names/tickers, Form 10-K/20-F exhibits, Forms 3/4/5, Schedules 13D/13G | EDGAR is primarily a filer/disclosure system, not a complete private-company or ownership registry. Exhibit 21 subsidiary lists may be incomplete, unstructured, omitted, or not assigned a CIK. | +| Internal Revenue Service (IRS) TEOS/EO BMF | EIN, legal name, exempt status, Form 990 XML/index, Schedule R related-organization disclosures | Primarily tax-exempt organizations; not a general corporate registry. IRS status, name, and filing data do not prove current operations or a facility relationship. | +| SAM.gov | UEI, legal business name, physical address, registration status, CAGE/award context where public | Federal-award registration is scoped to participating entities. Records can be private/opted out, expire or change, and do not prove corporate ownership or facility operation. | +| State Secretary of State registries | State-scoped entity number, formation/qualification filings, status, registered agent, and sometimes annual-report officers/directors | Fragmented by state, access and terms vary, some records are paid/manual, registered-agent data is not ownership, and there is no national state-registry identifier. | + +The existing graph foundation is compatible with this approach: keep each +native identifier source-qualified, store dated observations rather than +mutable edges, preserve contradictions, and keep factual review, privacy, and +publication states independent. Until a relationship is explicitly supported +by a source record and reviewed, it is a private candidate only. + +## Official source reconnaissance + +### FEC: committees, candidates, filings, and transactions + +Primary references: + +* [OpenFEC API documentation](https://api.open.fec.gov/developers/) +* [FEC browse/download data](https://www.fec.gov/data/browse-data/) +* [Candidate-committee linkage file description](https://www.fec.gov/campaign-finance-data/candidate-committee-linkage-file-description/) +* [Contributions by individuals file description](https://www.fec.gov/campaign-finance-data/contributions-individuals-file-description/) +* [FEC Form 1: Statement of Organization](https://www.fec.gov/resources/cms-content/documents/policy-guidance/fecfrm1.pdf) +* [FEC public-record research and use restrictions](https://www.fec.gov/introduction-campaign-finance/how-to-research-public-records/) + +Useful API route families (the API documentation and `/swagger/` schema are +the authority for fields and changes): + +| Route family | Stable keys / evidence | Suggested use | +| --- | --- | --- | +| `/v1/candidates/`, `/v1/candidates/search/` | `candidate_id` (`CAND_ID`), office, state, cycle, reported candidate data | Create a source-local candidate entity and link it only through the FEC's candidate/committee data. A candidate is a person, not a company. | +| `/v1/committees/`, `/v1/committees/{committee_id}/` | `committee_id` (`CMTE_ID`), committee type/designation, name, treasurer, address, connected organization/sponsor/affiliation fields where filed | Create a source-local committee organization. Preserve each Form 1 filing/version and distinguish connected organization, affiliated committee, sponsor, and treasurer roles. | +| `/v1/candidate/{candidate_id}/committees/` and linkage/bulk files | `CAND_ID` + `CMTE_ID` + election/designation/linkage ID | Emit an explicit FEC `authorized_by`/`candidate_committee` observation with its cycle and filing support. Do not infer a company relationship from the candidate's employer. | +| `/v1/committee/{committee_id}/filings/`, `/v1/filings/` | filing ID, report type, amendment indicator, receipt/coverage dates, image/file IDs | Keep filing versions append-only. Amendments and terminations are observations, not destructive updates. | +| `/v1/schedules/schedule_a/` | committee, contributor name/type, `other_id` when reported, date, amount, memo/transaction IDs, filing/image support | Model a reported financial transaction event. If `other_id` is a committee ID, it is a source-native committee reference; otherwise contributor text is not an organization identity. | +| `/v1/schedules/schedule_b/` and Schedule E/F families | committee, payee/recipient, `other_id` where available, date, amount, purpose/candidate references | Model disbursement or independent-expenditure events. Do not turn payee text into a parent, operator, or facility edge. | + +The FEC says API data are updated nightly, each API call is limited to 100 +results per page, and a normal key permits up to 1,000 calls per hour; the +documentation describes a possible 7,200-call/hour key by request. Use the +documented pagination fields and checkpoint every page. Treat those limits as +operational guidance that can change, not as a license to crawl. + +FEC bulk data is preferable for historical or broad pulls. The browse-data +page exposes candidate master, candidate-committee linkage, committee master, +committee summary, Form 1/Form 2, daily filing, and transaction/database dump +families. The page states that update cadence varies by file (daily to weekly) +and that transaction-level files can be very large. Record the exact linked +file URL, release/period label, retrieval time, content type, byte size, hash, +header description, and adapter version. Never treat an FEC page's current +result as a timeless fact. + +FEC matching hazards: + +* `CMTE_ID` and `CAND_ID` are useful source identifiers; names are not. + Committee names and connected-organization strings can change across Form 1 + amendments and may be abbreviated, sponsored, or affiliated rather than the + legal parent. +* Candidate-to-committee linkage is a filed electoral relationship, not an + ownership relationship. A candidate's employer field is a reported + individual attribute, not proof that the employer funded a committee or + operates a facility. +* Schedule A/B data contain amended reports, memo items, transfers, + earmarking, refunds, and different entity types. Deduplicate only within a + documented filing/transaction-version model; retain the original report and + all amendments. +* For an individual contributor, employer and occupation are reported text. + They are not a corporate identifier. A corporate-sounding contributor name + can still be a person, trade name, intermediary, or data-entry variant. +* FEC public-record pages state that copied contributor information may not be + sold or used for soliciting contributions or for commercial purposes. Raw + contributor names, addresses, occupations, employers, and officer contact + data must remain restricted pending policy and legal review; do not put them + in logs, fixtures, public graph responses, or exports. + +### SEC EDGAR: public issuers and disclosed corporate relationships + +Primary references: + +* [SEC EDGAR APIs](https://www.sec.gov/search-filings/edgar-application-programming-interfaces) +* [SEC developer resources / fair-access FAQ](https://www.sec.gov/about/webmaster-frequently-asked-questions) +* [SEC company ticker/CIK associations](https://www.sec.gov/files/company_tickers.json) +* [SEC submissions API](https://data.sec.gov/submissions/CIK##########.json) +* [SEC forms](https://www.sec.gov/forms) + +The submissions endpoint is keyed by a zero-padded 10-digit CIK and returns +current/former names, tickers/exchanges, and filing history. The SEC describes +the submissions and XBRL APIs as unauthenticated JSON services updated in real +time, with nightly bulk archives for `submissions.zip` and +`companyfacts.zip`. EDGAR does not support CORS; an automated client must use +the SEC's access rules and an honest identifying `User-Agent`. + +The SEC currently states a maximum access rate of 10 requests/second and asks +clients to declare a user agent. Use a lower bounded rate, conditional/cache +headers when available, retries with backoff for 429/5xx, and the nightly bulk +archives for wide historical work. Record an HTTP error or HTML challenge as a +quarantined acquisition, never as an empty dataset. + +Relationship-bearing filing evidence includes: + +* Form 10-K/20-F/40-F business and property disclosures and Exhibit 21 + subsidiary schedules; +* Forms 3, 4, and 5 for officer/director and beneficial-ownership filings; +* Schedules 13D/13G for certain beneficial ownership disclosures; and +* 8-K, merger, registration, and other filed exhibits when they explicitly + state a transaction or corporate relationship. + +These should be represented as `disclosed_subsidiary`, `disclosed_officer`, +`beneficial_owner_disclosure`, or another narrowly scoped observation with +form, accession, filing date, and exhibit/page/anchor support. An Exhibit 21 +list is evidence that the filer disclosed a subsidiary at a point in time; it +is not a complete, current, or independently verified ownership graph. An +officer/director filing connects a person to a filer for the filed role and +period; it does not connect that person's employer text to a facility. + +CIK is strong for an SEC registrant, but subsidiary names in exhibits often do +not have a CIK. Names may be historical, abbreviated, foreign, or omitted for +immaterial subsidiaries. Exact CIK/name evidence can create a candidate +crosswalk; automatic parent/subsidiary merging is prohibited. + +### IRS TEOS and EO BMF: tax-exempt identity and related organizations + +Primary references: + +* [IRS Tax Exempt Organization Search](https://www.irs.gov/charities-non-profits/search-for-tax-exempt-organizations) +* [IRS TEOS bulk downloads](https://www.irs.gov/charities-non-profits/tax-exempt-organization-search) +* [IRS Form 990 XML downloads](https://www.irs.gov/charities-non-profits/form-990-series-downloads) +* [IRS EO Business Master File extract](https://www.irs.gov/charities-non-profits/exempt-organizations-business-master-file-extract-eo-bmf) + +TEOS exposes exempt-organization status, filings, determination letters, and +bulk data. EO BMF files are CSV by state/region and are keyed/sorted by EIN; +the IRS describes them as the latest information it has for organizations with +an exempt determination. Form 990 XML and indices can provide related- +organization and officer/key-employee evidence (for example, Schedule R and +Part VII), subject to redaction and privacy review. + +This is a useful identity source for nonprofit political, trade, advocacy, and +animal-welfare organizations, but it is not a general business registry. The +IRS says its searchable name data are official names submitted to the IRS and +that DBA/common names may be absent in some datasets. Status can be revoked or +reinstated; an absence or stale row is not closure. Store EIN and form/index +IDs source-scoped, preserve raw XML privately, and never expose officer names +or addresses merely because a Form 990 is public. + +### SAM.gov: federal-award entity identity + +Primary references: + +* [SAM.gov entity information](https://sam.gov/entity-information) +* [SAM.gov entity registration and UEI](https://sam.gov/entity-registration) +* [SAM.gov about/data services](https://sam.gov/about/this-site) + +SAM.gov provides a governmentwide Unique Entity ID (UEI), legal business name, +physical address, registration status, and federal-award context for entities +that register or request an identifier. SAM says registration must be renewed +every 365 days and provides entity extracts and Entity Management APIs/system +connections. Public visibility is configurable, and some records are not +available to ordinary public search. + +Use `UEI` as a source-qualified identity only. A UEI is evidence of a SAM +entity record, not proof of a corporate parent, beneficial owner, operating +facility, or current activity. A SAM address is a validation/disclosure field, +not permission to publish a private or residential location. Acquisition must +record whether the observation came from a public search, official extract, or +authenticated API, and must preserve active/inactive/expired status and +observation dates. + +### State Secretary of State registries + +State registries are the primary source for many privately held US entities, +but they must be handled one jurisdiction at a time. For example, [Delaware's +Division of Corporations online services](https://corp.delaware.gov/services/) +offers business-entity search, status, annual-report, and document services; +its FAQ says a free search can return name, file number, formation date, +registered-agent information, entity kind/type, and residency, while more +detailed status/history may be fee-based. Delaware also states that current +officers/directors can be found on the most recent annual report, while +shareholder/owner information is not on file. Its search page prohibits data +mining and excessive/repeated searches. + +Accordingly: + +* use a state entity number plus jurisdiction as the native identifier; +* prefer official filing copies or explicitly permitted downloads/API access; +* do not automate a search form or mine a registry when its terms prohibit it; +* treat registered agent, principal address, annual-report officer, and owner + as different fields with different meanings; and +* model foreign qualification in another state as a source observation that + may support a reviewed crosswalk, not as a universal identity. + +There is no safe nationwide state-SOS bulk adapter implied by this document. +Start with one state only after a terms/access review and an operator-assisted +capture contract. A state filing can be strong evidence of formation, status, +or a filed officer/director role, but not automatically of current operation, +ownership, or facility control. + +## Proposed graph and evidence contracts + +### Source-native entities + +The current `graph-candidate-handoff-v1` contract should be reused. Add no +universal-ID field. Suggested identifier types are: + +| Entity | Identifier type | Example semantics | +| --- | --- | --- | +| FEC committee | `fec_committee_id` | Stable FEC `CMTE_ID`; source-scoped to FEC committee records. | +| FEC candidate | `fec_candidate_id` | Stable FEC `CAND_ID`; person entity, not a company. | +| SEC registrant | `sec_cik` | CIK for a registrant; accession is evidence-record identity, not entity identity. | +| IRS exempt organization | `irs_ein` | EIN in TEOS/EO BMF; availability is limited to applicable exempt records. | +| SAM entity | `sam_uei` | UEI in the observed extract/API/search result. | +| State entity | `state_entity_number` | Always paired with `jurisdiction` and filing/source ID. | +| facility source row | existing source-native facility ID | Never replace this with an organization name or address match. | + +Every identifier observation carries `source_id`, `source_record_id`, +`value_as_observed`, normalized value (if any), observed time, source URL, +artifact hash/bytes, record/form/file/image/accession ID, and independent +review/privacy/publication states. Raw names, addresses, coordinates, and +personal roles remain private until the applicable policy gates pass. + +### Relationship and event types + +Use existing `organization_relationship_observations` only for a narrowly +defined assertion that fits its direction and time semantics: + +* `parent` / `owner` / `operator` / `brand` for organization-to-organization + or organization-to-facility evidence; +* `regulatory_authority_for` only when the source explicitly describes that + authority scope; and +* a future source-specific role or claim for `fec_connected_organization`, + `fec_candidate_committee`, `sec_disclosed_subsidiary`, and + `officer_of` unless the schema is deliberately extended. + +Do not encode a contribution as `owner`, `parent`, `operator`, or `supplier`. +Use a separate append-only financial-event contract with: + +```text +financial_event { + source_id, source_record_id, filing_id, image_id, transaction_id, + amendment/version, observed_at, transaction_date, + payer_source_ref?, payee_source_ref?, recipient_committee_source_ref?, + candidate_source_ref?, amount, currency, schedule, line/category, + memo/earmark state, raw-party-name-private, review/privacy/publication state +} +``` + +`payer_source_ref` or `payee_source_ref` is populated only when the source +provides an explicit source-native identifier (for example FEC `other_id` for +another committee). A free-text contributor/payee name remains a claim or +unresolved candidate, never an organization edge. Each event must retain the +filed direction and not imply that a facility was involved. + +### Evidence state and confidence + +Confidence is scoped to one claim and one source observation; it is not a +probability that the whole graph is true. Store both a bounded numeric value +(nullable) and an explanation, and keep `assertion_status`, factual +`review_state`, `privacy_status`, and `publication_status` separate. + +Recommended evidence grades: + +| Grade | Permitted interpretation | Default result | +| --- | --- | --- | +| A — explicit | The source provides a stable ID or explicit filed relationship: FEC candidate/committee linkage, Form 1 connected organization, SEC filed exhibit/ownership form, IRS EIN/form relationship, SAM UEI record, or state filing. | Private review-required observation for that exact role/date. Never a universal merge or facility claim. | +| B — concordant | Two or more independent source records agree on a legal name plus scoped identifier/address/date, but no source explicitly states the relationship. | Candidate crosswalk only; human review required. | +| C — discovery | Name similarity, suffix-stripped match, website, shared address, registered agent, common officer name, employer string, proximity, or contribution alone. | Discovery queue only; no relationship row and no automatic crosswalk. | +| 0 — unresolved/contradicted | Missing IDs, conflicting filings, ambiguous same-name entities, or source disagreement. | Quarantine with explicit unknown/disputed/rejected state. | + +Do not average evidence grades or let a high-confidence identity claim upgrade a +low-confidence relationship. A direct source statement about a parent may be +Grade A for `disclosed_parent_at_time`, but only Grade C for a facility +operator edge unless the facility source itself names the organization or a +reviewer records a separate supported link. + +## Conservative matching and review rules + +1. Normalize only for candidate search. Preserve the original name, suffix, + punctuation, case, address, and source values beside any normalized key. +2. Require at least one explicit source-native key for an accepted crosswalk; + exact name alone is not enough. A legal-name match across FEC, SEC, IRS, + SAM, or a state registry remains source-scoped until reviewed. +3. Keep multiple same-name organizations alive. Never collapse different CIKs, + EINs, UEIs, state entity numbers, or FEC IDs because names are similar. +4. Require direction and time. Parent and subsidiary directions must not be + reversed; officer roles and filing periods are dated; source disappearance + means `not_observed`, never closure. +5. Keep committee/filer identity separate from the connected organization, + sponsor, treasurer, candidate, and contributor. A committee can be + sponsored or affiliated without being owned by that organization. +6. Do not create a facility edge from an FEC contribution, individual employer, + SEC officer, registered agent, mailing address, or shared name. An explicit + facility source ID or a separately reviewed source-supported observation is + required. +7. Preserve contradictions and amendments. A later filing is a new + observation; it does not erase an earlier claim or turn an absent row into a + closure event. +8. Keep names/addresses of individual contributors, officers, registered-agent + contacts, and private persons out of public projections and test fixtures. + Public release needs a privacy decision that covers maps, APIs, exports, + old releases, caches, raw previews, and reimports. + +## Reusable acquisition/query plan + +These are bounded private research queries, not public API promises: + +1. **FEC exact committee:** search `/v1/committees/` by a reviewed exact + `CMTE_ID`; retrieve filing history and current/previous Form 1 observations. +2. **FEC candidate linkage:** join only the official linkage file on + `CAND_ID + CMTE_ID + election year`; retain linkage ID, designation, and + period. +3. **FEC connected organization:** extract the filed Form 1 connected- + organization/sponsor/affiliation field into a private organization candidate + with source text and filing support; do not resolve it by name alone. +4. **FEC transaction:** retrieve Schedule A/B/E by committee, date window, and + filing/transaction ID; resolve `other_id` only to the matching FEC source + entity type; preserve memo/amendment state. +5. **SEC registrant:** resolve an already reviewed CIK using submissions JSON, + then fetch only selected filing accessions/exhibits. Store an exhibit-21 or + officer observation with accession and anchor/page support. +6. **IRS exempt organization:** resolve EIN in EO BMF/TEOS, then inspect the + relevant Form 990 XML/index (especially related-organization disclosures) + under the private retention policy. +7. **SAM entity:** query an exact UEI or approved bounded name search, record + whether the result is public search, extract, or API, and capture status and + observation date. +8. **State registry:** use an exact state entity number or a single reviewed + name search through the state-approved route; do not crawl or mine the + registry. +9. **Facility connection:** query existing facility source IDs only. A corporate + identity result can enter a facility relationship queue, but cannot create + the edge by proximity, name, address, contribution, or officer overlap. + +Every acquisition writes a source manifest with official URL, retrieval UTC, +publication/effective date if supplied, content type, byte size, SHA-256, +source terms/access notes, parser/configuration version, and a fail-closed +status. Raw and parsed layers stay in ignored private staging. A failed or +blocked refresh leaves the prior validated release available subject to +current suppression. + +## Test plan + +Use sanitized or synthetic fixtures only. Tests should cover: + +* FEC pagination, 100-row page boundaries, nightly/bulk manifest capture, + 429/backoff, malformed JSON, HTML/403 responses, missing IDs, and exact + source-record provenance; +* FEC amendment/version handling: the same transaction ID across an original + and amendment remains one event lineage with both filed observations, and + memo/earmarked/refund rows are not silently counted as ordinary donations; +* candidate-to-committee linkage keeps `CAND_ID`, `CMTE_ID`, election year, + designation, and linkage ID, and never emits an owner/operator edge; +* a Form 1 connected-organization string yields a review-required private + candidate; two committees with similar names do not merge; +* a Schedule A employer string and a corporate-looking contributor name never + create an organization or facility edge; an explicit `other_id` committee + reference may create only a typed FEC event reference; +* SEC CIK/ticker/former-name observations remain source-qualified; an Exhibit + 21 name without a CIK remains a candidate, and two CIKs with the same name + remain distinct; +* synthetic SEC exhibit, IRS Schedule R, state filing, and SAM UEI fixtures + preserve accession/form/entity numbers, observation dates, direction, and + unknown/withheld values; +* parent/subsidiary direction is validated, officer role periods do not become + ownership, and a source disappearance produces `not_observed`, not closure; +* state registry access is blocked when terms say no mining, and manual capture + records operator/context rather than inventing an API contract; +* graph candidate validation rejects `canonical_id`, `global_id`, or + `universal_identity`, rejects public/released state, requires source-local + identifiers, and preserves contradictory observations; +* public projection tests prove that FEC contributor/officer names, addresses, + raw filing payloads, facility coordinates, and restricted source values are + absent even when a related organization is otherwise eligible; and +* failed acquisition, privacy suppression, restore, and reimport tests never + resurrect a suppressed edge or release a private candidate. + +## Recommendation and next bounded slice + +Prioritize an FEC adapter for synthetic/test-only ingestion of committee master, +candidate master/linkage, Form 1 metadata, and a very small Schedule A/B +fixture. Add SEC CIK/submissions metadata next, without fetching broad filing +corpora. Defer state-registry automation until one state's terms and permitted +access path are documented. Treat IRS and SAM as scoped identity corroboration, +not as facility or corporate-master sources. + +The first implementation should add source-specific event/claim contracts and +fixtures rather than expanding the canonical graph relationship enum. No +production import, public projection, contribution-derived facility link, +automatic parent/subsidiary merge, or ownership inference is justified by this +reconnaissance. diff --git a/docs/countries/us/v1-field-crosswalk.json b/docs/countries/us/v1-field-crosswalk.json index 4ef3f54..ab127cc 100644 --- a/docs/countries/us/v1-field-crosswalk.json +++ b/docs/countries/us/v1-field-crosswalk.json @@ -3,17 +3,18 @@ "legacy_snapshot": "static_data/us/locations.csv", "legacy_rows": 7101, "legacy_columns": 269, - "status": "inventory-only; no current FSIS artifact was safely acquired", + "status": "mapping implemented; no current FSIS artifact was safely acquired", "groups": { - "identity": {"fields": ["establishment_id", "establishment_number", "establishment_name", "duns_number"], "current_evidence": "FSIS MPI directory identity fields are the intended source boundary; exact current header mapping pending authorized export", "state": "pending_current_artifact"}, - "location": {"fields": ["street", "city", "state", "zip", "county", "fips_code", "latitude", "longitude"], "current_evidence": "FSIS directory/demographic source describes physical establishments and geographic fields; source values remain private and coordinates require review", "state": "pending_privacy_and_current_artifact"}, - "contact": {"fields": ["phone"], "current_evidence": "May exist in legacy or current directory; not copied to normalized output", "state": "unresolved"}, - "administrative": {"fields": ["grant_date", "dbas", "district", "circuit", "size", "type"], "current_evidence": "FSIS demographic documentation describes district, HACCP size, and production activity categories; exact current field mapping pending export", "state": "partial"}, - "slaughter_categories": {"fields": ["slaughter", "meat_slaughter", "beef_cow_slaughter", "steer_slaughter", "heifer_slaughter", "bull_stag_slaughter", "dairy_cow_slaughter", "heavy_calf_slaughter", "bob_veal_slaughter", "formula_fed_veal_slaughter", "non_formula_fed_veal_slaughter", "market_swine_slaughter", "sow_slaughter", "roaster_swine_slaughter", "boar_stag_swine_slaughter", "stag_swine_slaughter", "feral_swine_slaughter", "goat_slaughter", "young_goat_slaughter", "adult_goat_slaughter", "sheep_slaughter", "lamb_slaughter", "deer_reindeer_slaughter", "antelope_slaughter", "elk_slaughter", "bison_slaughter", "buffalo_slaughter", "water_buffalo_slaughter", "cattalo_slaughter", "yak_slaughter", "other_voluntary_livestock_slaughter", "rabbit_slaughter", "poultry_slaughter", "young_chicken_slaughter", "light_fowl_slaughter", "heavy_fowl_slaughter", "capon_slaughter", "young_turkey_slaughter", "young_breeder_turkey_slaughter", "old_breeder_turkey_slaughter", "fryer_roaster_turkey_slaughter", "duck_slaughter", "goose_slaughter", "pheasant_slaughter", "quail_slaughter", "guinea_slaughter", "ostrich_slaughter", "emu_slaughter", "rhea_slaughter", "squab_slaughter", "other_voluntary_poultry_slaughter"], "current_evidence": "FSIS supplemental demographic dataset documents species slaughter subclasses; adapter preserves source activity vocabulary and does not infer absent species", "state": "partial_pending_current_artifact"}, - "processing_categories": {"fields": ["processing", "meat_processing", "poultry_processing", "egg_processing", "*_processing"], "current_evidence": "FSIS supplemental demographic dataset documents categorical processing, RTE/NRTE/raw-intact/raw-non-intact, species, cell-cultured, and exemption distinctions", "state": "partial_pending_current_artifact"}, - "inspection_systems": {"fields": ["inspection_system_*"], "current_evidence": "FSIS source documentation describes inspection activities; retain each source flag separately", "state": "partial_pending_current_artifact"}, - "derived_legacy_categories": {"fields": ["slaughter_or_processing_only", "slaughter_only_class", "slaughter_only_species", "meat_slaughter_only_species", "poultry_slaughter_only_species", "slaughter_volume_category", "processing_volume_category"], "current_evidence": "Project-derived from legacy FSIS columns; must be recomputed only from a reviewed current artifact and never treated as source facts", "state": "unresolved_current_reproduction"} + "identity": {"fields": ["establishment_id", "establishment_number", "establishment_name", "duns_number"], "current_evidence": "Bundle adapter maps source-native establishment ID/number and name; DUNS remains source-only when present", "state": "mapped_with_duns_source_only"}, + "location": {"fields": ["street", "city", "state", "zip", "county", "fips_code", "latitude", "longitude"], "current_evidence": "Directory fields are preserved in private source values; city/state/postal/county and source-provided coordinates map to normalized private fields, while address and coordinate publication remain review-gated", "state": "mapped_private_review_required"}, + "contact": {"fields": ["phone"], "current_evidence": "Preserved only in private directory source values; not copied into the normalized public-shaped projection", "state": "source_only_privacy_review_required"}, + "administrative": {"fields": ["grant_date", "dbas", "district", "circuit", "size", "type"], "current_evidence": "Directory values map to grant_date, district, circuit, size, and source_type when present; source values remain alongside interpretations", "state": "mapped_private"}, + "slaughter_categories": {"fields": ["slaughter", "meat_slaughter", "beef_cow_slaughter", "steer_slaughter", "heifer_slaughter", "bull_stag_slaughter", "dairy_cow_slaughter", "heavy_calf_slaughter", "bob_veal_slaughter", "formula_fed_veal_slaughter", "non_formula_fed_veal_slaughter", "market_swine_slaughter", "sow_slaughter", "roaster_swine_slaughter", "boar_stag_swine_slaughter", "stag_swine_slaughter", "feral_swine_slaughter", "goat_slaughter", "young_goat_slaughter", "adult_goat_slaughter", "sheep_slaughter", "lamb_slaughter", "deer_reindeer_slaughter", "antelope_slaughter", "elk_slaughter", "bison_slaughter", "buffalo_slaughter", "water_buffalo_slaughter", "cattalo_slaughter", "yak_slaughter", "other_voluntary_livestock_slaughter", "rabbit_slaughter", "poultry_slaughter", "young_chicken_slaughter", "light_fowl_slaughter", "heavy_fowl_slaughter", "capon_slaughter", "young_turkey_slaughter", "young_breeder_turkey_slaughter", "old_breeder_turkey_slaughter", "fryer_roaster_turkey_slaughter", "duck_slaughter", "goose_slaughter", "pheasant_slaughter", "quail_slaughter", "guinea_slaughter", "ostrich_slaughter", "emu_slaughter", "rhea_slaughter", "squab_slaughter", "other_voluntary_poultry_slaughter"], "current_evidence": "Supplemental demographic non-empty fields are preserved in normalized species_slaughtered without inferring absent species", "state": "mapped_by_source_vocabulary"}, + "processing_categories": {"fields": ["processing", "meat_processing", "poultry_processing", "egg_processing", "*_processing"], "current_evidence": "Supplemental demographic non-empty processing fields map to processing_activities with RTE/NRTE/raw-intact/raw-non-intact, species, cell-cultured, and exemption distinctions retained", "state": "mapped_by_source_vocabulary"}, + "inspection_systems": {"fields": ["inspection_system_*"], "current_evidence": "Non-empty inspection-system fields map separately to inspection_attributes; no inspection system is collapsed into a quality claim", "state": "mapped_separately"}, + "derived_legacy_categories": {"fields": ["slaughter_or_processing_only", "slaughter_only_class", "slaughter_only_species", "meat_slaughter_only_species", "poultry_slaughter_only_species", "slaughter_volume_category", "processing_volume_category"], "current_evidence": "Not copied as current source facts. Recompute only from a reviewed current artifact with an explicit derivation version; otherwise keep retired/unresolved", "state": "retired_until_reviewed_derivation"} }, + "adapter_contract": {"source_package": "pipeline/sources/us/fsis", "join_strategy": "exact_source_native_establishment_id_or_number_only", "fuzzy_matching": false, "geocoding": "disabled", "publication": "private_test_only_blocked"}, "legacy_aggregate_observations": {"slaughter_flag_presence": {"slaughter": 1340, "meat_slaughter": 1061, "poultry_slaughter": 320}, "processing_flag_presence": {"processing": 5755, "meat_processing": 4981, "poultry_processing": 3142, "egg_processing": 96}, "grant_flag_presence": {"active_meat_grant": 5550, "active_poultry_grant": 4462}, "warning": "presence counts overlap and are not mutually exclusive facility totals"}, "other_v1_surfaces": {"static_data/us/aphis_data_final.csv": "APHIS annual-use observations; not FSIS and not a laboratory census", "static_data/us/inspection_reports.csv": "APHIS registration/license-shaped legacy surface; current inspection exports remain separate observations", "policy": "No cross-source identity merge without an explicit reviewed identity/link event"} } diff --git a/docs/country-recon-us.md b/docs/country-recon-us.md index a8e7a98..756d3bf 100644 --- a/docs/country-recon-us.md +++ b/docs/country-recon-us.md @@ -41,7 +41,7 @@ No safe bounded private fetch was performed, so current hashes/bytes and determi ## 2026-09-15 recovery slice -The private implementation is in `pipeline/sources/us/`. FSIS now has a profile-aware adapter and refresh command with a sanctioned operator-assisted capture contract. APHIS now has one adapter with explicit `registrations`, `annual_reports`, and `inspections` profiles. All three APHIS populations remain observations, not a laboratory or facility master, and no identity merge with FSIS is performed. +The private implementation is in `pipeline/sources/us/`. FSIS now has a bundle adapter and refresh command with a sanctioned operator-assisted capture contract: one directory export plus the supplemental demographic export are reconciled only by exact source-native IDs/numbers, and source-provided coordinates, slaughter species/activity fields, processing fields, size, and inspection attributes remain private pending review. APHIS now has one adapter with explicit `registrations`, `annual_reports`, and `inspections` profiles. All three APHIS populations remain observations, not a laboratory or facility master, and no identity merge with FSIS is performed. The row-free V1 inventory and field/category crosswalk is [`docs/countries/us/v1-field-crosswalk.json`](countries/us/v1-field-crosswalk.json). It records 7,101 rows and 269 columns, maps identity/location/contact/administrative/slaughter/processing/inspection-system/derived fields, and records overlapping legacy field-presence counts. Since no authorized current FSIS artifact was available, current-versus-V1 reconciliation remains blocked; the existing exact-key crosswalk reports `not_observed`, never closure. diff --git a/pipeline/common/acquisition.py b/pipeline/common/acquisition.py index 77e8ac6..2b35813 100644 --- a/pipeline/common/acquisition.py +++ b/pipeline/common/acquisition.py @@ -18,7 +18,7 @@ import uuid from datetime import datetime, timezone from pathlib import Path -from typing import Any, Iterable +from typing import Any, Callable, Iterable class AcquisitionError(ValueError): @@ -137,6 +137,8 @@ def fetch_source( retry_delay_seconds: float = 0.0, max_retry_delay_seconds: float = 30.0, sleep_fn: Any = time.sleep, + query_context: dict[str, Any] | None = None, + artifact_validator: Callable[[Path, dict[str, str]], None] | None = None, ) -> dict[str, Any]: if not source_id or not url: raise AcquisitionError("source_id and url are required") @@ -182,6 +184,24 @@ def fetch_source( sha256, byte_size = archive_stream(response, download_path, max_bytes=max_bytes) headers = selected_headers(response.headers) final_url = response.geturl() + if headers.get("Content-Length") is not None: + try: + expected_size = int(headers["Content-Length"]) + except ValueError as error: + raise AcquisitionError( + "source returned an invalid Content-Length header", + failure_class="content-length", + action="inspect the private response evidence before retrying", + ) from error + if expected_size != byte_size: + raise AcquisitionError( + f"source Content-Length={expected_size} but received {byte_size} bytes", + failure_class="truncated-response", + retryable=True, + action="retry a bounded incomplete response; use the assisted capture route if it persists", + ) + if artifact_validator is not None: + artifact_validator(download_path, headers) if artifact_path.exists(): if artifact_path.read_bytes() != download_path.read_bytes(): raise AcquisitionError("existing run artifact differs from newly acquired bytes", failure_class="artifact-collision", action="use a new run_id and preserve both observations") @@ -201,27 +221,27 @@ def fetch_source( f"source returned HTTP {error.code}", failure_class=details["failure_class"], retryable=retryable, action="retry a bounded server/rate-limit failure" if retryable else "verify URL, authorization, and terms before another run", ) - _write_failure(run_dir, source_id, run_id, failure, attempts) + _write_failure(run_dir, source_id, run_id, failure, attempts, query_context, url, requested_at, effective_date, publication_date) raise failure from error except urllib.error.URLError as error: details = {"attempt": attempt_number, "outcome": "failed", "failure_class": "network", "retryable": True, "message": str(error)} attempts.append(details) if attempt_number == max_attempts: failure = AcquisitionError(f"network error: {error.reason}", failure_class="network", retryable=True, action="retry within the source bound; verify connectivity if it persists") - _write_failure(run_dir, source_id, run_id, failure, attempts) + _write_failure(run_dir, source_id, run_id, failure, attempts, query_context, url, requested_at, effective_date, publication_date) raise failure from error except (TimeoutError, socket.timeout) as error: details = {"attempt": attempt_number, "outcome": "failed", "failure_class": "timeout", "retryable": True, "message": str(error)} attempts.append(details) if attempt_number == max_attempts: failure = AcquisitionError(f"timeout: {error}", failure_class="timeout", retryable=True, action="retry within the source bound; use the manual capture route if it persists") - _write_failure(run_dir, source_id, run_id, failure, attempts) + _write_failure(run_dir, source_id, run_id, failure, attempts, query_context, url, requested_at, effective_date, publication_date) raise failure from error except AcquisitionError as error: download_path.unlink(missing_ok=True) attempts.append({"attempt": attempt_number, "outcome": "failed", "failure_class": error.failure_class, "retryable": error.retryable, "message": str(error)}) if not error.retryable or attempt_number == max_attempts: - _write_failure(run_dir, source_id, run_id, error, attempts) + _write_failure(run_dir, source_id, run_id, error, attempts, query_context, url, requested_at, effective_date, publication_date) raise if attempt_number < max_attempts: delay = min(max_retry_delay_seconds, retry_delay_seconds * (2 ** (attempt_number - 1))) @@ -257,11 +277,24 @@ def fetch_source( "artifact_state": artifact_state, "retention": {"class": "restricted-research-evidence", "public_exposure": False, "review_required": True}, } + if query_context is not None: + metadata["query_context"] = query_context _atomic_bytes(run_dir / "acquisition-metadata.json", (json.dumps(metadata, ensure_ascii=False, sort_keys=True, indent=2) + "\n").encode()) return metadata -def _write_failure(run_dir: Path, source_id: str, run_id: str, error: AcquisitionError, attempts: list[dict[str, Any]]) -> None: +def _write_failure( + run_dir: Path, + source_id: str, + run_id: str, + error: AcquisitionError, + attempts: list[dict[str, Any]], + query_context: dict[str, Any] | None = None, + requested_url: str | None = None, + requested_at_utc: str | None = None, + effective_date: str | None = None, + publication_date: str | None = None, +) -> None: """Leave a private, actionable failure record without creating an artifact.""" payload = { "schema_version": "acquisition-failure-v1", "source_id": source_id, "run_id": run_id, @@ -269,6 +302,16 @@ def _write_failure(run_dir: Path, source_id: str, run_id: str, error: Acquisitio "action": error.action, "attempts": attempts, "artifact_created": False, "public_exposure": False, } + if requested_url is not None: + payload["requested_url"] = requested_url + if requested_at_utc is not None: + payload["requested_at_utc"] = requested_at_utc + if effective_date is not None: + payload["effective_date"] = effective_date + if publication_date is not None: + payload["publication_date"] = publication_date + if query_context is not None: + payload["query_context"] = query_context try: _atomic_bytes(run_dir / "acquisition-failure.json", (json.dumps(payload, ensure_ascii=False, sort_keys=True, indent=2) + "\n").encode()) except OSError: diff --git a/pipeline/common/test_acquisition.py b/pipeline/common/test_acquisition.py index 4e43d32..e2349dd 100644 --- a/pipeline/common/test_acquisition.py +++ b/pipeline/common/test_acquisition.py @@ -41,7 +41,7 @@ def read(self, _size): def test_fetch_retries_network_failure_and_records_attempts(self): class Response: status = 200 - headers = {"Content-Type": "text/csv", "Content-Length": "7"} + headers = {"Content-Type": "text/csv", "Content-Length": "8"} def __enter__(self): return self diff --git a/pipeline/source-inventory.csv b/pipeline/source-inventory.csv index 8b0580d..4898a76 100644 --- a/pipeline/source-inventory.csv +++ b/pipeline/source-inventory.csv @@ -9,7 +9,7 @@ it.1069-2009,IT,,Italian Ministry of Health 1069/2009 animal by-products establi mx.locations,MX,static_data/mx/locations.csv;Old CSVs/mexico, Mexican official registers and INEGI-derived work,download/API or assisted export,source_candidate,Separate official source records from derived aquaculture research files. nz.locations,NZ,static_data/nz/locations.csv,New Zealand MPI approved premises/registers,download/API,source_candidate,MPI country listings expose identifier, address, processes, species, and expiry fields. uk.locations,GB,static_data/uk/locations.csv;static_data/uk/locations.csv.backup,Food Standards Agency approved food establishments,monthly CSV download,source_candidate,Current FSA publication is split across England/Wales, Northern Ireland, and Scotland. -us.fsis,US,static_data/us/locations.csv,USDA FSIS MPI directory and supplemental establishment-demographic data,operator-assisted official CSV export; bounded direct fetch with terms review,implemented_partial,Current route verified; direct links returned 403, so acquire an authorized edition and preserve provenance/schema/privacy review before test-only handoff. +us.fsis,US,static_data/us/locations.csv,USDA FSIS MPI directory and supplemental establishment-demographic data,operator-assisted official CSV export; bounded direct fetch with terms review,implemented_partial,Current route verified; direct links returned 403, so acquire an authorized edition and preserve per-file provenance, exact-key reconciliation, schema/drift/privacy review before test-only handoff. us.aphis,US,static_data/us/aphis_data_final.csv,USDA APHIS Animal Care Public Search Tool,operator-assisted profile-explicit export,implemented_partial,Registrations, annual reports, inspections, laboratories, and aggregate summaries remain separate evidence types. us.inspections,US,static_data/us/inspection_reports.csv,USDA APHIS inspection-reports public search,operator-assisted inspection export,implemented_partial,Inspection reports are observations, not facility master records; use explicit reviewable identity links only. nl.nvwa.approved-food,NL,,NVWA approved food establishment lists,public control XML plus SOAP POST,source_candidate,Recognition/activity observations retained separately; implement SOAP and review terms/privacy before release. diff --git a/pipeline/source_registry.json b/pipeline/source_registry.json index 7ccfb67..dd9041e 100644 --- a/pipeline/source_registry.json +++ b/pipeline/source_registry.json @@ -216,7 +216,7 @@ "cadence": "weekly replacement", "attribution_licensing_notes": "FSIS authority and directory scope verified; terms/attribution and current export URL must be recorded per run before publication", "adapter_status": "implemented_partial", - "expected_artifact_schema": "FSIS MPI directory CSV; source values preserved privately; supplemental demographic activity flags remain distinct", + "expected_artifact_schema": "FSIS MPI directory CSV plus supplemental establishment-demographic CSV; exact source-native ID/number reconciliation with source-provided coordinates, species slaughter, processing, size, and inspection attributes", "blockers": ["Current direct links returned HTTP 403 during reconnaissance; obtain an authorized current export and complete terms, schema, privacy, and project review."] }, { diff --git a/pipeline/sources/us/accountability/README.md b/pipeline/sources/us/accountability/README.md index 3d00ba2..842a597 100644 --- a/pipeline/sources/us/accountability/README.md +++ b/pipeline/sources/us/accountability/README.md @@ -67,3 +67,29 @@ and `publication_gate=blocked` are invariants. Geocoding is disabled. to FSIS facility candidates. It is row-free and observation-only: a missing current observation is `not-observed`, never closure; it creates no identity, suppression, or publication decision and does not inherit V1 assumptions. + +## Current identity integration + +`current_identity.py` consumes accepted records from the APHIS and FSIS +source-local adapters and emits a private, deterministic crosswalk handoff. +It links: + +* APHIS registrations to annual reports and inspections by certificate and/or + customer number; +* FSIS establishments to FSIS observation records by establishment and/or + approval number. + +Each emitted edge retains both source-record keys, profile-specific artifact +hash/URL/retrieval provenance, the matched identifier types, observation dates, +confidence, review state, and independent private/publication gates. It does +not emit canonical IDs, merge source entities, geocode, or write a database. + +Name/address agreement is limited to a bounded `candidate` with +`match_method=alternate_name_address_exact`, low confidence, and mandatory +review. Name-only, address-only, conflicting, ambiguous, missing-provenance, +and suppressed/restricted matches are quarantined. Alternate matching is +never attempted across APHIS and FSIS source families. + +The checked-in `fixtures/current_identity.json` is synthetic and test-only. +No current real source artifact is committed. Existing V1-derived real rows +remain legacy regression inputs and are not promoted to current evidence. diff --git a/pipeline/sources/us/accountability/current_identity.py b/pipeline/sources/us/accountability/current_identity.py new file mode 100644 index 0000000..3aac069 --- /dev/null +++ b/pipeline/sources/us/accountability/current_identity.py @@ -0,0 +1,590 @@ +"""Private current-US identity integration for APHIS and FSIS observations. + +This module consumes accepted records from the source-local adapters. It +creates source-scoped crosswalk candidates only; it never creates a canonical +identity, writes a graph database, or authorizes publication. + +The two supported link families are deliberately narrow: + +* APHIS registrations -> annual reports and inspections, using APHIS + certificate/customer identifiers. +* FSIS establishments -> FSIS observations, using establishment/approval + identifiers carried by both records. + +Names and addresses are a bounded review queue only. They are never a +fallback identity decision and are not used across APHIS and FSIS source +families. +""" +from __future__ import annotations + +import hashlib +import json +import re +from collections import Counter, defaultdict +from datetime import date, datetime +from pathlib import Path +from typing import Any, Iterable, Mapping + +from pipeline.contracts.source_lifecycle import atomic_json, atomic_jsonl + + +CONTRACT_VERSION = "us-current-identity-graph-v1" +PUBLICATION = { + "storage_state": "private", + "privacy_status": "pending", + "review_state": "review_required", + "publication_status": "not_eligible", + "release_id": None, +} +APHIS_PROFILES = frozenset({"registrations", "annual_reports", "inspections"}) +ALTERNATE_PAIRS = { + ("registrations", "annual_reports"), + ("registrations", "inspections"), + ("fsis_establishments", "fsis_observations"), +} +_HEX64 = re.compile(r"^[0-9a-f]{64}$") + + +class CurrentIdentityContractError(ValueError): + """Input records or provenance cannot satisfy the private contract.""" + + +def _text(value: Any) -> str | None: + if value is None: + return None + value = str(value).strip() + return value or None + + +def _profile(record: Mapping[str, Any], fallback: str | None = None) -> str: + normalized = record.get("normalized") + if isinstance(normalized, Mapping): + value = _text(normalized.get("evidence_type")) + if value: + return value + return _text(record.get("profile")) or fallback or "unknown" + + +def _source_key(record: Mapping[str, Any]) -> str: + value = _text(record.get("source_record_key")) + if not value: + raise CurrentIdentityContractError("record lacks source_record_key") + return value + + +def _normalized(record: Mapping[str, Any]) -> Mapping[str, Any]: + value = record.get("normalized") + if not isinstance(value, Mapping): + raise CurrentIdentityContractError("record lacks normalized evidence") + return value + + +def _identifiers(record: Mapping[str, Any], profile: str) -> dict[str, str]: + """Return source-qualified official identifiers without inventing one.""" + normalized = _normalized(record) + source_id = _text(record.get("source_id")) or "" + if source_id == "us.aphis" or profile in APHIS_PROFILES: + fields = ( + ("aphis_certificate_number", "certificate_number"), + ("aphis_customer_number", "customer_number"), + ) + elif source_id == "us.fsis" or profile == "fsis_establishments": + fields = ( + ("fsis_establishment_id", "establishment_id"), + ("fsis_establishment_number", "establishment_number"), + ) + else: + fields = ( + ("fsis_establishment_id", "establishment_id"), + ("fsis_establishment_number", "establishment_number"), + ("observation_id", "observation_id"), + ) + return {identifier_type: value for identifier_type, field in fields if (value := _text(normalized.get(field)))} + + +def _observation_date(record: Mapping[str, Any]) -> str | None: + normalized = _normalized(record) + for field in ("observed_at", "status_date", "grant_date"): + value = _text(normalized.get(field)) + if value: + return value + year = _text(normalized.get("report_year")) + if year and year.isdigit() and len(year) == 4: + return f"{year}-12-31" + value = _text(record.get("observed_at")) + return value + + +def _valid_datetime(value: str | None) -> bool: + if not value: + return False + try: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError: + return False + return parsed.tzinfo is not None + + +def _provenance( + provenance: Mapping[Any, Mapping[str, Any]], + *, + source_id: str, + profile: str, +) -> dict[str, str] | None: + """Resolve per-profile provenance, falling back to source-wide metadata.""" + values = ( + provenance.get((source_id, profile)) + or provenance.get(f"{source_id}:{profile}") + or provenance.get(source_id) + ) + if not isinstance(values, Mapping): + return None + digest = _text(values.get("artifact_sha256") or values.get("sha256")) + url = _text(values.get("source_url") or values.get("final_url")) + retrieved = _text(values.get("retrieved_at_utc")) + if not digest or not _HEX64.fullmatch(digest.lower()) or not url or not _valid_datetime(retrieved): + return None + return { + "artifact_sha256": digest.lower(), + "source_url": url, + "retrieved_at_utc": retrieved, + } + + +def _suppressed(record: Mapping[str, Any]) -> bool: + normalized = record.get("normalized") + values = [record.get("suppression_state"), record.get("privacy_status")] + if isinstance(normalized, Mapping): + values.extend((normalized.get("privacy_gate"), normalized.get("privacy_status"))) + return any(_text(value) in {"suppressed", "restricted", "failed", "suppressed_or_restricted"} for value in values) + + +def _stable_id(prefix: str, *parts: str) -> str: + material = "\0".join((prefix, *parts)) + return f"{prefix}-{hashlib.sha256(material.encode('utf-8')).hexdigest()[:24]}" + + +def _node(record: Mapping[str, Any], profile: str, provenance: Mapping[str, str] | None) -> dict[str, Any]: + source_id = _text(record.get("source_id")) or "unknown" + source_key = _source_key(record) + return { + "entity_id": _stable_id("us-identity-entity", source_id, source_key), + "entity_type": profile, + "source_id": source_id, + "source_record_key": source_key, + "official_identifiers": _identifiers(record, profile), + "observed_at": _observation_date(record), + "evidence": { + "source_record_key": source_key, + **(provenance or {}), + }, + "publication": dict(PUBLICATION), + "test_only": True, + } + + +def _name_address(record: Mapping[str, Any]) -> tuple[str | None, str | None, list[str]]: + normalized = _normalized(record) + raw = record.get("source_values") + source_values = raw if isinstance(raw, Mapping) else {} + + def value_for(field: str) -> str | None: + direct = _text(normalized.get(field)) + if direct: + return direct + wanted = _norm(field) + for source_field, source_value in source_values.items(): + if _norm(str(source_field)) == wanted: + return _text(source_value) + return None + + name = next( + (value for field in ("canonical_name", "account_name", "name", "facility_name", "establishment_name", "trading_name", "operator_name") if (value := value_for(field))), + None, + ) + address_parts: list[tuple[str, str]] = [] + for label, fields in ( + ("street", ("street", "Address Line 1", "address_line_1")), + ("city", ("city", "City")), + ("state", ("state", "State")), + ("postal_code", ("postal_code", "zip", "Zip")), + ): + value = next((value for field in fields if (value := value_for(field))), None) + if value: + address_parts.append((label, value)) + fields = [label for label, _ in address_parts] + address = "|".join(_norm(value) for _, value in address_parts) if address_parts else None + return (_norm(name) if name else None), address, fields + + +def _norm(value: str) -> str: + return re.sub(r"[^a-z0-9]+", " ", value.casefold()).strip() + + +def _candidate( + left: Mapping[str, Any], + left_profile: str, + right: Mapping[str, Any], + right_profile: str, + *, + method: str, + confidence: float, + matched_identifiers: Mapping[str, str] | None, + matched_fields: list[str] | None, + provenance: Mapping[Any, Mapping[str, Any]], + assertion_status: str = "candidate", + reason: str | None = None, +) -> dict[str, Any]: + left_source = _text(left.get("source_id")) or "unknown" + right_source = _text(right.get("source_id")) or "unknown" + left_key = _source_key(left) + right_key = _source_key(right) + left_provenance = _provenance(provenance, source_id=left_source, profile=left_profile) + right_provenance = _provenance(provenance, source_id=right_source, profile=right_profile) + provenance_ok = left_provenance is not None and right_provenance is not None + evidence = { + "source_record_keys": [left_key, right_key], + "source_profiles": [left_profile, right_profile], + "provenance": { + f"{left_source}:{left_profile}": left_provenance, + f"{right_source}:{right_profile}": right_provenance, + }, + "matched_identifier_types": sorted((matched_identifiers or {}).keys()), + "matched_fields": sorted(matched_fields or []), + } + identity = "|".join((left_source, left_key, right_source, right_key, method)) + result: dict[str, Any] = { + "candidate_id": _stable_id("us-identity-candidate", identity), + "candidate_type": "source_entity_crosswalk", + "relationship_type": "observation_of_registration" if left_profile == "registrations" else "observation_of_establishment", + "left": {"source_id": left_source, "source_record_key": left_key, "profile": left_profile}, + "right": {"source_id": right_source, "source_record_key": right_key, "profile": right_profile}, + "match_method": method, + "matched_identifiers": dict(sorted((matched_identifiers or {}).items())), + "confidence": confidence, + "review_state": "review_required", + "assertion_status": assertion_status, + "observation_dates": [_observation_date(left), _observation_date(right)], + "evidence": evidence, + "publication": dict(PUBLICATION), + "test_only": True, + } + if reason: + result["quarantine_reason"] = reason + result["reason"] = reason + if not provenance_ok and not reason: + result["quarantine_reason"] = "missing_or_invalid_provenance" + result["assertion_status"] = "quarantined" + return result + + +def _pair_key(profile: str, identifier_type: str, value: str, observed_at: str | None) -> tuple[str, str, str, str | None]: + # Multiple annual reports/inspections can legitimately share a certificate + # over time; duplicate observations on the same date are an ambiguity. + return profile, identifier_type, value, observed_at + + +def _records_for(records: Iterable[Mapping[str, Any]], profile: str) -> list[Mapping[str, Any]]: + if isinstance(records, Mapping): + records = records.get("accepted", ()) + return [record for record in records if _profile(record, profile) == profile] + + +def _official_pairs( + left_records: list[Mapping[str, Any]], + left_profile: str, + right_records: list[Mapping[str, Any]], + right_profile: str, +) -> tuple[list[tuple[Mapping[str, Any], Mapping[str, Any], dict[str, str]]], list[dict[str, Any]]]: + right_index: dict[tuple[str, str, str | None], list[Mapping[str, Any]]] = defaultdict(list) + collisions: Counter[tuple[str, str, str | None]] = Counter() + for record in right_records: + for identifier_type, value in _identifiers(record, right_profile).items(): + key = _pair_key(right_profile, identifier_type, value, _observation_date(record)) + right_index[key].append(record) + collisions[key] += 1 + + pairs: list[tuple[Mapping[str, Any], Mapping[str, Any], dict[str, str]]] = [] + quarantined: list[dict[str, Any]] = [] + seen: set[tuple[str, str, str]] = set() + for left in left_records: + left_ids = _identifiers(left, left_profile) + for right in right_records: + right_ids = _identifiers(right, right_profile) + shared = {kind: value for kind, value in left_ids.items() if right_ids.get(kind) == value} + conflicts = [kind for kind in set(left_ids) & set(right_ids) if left_ids[kind] != right_ids[kind]] + if conflicts: + quarantined.append({ + "reason": "conflicting_official_identifiers", + "left_source_record_key": _source_key(left), + "right_source_record_key": _source_key(right), + "identifier_types": sorted(conflicts), + "_left_record": left, + "_right_record": right, + }) + continue + if not shared: + continue + identity_key = (_source_key(left), _source_key(right), ",".join(sorted(shared))) + if identity_key in seen: + continue + seen.add(identity_key) + collision = any( + collisions[_pair_key(right_profile, kind, value, _observation_date(right))] > 1 + for kind, value in shared.items() + ) + if collision: + quarantined.append({ + "reason": "ambiguous_official_identifier", + "left_source_record_key": _source_key(left), + "right_source_record_key": _source_key(right), + "identifier_types": sorted(shared), + "_left_record": left, + "_right_record": right, + }) + else: + pairs.append((left, right, shared)) + return pairs, quarantined + + +def _alternate_pairs( + left_records: list[Mapping[str, Any]], + right_records: list[Mapping[str, Any]], +) -> tuple[list[tuple[Mapping[str, Any], Mapping[str, Any], list[str]]], list[dict[str, Any]]]: + left_index: dict[tuple[str, str], list[tuple[Mapping[str, Any], list[str]]]] = defaultdict(list) + right_index: dict[tuple[str, str], list[tuple[Mapping[str, Any], list[str]]]] = defaultdict(list) + for record in left_records: + name, address, fields = _name_address(record) + if name and address: + left_index[(name, address)].append((record, fields)) + for record in right_records: + name, address, fields = _name_address(record) + if name and address: + right_index[(name, address)].append((record, fields)) + + pairs: list[tuple[Mapping[str, Any], Mapping[str, Any], list[str]]] = [] + quarantined: list[dict[str, Any]] = [] + for key in sorted(set(left_index) & set(right_index)): + lefts = left_index[key] + rights = right_index[key] + if len(lefts) != 1 or len(rights) != 1: + for left, _ in lefts: + for right, _ in rights: + quarantined.append({ + "reason": "ambiguous_alternate_name_address", + "left_source_record_key": _source_key(left), + "right_source_record_key": _source_key(right), + "matched_fields": ["name", "address"], + "_left_record": left, + "_right_record": right, + }) + continue + left, left_fields = lefts[0] + right, right_fields = rights[0] + pairs.append((left, right, sorted(set(left_fields + right_fields)))) + return pairs, quarantined + + +def _quarantine_candidate( + left: Mapping[str, Any], + left_profile: str, + right: Mapping[str, Any], + right_profile: str, + *, + reason: str, + method: str = "unresolved", + provenance: Mapping[Any, Mapping[str, Any]], + matched_fields: list[str] | None = None, +) -> dict[str, Any]: + return _candidate( + left, + left_profile, + right, + right_profile, + method=method, + confidence=0.0, + matched_identifiers=None, + matched_fields=matched_fields, + provenance=provenance, + assertion_status="quarantined", + reason=reason, + ) + + +def build_current_identity_graph( + *, + aphis_records: Mapping[str, Iterable[Mapping[str, Any]]], + fsis_records: Iterable[Mapping[str, Any]], + fsis_observations: Iterable[Mapping[str, Any]] = (), + provenance: Mapping[Any, Mapping[str, Any]], +) -> dict[str, Any]: + """Build deterministic private crosswalk candidates from adapter records. + + ``aphis_records`` is keyed by ``registrations``, ``annual_reports``, and + ``inspections``. The values are the accepted record objects returned by + ``AphisPublicSearchAdapter.parse_bytes``. ``fsis_records`` contains the + accepted FSIS establishment objects; observations use the same record + shape and should expose ``establishment_id`` or ``establishment_number`` + in ``normalized``. + """ + registrations = _records_for(aphis_records.get("registrations", ()), "registrations") + annual_reports = _records_for(aphis_records.get("annual_reports", ()), "annual_reports") + inspections = _records_for(aphis_records.get("inspections", ()), "inspections") + establishments = _records_for(fsis_records, "fsis_establishments") + observations = _records_for(fsis_observations, "fsis_observations") + + all_records: list[tuple[Mapping[str, Any], str]] = [] + all_records.extend((record, "registrations") for record in registrations) + all_records.extend((record, "annual_reports") for record in annual_reports) + all_records.extend((record, "inspections") for record in inspections) + all_records.extend((record, "fsis_establishments") for record in establishments) + all_records.extend((record, "fsis_observations") for record in observations) + nodes = [] + for record, profile in all_records: + nodes.append(_node(record, profile, _provenance(provenance, source_id=_text(record.get("source_id")) or "unknown", profile=profile))) + nodes.sort(key=lambda node: node["entity_id"]) + + candidates: list[dict[str, Any]] = [] + quarantined: list[dict[str, Any]] = [] + + for left_profile, right_profile, left, right in ( + ("registrations", "annual_reports", registrations, annual_reports), + ("registrations", "inspections", registrations, inspections), + ("fsis_establishments", "fsis_observations", establishments, observations), + ): + pairs, pair_quarantine = _official_pairs(left, left_profile, right, right_profile) + conflicting_pairs = { + (_text(item.get("left_source_record_key")), _text(item.get("right_source_record_key"))) + for item in pair_quarantine + if item.get("reason") == "conflicting_official_identifiers" + } + for item in pair_quarantine: + quarantined.append( + _quarantine_candidate( + item.pop("_left_record"), + left_profile, + item.pop("_right_record"), + right_profile, + reason=item["reason"], + method="exact_official_identifier", + provenance=provenance, + ) + ) + exact_keys = set() + for left_record, right_record, identifiers in pairs: + exact_keys.add((_source_key(left_record), _source_key(right_record))) + if _suppressed(left_record) or _suppressed(right_record): + quarantined.append(_quarantine_candidate(left_record, left_profile, right_record, right_profile, reason="suppressed_or_restricted", method="exact_official_identifier", provenance=provenance)) + continue + candidate = _candidate( + left_record, + left_profile, + right_record, + right_profile, + method="exact_official_identifier", + confidence=1.0 if len(identifiers) > 1 else 0.9, + matched_identifiers=identifiers, + matched_fields=None, + provenance=provenance, + ) + (quarantined if candidate["assertion_status"] == "quarantined" else candidates).append(candidate) + + alternate, alternate_quarantine = _alternate_pairs(left, right) + for item in alternate_quarantine: + quarantined.append( + _quarantine_candidate( + item.pop("_left_record"), + left_profile, + item.pop("_right_record"), + right_profile, + reason=item["reason"], + method="alternate_name_address_exact", + provenance=provenance, + matched_fields=item.get("matched_fields"), + ) + ) + for left_record, right_record, fields in alternate: + key = (_source_key(left_record), _source_key(right_record)) + if key in exact_keys or key in conflicting_pairs: + continue + reason = "suppressed_or_restricted" if _suppressed(left_record) or _suppressed(right_record) else None + candidate = ( + _quarantine_candidate(left_record, left_profile, right_record, right_profile, reason=reason, method="alternate_name_address_exact", provenance=provenance, matched_fields=fields) + if reason + else _candidate(left_record, left_profile, right_record, right_profile, method="alternate_name_address_exact", confidence=0.45, matched_identifiers=None, matched_fields=fields, provenance=provenance) + ) + (quarantined if candidate["assertion_status"] == "quarantined" else candidates).append(candidate) + + # A record with neither an official key nor a complete bounded + # alternate key must remain visible to review, but cannot become an + # identity edge. + for left_record in left: + if not _identifiers(left_record, left_profile): + for right_record in right: + if not _identifiers(right_record, right_profile) and _source_key(left_record) != _source_key(right_record): + if not _name_address(left_record)[0] or not _name_address(left_record)[1] or not _name_address(right_record)[0] or not _name_address(right_record)[1]: + quarantined.append( + _quarantine_candidate( + left_record, + left_profile, + right_record, + right_profile, + reason="missing_official_identifier", + provenance=provenance, + ) + ) + + for item in quarantined: + item.pop("_left_record", None) + item.pop("_right_record", None) + item.setdefault("review_state", "review_required") + item.setdefault("assertion_status", "quarantined") + item.setdefault("publication", dict(PUBLICATION)) + item["test_only"] = True + candidates.sort(key=lambda item: item["candidate_id"]) + quarantined.sort(key=lambda item: (_text(item.get("left_source_record_key")) or _text(item.get("left", {}).get("source_record_key")) or "", _text(item.get("right_source_record_key")) or _text(item.get("right", {}).get("source_record_key")) or "", _text(item.get("reason")) or _text(item.get("quarantine_reason")) or "")) + source_hashes = {} + for value in provenance.values(): + if isinstance(value, Mapping): + digest = _text(value.get("artifact_sha256") or value.get("sha256")) + if digest: + source_hashes[digest] = digest + manifest = { + "schema_version": CONTRACT_VERSION, + "candidate_count": len(candidates), + "quarantined_count": len(quarantined), + "candidate_relationship_types": dict(sorted(Counter(item["relationship_type"] for item in candidates).items())), + "match_methods": dict(sorted(Counter(item["match_method"] for item in candidates).items())), + "source_artifact_sha256": sorted(source_hashes), + "storage_state": "private", + "privacy_status": "pending", + "review_state": "review_required", + "publication_status": "not_eligible", + "release_id": None, + "auto_merge": False, + "test_only": True, + } + return {"schema_version": CONTRACT_VERSION, "entities": nodes, "candidates": candidates, "quarantined": quarantined, "manifest": manifest} + + +def write_current_identity_graph(run_dir: str | Path, graph: Mapping[str, Any]) -> dict[str, Any]: + """Atomically write the row-private candidate handoff and manifest.""" + root = Path(run_dir) + candidates = list(graph.get("candidates", ())) + entities = list(graph.get("entities", ())) + quarantined = list(graph.get("quarantined", ())) + _, candidate_sha, _ = atomic_jsonl(root / "candidate" / "identity-links.jsonl", candidates) + _, entity_sha, _ = atomic_jsonl(root / "candidate" / "entities.jsonl", entities) + _, quarantine_sha, _ = atomic_jsonl(root / "quarantined" / "identity-links.jsonl", quarantined) + manifest = dict(graph["manifest"]) + manifest.update({ + "candidate_sha256": candidate_sha, + "entity_sha256": entity_sha, + "quarantine_sha256": quarantine_sha, + "candidate_rows": len(candidates), + "entity_rows": len(entities), + "quarantine_rows": len(quarantined), + "idempotency_key": hashlib.sha256((candidate_sha + entity_sha + quarantine_sha).encode()).hexdigest(), + }) + atomic_json(root / "identity-graph-manifest.json", manifest) + return manifest diff --git a/pipeline/sources/us/accountability/fixtures/current_identity.json b/pipeline/sources/us/accountability/fixtures/current_identity.json new file mode 100644 index 0000000..afeff69 --- /dev/null +++ b/pipeline/sources/us/accountability/fixtures/current_identity.json @@ -0,0 +1,46 @@ +{ + "provenance": { + "us.aphis:registrations": { + "artifact_sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "source_url": "https://example.invalid/aphis/registrations.csv", + "retrieved_at_utc": "2026-09-15T00:00:00Z" + }, + "us.aphis:annual_reports": { + "artifact_sha256": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "source_url": "https://example.invalid/aphis/annual-reports.csv", + "retrieved_at_utc": "2026-09-15T00:00:00Z" + }, + "us.aphis:inspections": { + "artifact_sha256": "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", + "source_url": "https://example.invalid/aphis/inspections.csv", + "retrieved_at_utc": "2026-09-15T00:00:00Z" + }, + "us.fsis": { + "artifact_sha256": "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", + "source_url": "https://example.invalid/fsis/establishments.csv", + "retrieved_at_utc": "2026-09-15T00:00:00Z" + }, + "us.fsis.observations": { + "artifact_sha256": "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", + "source_url": "https://example.invalid/fsis/observations.csv", + "retrieved_at_utc": "2026-09-15T00:00:00Z" + } + }, + "aphis": { + "registrations": [ + {"source_id": "us.aphis", "source_record_key": "registrations:00-B-TEST-001", "normalized": {"evidence_type": "registrations", "certificate_number": "00-B-TEST-001", "customer_number": "9001", "account_name": "Synthetic Shared Operator", "status_date": "2026-01-01"}, "source_values": {"Account Name": "Synthetic Shared Operator", "Address Line 1": "1 Test Way", "City": "Testville", "State": "TX", "Zip": "75001"}} + ], + "annual_reports": [ + {"source_id": "us.aphis", "source_record_key": "annual_reports:00-B-TEST-001:2025", "normalized": {"evidence_type": "annual_reports", "certificate_number": "00-B-TEST-001", "customer_number": "9001", "account_name": "Synthetic Shared Operator", "report_year": "2025"}, "source_values": {"Account Name": "Synthetic Shared Operator", "Address Line 1": "1 Test Way", "City": "Testville", "State": "TX", "Zip": "75001"}} + ], + "inspections": [ + {"source_id": "us.aphis", "source_record_key": "inspections:00-B-TEST-001:2026-02-01", "normalized": {"evidence_type": "inspections", "certificate_number": "00-B-TEST-001", "customer_number": "9001", "account_name": "Synthetic Shared Operator", "status_date": "2026-02-01"}, "source_values": {"Account Name": "Synthetic Shared Operator", "Address Line 1": "1 Test Way", "City": "Testville", "State": "TX", "Zip": "75001"}} + ] + }, + "fsis": [ + {"source_id": "us.fsis", "source_record_key": "FSIS-TEST-001", "normalized": {"evidence_type": "fsis_establishments", "establishment_id": "FSIS-TEST-001", "establishment_number": "M-TEST-001", "canonical_name": "Synthetic Shared Plant", "grant_date": "2026-01-01"}, "source_values": {"establishment_name": "Synthetic Shared Plant", "street": "2 Test Way", "city": "Testville", "state": "TX", "zip": "75001"}} + ], + "fsis_observations": [ + {"source_id": "us.fsis.observations", "source_record_key": "observation:FSIS-TEST-001:2026-02-01", "normalized": {"evidence_type": "fsis_observations", "establishment_id": "FSIS-TEST-001", "observation_id": "OBS-TEST-001", "observed_at": "2026-02-01", "name": "Synthetic Shared Plant"}, "source_values": {"name": "Synthetic Shared Plant", "street": "2 Test Way", "city": "Testville", "state": "TX", "zip": "75001"}} + ] +} diff --git a/pipeline/sources/us/accountability/test_current_identity.py b/pipeline/sources/us/accountability/test_current_identity.py new file mode 100644 index 0000000..6083552 --- /dev/null +++ b/pipeline/sources/us/accountability/test_current_identity.py @@ -0,0 +1,116 @@ +import copy +import json +import tempfile +import unittest +from pathlib import Path + +from .current_identity import build_current_identity_graph, write_current_identity_graph + + +ROOT = Path(__file__).parent + + +def load_fixture(): + return json.loads((ROOT / "fixtures" / "current_identity.json").read_text(encoding="utf-8")) + + +class CurrentIdentityGraphTests(unittest.TestCase): + def build(self, payload=None): + payload = payload or load_fixture() + return build_current_identity_graph( + aphis_records=payload["aphis"], + fsis_records=payload["fsis"], + fsis_observations=payload["fsis_observations"], + provenance=payload["provenance"], + ) + + def test_exact_official_ids_link_each_observation_family(self): + graph = self.build() + self.assertEqual(len(graph["candidates"]), 3) + self.assertEqual({candidate["match_method"] for candidate in graph["candidates"]}, {"exact_official_identifier"}) + self.assertEqual({candidate["relationship_type"] for candidate in graph["candidates"]}, {"observation_of_registration", "observation_of_establishment"}) + for candidate in graph["candidates"]: + self.assertEqual(candidate["publication"]["publication_status"], "not_eligible") + self.assertEqual(candidate["review_state"], "review_required") + self.assertTrue(candidate["evidence"]["provenance"]) + aphis_link = next(candidate for candidate in graph["candidates"] if candidate["right"]["profile"] == "annual_reports") + self.assertEqual(set(aphis_link["evidence"]["provenance"]), {"us.aphis:registrations", "us.aphis:annual_reports"}) + self.assertEqual(len(graph["entities"]), 5) + + def test_same_name_different_address_is_not_a_join(self): + payload = load_fixture() + payload["aphis"]["annual_reports"][0]["normalized"]["certificate_number"] = "" + payload["aphis"]["annual_reports"][0]["normalized"]["customer_number"] = "" + payload["aphis"]["annual_reports"][0]["source_values"]["Address Line 1"] = "99 Other Way" + graph = self.build(payload) + self.assertEqual(len([c for c in graph["candidates"] if c["right"]["profile"] == "annual_reports"]), 0) + + def test_conflicting_official_ids_block_name_address_rescue(self): + payload = load_fixture() + report = payload["aphis"]["annual_reports"][0] + report["normalized"]["certificate_number"] = "00-R-CONFLICT" + report["normalized"]["customer_number"] = "9999" + graph = self.build(payload) + self.assertFalse(any(candidate["match_method"] == "alternate_name_address_exact" for candidate in graph["candidates"])) + self.assertTrue(any(item["reason"] == "conflicting_official_identifiers" for item in graph["quarantined"])) + + def test_ambiguous_alternate_match_is_quarantined(self): + payload = load_fixture() + duplicate = copy.deepcopy(payload["aphis"]["annual_reports"][0]) + duplicate["source_record_key"] = "annual_reports:00-B-TEST-001:2024" + duplicate["normalized"]["certificate_number"] = "" + duplicate["normalized"]["customer_number"] = "" + payload["aphis"]["annual_reports"].append(duplicate) + payload["aphis"]["annual_reports"][0]["normalized"]["certificate_number"] = "" + payload["aphis"]["annual_reports"][0]["normalized"]["customer_number"] = "" + graph = self.build(payload) + self.assertTrue(any(item["reason"] == "ambiguous_alternate_name_address" for item in graph["quarantined"])) + self.assertFalse(any(candidate["match_method"] == "alternate_name_address_exact" for candidate in graph["candidates"])) + + def test_unique_name_and_address_is_only_a_review_candidate(self): + payload = load_fixture() + report = payload["aphis"]["annual_reports"][0] + report["normalized"]["certificate_number"] = "" + report["normalized"]["customer_number"] = "" + graph = self.build(payload) + alternate = [candidate for candidate in graph["candidates"] if candidate["match_method"] == "alternate_name_address_exact"] + self.assertEqual(len(alternate), 1) + self.assertEqual(alternate[0]["confidence"], 0.45) + self.assertEqual(alternate[0]["review_state"], "review_required") + self.assertEqual(alternate[0]["assertion_status"], "candidate") + + def test_missing_provenance_is_quarantined_without_raw_payload(self): + payload = load_fixture() + del payload["provenance"]["us.fsis.observations"] + graph = self.build(payload) + candidates = [c for c in graph["quarantined"] if c["right"]["profile"] == "fsis_observations"] + self.assertEqual(len(candidates), 1) + self.assertEqual(candidates[0]["assertion_status"], "quarantined") + self.assertEqual(candidates[0]["quarantine_reason"], "missing_or_invalid_provenance") + self.assertNotIn("source_values", json.dumps(candidates[0])) + + def test_suppressed_records_never_enter_accepted_candidates(self): + payload = load_fixture() + payload["aphis"]["inspections"][0]["normalized"]["privacy_gate"] = "suppressed" + graph = self.build(payload) + inspection = [c for c in graph["quarantined"] if c["right"]["profile"] == "inspections"][0] + self.assertEqual(inspection["assertion_status"], "quarantined") + self.assertEqual(inspection["quarantine_reason"], "suppressed_or_restricted") + + def test_write_is_idempotent_and_private(self): + graph = self.build() + with tempfile.TemporaryDirectory() as directory: + first = write_current_identity_graph(Path(directory) / "one", graph) + second = write_current_identity_graph(Path(directory) / "two", graph) + self.assertEqual(first["idempotency_key"], second["idempotency_key"]) + self.assertEqual(first["candidate_sha256"], second["candidate_sha256"]) + self.assertEqual(first["storage_state"], "private") + self.assertFalse(first["publication_status"] == "released") + self.assertEqual( + (Path(directory) / "one" / "candidate" / "identity-links.jsonl").read_bytes(), + (Path(directory) / "two" / "candidate" / "identity-links.jsonl").read_bytes(), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/sources/us/aphis/acquire.py b/pipeline/sources/us/aphis/acquire.py new file mode 100644 index 0000000..5e059b1 --- /dev/null +++ b/pipeline/sources/us/aphis/acquire.py @@ -0,0 +1,318 @@ +"""Bounded private acquisition for documented APHIS downloads. + +APHIS is an interactive public-search service. This module permits a direct +fetch only when an operator supplies a terms-reviewed, documented download +URL. The normal supported path remains saving the export or linked report in +an authorized browser and passing it to ``refresh --raw``. It never probes +hidden endpoints, paginates a UI, or bypasses a challenge or access control. +""" +from __future__ import annotations + +import csv +import hashlib +import json +from pathlib import Path +from typing import Any + +from pipeline.common.acquisition import AcquisitionError, fetch_source +from pipeline.contracts.source_lifecycle import atomic_json + +from .adapter import CONFIG + + +CSV_PROFILES = {"registrations", "annual_reports", "inspections"} +DOCUMENT_PROFILES = {"documents"} +ALL_PROFILES = CSV_PROFILES | DOCUMENT_PROFILES +CHALLENGE_MARKERS = ( + b" str: + if profile not in ALL_PROFILES: + raise ValueError(f"unsupported APHIS profile: {profile}") + return { + "registrations": CONFIG["public_search_url"], + "annual_reports": CONFIG["annual_reports_url"], + "inspections": CONFIG["inspection_reports_url"], + "documents": CONFIG["documents_url"], + }[profile] + + +def _csv_download_is_valid(path: Path) -> None: + raw = path.read_bytes() + if not raw: + raise AcquisitionError("APHIS export is empty", failure_class="empty-response") + if any(marker in raw[:65536].lower() for marker in CHALLENGE_MARKERS): + raise AcquisitionError( + "APHIS export appears to be HTML, a login page, or a challenge response", + failure_class="challenge-response", + action="use the documented browser export workflow; do not bypass the challenge", + ) + try: + rows = list(csv.reader(raw.decode("utf-8-sig").splitlines(), strict=True)) + except (UnicodeDecodeError, csv.Error) as error: + raise AcquisitionError( + "APHIS export is malformed or truncated", + failure_class="malformed-export", + action="repeat the documented export and inspect the saved file before retrying", + ) from error + if len(rows) < 2 or not rows[0] or not any(cell.strip() for cell in rows[0]): + raise AcquisitionError( + "APHIS export contains no data rows", + failure_class="empty-response", + action="check the selected profile and query in the authorized browser export", + ) + width = len(rows[0]) + if any(len(row) != width for row in rows[1:]): + raise AcquisitionError( + "APHIS export has inconsistent row widths", + failure_class="truncated-response", + action="repeat the documented export and retain the prior validated artifact", + ) + + +def _document_is_valid(path: Path) -> None: + raw = path.read_bytes() + if not raw: + raise AcquisitionError("APHIS document is empty", failure_class="empty-response") + head = raw[:65536].lower() + if any(marker in head for marker in CHALLENGE_MARKERS): + raise AcquisitionError( + "APHIS document appears to be HTML, a login page, or a challenge response", + failure_class="challenge-response", + action="use the documented browser download workflow; do not bypass the challenge", + ) + is_pdf = raw.startswith(b"%PDF-") and b"%%EOF" in raw[-8192:] + is_zip_container = raw.startswith(b"PK\x03\x04") + if not (is_pdf or is_zip_container): + raise AcquisitionError( + "APHIS document has no recognized PDF or Office-container signature", + failure_class="invalid-document", + action="retain the failed response for diagnosis and use the documented download control", + ) + + +def validate_download(profile: str, path: Path, headers: dict[str, str] | None = None) -> None: + """Validate bytes after download and before the artifact is committed.""" + if profile in CSV_PROFILES: + _csv_download_is_valid(path) + elif profile in DOCUMENT_PROFILES: + _document_is_valid(path) + else: + raise ValueError(f"unsupported APHIS profile: {profile}") + + +def _artifact_name(profile: str, requested_name: str | None) -> str: + if requested_name: + name = Path(requested_name).name + if name != requested_name or not name: + raise ValueError("artifact name must be a simple filename") + return name + return "source.pdf" if profile == "documents" else "source.csv" + + +def _write_manifest(metadata: dict[str, Any], profile: str) -> dict[str, Any]: + artifact_path = Path(metadata["artifact_path"]) + manifest = { + "manifest_version": "us-aphis-acquisition-v1", + "source_id": CONFIG["source_id"], + "profile": profile, + "artifact": metadata["artifact"], + "artifact_path": str(artifact_path), + "source_url": metadata["final_url"], + "requested_url": metadata["requested_url"], + "retrieved_at_utc": metadata["retrieved_at_utc"], + "effective_date": metadata.get("effective_date") or "unknown", + "publication_date": metadata.get("publication_date"), + "sha256": metadata["sha256"], + "byte_size": metadata["byte_size"], + "query_context": metadata.get("query_context") or {}, + "adapter_version": CONFIG["adapter_version"], + "config_version": CONFIG["contract_version"], + "release_state": "not-created", + "publication_state": "private-research-evidence", + "publication_gate": "blocked", + "retention": metadata["retention"], + "coverage": metadata.get("coverage"), + "document_policy": "documents and amendments remain source evidence; no automatic merge or release", + } + atomic_json(artifact_path.parent / "manifest.json", manifest) + return manifest + + +def fetch_profile( + *, + profile: str, + output_root: str | Path, + terms_review_path: str | Path, + source_url: str | None = None, + run_id: str | None = None, + query_context: dict[str, Any] | None = None, + publication_date: str | None = None, + effective_date: str | None = None, + timeout_seconds: float = 60.0, + max_bytes: int = 128 * 1024 * 1024, + artifact_name: str | None = None, +) -> dict[str, Any]: + if profile not in ALL_PROFILES: + raise ValueError(f"unsupported APHIS profile: {profile}") + context = {**(query_context or {}), "profile": profile} + url = source_url or profile_url(profile) + metadata = fetch_source( + source_id=CONFIG["source_id"], + url=url, + output_root=output_root, + artifact_name=_artifact_name(profile, artifact_name), + terms_review_path=terms_review_path, + run_id=run_id, + timeout_seconds=timeout_seconds, + max_bytes=max_bytes, + allowed_content_types=( + "text/csv", + "application/csv", + "application/pdf", + "application/vnd.ms-excel", + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + "application/octet-stream", + ), + effective_date=effective_date, + publication_date=publication_date, + code_version=CONFIG["adapter_version"], + config_version=CONFIG["contract_version"], + coverage=f"APHIS {profile} documented download only; no facility merge or completeness claim", + rights_caveat="APHIS source terms and attribution require operator review before publication", + privacy_caveat="restricted private staging; names, addresses, documents, and coordinates require review", + query_context=context, + artifact_validator=lambda path, headers: validate_download(profile, path, headers), + ) + metadata["profile"] = profile + atomic_json(Path(metadata["artifact_path"]).parent / "acquisition-metadata.json", metadata) + _write_manifest(metadata, profile) + return metadata + + +def preserve_local_document( + *, + raw_path: str | Path, + run_dir: str | Path, + source_url: str | None = None, + retrieved_at_utc: str | None = None, + effective_date: str | None = None, + publication_date: str | None = None, + query_context: dict[str, Any] | None = None, +) -> dict[str, Any]: + """Register an operator-saved document without copying or parsing it.""" + path = Path(raw_path) + if not path.is_file(): + raise ValueError(f"raw artifact does not exist: {path}") + validate_download("documents", path) + raw = path.read_bytes() + metadata = { + "acquisition_method": "operator_assisted_document_download", + "source_id": CONFIG["source_id"], + "profile": "documents", + "artifact": path.name, + "artifact_path": str(path), + "requested_url": source_url or CONFIG["documents_url"], + "final_url": source_url or CONFIG["documents_url"], + "retrieved_at_utc": retrieved_at_utc, + "effective_date": effective_date or "unknown", + "publication_date": publication_date, + "sha256": hashlib.sha256(raw).hexdigest(), + "byte_size": len(raw), + "query_context": {**(query_context or {}), "profile": "documents"}, + "adapter_version": CONFIG["adapter_version"], + "config_version": CONFIG["contract_version"], + "coverage": "APHIS downloadable document/amendment only; source evidence, no automatic row merge", + "rights_caveat": "APHIS source terms and attribution require operator review before publication", + "privacy_caveat": "restricted private staging; document contents require review", + "retention": {"class": "restricted-research-evidence", "public_exposure": False, "review_required": True}, + } + if not retrieved_at_utc: + raise ValueError("retrieved_at_utc is required for a preserved document") + root = Path(run_dir) + root.mkdir(parents=True, exist_ok=True) + atomic_json(root / "acquisition-metadata.json", metadata) + return _write_manifest(metadata, "documents") + + +def parse_query_context(value: str | None) -> dict[str, Any]: + if not value: + return {} + path = Path(value) + try: + payload = json.loads(path.read_text(encoding="utf-8") if path.is_file() else value) + except (OSError, json.JSONDecodeError) as error: + raise ValueError("query context must be JSON or a path to a JSON file") from error + if not isinstance(payload, dict): + raise ValueError("query context must be a JSON object") + return payload + + +def main() -> int: + import argparse + + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--profile", choices=sorted(ALL_PROFILES), required=True) + parser.add_argument("--terms-review", type=Path) + parser.add_argument("--raw", type=Path) + parser.add_argument("--run-dir", type=Path) + parser.add_argument("--output-root", type=Path, default=Path("data/raw")) + parser.add_argument("--run-id") + parser.add_argument("--source-url") + parser.add_argument("--query-context") + parser.add_argument("--publication-date") + parser.add_argument("--effective-date") + parser.add_argument("--retrieved-at-utc") + parser.add_argument("--artifact-name") + parser.add_argument("--timeout-seconds", type=float, default=60.0) + parser.add_argument("--max-bytes", type=int, default=128 * 1024 * 1024) + args = parser.parse_args() + try: + query_context = parse_query_context(args.query_context) + if args.raw: + if args.profile != "documents" or args.run_dir is None: + raise ValueError("--raw requires --profile documents and --run-dir") + metadata = preserve_local_document( + raw_path=args.raw, + run_dir=args.run_dir, + source_url=args.source_url, + retrieved_at_utc=args.retrieved_at_utc, + publication_date=args.publication_date, + effective_date=args.effective_date, + query_context=query_context, + ) + print(json.dumps({"status": "preserved", "manifest": str(Path(args.run_dir) / "manifest.json"), "sha256": metadata["sha256"]}, sort_keys=True)) + return 0 + if args.terms_review is None: + raise ValueError("--terms-review is required for documented network acquisition") + metadata = fetch_profile( + profile=args.profile, + output_root=args.output_root, + terms_review_path=args.terms_review, + source_url=args.source_url, + run_id=args.run_id, + query_context=query_context, + publication_date=args.publication_date, + effective_date=args.effective_date, + timeout_seconds=args.timeout_seconds, + max_bytes=args.max_bytes, + artifact_name=args.artifact_name, + ) + except (AcquisitionError, OSError, ValueError) as error: + print(json.dumps({"status": "failed", "error": str(error)}, sort_keys=True)) + return 2 + print(json.dumps({"status": "acquired", "artifact_path": metadata["artifact_path"], "sha256": metadata["sha256"]}, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/pipeline/sources/us/aphis/adapter.py b/pipeline/sources/us/aphis/adapter.py index 4f35ef0..01c28f6 100644 --- a/pipeline/sources/us/aphis/adapter.py +++ b/pipeline/sources/us/aphis/adapter.py @@ -1,78 +1,210 @@ -"""Private, profile-explicit APHIS Public Search Tool CSV adapter.""" +"""Private, profile-explicit APHIS Public Search Tool CSV adapter. + +APHIS exports are observations, not a facility master. This adapter keeps +registrations, inspections, annual animal-use reports, and amended report +versions source-local and separate. It never geocodes, merges identities, or +turns an APHIS row into an FSIS facility candidate. +""" from __future__ import annotations -import csv, hashlib, json + +import csv +import hashlib +import json from collections import Counter from pathlib import Path from typing import Any + from pipeline.contracts.adapter_contract import SourceArtifact from pipeline.contracts.source_lifecycle import atomic_json, atomic_jsonl, private_manifest + ROOT = Path(__file__).parent CONFIG = json.loads((ROOT / "config.json").read_text(encoding="utf-8")) + PROFILES = { - "registrations": ("Account Name", "Customer Number", "Certificate Number", "License Type", "Certificate Status", "Status Date"), - "annual_reports": ("Account Name", "Customer Number_y", "Certificate Number", "Registration Type", "Certificate Status", "Year"), - "inspections": ("Account Name", "Customer Number", "Certificate Number", "Certificate Status", "Status Date"), + "registrations": ("Account Name", "Certificate Number", "Certificate Status"), + "annual_reports": ("Account Name", "Certificate Number", "Registration Type", "Year"), + "inspections": ("Account Name", "Certificate Number", "Certificate Status"), } +CUSTOMER_COLUMNS = ("Customer Number", "Customer Number_x", "Customer Number_y") +AMENDMENT_COLUMNS = ( + "Amendment Number", "Amendment ID", "Amendment Date", "Amended", + "Amendment", "Report Version", "Version", +) +NON_ANIMAL_COLUMNS = { + "Account Name", "Certificate Number", "Certificate Status", "Status Date", + "Registration Type", "License Type", "Year", *CUSTOMER_COLUMNS, + *AMENDMENT_COLUMNS, "Address Line 1", "Address Line 2", "City-State-Zip", + "County", "City", "State", "Zip", "latitude", "longitude", + "Geocodio Latitude", "Geocodio Longitude", "Exception Report", +} + + class AphisContractError(ValueError): - pass + """The captured artifact is not a supported APHIS profile.""" + def _clean(value: Any) -> str | None: - if value is None: return None - value = str(value).strip() - return value or None + if value is None: + return None + text = str(value).strip() + return text or None + + +def _schema_fingerprint(headers: tuple[str, ...]) -> str: + return hashlib.sha256(json.dumps(headers, separators=(",", ":")).encode()).hexdigest() + def _read(content: bytes) -> tuple[tuple[str, ...], list[dict[str, Any]]]: try: - reader = csv.DictReader(content.decode("utf-8-sig").splitlines(), strict=True) - headers = tuple(reader.fieldnames or ()); rows = list(reader) + text = content.decode("utf-8-sig") + reader = csv.DictReader(text.splitlines(), strict=True) + headers = tuple(reader.fieldnames or ()) + rows = list(reader) except (UnicodeDecodeError, csv.Error) as exc: - raise AphisContractError("malformed or unsupported APHIS CSV") from exc - if not headers or None in headers or len(headers) != len(set(headers)) or any(None in row for row in rows): + raise AphisContractError("malformed or unsupported UTF-8 APHIS CSV") from exc + if ( + not headers + or any(not _clean(header) for header in headers) + or None in headers + or len(headers) != len(set(headers)) + ): + raise AphisContractError("APHIS schema drift or malformed header") + # DictReader represents short rows with None-valued cells and extra + # columns with a None key. Both are schema failures, not missing source + # values: a genuinely blank cell is represented by an empty string. + if any(None in row or any(value is None for value in row.values()) for row in rows): raise AphisContractError("APHIS schema drift or malformed row") return headers, rows + +def _unsupported() -> str: + raise AphisContractError("profile is unsupported") + + def _profile(headers: tuple[str, ...]) -> str: - if "License Type" in headers and set(PROFILES["registrations"]).issubset(headers): - return "registrations" - if "Registration Type" in headers and set(PROFILES["annual_reports"]).issubset(headers): - return "annual_reports" - if set(PROFILES["inspections"]).issubset(headers): + matches = [profile for profile, required in PROFILES.items() if set(required).issubset(headers)] + # License Type and Registration Type are the meaningful discriminator when + # an export happens to contain the common identity/status columns. + if "License Type" in headers and "Registration Type" not in headers: + return "registrations" if "registrations" in matches else _unsupported() + if "Registration Type" in headers: + return "annual_reports" if "annual_reports" in matches else _unsupported() + if "inspections" in matches: return "inspections" - raise AphisContractError("profile is unsupported") + return _unsupported() + + +def _customer_values(row: dict[str, Any]) -> dict[str, str | None]: + return { + "customer_number": _clean(row.get("Customer Number")), + "customer_number_x": _clean(row.get("Customer Number_x")), + "customer_number_y": _clean(row.get("Customer Number_y")), + } + + +def _certificate_or_customer(row: dict[str, Any]) -> str | None: + certificate = _clean(row.get("Certificate Number")) + customers = _customer_values(row) + return certificate or customers["customer_number"] or customers["customer_number_y"] or customers["customer_number_x"] + + +def _year(row: dict[str, Any]) -> str | None: + return _clean(row.get("Year")) + + +def _amendment_version(row: dict[str, Any]) -> str | None: + """Return an explicit amendment/version token, never an inferred one.""" + for column in AMENDMENT_COLUMNS: + value = _clean(row.get(column)) + if value: + return f"{column}={value}" + return None + + +def _is_amendment(row: dict[str, Any]) -> bool: + marker = _amendment_version(row) + if not marker: + return False + # A version/date/number is explicit. Boolean-ish flags only count when + # the source says the row was amended; false remains a base report. + column, value = marker.split("=", 1) + if column in {"Amended", "Amendment"}: + return value.casefold() in {"true", "yes", "y", "1", "amended"} + return True + def _observation_key(profile: str, row: dict[str, Any]) -> str | None: - key = _clean(row.get("Certificate Number")) or _clean(row.get("Customer Number")) or _clean(row.get("Customer Number_y")) - if key and profile == "annual_reports": - return f"{key}:{_clean(row.get('Year'))}" - return key + identity = _certificate_or_customer(row) + if not identity: + return None + customers = _customer_values(row) + parts = [ + f"certificate={_clean(row.get('Certificate Number')) or 'unknown'}", + f"customer={customers['customer_number'] or 'unknown'}", + f"customer_x={customers['customer_number_x'] or 'unknown'}", + f"customer_y={customers['customer_number_y'] or 'unknown'}", + ] + if profile in {"annual_reports", "amendments"}: + parts.append(f"year={_year(row) or 'unknown'}") + parts.append(f"version={_amendment_version(row) or 'original'}") + return f"{profile}|" + "|".join(parts) + + +def _evidence_type(profile: str, row: dict[str, Any]) -> str: + return "amendments" if profile == "annual_reports" and _is_amendment(row) else profile + def _record(profile: str, row: dict[str, Any], line: int) -> dict[str, Any]: certificate = _clean(row.get("Certificate Number")) - customer = _clean(row.get("Customer Number")) or _clean(row.get("Customer Number_y")) - key = certificate or customer + customers = _customer_values(row) + customer = customers["customer_number"] or customers["customer_number_y"] or customers["customer_number_x"] observation_key = _observation_key(profile, row) + evidence_type = _evidence_type(profile, row) + animal_use_fields = tuple(sorted(key for key, value in row.items() if key not in NON_ANIMAL_COLUMNS and _clean(value))) normalized = { + # APHIS identifiers are deliberately source-native. The generic + # establishment_id field remains null so a facility importer cannot + # silently reinterpret this evidence. "establishment_id": None, "source_observation_key": observation_key, "country_code": "US", - "evidence_type": profile, + "evidence_type": evidence_type, + "profile": profile, "account_name": _clean(row.get("Account Name")), "certificate_number": certificate, "customer_number": customer, + "customer_number_x": customers["customer_number_x"], + "customer_number_y": customers["customer_number_y"], "registration_or_license_type": _clean(row.get("Registration Type")) or _clean(row.get("License Type")), "status": _clean(row.get("Certificate Status")), "status_date": _clean(row.get("Status Date")), - "report_year": _clean(row.get("Year")), - "animal_use_fields_present": tuple(sorted(key for key, value in row.items() if key not in {"Account Name", "Customer Number", "Customer Number_y", "Certificate Number", "Registration Type", "License Type", "Certificate Status", "Status Date", "Year", "Address Line 1", "Address Line 2", "City-State-Zip", "County", "City", "State", "Zip", "latitude", "longitude", "Geocodio Latitude", "Geocodio Longitude", "Exception Report"} and _clean(value))), + "report_year": _year(row), + "amendment_version": _amendment_version(row), + "amendment_state": "amended" if evidence_type == "amendments" else "original_or_not_supplied", + "animal_use_fields_present": animal_use_fields, + "awa_coverage_state": "source_profile_only_unknown_completeness", + "awa_coverage_limitations": ( + "APHIS Animal Welfare Act public-search evidence only; not a census of all animal-use activity", + "annual reports describe reported use for the supplied year and may be amended", + "absence is not closure, non-use, or non-coverage", + "registrations, inspections, annual reports, and amendments are separate evidence types", + ), "coordinates": None, "address_state": "source-address-retained-private-pending-review", "privacy_gate": "pending-review", "coordinate_gate": "review_required", "publication_gate": "blocked", } - return {"source_id": CONFIG["source_id"], "source_row": line, "source_record_key": f"{profile}:{observation_key or 'unknown'}", "source_values": {str(k): v for k, v in row.items()}, "normalized": normalized} + return { + "source_id": CONFIG["source_id"], + "source_row": line, + "source_record_key": f"{evidence_type}:{observation_key or 'unknown'}", + "source_values": {str(key): value for key, value in row.items()}, + "normalized": normalized, + } + class AphisPublicSearchAdapter: source_id = CONFIG["source_id"] @@ -80,26 +212,78 @@ class AphisPublicSearchAdapter: schema_version = CONFIG["contract_version"] def parse_bytes(self, content: bytes) -> dict[str, Any]: - digest = hashlib.sha256(content).hexdigest(); headers, rows = _read(content); profile = _profile(headers) - keys = [_observation_key(profile, row) for row in rows] + digest = hashlib.sha256(content).hexdigest() + headers, rows = _read(content) + profile = _profile(headers) + records = [_record(profile, row, line) for line, row in enumerate(rows, 2)] + keys = [record["normalized"]["source_observation_key"] for record in records] duplicates = {key for key, count in Counter(key for key in keys if key).items() if count > 1} - accepted=[]; quarantined=[] - for line, row in enumerate(rows, 2): - key = (_clean(row.get("Certificate Number")) or _clean(row.get("Customer Number")) or _clean(row.get("Customer Number_y"))) - duplicate_key = _observation_key(profile, row) - reasons=[] - if not key: reasons.append("missing_certificate_or_customer_id") - if duplicate_key is not None and duplicate_key in duplicates: reasons.append("duplicate_observation_id") - if profile == "annual_reports" and not _clean(row.get("Year")): reasons.append("missing_report_year") - record = _record(profile, row, line); (quarantined if reasons else accepted).append({"reasons": tuple(dict.fromkeys(reasons)), "record": record} if reasons else record) - return {"accepted": accepted, "quarantined": quarantined, "profile": profile, "headers": headers, "schema_fingerprint": hashlib.sha256(json.dumps(headers, separators=(",", ":")).encode()).hexdigest(), "source_sha256": digest, "input_rows": len(rows)} + accepted: list[dict[str, Any]] = [] + quarantined: list[dict[str, Any]] = [] + for record, row in zip(records, rows): + normalized = record["normalized"] + reasons: list[str] = [] + if not _certificate_or_customer(row): + reasons.append("missing_certificate_or_customer_id") + if normalized["source_observation_key"] in duplicates: + reasons.append("duplicate_observation_id") + if profile == "annual_reports" and not _year(row): + reasons.append("missing_report_year") + if reasons: + quarantined.append({"reasons": tuple(dict.fromkeys(reasons)), "record": record}) + else: + accepted.append(record) + return { + "accepted": accepted, + "quarantined": quarantined, + "profile": profile, + "headers": headers, + "schema_fingerprint": _schema_fingerprint(headers), + "source_sha256": digest, + "input_rows": len(rows), + "evidence_type_counts": dict(sorted(Counter(record["normalized"]["evidence_type"] for record in accepted).items())), + } def run(self, raw_path: str | Path, run_dir: str | Path, artifact: SourceArtifact) -> dict[str, Any]: - raw=Path(raw_path).read_bytes(); digest=hashlib.sha256(raw).hexdigest() - if artifact.sha256 != digest or artifact.byte_size != len(raw): raise ValueError("artifact provenance mismatch") - result=self.parse_bytes(raw); accepted=result["accepted"]; quarantined=result["quarantined"]; parsed=accepted+[item["record"] for item in quarantined] - _, parsed_sha, _=atomic_jsonl(Path(run_dir)/"parsed/records.jsonl", parsed); _, normalized_sha, _=atomic_jsonl(Path(run_dir)/"normalized/records.jsonl", accepted); atomic_jsonl(Path(run_dir)/"quarantined/records.jsonl", quarantined) - anomalies=Counter(reason for item in quarantined for reason in item["reasons"]) - manifest=private_manifest(source_id=self.source_id, adapter_version=self.adapter_version, schema_version=self.schema_version, artifact=artifact, input_rows=result["input_rows"], normalized_rows=len(accepted), quarantined_rows=len(quarantined), normalized_sha256=normalized_sha, parsed_sha256=parsed_sha, anomaly_counts=dict(sorted(anomalies.items()))) - manifest.update({"source_profile": result["profile"], "schema_fingerprint": result["schema_fingerprint"], "entity_policy": CONFIG["entity_policy"], "geocoding": "disabled", "coverage": f"APHIS Public Search Tool {result['profile']} observations only; other APHIS profiles excluded"}) - atomic_json(Path(run_dir)/"manifest.json", manifest); return manifest + raw = Path(raw_path).read_bytes() + digest = hashlib.sha256(raw).hexdigest() + if artifact.sha256 != digest or artifact.byte_size != len(raw): + raise ValueError("artifact provenance mismatch") + result = self.parse_bytes(raw) + accepted = result["accepted"] + quarantined = result["quarantined"] + parsed = accepted + [item["record"] for item in quarantined] + _, parsed_sha, _ = atomic_jsonl(Path(run_dir) / "parsed/records.jsonl", parsed) + _, normalized_sha, _ = atomic_jsonl(Path(run_dir) / "normalized/records.jsonl", accepted) + atomic_jsonl(Path(run_dir) / "quarantined/records.jsonl", quarantined) + anomalies = Counter(reason for item in quarantined for reason in item["reasons"]) + manifest = private_manifest( + source_id=self.source_id, + adapter_version=self.adapter_version, + schema_version=self.schema_version, + artifact=artifact, + input_rows=result["input_rows"], + normalized_rows=len(accepted), + quarantined_rows=len(quarantined), + normalized_sha256=normalized_sha, + parsed_sha256=parsed_sha, + anomaly_counts=dict(sorted(anomalies.items())), + ) + manifest.update({ + "source_profile": result["profile"], + "evidence_type_counts": result["evidence_type_counts"], + "schema_fingerprint": result["schema_fingerprint"], + "entity_policy": CONFIG["entity_policy"], + "geocoding": "disabled", + "publication_gate": "blocked", + "test_only": True, + "coverage": "APHIS Animal Care public-search observations for the captured profile only; AWA coverage, currentness, completeness, and facility equivalence remain unknown; no FSIS/facility merge", + "coverage_limitations": [ + "The public-search export is not a complete census of animal use or all AWA-regulated entities.", + "An annual report covers the supplied reporting year and may be amended; missing years are not closure or non-use.", + "Registrations, inspections, annual reports, and amendments remain separate evidence types.", + "Addresses and coordinates are retained only in private source evidence pending privacy review.", + ], + }) + atomic_json(Path(run_dir) / "manifest.json", manifest) + return manifest diff --git a/pipeline/sources/us/aphis/config.json b/pipeline/sources/us/aphis/config.json index 491673b..af3d093 100644 --- a/pipeline/sources/us/aphis/config.json +++ b/pipeline/sources/us/aphis/config.json @@ -1,12 +1,13 @@ { "source_id": "us.aphis", - "contract_version": "us-aphis-public-search-v1", - "adapter_version": "us-aphis-candidate-v1", + "contract_version": "us-aphis-public-search-v2", + "adapter_version": "us-aphis-candidate-v2", "authority": "USDA Animal and Plant Health Inspection Service, Animal Care", "public_search_url": "https://www.aphis.usda.gov/animal-care/awa-services/usda-animal-care-public-search-tool", "annual_reports_url": "https://direct.aphis.usda.gov/awa/research-facility-report/annual-summary", "inspection_reports_url": "https://direct.aphis.usda.gov/awa/annual-inspection-reports", - "profiles": ["registrations", "annual_reports", "inspections"], + "documents_url": "https://www.aphis.usda.gov/animal-care/awa-services/usda-animal-care-public-search-tool", + "profiles": ["registrations", "annual_reports", "inspections", "documents"], "acquisition": "operator-assisted-public-search-export", "release_allowed_by_default": false, "entity_policy": "registrations, annual reports, inspections, laboratories, and aggregates remain separate evidence types; no facility merge" diff --git a/pipeline/sources/us/aphis/handoff.py b/pipeline/sources/us/aphis/handoff.py new file mode 100644 index 0000000..f3f14f3 --- /dev/null +++ b/pipeline/sources/us/aphis/handoff.py @@ -0,0 +1,115 @@ +"""Private APHIS observation handoff and disposable import packet. + +The shared facility handoff is intentionally not used here: APHIS rows have +source-native certificate/customer identities and must not become facilities or +graph ownership edges without a separate reviewed link event. +""" +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +from typing import Any + +from pipeline.contracts.adapter_contract import SourceArtifact +from pipeline.contracts.source_lifecycle import atomic_bytes, atomic_json + + +HANDOFF_VERSION = "us-aphis-observation-handoff-v1" +IMPORT_VERSION = "us-aphis-private-candidate-import-v1" + + +def _jsonl(rows: list[dict[str, Any]]) -> bytes: + return b"".join((json.dumps(row, ensure_ascii=False, sort_keys=True, default=list) + "\n").encode("utf-8") for row in rows) + + +def write_private_handoff( + run_dir: str | Path, + rows: list[dict[str, Any]], + artifact: SourceArtifact, + *, + profile: str, + source_sha256: str, +) -> dict[str, Any]: + """Write a source-specific private candidate handoff. + + This is a row-bearing restricted artifact. Its explicit entity scope keeps + downstream importers from applying the facility-master contract. + """ + payload = _jsonl(rows) + root = Path(run_dir) + atomic_bytes(root / "records.jsonl", payload) + manifest = { + "contract_version": HANDOFF_VERSION, + "source_id": "us.aphis", + "profile": profile, + "source_url": artifact.source_url, + "retrieved_at_utc": artifact.retrieved_at_utc, + "checksum_sha256": artifact.sha256, + "source_artifact_sha256": source_sha256, + "byte_size": artifact.byte_size, + "code_version": artifact.code_version, + "config_version": artifact.config_version, + "normalized_rows": len(rows), + "normalized_sha256": hashlib.sha256(payload).hexdigest(), + "entity_scope": "aphis_observation", + "source_native_identity": ["certificate_number", "customer_number", "customer_number_x", "customer_number_y"], + "graph_candidate_emission": False, + "auto_merge": False, + "release_state": "not-created", + "publication_state": "private-candidate", + "review_state": "review_required", + "privacy_gate": "pending", + "coordinate_gate": "review_required", + "publication_eligible_rows": 0, + "test_only": True, + "row_payloads_included": True, + "coverage": "APHIS profile observations only; no facility-master, laboratory, ownership, or cross-source identity claim", + } + atomic_json(root / "manifest.json", manifest) + return manifest + + +def import_private_candidate(handoff_dir: str | Path, output_dir: str | Path | None = None) -> dict[str, Any]: + """Materialize a deterministic private import packet, never a release. + + APHIS needs a source-specific importer because the generic facility + importer requires ``normalized.establishment_id`` and would create an + unintended facility. This packet is the explicit disposable/test-only + import boundary for later review tooling and has no public/API side effect. + """ + root = Path(handoff_dir) + manifest = json.loads((root / "manifest.json").read_text(encoding="utf-8")) + payload = (root / "records.jsonl").read_bytes() + if manifest.get("contract_version") != HANDOFF_VERSION: + raise ValueError("unsupported APHIS handoff contract") + if manifest.get("release_state") != "not-created" or manifest.get("publication_state") != "private-candidate": + raise ValueError("APHIS handoff is not an unpromoted private candidate") + if hashlib.sha256(payload).hexdigest() != manifest.get("normalized_sha256"): + raise ValueError("APHIS handoff checksum mismatch") + rows = [json.loads(line) for line in payload.decode("utf-8").splitlines() if line] + if len(rows) != int(manifest.get("normalized_rows", -1)): + raise ValueError("APHIS handoff row count mismatch") + target = Path(output_dir) if output_dir is not None else root.parent / "candidate-import" + atomic_bytes(target / "records.jsonl", payload) + imported = { + "contract_version": IMPORT_VERSION, + "source_id": manifest["source_id"], + "profile": manifest["profile"], + "handoff_sha256": manifest["normalized_sha256"], + "status": "completed", + "imported_rows": len(rows), + "rerun_imported_rows": 0, + "default_visible_rows": 0, + "publication_eligible_rows": 0, + "public_exposure": False, + "test_only": True, + "database_import": False, + "disposable_target": "source-specific private staging only", + "identity_merge": False, + "graph_edges_created": 0, + "release_state": "not-created", + "publication_state": "private-candidate", + } + atomic_json(target / "manifest.json", imported) + return imported diff --git a/pipeline/sources/us/aphis/refresh.py b/pipeline/sources/us/aphis/refresh.py index ce3c5b6..6b69751 100644 --- a/pipeline/sources/us/aphis/refresh.py +++ b/pipeline/sources/us/aphis/refresh.py @@ -1,30 +1,229 @@ -"""Assisted private refresh for APHIS public-search exports.""" +"""Private APHIS refresh with a documented-download or assisted-capture boundary.""" from __future__ import annotations -import argparse, hashlib, json + +import argparse +import hashlib +import json from pathlib import Path +from typing import Any + +from pipeline.common.acquisition import AcquisitionError, utc_now from pipeline.common.orchestrator import run_private_lifecycle -from pipeline.common.acquisition import utc_now from pipeline.contracts.adapter_contract import SourceArtifact +from pipeline.contracts.source_health import build_health_snapshot, write_health_snapshot from pipeline.contracts.source_lifecycle import atomic_json + +from .acquire import CSV_PROFILES, ALL_PROFILES, fetch_profile, parse_query_context, profile_url, validate_download from .adapter import CONFIG, AphisPublicSearchAdapter +from .handoff import import_private_candidate, write_private_handoff + + +def assisted_capture_contract(profile: str) -> dict[str, Any]: + if profile not in ALL_PROFILES: + raise ValueError("unsupported APHIS profile") + return { + "source_id": CONFIG["source_id"], + "profile": profile, + "method": "operator-assisted-public-search-export", + "source_url": profile_url(profile), + "steps": [ + "Open the APHIS Animal Care Public Search Tool or the documented annual-summary/inspection page in an authorized browser session.", + f"Select the {profile.replace('_', ' ')} view and use only its documented export or download control.", + "For annual reports, retain the selected fiscal year and any amended-report indicator; for documents, retain the report/document identifier and displayed source date.", + "Save the export or linked document without editing it; record query parameters, displayed date/year, final URL, and the retrieval time in the run metadata.", + "Run the private dry-run and review profile, schema/signature, duplicate IDs, missing dates, privacy, and coverage before any test-only handoff.", + ], + "boundaries": [ + "No automation against hidden endpoints, browser internals, credentials, rate limits, or access-control challenges.", + "Registrations/licenses, annual reports, inspections, downloadable documents/amendments, laboratories, and aggregate summaries are separate evidence types.", + "A failed or empty response is not a valid zero-row observation; keep the previous validated artifact available.", + "Absence is not closure and an inspection or document is not a facility-master assertion.", + ], + } -def assisted_capture_contract(profile: str) -> dict: - if profile not in CONFIG["profiles"]: raise ValueError("unsupported APHIS profile") - urls={"registrations":CONFIG["public_search_url"],"annual_reports":CONFIG["annual_reports_url"],"inspections":CONFIG["inspection_reports_url"]} - return {"source_id": CONFIG["source_id"], "profile": profile, "method": "operator-assisted-public-search-export", "source_url": urls[profile], "steps": ["Open the APHIS Animal Care Public Search Tool or the documented annual-summary page.", f"Select the {profile.replace('_',' ')} view and use only its documented export/download control.", "Save the export without editing it; retain the query parameters, displayed date/year, and final URL in the run metadata.", "Run dry-run and review profile, schema, duplicate IDs, missing dates, privacy, and coverage before private test-only import."], "boundaries": ["No automation against hidden endpoints or access-control bypass", "Registrations/licenses, annual reports, inspections, laboratories, and aggregate summaries are separate evidence types", "Absence is not closure and an inspection is not a facility-master assertion"]} -def refresh(*, run_dir: str | Path, raw_path: str | Path, profile: str, source_url: str | None = None, retrieved_at_utc: str | None = None, effective_date: str | None = None) -> dict: - path=Path(raw_path); raw=path.read_bytes(); adapter=AphisPublicSearchAdapter(); metadata={"acquisition_method":"preserved_local_artifact","source_id":CONFIG["source_id"],"profile":profile,"artifact":path.name,"artifact_path":str(path),"requested_url":source_url or CONFIG["public_search_url"],"final_url":source_url or CONFIG["public_search_url"],"retrieved_at_utc":retrieved_at_utc or utc_now(),"effective_date":effective_date or "unknown","sha256":hashlib.sha256(raw).hexdigest(),"byte_size":len(raw),"code_version":adapter.adapter_version,"config_version":adapter.schema_version,"rights_caveat":"APHIS export terms and attribution require operator review","privacy_caveat":"restricted private staging; address and coordinate review pending","coverage":f"APHIS {profile} export only; no facility merge","terms_review":"required before publication"} - result=adapter.parse_bytes(raw) - if result["profile"] != profile: raise ValueError(f"captured APHIS profile is {result['profile']}, expected {profile}") - root=Path(run_dir); atomic_json(root/"acquisition-metadata.json",metadata) - artifact=SourceArtifact(source_url=metadata["final_url"],retrieved_at_utc=metadata["retrieved_at_utc"],sha256=metadata["sha256"],byte_size=metadata["byte_size"],effective_date=effective_date or "unknown",code_version=adapter.adapter_version,config_version=adapter.schema_version,rights_caveat=metadata["rights_caveat"],privacy_caveat=metadata["privacy_caveat"],coverage=metadata["coverage"]) - status=run_private_lifecycle(path,root,artifact,adapter,health_as_of_utc=metadata["retrieved_at_utc"]); contract=assisted_capture_contract(profile); status["assisted_capture_contract"]=contract; atomic_json(root/"assisted-capture-contract.json",contract); return status +def _local_metadata( + path: Path, + *, + profile: str, + source_url: str | None, + retrieved_at_utc: str | None, + effective_date: str | None, + publication_date: str | None, + query_context: dict[str, Any] | None, +) -> dict[str, Any]: + raw = path.read_bytes() + validate_download(profile, path) + return { + "acquisition_method": "preserved_local_artifact", + "source_id": CONFIG["source_id"], + "profile": profile, + "artifact": path.name, + "artifact_path": str(path), + "requested_url": source_url or profile_url(profile), + "final_url": source_url or profile_url(profile), + "retrieved_at_utc": retrieved_at_utc or utc_now(), + "effective_date": effective_date or "unknown", + "publication_date": publication_date, + "sha256": hashlib.sha256(raw).hexdigest(), + "byte_size": len(raw), + "code_version": CONFIG["adapter_version"], + "config_version": CONFIG["contract_version"], + "rights_caveat": "APHIS export terms and attribution require operator review before publication", + "privacy_caveat": "restricted private staging; address, names, documents, and coordinates require review", + "coverage": f"APHIS {profile} export only; no facility merge or completeness claim", + "terms_review": "required before publication", + "query_context": {**(query_context or {}), "profile": profile}, + "retention": {"class": "restricted-research-evidence", "public_exposure": False, "review_required": True}, + } + + +def refresh( + *, + run_dir: str | Path, + profile: str, + raw_path: str | Path | None = None, + fetch: bool = False, + source_url: str | None = None, + terms_review_path: str | Path | None = None, + output_root: str | Path = "data/raw", + run_id: str | None = None, + retrieved_at_utc: str | None = None, + effective_date: str | None = None, + publication_date: str | None = None, + query_context: dict[str, Any] | None = None, + timeout_seconds: float = 60.0, + max_bytes: int = 128 * 1024 * 1024, +) -> dict[str, Any]: + if profile not in CSV_PROFILES: + raise ValueError("refresh parses registrations, annual_reports, or inspections; use acquire.py for documents/amendments") + if fetch == (raw_path is not None): + raise ValueError("specify exactly one of raw_path or fetch") + root = Path(run_dir) + if fetch: + if terms_review_path is None: + raise ValueError("terms_review_path is required for network acquisition") + acquisition = fetch_profile( + profile=profile, + output_root=Path(output_root), + terms_review_path=terms_review_path, + source_url=source_url, + run_id=run_id, + query_context=query_context, + publication_date=publication_date, + effective_date=effective_date, + timeout_seconds=timeout_seconds, + max_bytes=max_bytes, + ) + path = Path(acquisition["artifact_path"]) + else: + path = Path(raw_path) # type: ignore[arg-type] + if not path.is_file(): + raise ValueError(f"raw artifact does not exist: {path}") + acquisition = _local_metadata( + path, + profile=profile, + source_url=source_url, + retrieved_at_utc=retrieved_at_utc, + effective_date=effective_date, + publication_date=publication_date, + query_context=query_context, + ) + + raw = path.read_bytes() + artifact = SourceArtifact( + source_url=str(acquisition["final_url"]), + retrieved_at_utc=str(acquisition["retrieved_at_utc"]), + sha256=hashlib.sha256(raw).hexdigest(), + byte_size=len(raw), + publication_date=acquisition.get("publication_date"), + effective_date=acquisition.get("effective_date") or "unknown", + code_version=CONFIG["adapter_version"], + config_version=CONFIG["contract_version"], + rights_caveat=acquisition["rights_caveat"], + privacy_caveat=acquisition["privacy_caveat"], + coverage=acquisition["coverage"], + ) + atomic_json(root / "acquisition-metadata.json", acquisition) + adapter = AphisPublicSearchAdapter() + result = adapter.parse_bytes(raw) + if result["profile"] != profile: + raise ValueError(f"captured APHIS profile is {result['profile']}, expected {profile}") + status = run_private_lifecycle(path, root, artifact, adapter, health_as_of_utc=artifact.retrieved_at_utc) + contract = assisted_capture_contract(profile) + status["assisted_capture_contract"] = contract + status["query_context"] = acquisition.get("query_context") or {} + atomic_json(root / "assisted-capture-contract.json", contract) + if status.get("status") == "candidate-ready": + lifecycle_root = Path(status["run_dir"]) + rows = [ + json.loads(line) + for line in (lifecycle_root / "normalized/records.jsonl").read_text(encoding="utf-8").splitlines() + if line + ] + handoff = write_private_handoff( + lifecycle_root / "candidate-handoff", + rows, + artifact, + profile=profile, + source_sha256=acquisition["sha256"], + ) + imported = import_private_candidate(lifecycle_root / "candidate-handoff") + atomic_json(lifecycle_root / "candidate-import.json", imported) + write_health_snapshot( + lifecycle_root / "source-health.json", + build_health_snapshot( + lifecycle_root, + as_of_utc=artifact.retrieved_at_utc, + import_evidence_path=lifecycle_root / "candidate-import.json", + ), + ) + status["candidate_handoff"] = handoff + status["candidate_import"] = imported + atomic_json(root / "run-status.json", status) + return status def main() -> int: - parser=argparse.ArgumentParser(description=__doc__); parser.add_argument("--raw",type=Path,required=True); parser.add_argument("--run-dir",type=Path,required=True); parser.add_argument("--profile",choices=CONFIG["profiles"],required=True); parser.add_argument("--source-url"); parser.add_argument("--retrieved-at-utc"); parser.add_argument("--effective-date"); args=parser.parse_args() - try: result=refresh(run_dir=args.run_dir,raw_path=args.raw,profile=args.profile,source_url=args.source_url,retrieved_at_utc=args.retrieved_at_utc,effective_date=args.effective_date) - except (OSError,ValueError) as exc: print(json.dumps({"status":"failed","error":str(exc)})); return 2 - print(json.dumps({"status":result.get("status"),"run_dir":result.get("run_dir")},sort_keys=True)); return 0 + parser = argparse.ArgumentParser(description=__doc__) + source = parser.add_mutually_exclusive_group(required=True) + source.add_argument("--raw", type=Path) + source.add_argument("--fetch", action="store_true") + parser.add_argument("--run-dir", type=Path, required=True) + parser.add_argument("--profile", choices=sorted(CSV_PROFILES), required=True) + parser.add_argument("--source-url") + parser.add_argument("--terms-review", type=Path) + parser.add_argument("--output-root", type=Path, default=Path("data/raw")) + parser.add_argument("--run-id") + parser.add_argument("--retrieved-at-utc") + parser.add_argument("--effective-date") + parser.add_argument("--publication-date") + parser.add_argument("--query-context") + parser.add_argument("--timeout-seconds", type=float, default=60.0) + parser.add_argument("--max-bytes", type=int, default=128 * 1024 * 1024) + args = parser.parse_args() + try: + result = refresh( + run_dir=args.run_dir, + profile=args.profile, + raw_path=args.raw, + fetch=args.fetch, + source_url=args.source_url, + terms_review_path=args.terms_review, + output_root=args.output_root, + run_id=args.run_id, + retrieved_at_utc=args.retrieved_at_utc, + effective_date=args.effective_date, + publication_date=args.publication_date, + query_context=parse_query_context(args.query_context), + timeout_seconds=args.timeout_seconds, + max_bytes=args.max_bytes, + ) + except (AcquisitionError, OSError, ValueError) as error: + print(json.dumps({"status": "failed", "error": str(error)}, sort_keys=True)) + return 2 + print(json.dumps({"status": result.get("status"), "run_dir": result.get("run_dir")}, sort_keys=True)) + return 0 if result.get("status") in {"candidate-ready", "staged-restricted"} else 1 + -if __name__ == "__main__": raise SystemExit(main()) +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/pipeline/sources/us/aphis/test_acquire.py b/pipeline/sources/us/aphis/test_acquire.py new file mode 100644 index 0000000..31da1c2 --- /dev/null +++ b/pipeline/sources/us/aphis/test_acquire.py @@ -0,0 +1,156 @@ +import json +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +from pipeline.common.acquisition import AcquisitionError + +from .acquire import fetch_profile, preserve_local_document, validate_download + + +class AphisAcquisitionTests(unittest.TestCase): + def test_csv_validator_rejects_empty_challenge_and_inconsistent_rows(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + cases = { + "empty.csv": b"Account Name,Certificate Number\n", + "challenge.csv": b"Just a moment...\n", + "truncated.csv": b"Account Name,Certificate Number\nSynthetic,1,unexpected\n", + } + for name, payload in cases.items(): + path = root / name + path.write_bytes(payload) + with self.subTest(name=name): + with self.assertRaises(AcquisitionError): + validate_download("registrations", path) + + def test_document_validator_requires_pdf_or_office_signature(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + invalid = root / "invalid.pdf" + invalid.write_bytes(b"not a document") + with self.assertRaisesRegex(AcquisitionError, "signature"): + validate_download("documents", invalid) + + valid = root / "report.pdf" + valid.write_bytes(b"%PDF-1.7\nsynthetic\n%%EOF\n") + validate_download("documents", valid) + + def test_operator_saved_document_gets_private_manifest_without_parsing(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + document = root / "amendment.pdf" + document.write_bytes(b"%PDF-1.7\nsynthetic amendment\n%%EOF\n") + manifest = preserve_local_document( + raw_path=document, + run_dir=root / "run", + source_url="https://example.test/amendment.pdf", + retrieved_at_utc="2026-09-18T00:00:00Z", + effective_date="2025", + query_context={"document_id": "synthetic"}, + ) + self.assertEqual(manifest["profile"], "documents") + self.assertEqual(manifest["publication_gate"], "blocked") + self.assertEqual(manifest["query_context"]["document_id"], "synthetic") + self.assertEqual(manifest["artifact_path"], str(document)) + + def test_fetch_preserves_query_context_and_writes_manifest(self): + payload = b"Account Name,Customer Number,Certificate Number,License Type,Certificate Status,Status Date\nSynthetic,1,00-B-0001,Class B,Active,2026-01-01\n" + + class Response: + status = 200 + headers = {"Content-Type": "text/csv", "Content-Length": str(len(payload)), "Last-Modified": "Tue, 15 Sep 2026 00:00:00 GMT"} + + def __enter__(self): + return self + + def __exit__(self, *_args): + return False + + def read(self, _size): + if getattr(self, "done", False): + return b"" + self.done = True + return payload + + def geturl(self): + return "https://download.example.test/aphis.csv" + + class Opener: + def open(self, _request, timeout): + return Response() + + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + terms = root / "terms.json" + terms.write_text(json.dumps({"reviewer": "operator", "reference": "synthetic", "reviewed_at": "2026-09-15T00:00:00Z", "decision": "approved", "notes": "test"}), encoding="utf-8") + with patch("pipeline.common.acquisition.urllib.request.build_opener", return_value=Opener()): + metadata = fetch_profile( + profile="registrations", + output_root=root / "raw", + terms_review_path=terms, + run_id="run-1", + query_context={"selected_year": "2026", "amended": False}, + ) + run = Path(metadata["artifact_path"]).parent + manifest = json.loads((run / "manifest.json").read_text(encoding="utf-8")) + self.assertEqual(manifest["query_context"]["selected_year"], "2026") + self.assertEqual(manifest["byte_size"], len(payload)) + self.assertEqual(manifest["effective_date"], "Tue, 15 Sep 2026 00:00:00 GMT") + + def test_challenge_failure_keeps_prior_artifact_and_records_context(self): + previous = b"Account Name,Customer Number,Certificate Number,License Type,Certificate Status,Status Date\nSynthetic,1,00-B-0001,Class B,Active,2026-01-01\n" + challenge = b"Access denied\n" + + class Response: + status = 200 + headers = {"Content-Type": "text/csv", "Content-Length": str(len(challenge))} + + def __enter__(self): + return self + + def __exit__(self, *_args): + return False + + def read(self, _size): + if getattr(self, "done", False): + return b"" + self.done = True + return challenge + + def geturl(self): + return "https://download.example.test/challenge.csv" + + class Opener: + def open(self, _request, timeout): + return Response() + + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + prior = root / "prior.csv" + prior.write_bytes(previous) + terms = root / "terms.json" + terms.write_text(json.dumps({"reviewer": "operator", "reference": "synthetic", "reviewed_at": "2026-09-15T00:00:00Z", "decision": "approved", "notes": "test"}), encoding="utf-8") + with patch("pipeline.common.acquisition.urllib.request.build_opener", return_value=Opener()): + with self.assertRaisesRegex(AcquisitionError, "challenge"): + fetch_profile( + profile="annual_reports", + output_root=root / "raw", + terms_review_path=terms, + run_id="run-2", + source_url="https://example.test/annual-reports.csv", + effective_date="2025", + query_context={"selected_year": "2025"}, + ) + self.assertEqual(prior.read_bytes(), previous) + failure = json.loads((root / "raw/us.aphis/run-2/acquisition-failure.json").read_text(encoding="utf-8")) + self.assertEqual(failure["failure_class"], "challenge-response") + self.assertEqual(failure["requested_url"], "https://example.test/annual-reports.csv") + self.assertEqual(failure["effective_date"], "2025") + self.assertTrue(failure["requested_at_utc"]) + self.assertEqual(failure["query_context"]["selected_year"], "2025") + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/sources/us/aphis/test_adapter.py b/pipeline/sources/us/aphis/test_adapter.py index 0baf565..c827f03 100644 --- a/pipeline/sources/us/aphis/test_adapter.py +++ b/pipeline/sources/us/aphis/test_adapter.py @@ -1,5 +1,9 @@ import unittest +import hashlib +import tempfile from pathlib import Path + +from pipeline.contracts.adapter_contract import SourceArtifact from .adapter import AphisContractError, AphisPublicSearchAdapter ROOT=Path(__file__).parent @@ -31,4 +35,39 @@ def test_annual_reports_use_year_in_observation_identity(self): def test_unsupported_profile_fails_closed(self): with self.assertRaises(AphisContractError): AphisPublicSearchAdapter().parse_bytes(b"Name,Value\nA,B\n") + def test_customer_variants_and_source_values_are_preserved(self): + result = AphisPublicSearchAdapter().parse_bytes((ROOT / "fixtures/annual_reports.csv").read_bytes()) + record = result["accepted"][0] + normalized = record["normalized"] + self.assertEqual(normalized["customer_number_x"], "Synthetic Laboratory") + self.assertEqual(normalized["customer_number_y"], "2") + self.assertEqual(record["source_values"]["Customer Number_x"], "Synthetic Laboratory") + self.assertIn("Cats", normalized["animal_use_fields_present"]) + self.assertEqual(normalized["awa_coverage_state"], "source_profile_only_unknown_completeness") + + def test_amendment_versions_are_distinct_evidence(self): + lines = (ROOT / "fixtures/annual_reports.csv").read_text(encoding="utf-8").splitlines() + lines[0] += ",Amendment Number" + lines[1] += ",1" + lines.append(lines[1].rsplit(",1", 1)[0] + ",2") + result = AphisPublicSearchAdapter().parse_bytes(("\n".join(lines) + "\n").encode()) + self.assertEqual(len(result["accepted"]), 2) + self.assertEqual({row["normalized"]["evidence_type"] for row in result["accepted"]}, {"amendments"}) + self.assertNotEqual(result["accepted"][0]["source_record_key"], result["accepted"][1]["source_record_key"]) + + def test_short_rows_fail_closed_as_schema_drift(self): + with self.assertRaises(AphisContractError): + AphisPublicSearchAdapter().parse_bytes(b"Account Name,Certificate Number,Certificate Status\nOnly One Cell\n") + + def test_run_is_idempotent_and_reconciles_every_row(self): + raw = (ROOT / "fixtures/inspections.csv").read_bytes() + artifact = SourceArtifact("https://example.invalid/aphis.csv", "2026-09-15T00:00:00Z", hashlib.sha256(raw).hexdigest(), len(raw), code_version="test", config_version="test") + with tempfile.TemporaryDirectory() as directory: + first = AphisPublicSearchAdapter().run(ROOT / "fixtures/inspections.csv", Path(directory) / "one", artifact) + second = AphisPublicSearchAdapter().run(ROOT / "fixtures/inspections.csv", Path(directory) / "two", artifact) + self.assertEqual(first["normalized_sha256"], second["normalized_sha256"]) + self.assertEqual(first["parsed_sha256"], second["parsed_sha256"]) + self.assertEqual(first["input_rows"], first["normalized_rows"] + first["quarantined_rows"]) + self.assertEqual(first["release_state"], "not-created") + if __name__ == "__main__": unittest.main() diff --git a/pipeline/sources/us/aphis/test_refresh.py b/pipeline/sources/us/aphis/test_refresh.py new file mode 100644 index 0000000..a30d00b --- /dev/null +++ b/pipeline/sources/us/aphis/test_refresh.py @@ -0,0 +1,38 @@ +import json +import tempfile +import unittest +from pathlib import Path + +from .refresh import refresh + + +ROOT = Path(__file__).parent + + +class AphisRefreshTests(unittest.TestCase): + def test_private_handoff_and_import_are_emitted_without_graph_or_release(self): + with tempfile.TemporaryDirectory() as directory: + result = refresh( + run_dir=Path(directory) / "refresh", + raw_path=ROOT / "fixtures/annual_reports.csv", + profile="annual_reports", + retrieved_at_utc="2026-09-15T00:00:00Z", + ) + self.assertEqual(result["status"], "candidate-ready") + run = Path(result["run_dir"]) + handoff = json.loads((run / "candidate-handoff/manifest.json").read_text(encoding="utf-8")) + imported = json.loads((run / "candidate-import/manifest.json").read_text(encoding="utf-8")) + health = json.loads((run / "source-health.json").read_text(encoding="utf-8")) + self.assertEqual(handoff["entity_scope"], "aphis_observation") + self.assertFalse(handoff["graph_candidate_emission"]) + self.assertEqual(imported["imported_rows"], 1) + self.assertFalse(imported["public_exposure"]) + self.assertEqual(imported["publication_eligible_rows"], 0) + self.assertEqual(imported["graph_edges_created"], 0) + self.assertEqual(health["import"]["state"], "completed") + self.assertEqual(health["import"]["publication_eligible_rows"], 0) + self.assertFalse((run / "candidate-handoff/graph-candidates").exists()) + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/sources/us/fsis/README.md b/pipeline/sources/us/fsis/README.md new file mode 100644 index 0000000..e3f5781 --- /dev/null +++ b/pipeline/sources/us/fsis/README.md @@ -0,0 +1,62 @@ +# FSIS MPI private refresh + +This source package is a private, test-only acquisition and transformation +boundary for the USDA FSIS Meat, Poultry and Egg Product Inspection (MPI) +Directory. It does not create a public release. State inspection programs, +APHIS observations, and non-FSIS populations remain outside this source. + +## Capture boundary + +The official page currently exposes a directory export by establishment name, +a directory export by establishment number, and a supplemental establishment- +demographic CSV. Direct links may return HTTP 403. The refresh command never +bypasses that control: use an authorized operator-assisted capture or a +terms-reviewed bounded fetch. HTML, login pages, 403 responses, unsupported +content types, malformed CSV, and schema drift fail closed. + +For an operator-assisted capture: + +```text +python -m pipeline.sources.us.fsis.refresh \ + --directory \ + --demographics \ + --run-dir --mode dry-run +``` + +Use `--mode handoff` only after reviewing the private manifest and quarantine. +`--raw` remains a directory-only compatibility alias. `--fetch` requires a +terms-review JSON and fetches the configured directory-by-number and +demographic routes through the shared bounded acquisition primitive. + +## Transformation contract + +The directory is the facility identity/location spine. Demographic rows are +joined only by exact source-native establishment ID or establishment number; +names, addresses, phones, and coordinates are never identity keys. Duplicate, +ambiguous, and orphan demographic rows are quarantined. An unmatched current +observation is not evidence of closure. + +Each accepted private normalized row retains: + +- source values separately for the directory and demographics files; +- source-native identity and row numbers; +- source-provided coordinates with provider, precision, and pending review + state (no geocoding); and +- source activity fields grouped as species slaughtered, processing + activities, and inspection attributes, without collapsing species, + inspection systems, exemptions, or volume categories. + +The manifest records a hash and byte size for every source file, schema +fingerprints, exact-key reconciliation counts, drift alarms, and the common +`release_state=not-created`, `publication_state=private-candidate` gates. +Private handoff output remains blocked from public API, map, export, cache, and +history surfaces. + +## V1 disposition + +The field crosswalk in `docs/countries/us/v1-field-crosswalk.json` maps +identity, location, administrative, species/activity, and inspection fields +where a current source field is available. Phone/address/DUNS values remain +source-only pending privacy review. Legacy derived categories are not asserted +as current facts; they are recomputed only from an approved current artifact or +explicitly retired when the current source does not supply the needed inputs. diff --git a/pipeline/sources/us/fsis/adapter.py b/pipeline/sources/us/fsis/adapter.py index 5969083..6a26c61 100644 --- a/pipeline/sources/us/fsis/adapter.py +++ b/pipeline/sources/us/fsis/adapter.py @@ -1,12 +1,19 @@ -"""Fail-closed private adapter for the FSIS MPI directory CSV.""" +"""Fail-closed private adapter for the FSIS MPI directory bundle. + +The official page exposes directory and supplemental demographic exports as +separate artifacts. This adapter preserves each artifact's source values and +joins them only on exact source-native establishment identifiers. It never +uses names, addresses, phones, or coordinates as identity keys. +""" from __future__ import annotations import csv import hashlib import json +import re from collections import Counter from pathlib import Path -from typing import Any +from typing import Any, Iterable from pipeline.contracts.adapter_contract import SourceArtifact from pipeline.contracts.candidate_handoff import write_handoff @@ -14,11 +21,9 @@ ROOT = Path(__file__).parent CONFIG = json.loads((ROOT / "config.json").read_text(encoding="utf-8")) -V1_HEADER = Path(__file__).resolve().parents[4] / "static_data/us/locations.csv" -V1_COLUMNS = tuple(next(csv.reader([V1_HEADER.read_text(encoding="utf-8-sig").splitlines()[0]]))) -CORE_COLUMNS = ("establishment_id", "establishment_number", "establishment_name", "street", "city", "state", "zip", "phone", "grant_date", "type", "dbas", "district", "circuit", "size", "latitude", "longitude", "county", "fips_code") -ACTIVITY_COLUMNS = frozenset(column for column in V1_COLUMNS if column.endswith("_slaughter") or column.endswith("_processing") or column in {"slaughter", "processing", "egg_processing", "ratite_processing", "siluriformes_processing"}) -ALLOWED_STATES = frozenset("AL AK AZ AR CA CO CT DE FL GA HI ID IL IN IA KS KY LA ME MD MA MI MN MS MO MT NE NV NH NJ NM NY NC ND OH OK OR PA RI SC SD TN TX UT VT VA WA WV WI WY DC PR VI GU AS MP".split()) +ALLOWED_STATES = frozenset( + "AL AK AZ AR CA CO CT DE FL GA HI ID IL IN IA KS KY LA ME MD MA MI MN MS MO MT NE NV NH NJ NM NY NC ND OH OK OR PA RI SC SD TN TX UT VT VA WA WV WI WY DC PR VI GU AS MP".split() +) class FsisContractError(ValueError): @@ -32,46 +37,198 @@ def _clean(value: Any) -> str | None: return value or None +def _header_key(value: str) -> str: + """Make header matching tolerant of punctuation/case, not row identity.""" + return re.sub(r"[^a-z0-9]+", "_", value.strip().lower()).strip("_") + + def _schema_fingerprint(headers: tuple[str, ...]) -> str: return hashlib.sha256(json.dumps(headers, ensure_ascii=False, separators=(",", ":")).encode()).hexdigest() -def _csv(content: bytes) -> tuple[tuple[str, ...], list[dict[str, Any]]]: +def _csv(content: bytes, *, role: str) -> tuple[tuple[str, ...], list[dict[str, Any]]]: try: reader = csv.DictReader(content.decode("utf-8-sig").splitlines(), strict=True) headers = tuple(reader.fieldnames or ()) rows = list(reader) except (UnicodeDecodeError, csv.Error) as exc: - raise FsisContractError("malformed or unsupported UTF-8 CSV") from exc + raise FsisContractError(f"malformed or unsupported UTF-8 {role} CSV") from exc if not headers: - raise FsisContractError("missing FSIS header") + raise FsisContractError(f"missing FSIS {role} header") if None in headers or len(set(headers)) != len(headers): - raise FsisContractError("duplicate or unnamed FSIS columns") + raise FsisContractError(f"duplicate or unnamed FSIS {role} columns") if any(None in row for row in rows): - raise FsisContractError("schema drift: row has extra columns") - if not {"establishment_id", "establishment_name", "state"}.issubset(headers): - raise FsisContractError("unsupported FSIS profile: missing facility identity fields") + raise FsisContractError(f"FSIS {role} schema drift: row has extra columns") + normalized_headers = {_header_key(header) for header in headers} + if role == "directory" and not ({"establishment_id", "establishment_number"} & normalized_headers): + raise FsisContractError("unsupported FSIS directory: missing establishment identity fields") + if role == "demographics" and not ({"establishment_id", "establishment_number", "number"} & normalized_headers): + raise FsisContractError("unsupported FSIS demographics: missing establishment identity fields") return headers, rows -def _record(row: dict[str, Any], line: int) -> dict[str, Any]: - source_values = {str(key): value for key, value in row.items()} - activities = tuple(key for key in sorted(ACTIVITY_COLUMNS) if _clean(row.get(key))) +def _field(row: dict[str, Any], *aliases: str) -> str | None: + wanted = {_header_key(alias) for alias in aliases} + for key, value in row.items(): + if _header_key(str(key)) in wanted: + return _clean(value) + return None + + +def _key_candidates(row: dict[str, Any]) -> tuple[str, ...]: + """Return source-native aliases, preserving ID-vs-number distinctions.""" + values: list[str] = [] + for alias in ( + "establishment_id", "establishment id", "mpi id", "establishment_number", + "establishment number", "establishment no", "establishment no.", "number", + ): + value = _field(row, alias) + if value and value not in values: + values.append(value) + return tuple(values) + + +def _identity_key(row: dict[str, Any]) -> str | None: + candidates = _key_candidates(row) + return candidates[0] if candidates else None + + +def _coordinate(row: dict[str, Any]) -> tuple[dict[str, Any] | None, str, str | None]: + latitude = _field(row, "latitude", "lat", "y") + longitude = _field(row, "longitude", "lon", "lng", "long", "x") + if latitude is None and longitude is None: + return None, "unknown", None + try: + lat = float(latitude) if latitude is not None else None + lon = float(longitude) if longitude is not None else None + except ValueError: + return None, "source-value-invalid", "invalid_coordinates" + if lat is None or lon is None or not -90 <= lat <= 90 or not -180 <= lon <= 180: + return None, "source-value-invalid", "invalid_coordinates" + return ( + { + "latitude": lat, + "longitude": lon, + "provider": "FSIS source-provided coordinate", + "query": None, + "precision": "source-provided", + "review_state": "pending-review", + }, + "source-value-present-pending-review", + None, + ) + + +def _activity_maps(rows: Iterable[dict[str, Any]]) -> tuple[dict[str, str], dict[str, str], dict[str, str], dict[str, str]]: + slaughter: dict[str, str] = {} + processing: dict[str, str] = {} + inspection: dict[str, str] = {} + all_activities: dict[str, str] = {} + for row in rows: + for raw_key, raw_value in row.items(): + value = _clean(raw_value) + key = _header_key(str(raw_key)) + if not value or key in {"establishment_id", "establishment_number", "establishment_name", "name"}: + continue + if "slaughter" in key: + slaughter[key] = value + all_activities[key] = value + elif "processing" in key or key in {"egg_product", "egg_products"}: + processing[key] = value + all_activities[key] = value + elif key.startswith("inspection") or "inspection_system" in key: + inspection[key] = value + return ( + dict(sorted(slaughter.items())), + dict(sorted(processing.items())), + dict(sorted(inspection.items())), + dict(sorted(all_activities.items())), + ) + + +def _record(directory: dict[str, Any], line: int, demographic: dict[str, Any] | None = None, demographic_line: int | None = None) -> dict[str, Any]: + source_rows: list[dict[str, Any]] = [directory] + if demographic: + source_rows.append(demographic) + slaughter, processing, inspection, all_activities = _activity_maps(source_rows) + inspection_attributes = dict(inspection) + for attribute, aliases in { + "establishment_type": ("type", "establishment_type"), + "district": ("district",), + "circuit": ("circuit",), + "size": ("size", "haccp_size", "establishment_size"), + "grant_date": ("grant_date", "grant date"), + }.items(): + value = _field(directory, *aliases) + if value: + inspection_attributes[attribute] = value + coordinates, coordinate_state, coordinate_reason = _coordinate(directory) + # Editions may place coordinates in the supplemental file. Use them only + # when the directory has no coordinate value; a malformed directory value + # remains an anomaly rather than being silently repaired. + if coordinates is None and coordinate_reason is None and demographic: + coordinates, coordinate_state, coordinate_reason = _coordinate(demographic) + establishment_id = _field(directory, "establishment_id", "establishment id", "mpi id") + establishment_number = _field(directory, "establishment_number", "establishment number", "establishment no", "establishment no.", "number") + # Some official directory editions expose only the establishment number. + # It is source-native, so retain it as the fallback ID rather than + # inventing a project UUID or dropping the row. + identity = establishment_id or establishment_number normalized = { - "establishment_id": _clean(row.get("establishment_id")), - "establishment_number": _clean(row.get("establishment_number")), - "canonical_name": _clean(row.get("establishment_name")), + "establishment_id": identity, + "establishment_number": establishment_number, + "canonical_name": _field(directory, "establishment_name", "establishment name", "name", "facility_name"), "country_code": "US", - "city": _clean(row.get("city")), "state": _clean(row.get("state")), "postal_code": _clean(row.get("zip")), - "county": _clean(row.get("county")), "district": _clean(row.get("district")), "circuit": _clean(row.get("circuit")), - "size": _clean(row.get("size")), "source_type": _clean(row.get("type")), "activities": activities, - "activity_categories": tuple(sorted({"slaughter" if key.endswith("_slaughter") or key == "slaughter" else "processing" for key in activities})), - "grant_date": _clean(row.get("grant_date")), "coordinates": None, - "coordinate_state": "source-value-present-pending-review" if _clean(row.get("latitude")) or _clean(row.get("longitude")) else "unknown", - "address_state": "source-address-retained-private-pending-review" if _clean(row.get("street")) else "unknown", - "privacy_gate": "pending-review", "coordinate_gate": "review_required", "publication_gate": "blocked", + "city": _field(directory, "city", "town"), + "state": (_field(directory, "state", "st") or "").upper() or None, + "postal_code": _field(directory, "zip", "zipcode", "zip_code", "postal_code"), + "county": _field(directory, "county"), + "fips_code": _field(directory, "fips_code", "fips"), + "district": _field(directory, "district"), + "circuit": _field(directory, "circuit"), + "size": _field(directory, "size", "haccp_size", "establishment_size"), + "source_type": _field(directory, "type", "establishment_type", "activities", "activity"), + "grant_date": _field(directory, "grant_date", "grant date"), + "coordinates": coordinates, + "coordinate_state": coordinate_state, + "address_state": "source-address-retained-private-pending-review" if _field(directory, "street", "address", "address_line_1") else "unknown", + "species_slaughtered": slaughter, + "processing_activities": processing, + "inspection_attributes": dict(sorted(inspection_attributes.items())), + "activities": tuple(sorted(all_activities)), + "activity_categories": tuple(category for category, values in (("slaughter", slaughter), ("processing", processing)) if values), + "privacy_gate": "pending-review", + "coordinate_gate": "review_required", + "publication_gate": "blocked", } - return {"source_id": CONFIG["source_id"], "source_row": line, "source_record_key": normalized["establishment_id"], "source_values": source_values, "normalized": normalized} + directory_values = {str(key): value for key, value in directory.items()} + return { + "source_id": CONFIG["source_id"], + "source_row": line, + "source_record_key": identity, + "source_values": { + # Keep the directory-only shape available to existing private + # rehearsals while the role-qualified keys make bundle provenance + # explicit for new consumers. + **directory_values, + "directory": directory_values, + "demographics": {str(key): value for key, value in demographic.items()} if demographic else None, + }, + "source_rows": {"directory": line, "demographics": demographic_line}, + "normalized": normalized, + "coordinate_anomaly": coordinate_reason, + } + + +def _quarantine(record: dict[str, Any], reasons: Iterable[str]) -> dict[str, Any]: + record = dict(record) + record.pop("coordinate_anomaly", None) + return {"reasons": tuple(dict.fromkeys(reasons)), "record": record} + + +def _demo_matches(row: dict[str, Any], directory: dict[str, Any]) -> bool: + demo_keys = set(_key_candidates(row)) + return bool(demo_keys & set(_key_candidates(directory))) class FsisMpiAdapter: @@ -80,38 +237,165 @@ class FsisMpiAdapter: schema_version = CONFIG["contract_version"] def parse_bytes(self, content: bytes) -> dict[str, Any]: - digest = hashlib.sha256(content).hexdigest() - headers, rows = _csv(content) + """Parse a directory-only capture for compatibility with old callers.""" + return self.parse_sources(content) + + def parse_sources(self, directory: bytes, demographics: bytes | None = None) -> dict[str, Any]: + directory_headers, directory_rows = _csv(directory, role="directory") + demographic_headers: tuple[str, ...] = () + demographic_rows: list[dict[str, Any]] = [] + if demographics is not None: + demographic_headers, demographic_rows = _csv(demographics, role="demographics") + directory_alias_counts = Counter(alias for row in directory_rows for alias in _key_candidates(row)) + duplicate_directory_aliases = {alias for alias, count in directory_alias_counts.items() if count > 1} + demographic_alias_counts = Counter(alias for row in demographic_rows for alias in _key_candidates(row)) + duplicate_demographic_aliases = {alias for alias, count in demographic_alias_counts.items() if count > 1} + accepted: list[dict[str, Any]] = [] quarantined: list[dict[str, Any]] = [] - keys = [_clean(row.get("establishment_id")) for row in rows] - duplicates = {key for key, count in Counter(key for key in keys if key).items() if count > 1} - for line, row in enumerate(rows, 2): + matched_demographics: set[int] = set() + identity_conflicts = 0 + for index, row in enumerate(directory_rows): + line = index + 2 + aliases = set(_key_candidates(row)) reasons: list[str] = [] - identifier = _clean(row.get("establishment_id")); state = (_clean(row.get("state")) or "").upper() - if not identifier: reasons.append("missing_establishment_id") - if identifier in duplicates: reasons.append("duplicate_establishment_id") - if state and state not in ALLOWED_STATES: reasons.append("unknown_state") - if not _clean(row.get("establishment_name")): reasons.append("missing_establishment_name") - record = _record(row, line) - (quarantined if reasons else accepted).append({"reasons": tuple(dict.fromkeys(reasons)), "record": record} if reasons else record) - return {"accepted": accepted, "quarantined": quarantined, "source_sha256": digest, "schema_fingerprint": _schema_fingerprint(headers), "headers": headers, "input_rows": len(rows)} + if not aliases: + reasons.append("missing_establishment_id") + reasons.append("missing_establishment_identity") + if aliases & duplicate_directory_aliases: + reasons.append("duplicate_establishment_id") + reasons.append("duplicate_establishment_identity") + state = (_field(row, "state", "st") or "").upper() + if state and state not in ALLOWED_STATES: + reasons.append("unknown_state") + if not _field(row, "establishment_name", "establishment name", "name", "facility_name"): + reasons.append("missing_establishment_name") + matching_demo = [demo_index for demo_index, demo in enumerate(demographic_rows) if _demo_matches(demo, row)] + demographic: dict[str, Any] | None = None + demographic_line: int | None = None + if len(matching_demo) > 1: + reasons.append("ambiguous_demographic_identity") + identity_conflicts += 1 + elif matching_demo: + demo_index = matching_demo[0] + matched_demographics.add(demo_index) + demographic = demographic_rows[demo_index] + demographic_line = demo_index + 2 + if set(_key_candidates(demographic)) & duplicate_demographic_aliases: + reasons.append("duplicate_demographic_identity") + record = _record(row, line, demographic, demographic_line) + if record.pop("coordinate_anomaly", None): + reasons.append("invalid_coordinates") + (quarantined if reasons else accepted).append(_quarantine(record, reasons) if reasons else record) + + orphan_demographics = 0 + for index, row in enumerate(demographic_rows): + if index in matched_demographics: + continue + orphan_demographics += 1 + record = _record({}, index + 2, row, index + 2) + record["source_record_key"] = _identity_key(row) + record["normalized"]["establishment_id"] = _identity_key(row) + record["source_values"] = { + "directory": None, + "demographics": {str(key): value for key, value in row.items()}, + } + record["source_rows"] = {"directory": None, "demographics": index + 2} + quarantined.append(_quarantine(record, ("unmatched_demographic_identity",))) + + return { + "accepted": accepted, + "quarantined": quarantined, + "source_sha256": hashlib.sha256(directory).hexdigest(), + "schema_fingerprint": _schema_fingerprint(directory_headers), + "demographic_schema_fingerprint": _schema_fingerprint(demographic_headers) if demographic_headers else None, + "headers": directory_headers, + "demographic_headers": demographic_headers, + "input_rows": len(accepted) + len(quarantined), + "directory_rows": len(directory_rows), + "demographic_rows": len(demographic_rows), + "matched_demographic_rows": len(matched_demographics), + "orphan_demographic_rows": orphan_demographics, + "identity_conflicts": identity_conflicts, + } def run(self, raw_path: str | Path, run_dir: str | Path, artifact: SourceArtifact) -> dict[str, Any]: - raw = Path(raw_path).read_bytes(); digest = hashlib.sha256(raw).hexdigest() - if artifact.sha256 != digest or artifact.byte_size != len(raw): raise ValueError("artifact provenance mismatch") - result = self.parse_bytes(raw); accepted = result["accepted"]; quarantined = result["quarantined"] + raw = Path(raw_path).read_bytes() + digest = hashlib.sha256(raw).hexdigest() + if artifact.sha256 != digest or artifact.byte_size != len(raw): + raise ValueError("artifact provenance mismatch") + return self.run_sources({"directory": raw}, run_dir, {"directory": artifact}) + + def run_sources(self, raw_paths: dict[str, bytes | str | Path], run_dir: str | Path, artifacts: dict[str, SourceArtifact]) -> dict[str, Any]: + if "directory" not in raw_paths or "directory" not in artifacts: + raise ValueError("FSIS bundle requires a directory artifact") + directory = _bytes(raw_paths["directory"]) + demographics = _bytes(raw_paths["demographics"]) if "demographics" in raw_paths else None + for role, content in (("directory", directory), ("demographics", demographics)): + if content is None: + continue + artifact = artifacts.get(role) + if artifact is None or artifact.sha256 != hashlib.sha256(content).hexdigest() or artifact.byte_size != len(content): + raise ValueError(f"{role} artifact provenance mismatch") + result = self.parse_sources(directory, demographics) + accepted = result["accepted"] + quarantined = result["quarantined"] parsed = accepted + [item["record"] for item in quarantined] - _, parsed_sha, _ = atomic_jsonl(Path(run_dir) / "parsed/records.jsonl", parsed) - _, normalized_sha, _ = atomic_jsonl(Path(run_dir) / "normalized/records.jsonl", accepted) - atomic_jsonl(Path(run_dir) / "quarantined/records.jsonl", quarantined) + root = Path(run_dir) + _, parsed_sha, _ = atomic_jsonl(root / "parsed/records.jsonl", parsed) + _, normalized_sha, _ = atomic_jsonl(root / "normalized/records.jsonl", accepted) + atomic_jsonl(root / "quarantined/records.jsonl", quarantined) anomalies = Counter(reason for item in quarantined for reason in item["reasons"]) - manifest = private_manifest(source_id=self.source_id, adapter_version=self.adapter_version, schema_version=self.schema_version, artifact=artifact, input_rows=result["input_rows"], normalized_rows=len(accepted), quarantined_rows=len(quarantined), normalized_sha256=normalized_sha, parsed_sha256=parsed_sha, anomaly_counts=dict(sorted(anomalies.items()))) - manifest.update({"schema_fingerprint": result["schema_fingerprint"], "source_profile": "fsis-mpi-directory-plus-demographics", "geocoding": "disabled", "coverage": "FSIS-regulated meat, poultry, and egg establishments in the captured edition; state-inspection programs and non-FSIS populations excluded"}) - atomic_json(Path(run_dir) / "manifest.json", manifest) + directory_artifact = artifacts["directory"] + bundle_digest = hashlib.sha256("".join(f"{role}:{artifacts[role].sha256}\n" for role in sorted(artifacts)).encode()).hexdigest() + bundle_artifact = SourceArtifact( + source_url=directory_artifact.source_url, + retrieved_at_utc=directory_artifact.retrieved_at_utc, + sha256=bundle_digest, + byte_size=sum(artifact.byte_size for artifact in artifacts.values()), + publication_date=directory_artifact.publication_date, + effective_date=directory_artifact.effective_date, + code_version=directory_artifact.code_version, + config_version=directory_artifact.config_version, + rights_caveat=directory_artifact.rights_caveat, + privacy_caveat=directory_artifact.privacy_caveat, + coverage=directory_artifact.coverage, + redirects=directory_artifact.redirects, + ) + manifest = private_manifest( + source_id=self.source_id, adapter_version=self.adapter_version, schema_version=self.schema_version, + artifact=bundle_artifact, input_rows=result["input_rows"], normalized_rows=len(accepted), + quarantined_rows=len(quarantined), normalized_sha256=normalized_sha, parsed_sha256=parsed_sha, + anomaly_counts=dict(sorted(anomalies.items())), + ) + manifest.update({ + "source_profile": "fsis-mpi-directory-plus-demographics" if demographics is not None else "fsis-mpi-directory-only", + "schema_fingerprint": result["schema_fingerprint"], + "demographic_schema_fingerprint": result["demographic_schema_fingerprint"], + "source_artifacts": { + role: {"source_url": artifacts[role].source_url, "retrieved_at_utc": artifacts[role].retrieved_at_utc, + "sha256": artifacts[role].sha256, "byte_size": artifacts[role].byte_size, + "effective_date": artifacts[role].effective_date} + for role in sorted(artifacts) + }, + "row_reconciliation": { + "directory_rows": result["directory_rows"], "demographic_rows": result["demographic_rows"], + "matched_demographic_rows": result["matched_demographic_rows"], + "orphan_demographic_rows": result["orphan_demographic_rows"], + "identity_conflicts": result["identity_conflicts"], "unmatched_demographic_is_not_closure": True, + }, + "geocoding": "disabled", + "coverage": "FSIS-regulated meat, poultry, and egg establishments in the captured edition; state-inspection programs and non-FSIS populations excluded", + "publication_state": "private-candidate", + }) + atomic_json(root / "manifest.json", manifest) return manifest def write_candidate_handoff(self, run_dir: str | Path, artifact: SourceArtifact, *, output_dir: str | Path | None = None) -> dict[str, Any]: root = Path(run_dir) rows = [json.loads(line) for line in (root / "normalized/records.jsonl").read_text(encoding="utf-8").splitlines() if line] return write_handoff(output_dir or root, rows, artifact, source_id=self.source_id, profile="us-fsis-test-only") + + +def _bytes(value: bytes | str | Path) -> bytes: + return value if isinstance(value, bytes) else Path(value).read_bytes() diff --git a/pipeline/sources/us/fsis/config.json b/pipeline/sources/us/fsis/config.json index daeb12e..85d8719 100644 --- a/pipeline/sources/us/fsis/config.json +++ b/pipeline/sources/us/fsis/config.json @@ -1,10 +1,13 @@ { "source_id": "us.fsis", "contract_version": "us-fsis-mpi-v1", - "adapter_version": "us-fsis-candidate-v1", + "adapter_version": "us-fsis-candidate-v2", "authority": "USDA Food Safety and Inspection Service", "directory_url": "https://www.fsis.usda.gov/inspection/establishments/meat-poultry-and-egg-product-inspection-directory", - "data_url": "https://www.fsis.usda.gov/inspection/establishments/meat-poultry-and-egg-product-inspection-directory", + "data_url": "https://www.fsis.usda.gov/sites/default/files/media_file/documents/MPI_Directory_by_Establishment_Number.csv", + "directory_by_name_url": "https://www.fsis.usda.gov/sites/default/files/media_file/documents/MPI_Directory_by_Establishment_Name.csv", + "directory_by_number_url": "https://www.fsis.usda.gov/sites/default/files/media_file/documents/MPI_Directory_by_Establishment_Number.csv", + "demographics_url": "https://www.fsis.usda.gov/sites/default/files/media_file/documents/Dataset_Establishment_Demographic_Data.csv", "cadence": "weekly replacement; verify current edition before each run", "format": "CSV directory and supplemental establishment-demographic CSV", "release_allowed_by_default": false, diff --git a/pipeline/sources/us/fsis/fixtures/demographics.csv b/pipeline/sources/us/fsis/fixtures/demographics.csv new file mode 100644 index 0000000..b9eebe7 --- /dev/null +++ b/pipeline/sources/us/fsis/fixtures/demographics.csv @@ -0,0 +1,3 @@ +establishment_id,establishment_number,beef_cow_slaughter,young_chicken_slaughter,raw_intact_beef_processing,inspection_system_nsis,inspection_system_sis +FSIS-001,M001,Yes,,Yes,Yes, +FSIS-002,P002,,Yes,,Yes,Yes diff --git a/pipeline/sources/us/fsis/refresh.py b/pipeline/sources/us/fsis/refresh.py index 772868c..48c4772 100644 --- a/pipeline/sources/us/fsis/refresh.py +++ b/pipeline/sources/us/fsis/refresh.py @@ -1,51 +1,239 @@ """Private FSIS refresh with a sanctioned assisted-acquisition boundary.""" from __future__ import annotations -import argparse, hashlib, json + +import argparse +import hashlib +import json from pathlib import Path +from typing import Any + from pipeline.common.acquisition import AcquisitionError, fetch_source, utc_now -from pipeline.common.orchestrator import run_private_lifecycle from pipeline.contracts.adapter_contract import SourceArtifact from pipeline.contracts.source_lifecycle import atomic_json + from .adapter import CONFIG, FsisMpiAdapter -def assisted_capture_contract(*, source_url: str = CONFIG["directory_url"]) -> dict: - return {"source_id": CONFIG["source_id"], "method": "operator-assisted-official-export", "steps": ["Open the official FSIS MPI Directory page in an authorized browser session.", "Select the current downloadable MPI Directory and Establishment Demographic CSV files shown on that page.", "Save the files without editing them; record the displayed edition/publication date and the final download URLs.", "Place the selected directory CSV at the --raw path or use --fetch only when a terms-reviewed direct URL is authorized.", "Run dry-run first and review schema, count, duplicate, privacy, and category results before any test-only handoff."], "controls": ["No credential or access-control bypass", "HTML, login, 403, and schema-drift responses fail closed", "No raw artifact in Git", "FSIS facility evidence is not joined to APHIS rows"], "source_url": source_url} -def _local_facts(path: Path, *, retrieved_at_utc: str, effective_date: str | None) -> dict: +def assisted_capture_contract(*, source_url: str = CONFIG["directory_url"]) -> dict[str, Any]: + return { + "source_id": CONFIG["source_id"], + "method": "operator-assisted-official-export", + "steps": [ + "Open the official FSIS MPI Directory page in an authorized browser session.", + "Save one current MPI Directory CSV (by establishment name or number) and the supplemental Establishment Demographic CSV without editing them.", + "Record the displayed edition/publication date and the final URL for each saved file.", + "Place the directory CSV at --directory and, when available, the demographic CSV at --demographics; --raw remains a directory-only compatibility alias.", + "Run dry-run first and review schema fingerprints, count/drift, duplicate identities, exact-key reconciliation, privacy, coordinates, and category results before any test-only handoff.", + ], + "controls": [ + "No credential or access-control bypass", + "HTML, login, 403, and schema-drift responses fail closed", + "No raw artifact in Git", + "Directory and demographics join only on exact source-native IDs/numbers", + "FSIS facility evidence is not joined to APHIS rows", + ], + "source_url": source_url, + "current_routes": { + "directory_by_name": CONFIG.get("directory_by_name_url"), + "directory_by_number": CONFIG.get("directory_by_number_url"), + "demographics": CONFIG.get("demographics_url"), + }, + } + + +def _local_facts(path: Path, *, role: str, source_url: str, retrieved_at_utc: str, effective_date: str | None) -> dict[str, Any]: raw = path.read_bytes() - return {"acquisition_method": "preserved_local_artifact", "source_id": CONFIG["source_id"], "artifact": path.name, "artifact_path": str(path), "requested_url": CONFIG["directory_url"], "final_url": CONFIG["directory_url"], "retrieved_at_utc": retrieved_at_utc, "effective_date": effective_date or "unknown", "publication_date": None, "sha256": hashlib.sha256(raw).hexdigest(), "byte_size": len(raw), "code_version": CONFIG["adapter_version"], "config_version": CONFIG["contract_version"], "rights_caveat": "FSIS source terms and attribution require operator review before publication", "privacy_caveat": "private staging; address, phone, DUNS, and coordinate review pending", "coverage": "FSIS MPI edition only; state-inspection and APHIS populations excluded", "terms_review": "required before network acquisition or handoff"} + digest = hashlib.sha256(raw).hexdigest() + return { + "acquisition_method": "preserved_local_artifact", + "source_id": f"{CONFIG['source_id']}.{role}", + "artifact_role": role, + "artifact": path.name, + "artifact_path": str(path), + "requested_url": source_url, + "final_url": source_url, + "retrieved_at_utc": retrieved_at_utc, + "effective_date": effective_date or "unknown", + "publication_date": None, + "sha256": digest, + "byte_size": len(raw), + "code_version": CONFIG["adapter_version"], + "config_version": CONFIG["contract_version"], + "rights_caveat": "FSIS source terms and attribution require operator review before publication", + "privacy_caveat": "private staging; address, phone, DUNS, and coordinate review pending", + "coverage": "FSIS MPI edition only; state-inspection and APHIS populations excluded", + "terms_review": "required before network acquisition or handoff", + } + + +def _artifact(metadata: dict[str, Any]) -> SourceArtifact: + return SourceArtifact( + source_url=str(metadata.get("final_url") or metadata.get("requested_url")), + retrieved_at_utc=str(metadata["retrieved_at_utc"]), + sha256=str(metadata["sha256"]), + byte_size=int(metadata["byte_size"]), + publication_date=metadata.get("publication_date"), + effective_date=metadata.get("effective_date"), + code_version=str(metadata.get("code_version") or CONFIG["adapter_version"]), + config_version=str(metadata.get("config_version") or CONFIG["contract_version"]), + rights_caveat=metadata.get("rights_caveat"), + privacy_caveat=metadata.get("privacy_caveat"), + coverage=metadata.get("coverage"), + redirects=tuple(metadata.get("redirects") or ()), + ) + + +def _drift(manifest: dict[str, Any], previous_manifest: str | Path | None) -> dict[str, Any]: + if previous_manifest is None: + return {"checked": False, "blocked": False, "alarms": []} + previous = json.loads(Path(previous_manifest).read_text(encoding="utf-8")) + alarms: list[str] = [] + if previous.get("schema_fingerprint") and previous["schema_fingerprint"] != manifest.get("schema_fingerprint"): + alarms.append("directory_schema_fingerprint_changed") + if previous.get("demographic_schema_fingerprint") and previous.get("demographic_schema_fingerprint") != manifest.get("demographic_schema_fingerprint"): + alarms.append("demographic_schema_fingerprint_changed") + before = previous.get("row_reconciliation", {}).get("directory_rows") + after = manifest.get("row_reconciliation", {}).get("directory_rows") + if isinstance(before, int) and before and isinstance(after, int) and abs(after - before) > max(100, before // 10): + alarms.append("directory_row_count_changed_gt_10_percent") + return {"checked": True, "blocked": bool(alarms), "alarms": alarms, "previous_manifest": str(previous_manifest)} + + +def refresh( + *, + run_dir: str | Path, + raw_path: str | Path | None = None, + directory_path: str | Path | None = None, + demographics_path: str | Path | None = None, + fetch: bool = False, + source_url: str = CONFIG["directory_url"], + retrieved_at_utc: str | None = None, + effective_date: str | None = None, + mode: str = "dry-run", + terms_review_path: str | Path | None = None, + previous_manifest: str | Path | None = None, + max_bytes: int = 128 * 1024 * 1024, +) -> dict[str, Any]: + if raw_path is not None and directory_path is not None: + raise ValueError("specify raw_path or directory_path, not both") + if raw_path is not None and demographics_path is not None: + raise ValueError("--demographics requires --directory") + if fetch and (raw_path is not None or directory_path is not None or demographics_path is not None): + raise ValueError("fetch cannot be combined with local artifacts") + if not fetch and raw_path is None and directory_path is None: + raise ValueError("specify a directory artifact or fetch") + if mode not in {"dry-run", "handoff"}: + raise ValueError("mode must be dry-run or handoff") -def refresh(*, run_dir: str | Path, raw_path: str | Path | None = None, fetch: bool = False, source_url: str = CONFIG["directory_url"], terms_review_path: str | Path | None = None, retrieved_at_utc: str | None = None, effective_date: str | None = None, mode: str = "dry-run", max_bytes: int = 128 * 1024 * 1024) -> dict: - if fetch == (raw_path is not None): raise ValueError("specify exactly one of raw_path or fetch") - if mode not in {"dry-run", "handoff"}: raise ValueError("mode must be dry-run or handoff") root = Path(run_dir) + metadata: dict[str, Any] = {} + paths: dict[str, Path] = {} if fetch: - if terms_review_path is None: raise ValueError("terms_review_path is required for network acquisition") - try: acquisition = fetch_source(source_id=CONFIG["source_id"], url=source_url, output_root=root / "acquisition", artifact_name="source.csv", terms_review_path=terms_review_path, max_bytes=max_bytes, allowed_content_types=("text/csv", "application/csv", "application/octet-stream"), code_version=CONFIG["adapter_version"], config_version=CONFIG["contract_version"], coverage="FSIS MPI edition only; state-inspection and APHIS populations excluded", rights_caveat="terms review retained with run", privacy_caveat="private staging; privacy review pending", effective_date=effective_date) - except AcquisitionError as exc: raise ValueError(str(exc)) from exc - path = Path(acquisition["artifact_path"]) + if terms_review_path is None: + raise ValueError("terms_review_path is required for network acquisition") + routes = { + "directory": CONFIG.get("directory_by_number_url") or CONFIG["data_url"], + "demographics": CONFIG.get("demographics_url"), + } + for role, url in routes.items(): + if not url: + raise ValueError(f"missing configured FSIS {role} URL") + try: + acquired = fetch_source( + source_id=f"{CONFIG['source_id']}.{role}", url=url, output_root=root / "acquisition", + artifact_name=f"{role}.csv", terms_review_path=terms_review_path, max_bytes=max_bytes, + allowed_content_types=("text/csv", "application/csv", "application/octet-stream"), + code_version=CONFIG["adapter_version"], config_version=CONFIG["contract_version"], + coverage="FSIS MPI edition only; state-inspection and APHIS populations excluded", + rights_caveat="terms review retained with run", privacy_caveat="private staging; privacy review pending", + effective_date=effective_date, + ) + except AcquisitionError as exc: + raise ValueError(f"{role} acquisition failed closed: {exc}") from exc + paths[role] = Path(acquired["artifact_path"]) + metadata[role] = acquired else: - path = Path(raw_path) # type: ignore[arg-type] - if not path.is_file(): raise ValueError(f"raw artifact does not exist: {path}") - acquisition = _local_facts(path, retrieved_at_utc=retrieved_at_utc or utc_now(), effective_date=effective_date) - raw = path.read_bytes(); acquired_at = acquisition.get("retrieved_at_utc") or retrieved_at_utc or utc_now() - artifact = SourceArtifact(source_url=str(acquisition.get("final_url") or source_url), retrieved_at_utc=str(acquired_at), sha256=hashlib.sha256(raw).hexdigest(), byte_size=len(raw), publication_date=acquisition.get("publication_date"), effective_date=acquisition.get("effective_date") or effective_date, code_version=CONFIG["adapter_version"], config_version=CONFIG["contract_version"], rights_caveat=acquisition.get("rights_caveat"), privacy_caveat=acquisition.get("privacy_caveat"), coverage=acquisition.get("coverage"), redirects=tuple(acquisition.get("redirects") or ())) - atomic_json(root / "acquisition-metadata.json", acquisition) - status = run_private_lifecycle(path, root, artifact, FsisMpiAdapter(), health_as_of_utc=str(acquired_at)) - if mode == "handoff" and status.get("status") in {"candidate-ready", "success"}: - lifecycle_run = Path(status["run_dir"]) - status["handoff"] = FsisMpiAdapter().write_candidate_handoff(lifecycle_run, artifact, output_dir=lifecycle_run / "handoff") - status["mode"] = mode - if status.get("run_dir"): - atomic_json(Path(status["run_dir"]) / "run-status.json", status) - status["assisted_capture_contract"] = assisted_capture_contract(source_url=source_url); atomic_json(root / "assisted-capture-contract.json", status["assisted_capture_contract"]) + paths["directory"] = Path(directory_path or raw_path) # type: ignore[arg-type] + if not paths["directory"].is_file(): + raise ValueError(f"directory artifact does not exist: {paths['directory']}") + observed_at = retrieved_at_utc or utc_now() + metadata["directory"] = _local_facts(paths["directory"], role="directory", source_url=source_url, retrieved_at_utc=observed_at, effective_date=effective_date) + if demographics_path is not None: + paths["demographics"] = Path(demographics_path) + if not paths["demographics"].is_file(): + raise ValueError(f"demographics artifact does not exist: {paths['demographics']}") + metadata["demographics"] = _local_facts(paths["demographics"], role="demographics", source_url=CONFIG.get("demographics_url", source_url), retrieved_at_utc=observed_at, effective_date=effective_date) + + atomic_json(root / "acquisition-metadata.json", metadata) + artifacts = {role: _artifact(facts) for role, facts in metadata.items()} + adapter = FsisMpiAdapter() + lifecycle_root = root / "lifecycle" + manifest = adapter.run_sources(paths, lifecycle_root, artifacts) + drift = _drift(manifest, previous_manifest) + manifest["drift"] = drift + atomic_json(lifecycle_root / "manifest.json", manifest) + if drift["blocked"] and mode == "handoff": + raise ValueError("refresh drift alarm blocks handoff: " + ", ".join(drift["alarms"])) + + handoff = None + if mode == "handoff": + bundle_artifact = SourceArtifact( + source_url=manifest["source_url"], retrieved_at_utc=manifest["retrieved_at_utc"], + sha256=manifest["sha256"], byte_size=manifest["byte_size"], publication_date=manifest.get("publication_date"), + effective_date=manifest.get("effective_date"), code_version=manifest["code_version"], + config_version=manifest["config_version"], rights_caveat=manifest.get("acquisition", {}).get("rights_caveat"), + privacy_caveat=manifest.get("acquisition", {}).get("privacy_caveat"), coverage=manifest.get("coverage"), + ) + handoff = adapter.write_candidate_handoff(lifecycle_root, bundle_artifact, output_dir=lifecycle_root / "handoff") + + status = { + "status": "candidate-ready" if handoff else "staged-restricted", + "mode": mode, + "candidate_created": bool(handoff), + "release_promoted": False, + "release_state": "not-created", + "publication_state": "private-candidate" if handoff else "terms-gate-blocked", + "public_surfaces": {"api": False, "map": False, "export": False, "cache": False, "history": False}, + "geocoding": "disabled", + "run_dir": str(lifecycle_root), + "manifest": manifest, + "drift": drift, + "assisted_capture_contract": assisted_capture_contract(source_url=source_url), + } + atomic_json(lifecycle_root / "run-status.json", status) + atomic_json(root / "assisted-capture-contract.json", status["assisted_capture_contract"]) return status + def main() -> int: - parser = argparse.ArgumentParser(description=__doc__); source = parser.add_mutually_exclusive_group(required=True); source.add_argument("--raw", type=Path); source.add_argument("--fetch", action="store_true") - parser.add_argument("--run-dir", type=Path, required=True); parser.add_argument("--source-url", default=CONFIG["directory_url"]); parser.add_argument("--terms-review", type=Path); parser.add_argument("--retrieved-at-utc"); parser.add_argument("--effective-date"); parser.add_argument("--mode", choices=("dry-run", "handoff"), default="dry-run"); parser.add_argument("--max-bytes", type=int, default=128 * 1024 * 1024); args = parser.parse_args() - try: result = refresh(run_dir=args.run_dir, raw_path=args.raw, fetch=args.fetch, source_url=args.source_url, terms_review_path=args.terms_review, retrieved_at_utc=args.retrieved_at_utc, effective_date=args.effective_date, mode=args.mode, max_bytes=args.max_bytes) - except (OSError, ValueError) as exc: print(json.dumps({"status": "failed", "error": str(exc)})); return 2 - print(json.dumps({"status": result.get("status"), "run_dir": result.get("run_dir"), "manifest": result.get("manifest", {}).get("source_id")}, sort_keys=True)); return 0 + parser = argparse.ArgumentParser(description=__doc__) + source = parser.add_mutually_exclusive_group(required=True) + source.add_argument("--raw", type=Path, help="legacy alias for a directory CSV") + source.add_argument("--directory", type=Path) + source.add_argument("--fetch", action="store_true") + parser.add_argument("--demographics", type=Path) + parser.add_argument("--run-dir", type=Path, required=True) + parser.add_argument("--source-url", default=CONFIG["directory_url"]) + parser.add_argument("--retrieved-at-utc") + parser.add_argument("--effective-date") + parser.add_argument("--previous-manifest", type=Path) + parser.add_argument("--terms-review", type=Path) + parser.add_argument("--mode", choices=("dry-run", "handoff"), default="dry-run") + parser.add_argument("--max-bytes", type=int, default=128 * 1024 * 1024) + args = parser.parse_args() + try: + result = refresh( + run_dir=args.run_dir, raw_path=args.raw, directory_path=args.directory, demographics_path=args.demographics, + fetch=args.fetch, source_url=args.source_url, retrieved_at_utc=args.retrieved_at_utc, + effective_date=args.effective_date, mode=args.mode, terms_review_path=args.terms_review, + previous_manifest=args.previous_manifest, max_bytes=args.max_bytes, + ) + except (OSError, ValueError) as exc: + print(json.dumps({"status": "failed", "error": str(exc)})) + return 2 + print(json.dumps({"status": result.get("status"), "run_dir": result.get("run_dir"), "manifest": result.get("manifest", {}).get("source_id")}, sort_keys=True)) + return 0 + -if __name__ == "__main__": raise SystemExit(main()) +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/pipeline/sources/us/fsis/test_adapter.py b/pipeline/sources/us/fsis/test_adapter.py index b57b202..43543f3 100644 --- a/pipeline/sources/us/fsis/test_adapter.py +++ b/pipeline/sources/us/fsis/test_adapter.py @@ -13,14 +13,35 @@ class FsisAdapterTests(unittest.TestCase): def test_valid_fixture_preserves_source_values_and_blocks_coordinates(self): raw = (ROOT / "fixtures/valid.csv").read_bytes(); adapter = FsisMpiAdapter(); result = adapter.parse_bytes(raw) self.assertEqual(result["input_rows"], 2); self.assertEqual(len(result["accepted"]), 2); self.assertFalse(result["quarantined"]) - row = result["accepted"][0]; self.assertEqual(row["source_values"]["phone"], "555-0100") - self.assertIsNone(row["normalized"]["coordinates"]); self.assertEqual(row["normalized"]["country_code"], "US") + row = result["accepted"][0]; self.assertEqual(row["source_values"]["directory"]["phone"], "555-0100") + self.assertEqual(row["normalized"]["coordinates"]["latitude"], 32.1); self.assertEqual(row["normalized"]["country_code"], "US") + self.assertEqual(row["normalized"]["coordinate_gate"], "review_required") + + def test_directory_and_demographics_reconcile_on_exact_native_keys(self): + result = FsisMpiAdapter().parse_sources( + (ROOT / "fixtures/valid.csv").read_bytes(), + (ROOT / "fixtures/demographics.csv").read_bytes(), + ) + self.assertEqual(len(result["accepted"]), 2) + self.assertEqual(result["matched_demographic_rows"], 2) + row = result["accepted"][0]["normalized"] + self.assertEqual(row["species_slaughtered"]["beef_cow_slaughter"], "Yes") + self.assertEqual(row["processing_activities"]["raw_intact_beef_processing"], "Yes") + self.assertEqual(row["inspection_attributes"]["inspection_system_nsis"], "Yes") + self.assertEqual(result["accepted"][0]["source_values"]["demographics"]["establishment_id"], "FSIS-001") + + def test_unmatched_demographics_are_quarantined_not_dropped(self): + demographic = b"establishment_number,goat_slaughter\nNOT-IN-DIRECTORY,Yes\n" + result = FsisMpiAdapter().parse_sources((ROOT / "fixtures/valid.csv").read_bytes(), demographic) + self.assertEqual(len(result["accepted"]), 2) + self.assertEqual(result["orphan_demographic_rows"], 1) + self.assertIn("unmatched_demographic_identity", result["quarantined"][-1]["reasons"]) def test_duplicates_missing_name_and_unknown_state_quarantine(self): result = FsisMpiAdapter().parse_bytes((ROOT / "fixtures/malformed.csv").read_bytes()) self.assertEqual(len(result["accepted"]), 0); self.assertEqual(len(result["quarantined"]), 3) reasons = [set(item["reasons"]) for item in result["quarantined"]] - self.assertTrue(all("duplicate_establishment_id" in reason for reason in reasons[:2])) + self.assertTrue(all("duplicate_establishment_identity" in reason for reason in reasons[:2])) self.assertIn("unknown_state", reasons[2]); self.assertIn("missing_establishment_name", reasons[2]) def test_schema_drift_fails_closed(self): @@ -33,5 +54,6 @@ def test_run_is_deterministic_and_private(self): self.assertEqual(manifest["release_state"],"not-created"); self.assertEqual(manifest["publication_state"],"private-candidate") self.assertEqual(manifest["input_rows"],manifest["normalized_rows"]+manifest["quarantined_rows"]) self.assertTrue((Path(directory)/"parsed/records.jsonl").exists()); self.assertEqual(json.loads((Path(directory)/"normalized/records.jsonl").read_text().splitlines()[0])["normalized"]["publication_gate"],"blocked") + self.assertEqual(manifest["source_profile"], "fsis-mpi-directory-only") if __name__ == "__main__": unittest.main() diff --git a/pipeline/sources/us/fsis/test_refresh.py b/pipeline/sources/us/fsis/test_refresh.py new file mode 100644 index 0000000..953f1b5 --- /dev/null +++ b/pipeline/sources/us/fsis/test_refresh.py @@ -0,0 +1,48 @@ +import hashlib +import json +import tempfile +import unittest +from pathlib import Path + +from .refresh import refresh + + +ROOT = Path(__file__).parent + + +class FsisRefreshTests(unittest.TestCase): + def test_bundle_refresh_writes_private_handoff_and_provenance_per_file(self): + with tempfile.TemporaryDirectory() as directory: + result = refresh( + run_dir=Path(directory) / "run", + directory_path=ROOT / "fixtures/valid.csv", + demographics_path=ROOT / "fixtures/demographics.csv", + retrieved_at_utc="2026-09-18T00:00:00Z", + effective_date="2026-09-14", + mode="handoff", + ) + manifest = result["manifest"] + self.assertTrue(result["candidate_created"]) + self.assertEqual(manifest["release_state"], "not-created") + self.assertEqual(manifest["publication_state"], "private-candidate") + self.assertEqual(manifest["source_artifacts"]["demographics"]["byte_size"], len((ROOT / "fixtures/demographics.csv").read_bytes())) + self.assertEqual(manifest["row_reconciliation"]["matched_demographic_rows"], 2) + self.assertTrue((Path(directory) / "run/lifecycle/handoff/manifest.json").exists()) + + def test_schema_drift_blocks_handoff_after_previous_manifest(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + first = refresh(run_dir=root / "first", directory_path=ROOT / "fixtures/valid.csv", mode="dry-run") + previous = root / "first/lifecycle/manifest.json" + changed = root / "changed.csv" + original = (ROOT / "fixtures/valid.csv").read_text(encoding="utf-8").splitlines() + changed.write_text("\n".join([original[0] + ",new_column"] + [line + ",new" for line in original[1:]]) + "\n", encoding="utf-8") + # A changed but parseable header is detected against the prior + # manifest and blocks handoff. + with self.assertRaises(ValueError): + refresh(run_dir=root / "second", directory_path=changed, previous_manifest=previous, mode="handoff") + self.assertFalse((root / "second/lifecycle/handoff/manifest.json").exists()) + + +if __name__ == "__main__": + unittest.main() From f8d9f13a283d9d090c6c323b5aa7b48db77cdf9b Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Fri, 18 Sep 2026 09:15:16 -0700 Subject: [PATCH 243/311] Harden US current-data parity contracts --- pipeline/sources/us/accountability/README.md | 4 +- pipeline/sources/us/accountability/adapter.py | 25 ++++++++-- .../us/accountability/current_identity.py | 48 ++++++++++++++++--- .../sources/us/accountability/test_adapter.py | 23 +++++++++ .../accountability/test_current_identity.py | 20 ++++++++ pipeline/sources/us/aphis/acquire.py | 3 +- pipeline/sources/us/aphis/adapter.py | 10 +++- pipeline/sources/us/aphis/test_adapter.py | 19 ++++++++ pipeline/sources/us/fsis/adapter.py | 47 ++++++++++++++++-- pipeline/sources/us/fsis/test_adapter.py | 19 ++++++++ 10 files changed, 199 insertions(+), 19 deletions(-) diff --git a/pipeline/sources/us/accountability/README.md b/pipeline/sources/us/accountability/README.md index 842a597..8e1ae5b 100644 --- a/pipeline/sources/us/accountability/README.md +++ b/pipeline/sources/us/accountability/README.md @@ -74,8 +74,8 @@ suppression, or publication decision and does not inherit V1 assumptions. source-local adapters and emits a private, deterministic crosswalk handoff. It links: -* APHIS registrations to annual reports and inspections by certificate and/or - customer number; +* APHIS registrations to annual reports, explicit amended-report versions, and + inspections by certificate and/or customer number; * FSIS establishments to FSIS observation records by establishment and/or approval number. diff --git a/pipeline/sources/us/accountability/adapter.py b/pipeline/sources/us/accountability/adapter.py index fbb5c84..c071631 100644 --- a/pipeline/sources/us/accountability/adapter.py +++ b/pipeline/sources/us/accountability/adapter.py @@ -9,6 +9,7 @@ import csv import hashlib +import io import json from collections import Counter from datetime import date, datetime @@ -125,7 +126,16 @@ def _relationship(row: dict[str, str], *, observed_at: str, retrieved_at: str, s "url": row["evidence_url"], "excerpt": row["evidence_excerpt"], } - identity = {"subject": subject, "object": object_, "relationship_type": row["relationship_type"], "source_native_id": row["evidence_source_native_id"], "observation_date": observed_at} + identity = { + "subject": subject, + "object": object_, + "relationship_type": row["relationship_type"], + "evidence_source_id": row["evidence_source_id"], + "source_native_id": row["evidence_source_native_id"], + "observation_date": observed_at, + "valid_from": row.get("valid_from") or None, + "valid_to": row.get("valid_to") or None, + } relation_id = "candidate-relationship-" + hashlib.sha256(json.dumps(identity, sort_keys=True).encode()).hexdigest()[:24] relation = { "relationship_id": relation_id, @@ -170,7 +180,7 @@ def __init__(self, *, stale_after_days: int = CONFIG["stale_after_days"], refere def parse_bytes(self, content: bytes) -> dict[str, Any]: digest = hashlib.sha256(content).hexdigest() try: - reader = csv.DictReader(content.decode("utf-8-sig").splitlines(), strict=True) + reader = csv.DictReader(io.StringIO(content.decode("utf-8-sig"), newline=""), strict=True) headers = tuple(reader.fieldnames or ()) rows = list(reader) except (UnicodeDecodeError, csv.Error) as exc: @@ -179,8 +189,10 @@ def parse_bytes(self, content: bytes) -> dict[str, Any]: missing = [header for header in REQUIRED_HEADERS if header not in headers] extra = [header for header in headers if header not in REQUIRED_HEADERS] raise AccountabilityContractError(f"unsupported link-ledger schema; missing={missing}, extra={extra}") - if len(headers) != len(set(headers)) or any(None in row for row in rows): + if len(headers) != len(set(headers)) or any(None in row or any(value is None for value in row.values()) for row in rows): raise AccountabilityContractError("schema drift: duplicate or extra link-ledger columns") + if not rows: + raise AccountabilityContractError("link-ledger contains no data rows") accepted: list[dict[str, Any]] = [] quarantined: list[dict[str, Any]] = [] @@ -238,8 +250,11 @@ def parse_bytes(self, content: bytes) -> dict[str, Any]: reasons.append("ambiguous_duplicate_name") relationship_key = ( row["subject_type"], row["subject_source_id"], - row["subject_source_native_id"], row["object_source_native_id"], - row["relationship_type"], + row["subject_source_native_id"], row["object_type"], + row["object_source_id"], row["object_source_native_id"], + row["relationship_type"], row["evidence_source_id"], + row["evidence_source_native_id"], row["observation_date"], + row.get("valid_from") or "", row.get("valid_to") or "", ) if relationship_key in relationship_keys: reasons.append("duplicate_relationship_observation") diff --git a/pipeline/sources/us/accountability/current_identity.py b/pipeline/sources/us/accountability/current_identity.py index 3aac069..b3b5f98 100644 --- a/pipeline/sources/us/accountability/current_identity.py +++ b/pipeline/sources/us/accountability/current_identity.py @@ -36,9 +36,10 @@ "publication_status": "not_eligible", "release_id": None, } -APHIS_PROFILES = frozenset({"registrations", "annual_reports", "inspections"}) +APHIS_PROFILES = frozenset({"registrations", "annual_reports", "amendments", "inspections"}) ALTERNATE_PAIRS = { ("registrations", "annual_reports"), + ("registrations", "amendments"), ("registrations", "inspections"), ("fsis_establishments", "fsis_observations"), } @@ -132,11 +133,20 @@ def _provenance( profile: str, ) -> dict[str, str] | None: """Resolve per-profile provenance, falling back to source-wide metadata.""" - values = ( - provenance.get((source_id, profile)) - or provenance.get(f"{source_id}:{profile}") - or provenance.get(source_id) - ) + profile_keys = [profile] + # Amendments are versioned rows in the annual-report capture unless a + # separate amendment artifact was explicitly supplied. + if profile == "amendments": + profile_keys.append("annual_reports") + values = None + for profile_key in profile_keys: + values = ( + provenance.get((source_id, profile_key)) + or provenance.get(f"{source_id}:{profile_key}") + ) + if values: + break + values = values or provenance.get(source_id) if not isinstance(values, Mapping): return None digest = _text(values.get("artifact_sha256") or values.get("sha256")) @@ -272,9 +282,17 @@ def _candidate( if reason: result["quarantine_reason"] = reason result["reason"] = reason + elif method == "exact_official_identifier": + matched = ", ".join(sorted((matched_identifiers or {}).keys())) or "source-native identifiers" + result["confidence_explanation"] = f"Exact agreement on {matched}; source records remain separate and require review." + elif method == "alternate_name_address_exact": + result["confidence_explanation"] = "Exact normalized name and complete address agreement is a review candidate only; no official identifier matched." + else: + result["confidence_explanation"] = "No defensible identity evidence was accepted; retained for private review only." if not provenance_ok and not reason: result["quarantine_reason"] = "missing_or_invalid_provenance" result["assertion_status"] = "quarantined" + result["confidence_explanation"] = "Candidate lacks complete artifact provenance and is quarantined." return result @@ -429,6 +447,8 @@ def build_current_identity_graph( """ registrations = _records_for(aphis_records.get("registrations", ()), "registrations") annual_reports = _records_for(aphis_records.get("annual_reports", ()), "annual_reports") + amendments = _records_for(aphis_records.get("annual_reports", ()), "amendments") + amendments.extend(_records_for(aphis_records.get("amendments", ()), "amendments")) inspections = _records_for(aphis_records.get("inspections", ()), "inspections") establishments = _records_for(fsis_records, "fsis_establishments") observations = _records_for(fsis_observations, "fsis_observations") @@ -436,6 +456,7 @@ def build_current_identity_graph( all_records: list[tuple[Mapping[str, Any], str]] = [] all_records.extend((record, "registrations") for record in registrations) all_records.extend((record, "annual_reports") for record in annual_reports) + all_records.extend((record, "amendments") for record in amendments) all_records.extend((record, "inspections") for record in inspections) all_records.extend((record, "fsis_establishments") for record in establishments) all_records.extend((record, "fsis_observations") for record in observations) @@ -449,6 +470,7 @@ def build_current_identity_graph( for left_profile, right_profile, left, right in ( ("registrations", "annual_reports", registrations, annual_reports), + ("registrations", "amendments", registrations, amendments), ("registrations", "inspections", registrations, inspections), ("fsis_establishments", "fsis_observations", establishments, observations), ): @@ -570,9 +592,23 @@ def build_current_identity_graph( def write_current_identity_graph(run_dir: str | Path, graph: Mapping[str, Any]) -> dict[str, Any]: """Atomically write the row-private candidate handoff and manifest.""" root = Path(run_dir) + source_manifest = graph.get("manifest") + if not isinstance(source_manifest, Mapping) or any( + ( + source_manifest.get("storage_state") != "private", + source_manifest.get("publication_status") != "not_eligible", + source_manifest.get("test_only") is not True, + source_manifest.get("auto_merge") is not False, + ) + ): + raise ValueError("current identity graph is not a blocked private test-only candidate") candidates = list(graph.get("candidates", ())) entities = list(graph.get("entities", ())) quarantined = list(graph.get("quarantined", ())) + for row in (*candidates, *entities, *quarantined): + publication = row.get("publication") + if row.get("test_only") is not True or not isinstance(publication, Mapping) or publication.get("publication_status") != "not_eligible": + raise ValueError("current identity row is not a blocked private test-only candidate") _, candidate_sha, _ = atomic_jsonl(root / "candidate" / "identity-links.jsonl", candidates) _, entity_sha, _ = atomic_jsonl(root / "candidate" / "entities.jsonl", entities) _, quarantine_sha, _ = atomic_jsonl(root / "quarantined" / "identity-links.jsonl", quarantined) diff --git a/pipeline/sources/us/accountability/test_adapter.py b/pipeline/sources/us/accountability/test_adapter.py index ea16cbf..4a159ee 100644 --- a/pipeline/sources/us/accountability/test_adapter.py +++ b/pipeline/sources/us/accountability/test_adapter.py @@ -131,6 +131,29 @@ def test_exact_duplicate_relationship_is_quarantined(self): self.assertEqual(len(result["quarantined"]), 1) self.assertEqual(result["quarantined"][0]["reasons"], ("duplicate_relationship_observation",)) + def test_distinct_source_observations_are_not_collapsed_as_duplicates(self): + rows = rows_from_fixture() + observation = dict(rows[0]) + observation.update({ + "object_source_id": "us.fsis.snapshot-2", + "evidence_source_id": "us.fsis.snapshot-2", + "evidence_source_native_id": "legacy-row:FSIS-001:second-observation", + "observation_date": "2026-09-02", + }) + rows.append(observation) + result = UsAccountabilityAdapter().parse_bytes(content_for(rows)) + self.assertEqual(len(result["accepted"]), 13) + self.assertFalse(result["quarantined"]) + + def test_multiline_fields_and_short_rows_fail_closed(self): + rows = rows_from_fixture() + rows[0]["evidence_excerpt"] = "First line\nSecond line" + result = UsAccountabilityAdapter().parse_bytes(content_for(rows)) + self.assertEqual(len(result["accepted"]), 12) + short = (",".join(REQUIRED_HEADERS) + "\nfacility,us.fsis,FSIS-001\n").encode() + with self.assertRaises(AccountabilityContractError): + UsAccountabilityAdapter().parse_bytes(short) + if __name__ == "__main__": unittest.main() diff --git a/pipeline/sources/us/accountability/test_current_identity.py b/pipeline/sources/us/accountability/test_current_identity.py index 6083552..96363c5 100644 --- a/pipeline/sources/us/accountability/test_current_identity.py +++ b/pipeline/sources/us/accountability/test_current_identity.py @@ -36,6 +36,19 @@ def test_exact_official_ids_link_each_observation_family(self): aphis_link = next(candidate for candidate in graph["candidates"] if candidate["right"]["profile"] == "annual_reports") self.assertEqual(set(aphis_link["evidence"]["provenance"]), {"us.aphis:registrations", "us.aphis:annual_reports"}) self.assertEqual(len(graph["entities"]), 5) + self.assertTrue(all(candidate["confidence_explanation"] for candidate in graph["candidates"])) + + def test_amended_annual_reports_remain_source_versioned_and_linkable(self): + payload = load_fixture() + amendment = copy.deepcopy(payload["aphis"]["annual_reports"][0]) + amendment["source_record_key"] = "amendments:00-B-TEST-001:2025:2" + amendment["normalized"]["evidence_type"] = "amendments" + payload["aphis"]["annual_reports"].append(amendment) + graph = self.build(payload) + amendment_links = [candidate for candidate in graph["candidates"] if candidate["right"]["profile"] == "amendments"] + self.assertEqual(len(amendment_links), 1) + self.assertEqual(amendment_links[0]["assertion_status"], "candidate") + self.assertTrue(any(entity["entity_type"] == "amendments" for entity in graph["entities"])) def test_same_name_different_address_is_not_a_join(self): payload = load_fixture() @@ -111,6 +124,13 @@ def test_write_is_idempotent_and_private(self): (Path(directory) / "two" / "candidate" / "identity-links.jsonl").read_bytes(), ) + def test_write_rejects_a_promoted_or_non_test_only_graph(self): + graph = self.build() + graph["manifest"]["test_only"] = False + with tempfile.TemporaryDirectory() as directory: + with self.assertRaises(ValueError): + write_current_identity_graph(Path(directory) / "blocked", graph) + if __name__ == "__main__": unittest.main() diff --git a/pipeline/sources/us/aphis/acquire.py b/pipeline/sources/us/aphis/acquire.py index 5e059b1..3abdedf 100644 --- a/pipeline/sources/us/aphis/acquire.py +++ b/pipeline/sources/us/aphis/acquire.py @@ -10,6 +10,7 @@ import csv import hashlib +import io import json from pathlib import Path from typing import Any @@ -57,7 +58,7 @@ def _csv_download_is_valid(path: Path) -> None: action="use the documented browser export workflow; do not bypass the challenge", ) try: - rows = list(csv.reader(raw.decode("utf-8-sig").splitlines(), strict=True)) + rows = list(csv.reader(io.StringIO(raw.decode("utf-8-sig"), newline=""), strict=True)) except (UnicodeDecodeError, csv.Error) as error: raise AcquisitionError( "APHIS export is malformed or truncated", diff --git a/pipeline/sources/us/aphis/adapter.py b/pipeline/sources/us/aphis/adapter.py index 01c28f6..ceb2aea 100644 --- a/pipeline/sources/us/aphis/adapter.py +++ b/pipeline/sources/us/aphis/adapter.py @@ -9,6 +9,7 @@ import csv import hashlib +import io import json from collections import Counter from pathlib import Path @@ -59,7 +60,7 @@ def _schema_fingerprint(headers: tuple[str, ...]) -> str: def _read(content: bytes) -> tuple[tuple[str, ...], list[dict[str, Any]]]: try: text = content.decode("utf-8-sig") - reader = csv.DictReader(text.splitlines(), strict=True) + reader = csv.DictReader(io.StringIO(text, newline=""), strict=True) headers = tuple(reader.fieldnames or ()) rows = list(reader) except (UnicodeDecodeError, csv.Error) as exc: @@ -74,6 +75,8 @@ def _read(content: bytes) -> tuple[tuple[str, ...], list[dict[str, Any]]]: # DictReader represents short rows with None-valued cells and extra # columns with a None key. Both are schema failures, not missing source # values: a genuinely blank cell is represented by an empty string. + if not rows: + raise AphisContractError("APHIS export contains no data rows") if any(None in row or any(value is None for value in row.values()) for row in rows): raise AphisContractError("APHIS schema drift or malformed row") return headers, rows @@ -149,6 +152,11 @@ def _observation_key(profile: str, row: dict[str, Any]) -> str | None: if profile in {"annual_reports", "amendments"}: parts.append(f"year={_year(row) or 'unknown'}") parts.append(f"version={_amendment_version(row) or 'original'}") + elif profile == "inspections": + # A certificate/customer can have multiple inspection observations over + # time. Keep those observations distinct when the source supplies its + # observation date; an undated duplicate remains quarantine-worthy. + parts.append(f"status_date={_clean(row.get('Status Date')) or 'unknown'}") return f"{profile}|" + "|".join(parts) diff --git a/pipeline/sources/us/aphis/test_adapter.py b/pipeline/sources/us/aphis/test_adapter.py index c827f03..1d1f438 100644 --- a/pipeline/sources/us/aphis/test_adapter.py +++ b/pipeline/sources/us/aphis/test_adapter.py @@ -32,6 +32,13 @@ def test_annual_reports_use_year_in_observation_identity(self): self.assertEqual(len(result["accepted"]),2) self.assertNotEqual(result["accepted"][0]["source_record_key"], result["accepted"][1]["source_record_key"]) + def test_inspections_use_status_date_in_observation_identity(self): + raw = (ROOT / "fixtures/inspections.csv").read_text(encoding="utf-8") + second = raw.splitlines()[1].replace("2026-02-01", "2026-03-01") + result = AphisPublicSearchAdapter().parse_bytes((raw + second + "\n").encode()) + self.assertEqual(len(result["accepted"]), 2) + self.assertNotEqual(result["accepted"][0]["source_record_key"], result["accepted"][1]["source_record_key"]) + def test_unsupported_profile_fails_closed(self): with self.assertRaises(AphisContractError): AphisPublicSearchAdapter().parse_bytes(b"Name,Value\nA,B\n") @@ -59,6 +66,18 @@ def test_short_rows_fail_closed_as_schema_drift(self): with self.assertRaises(AphisContractError): AphisPublicSearchAdapter().parse_bytes(b"Account Name,Certificate Number,Certificate Status\nOnly One Cell\n") + def test_multiline_csv_fields_and_header_only_exports_fail_closed(self): + multiline = ( + b"Account Name,Customer Number,Certificate Number,License Type,Certificate Status,Status Date\n" + b"\"Synthetic\nRegistrant\",1,00-B-0001,Class B,Active,2026-01-01\n" + ) + result = AphisPublicSearchAdapter().parse_bytes(multiline) + self.assertEqual(result["accepted"][0]["normalized"]["account_name"], "Synthetic\nRegistrant") + with self.assertRaises(AphisContractError): + AphisPublicSearchAdapter().parse_bytes( + b"Account Name,Customer Number,Certificate Number,License Type,Certificate Status,Status Date\n" + ) + def test_run_is_idempotent_and_reconciles_every_row(self): raw = (ROOT / "fixtures/inspections.csv").read_bytes() artifact = SourceArtifact("https://example.invalid/aphis.csv", "2026-09-15T00:00:00Z", hashlib.sha256(raw).hexdigest(), len(raw), code_version="test", config_version="test") diff --git a/pipeline/sources/us/fsis/adapter.py b/pipeline/sources/us/fsis/adapter.py index 6a26c61..74dc774 100644 --- a/pipeline/sources/us/fsis/adapter.py +++ b/pipeline/sources/us/fsis/adapter.py @@ -9,6 +9,7 @@ import csv import hashlib +import io import json import re from collections import Counter @@ -48,7 +49,7 @@ def _schema_fingerprint(headers: tuple[str, ...]) -> str: def _csv(content: bytes, *, role: str) -> tuple[tuple[str, ...], list[dict[str, Any]]]: try: - reader = csv.DictReader(content.decode("utf-8-sig").splitlines(), strict=True) + reader = csv.DictReader(io.StringIO(content.decode("utf-8-sig"), newline=""), strict=True) headers = tuple(reader.fieldnames or ()) rows = list(reader) except (UnicodeDecodeError, csv.Error) as exc: @@ -57,7 +58,9 @@ def _csv(content: bytes, *, role: str) -> tuple[tuple[str, ...], list[dict[str, raise FsisContractError(f"missing FSIS {role} header") if None in headers or len(set(headers)) != len(headers): raise FsisContractError(f"duplicate or unnamed FSIS {role} columns") - if any(None in row for row in rows): + if not rows: + raise FsisContractError(f"FSIS {role} CSV contains no data rows") + if any(None in row or any(value is None for value in row.values()) for row in rows): raise FsisContractError(f"FSIS {role} schema drift: row has extra columns") normalized_headers = {_header_key(header) for header in headers} if role == "directory" and not ({"establishment_id", "establishment_number"} & normalized_headers): @@ -88,6 +91,16 @@ def _key_candidates(row: dict[str, Any]) -> tuple[str, ...]: return tuple(values) +def _identity_values(row: dict[str, Any]) -> dict[str, str | None]: + """Return identity values by source-native kind, not just raw value.""" + return { + "establishment_id": _field(row, "establishment_id", "establishment id", "mpi id"), + "establishment_number": _field( + row, "establishment_number", "establishment number", "establishment no", "establishment no.", "number" + ), + } + + def _identity_key(row: dict[str, Any]) -> str | None: candidates = _key_candidates(row) return candidates[0] if candidates else None @@ -227,8 +240,31 @@ def _quarantine(record: dict[str, Any], reasons: Iterable[str]) -> dict[str, Any def _demo_matches(row: dict[str, Any], directory: dict[str, Any]) -> bool: - demo_keys = set(_key_candidates(row)) - return bool(demo_keys & set(_key_candidates(directory))) + demo_values = _identity_values(row) + directory_values = _identity_values(directory) + shared = any( + demo_values[k] and directory_values[k] and demo_values[k] == directory_values[k] + for k in demo_values + ) + conflicting = any( + demo_values[k] and directory_values[k] and demo_values[k] != directory_values[k] + for k in demo_values + ) + return shared and not conflicting + + +def _demo_identity_conflict(row: dict[str, Any], directory: dict[str, Any]) -> bool: + demo_values = _identity_values(row) + directory_values = _identity_values(directory) + shared = any( + demo_values[k] and directory_values[k] and demo_values[k] == directory_values[k] + for k in demo_values + ) + conflicting = any( + demo_values[k] and directory_values[k] and demo_values[k] != directory_values[k] + for k in demo_values + ) + return shared and conflicting class FsisMpiAdapter: @@ -273,6 +309,9 @@ def parse_sources(self, directory: bytes, demographics: bytes | None = None) -> matching_demo = [demo_index for demo_index, demo in enumerate(demographic_rows) if _demo_matches(demo, row)] demographic: dict[str, Any] | None = None demographic_line: int | None = None + if any(_demo_identity_conflict(demo, row) for demo in demographic_rows): + reasons.append("conflicting_demographic_identity") + identity_conflicts += 1 if len(matching_demo) > 1: reasons.append("ambiguous_demographic_identity") identity_conflicts += 1 diff --git a/pipeline/sources/us/fsis/test_adapter.py b/pipeline/sources/us/fsis/test_adapter.py index 43543f3..a24381b 100644 --- a/pipeline/sources/us/fsis/test_adapter.py +++ b/pipeline/sources/us/fsis/test_adapter.py @@ -37,6 +37,25 @@ def test_unmatched_demographics_are_quarantined_not_dropped(self): self.assertEqual(result["orphan_demographic_rows"], 1) self.assertIn("unmatched_demographic_identity", result["quarantined"][-1]["reasons"]) + def test_demographic_identifier_conflict_does_not_join_on_one_matching_alias(self): + demographic = ( + b"establishment_id,establishment_number,beef_cow_slaughter\n" + b"FSIS-001,WRONG-NUMBER,Yes\n" + b"FSIS-002,P002,Yes\n" + ) + result = FsisMpiAdapter().parse_sources((ROOT / "fixtures/valid.csv").read_bytes(), demographic) + self.assertEqual(len(result["accepted"]), 1) + self.assertEqual(result["matched_demographic_rows"], 1) + reasons = [reason for item in result["quarantined"] for reason in item["reasons"]] + self.assertIn("conflicting_demographic_identity", reasons) + + def test_multiline_csv_fields_and_header_only_exports_fail_closed(self): + multiline = b"establishment_id,establishment_name,state\nFSIS-100,\"Plant\nNorth\",TX\n" + result = FsisMpiAdapter().parse_bytes(multiline) + self.assertEqual(result["accepted"][0]["normalized"]["canonical_name"], "Plant\nNorth") + with self.assertRaises(FsisContractError): + FsisMpiAdapter().parse_bytes(b"establishment_id,establishment_name,state\n") + def test_duplicates_missing_name_and_unknown_state_quarantine(self): result = FsisMpiAdapter().parse_bytes((ROOT / "fixtures/malformed.csv").read_bytes()) self.assertEqual(len(result["accepted"]), 0); self.assertEqual(len(result["quarantined"]), 3) From 1fa2d2d69ddcad7a5404cdf089430d604e1ad200 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Fri, 18 Sep 2026 11:58:19 -0700 Subject: [PATCH 244/311] Consolidate US real-data proof Wave 1 --- ...phis-wave1-real-data-proof-2026-09-18.json | 1139 +++++++++++++++++ data/manifests/us-fsis-proof-2026-09-18.json | 57 + docs/countries/us/README.md | 56 + docs/countries/us/operator-refresh.md | 119 ++ .../us/research-animal-coverage-recon.md | 127 ++ docs/countries/us/state-mpi-source-recon.md | 192 +++ docs/country-recon-us.md | 53 + docs/source-status.json | 10 + docs/source-status.md | 10 + pipeline/common/acquisition.py | 14 + pipeline/common/test_acquisition.py | 48 + pipeline/common/test_source_operations.py | 4 +- .../maintenance/rehearse_fsis_candidate.py | 158 +++ pipeline/source_operations.json | 4 +- pipeline/source_registry.json | 120 ++ pipeline/sources/us/aphis/acquire.py | 12 + pipeline/sources/us/aphis/adapter.py | 38 +- pipeline/sources/us/aphis/refresh.py | 12 + pipeline/sources/us/aphis/test_adapter.py | 32 + pipeline/sources/us/fsis/adapter.py | 47 +- pipeline/sources/us/fsis/refresh.py | 32 +- pipeline/sources/us/fsis/test_refresh.py | 7 +- pipeline/sources/us/refresh.py | 347 +++++ pipeline/sources/us/test_refresh.py | 98 ++ pipeline/tests/test_source_registry.py | 5 +- 25 files changed, 2720 insertions(+), 21 deletions(-) create mode 100644 data/manifests/us-aphis-wave1-real-data-proof-2026-09-18.json create mode 100644 data/manifests/us-fsis-proof-2026-09-18.json create mode 100644 docs/countries/us/operator-refresh.md create mode 100644 docs/countries/us/research-animal-coverage-recon.md create mode 100644 docs/countries/us/state-mpi-source-recon.md create mode 100644 pipeline/scripts/maintenance/rehearse_fsis_candidate.py create mode 100644 pipeline/sources/us/refresh.py create mode 100644 pipeline/sources/us/test_refresh.py diff --git a/data/manifests/us-aphis-wave1-real-data-proof-2026-09-18.json b/data/manifests/us-aphis-wave1-real-data-proof-2026-09-18.json new file mode 100644 index 0000000..c1cb6c6 --- /dev/null +++ b/data/manifests/us-aphis-wave1-real-data-proof-2026-09-18.json @@ -0,0 +1,1139 @@ +{ + "manifest_version": "us-aphis-real-data-proof-v1", + "source_id": "us.aphis", + "run_id": "20260918T000000Z-aphis-wave1", + "captured_for": "private/test-only", + "release_state": "not-created", + "publication_gate": "blocked", + "official_routes": { + "public_search_tool": "https://aphis.my.site.com/PublicSearchTool/s/", + "annual_reports": "https://aphis.my.site.com/PublicSearchTool/s/annual-reports", + "inspection_reports": "https://aphis.my.site.com/PublicSearchTool/s/inspection-reports" + }, + "retrieval": { + "annual_reports": { + "retrieved_at_utc": "2026-09-18T17:54:55Z", + "selected_year": "2025", + "displayed_result_count": 995, + "raw_pages": 10, + "source_rows": 995, + "refresh_pages_succeeded": 10 + }, + "registrations": { + "retrieval_windows_utc": [ + "2026-09-18T17:58:45Z", + "2026-09-18T18:06:26Z" + ], + "displayed_result_count": 2841, + "raw_pages": 81, + "raw_source_rows": 4941, + "unfiltered_pages": 21, + "state_partition_pages": 60, + "refresh_pages_succeeded": 78, + "refresh_pages_failed": 3, + "state_partitioning_used_to_work_around_unfiltered_pagination_boundary": true + }, + "inspections": { + "retrieved_at_utc": "2026-09-18T18:17:14Z", + "view": "View Inspection Reports", + "selected_license_registration_type": "RESEARCH FACILITY", + "displayed_result_count": 15726, + "raw_pages": 21, + "source_rows": 2100, + "refresh_pages_succeeded": 21, + "pagination_boundary_remaining_rows": 13626 + } + }, + "normalized": { + "annual_reports": { + "accepted_page_rows": 995, + "quarantined_rows": 0, + "distinct_source_observation_keys": 995, + "coordinates_present": 0, + "coordinate_state": "unknown_not_supplied" + }, + "registrations": { + "accepted_page_rows": 4763, + "quarantined_rows": 0, + "distinct_source_observation_keys": 2811, + "coordinates_present": 0, + "coordinate_state": "unknown_not_supplied", + "note": "Unfiltered and state-partition exports intentionally remain separate evidence pages; duplicate identities are not silently merged." + }, + "inspections": { + "accepted_page_rows": 1917, + "quarantined_rows": 183, + "distinct_source_observation_keys": 1917, + "quarantine_reason_counts": { + "duplicate_observation_id": 183 + }, + "coordinates_present": 0, + "coordinate_state": "unknown_not_supplied" + } + }, + "documents_and_amendments": { + "annual_page_001_visible_links": { + "view_report": 100, + "download_exception": 21 + }, + "attempted_visible_downloads": 2, + "acquired_document_artifacts": 0, + "limitation": "Observed report links resolved to the visible APHIS Force.com download URLs, but the authorized browser returned ERR_BLOCKED_BY_CLIENT. No hidden endpoint, bypass, or alternate downloader was used; signed URLs are not retained in this tracked report.", + "amendment_policy": "Annual/amended evidence remains a separate profile/version; no automatic merge." + }, + "failures": [ + { + "artifact": "registrations_unfiltered_page_020.csv", + "reason": "malformed_or_truncated_csv", + "retry": "same failure reproduced by a second official Export To CSV action" + }, + { + "artifact": "registrations_state_partition_019.csv", + "reason": "malformed_or_truncated_csv", + "retry": "not_retried_after equivalent source-format failure preserved" + }, + { + "artifact": "registrations_state_partition_033.csv", + "reason": "malformed_or_truncated_csv", + "retry": "not_retried_after equivalent source-format failure preserved" + } + ], + "raw_artifacts": [ + { + "artifact": "annual_reports_page_001.csv", + "profile": "annual_reports", + "bytes": 6218, + "sha256": "896257ad067600430bebe64ddad880b19f5a0aa96c95ed254a623bcbec3e47a9", + "line_count": 101, + "source_row_count": 100, + "validation": "validated_csv" + }, + { + "artifact": "annual_reports_page_002.csv", + "profile": "annual_reports", + "bytes": 6204, + "sha256": "99b2420c539711e139c361ca5af88a8927ebd7c30bf8240f914161ee876bf103", + "line_count": 101, + "source_row_count": 100, + "validation": "validated_csv" + }, + { + "artifact": "annual_reports_page_003.csv", + "profile": "annual_reports", + "bytes": 6184, + "sha256": "8bc90f580feff2da5557a169d076a063b3dd83e83ed9741bb7b4dfb5a777cefd", + "line_count": 101, + "source_row_count": 100, + "validation": "validated_csv" + }, + { + "artifact": "annual_reports_page_004.csv", + "profile": "annual_reports", + "bytes": 6195, + "sha256": "4ad684075cb24d22b096894262e9c272b7fdcb66ee8e35f6ea55be690282da22", + "line_count": 101, + "source_row_count": 100, + "validation": "validated_csv" + }, + { + "artifact": "annual_reports_page_005.csv", + "profile": "annual_reports", + "bytes": 6181, + "sha256": "5095c0407a34571f765f963de872cd96ee6c94b09a85ce116a598a889360c830", + "line_count": 101, + "source_row_count": 100, + "validation": "validated_csv" + }, + { + "artifact": "annual_reports_page_006.csv", + "profile": "annual_reports", + "bytes": 6256, + "sha256": "7e5a93abaafded4781dca7026f1aac81d491711bea3d17ba33ee6ee4109e2342", + "line_count": 101, + "source_row_count": 100, + "validation": "validated_csv" + }, + { + "artifact": "annual_reports_page_007.csv", + "profile": "annual_reports", + "bytes": 6216, + "sha256": "ba39a3aa8b2c70e63988403ee6f61e5ef4f1481ee340766a33da97af546f5c4d", + "line_count": 101, + "source_row_count": 100, + "validation": "validated_csv" + }, + { + "artifact": "annual_reports_page_008.csv", + "profile": "annual_reports", + "bytes": 6293, + "sha256": "bfa20ad6fd9056d6e51579c026d16afa545e444dab4a30c7ba25ba32d7725cb3", + "line_count": 101, + "source_row_count": 100, + "validation": "validated_csv" + }, + { + "artifact": "annual_reports_page_009.csv", + "profile": "annual_reports", + "bytes": 6382, + "sha256": "b6b91a824ffe8c5b95b8e896d0777bf468d3aac7aadf176e221396cfd18bb5b8", + "line_count": 101, + "source_row_count": 100, + "validation": "validated_csv" + }, + { + "artifact": "annual_reports_page_010.csv", + "profile": "annual_reports", + "bytes": 6046, + "sha256": "dfd982c0457ecee4f953b98864687fcb17cdebcdc27fabd09ebb3641e15080b0", + "line_count": 96, + "source_row_count": 95, + "validation": "validated_csv" + }, + { + "artifact": "inspections_research_facility_unfiltered_page_001.csv", + "profile": "inspections", + "bytes": 16745, + "sha256": "7732fa51158e64772dc9435be4e3e9023d6d4f104bb2e4dbfadefcaf7f6cd44d", + "line_count": 101, + "source_row_count": 100, + "validation": "validated_csv" + }, + { + "artifact": "inspections_research_facility_unfiltered_page_002.csv", + "profile": "inspections", + "bytes": 16766, + "sha256": "2c2ef0a21b6e0b2422aab65328f1ae6c58798719011b075cd696c8dde12dc8f8", + "line_count": 101, + "source_row_count": 100, + "validation": "validated_csv" + }, + { + "artifact": "inspections_research_facility_unfiltered_page_003.csv", + "profile": "inspections", + "bytes": 16541, + "sha256": "b30d40edc45bef9c5d3cb7c9d1b254d0d204a56b225b952429eb1f4c99de090c", + "line_count": 101, + "source_row_count": 100, + "validation": "validated_csv" + }, + { + "artifact": "inspections_research_facility_unfiltered_page_004.csv", + "profile": "inspections", + "bytes": 16912, + "sha256": "6fd23e74981ccb7372cb8668d59c2c22ba595f7d32563b517bf1d5a310dc8427", + "line_count": 101, + "source_row_count": 100, + "validation": "validated_csv" + }, + { + "artifact": "inspections_research_facility_unfiltered_page_005.csv", + "profile": "inspections", + "bytes": 16500, + "sha256": "0fea091c3811c009dd53f6aeeb08f5693c82ad998cf5a69c17ab4b26f5977a04", + "line_count": 101, + "source_row_count": 100, + "validation": "validated_csv" + }, + { + "artifact": "inspections_research_facility_unfiltered_page_006.csv", + "profile": "inspections", + "bytes": 16622, + "sha256": "9f6a6b08e3f36cb2203416095feb5dc9262ca1ab1aff742cbed3a2184c05b20a", + "line_count": 101, + "source_row_count": 100, + "validation": "validated_csv" + }, + { + "artifact": "inspections_research_facility_unfiltered_page_007.csv", + "profile": "inspections", + "bytes": 16540, + "sha256": "41e92b98f3d4be81de120011734c7ee29902c97faa3b2b4202d6522bacab1acd", + "line_count": 101, + "source_row_count": 100, + "validation": "validated_csv" + }, + { + "artifact": "inspections_research_facility_unfiltered_page_008.csv", + "profile": "inspections", + "bytes": 16676, + "sha256": "ee3dfe7eafe8e766971a3207d115fab23d599f3bf46abdc59b648dcffe23e4e2", + "line_count": 101, + "source_row_count": 100, + "validation": "validated_csv" + }, + { + "artifact": "inspections_research_facility_unfiltered_page_009.csv", + "profile": "inspections", + "bytes": 16480, + "sha256": "0f55f7bd588615b62b2b16ca10a1db04424c7da354e7babb269143d021b1852e", + "line_count": 101, + "source_row_count": 100, + "validation": "validated_csv" + }, + { + "artifact": "inspections_research_facility_unfiltered_page_010.csv", + "profile": "inspections", + "bytes": 16205, + "sha256": "53ffc0bec754fae8cd5d8ccbd02b7ee7b89ff5ab94f62940f0be935ff000150b", + "line_count": 101, + "source_row_count": 100, + "validation": "validated_csv" + }, + { + "artifact": "inspections_research_facility_unfiltered_page_011.csv", + "profile": "inspections", + "bytes": 16833, + "sha256": "f54160c9e8583d1ca3ec9c275672b8c748f83b326e837d188b2e1ca1a343865c", + "line_count": 101, + "source_row_count": 100, + "validation": "validated_csv" + }, + { + "artifact": "inspections_research_facility_unfiltered_page_012.csv", + "profile": "inspections", + "bytes": 16623, + "sha256": "81d073d40c00f64e7402b991815f17625f0034e395bcff2e55531dee97366d64", + "line_count": 101, + "source_row_count": 100, + "validation": "validated_csv" + }, + { + "artifact": "inspections_research_facility_unfiltered_page_013.csv", + "profile": "inspections", + "bytes": 16939, + "sha256": "ef9784b55d299a33f74b3b7c477e599bad7678d6672d9b3c08c0246cdad3b52f", + "line_count": 101, + "source_row_count": 100, + "validation": "validated_csv" + }, + { + "artifact": "inspections_research_facility_unfiltered_page_014.csv", + "profile": "inspections", + "bytes": 16491, + "sha256": "4135d513b28b5ff2ba3f2522f0c83076ec45c6b18b35855a8f366522a3d7e7b4", + "line_count": 101, + "source_row_count": 100, + "validation": "validated_csv" + }, + { + "artifact": "inspections_research_facility_unfiltered_page_015.csv", + "profile": "inspections", + "bytes": 16792, + "sha256": "785a9bacba504738f3019412ac26c5c2e7e3f6bd3703bb2e3f73ade3041ea663", + "line_count": 101, + "source_row_count": 100, + "validation": "validated_csv" + }, + { + "artifact": "inspections_research_facility_unfiltered_page_016.csv", + "profile": "inspections", + "bytes": 16689, + "sha256": "aff7b8aa99570517dd93336c6ea4b57b47d9e288d57830ef20435c3d01b3b912", + "line_count": 101, + "source_row_count": 100, + "validation": "validated_csv" + }, + { + "artifact": "inspections_research_facility_unfiltered_page_017.csv", + "profile": "inspections", + "bytes": 16219, + "sha256": "c1c58f902d3d8553f20d2da380b53b837a06c0dd80bea17557f19b5aac08735f", + "line_count": 101, + "source_row_count": 100, + "validation": "validated_csv" + }, + { + "artifact": "inspections_research_facility_unfiltered_page_018.csv", + "profile": "inspections", + "bytes": 16888, + "sha256": "8bac2115d825642621e56067545f9ad51e220515e6a43384e9ccc92c69d7c594", + "line_count": 101, + "source_row_count": 100, + "validation": "validated_csv" + }, + { + "artifact": "inspections_research_facility_unfiltered_page_019.csv", + "profile": "inspections", + "bytes": 16292, + "sha256": "6dec7faf32c3e61cffcf1686c5466196cf669f646694a72a58a365d64b355e5a", + "line_count": 101, + "source_row_count": 100, + "validation": "validated_csv" + }, + { + "artifact": "inspections_research_facility_unfiltered_page_020.csv", + "profile": "inspections", + "bytes": 16464, + "sha256": "1856b5f7a463d62bb31f810003a8c947e78726f5a65009272ec8422d58a747ce", + "line_count": 101, + "source_row_count": 100, + "validation": "validated_csv" + }, + { + "artifact": "inspections_research_facility_unfiltered_page_021.csv", + "profile": "inspections", + "bytes": 16634, + "sha256": "2f072d935eaa2e864e39ce339d56838f8d1acc03e340a7e399ff6b8a077c8ffd", + "line_count": 101, + "source_row_count": 100, + "validation": "validated_csv" + }, + { + "artifact": "registrations_state_partition_001.csv", + "profile": "registrations", + "bytes": 4587, + "sha256": "3fe4a1391dfd031c414d2d8e757137d72aa39d223e7eca67f5da9a89f1096f19", + "line_count": 28, + "source_row_count": 27, + "validation": "validated_csv" + }, + { + "artifact": "registrations_state_partition_002.csv", + "profile": "registrations", + "bytes": 1940, + "sha256": "dc2a4da413b73fd10e8810669d950ef20d50beb8a165b60e8995b02dc822ebba", + "line_count": 11, + "source_row_count": 10, + "validation": "validated_csv" + }, + { + "artifact": "registrations_state_partition_003.csv", + "profile": "registrations", + "bytes": 4349, + "sha256": "e1864d4ffd1d560d6c5b71bf0e768ed3630f63a8c603c9995c20b70a8c82b168", + "line_count": 26, + "source_row_count": 25, + "validation": "validated_csv" + }, + { + "artifact": "registrations_state_partition_004.csv", + "profile": "registrations", + "bytes": 6629, + "sha256": "f614e6444bd3d8b69de6b604b854318e9e16cbc691d3cd8a59e49ec08d32e8a0", + "line_count": 39, + "source_row_count": 38, + "validation": "validated_csv" + }, + { + "artifact": "registrations_state_partition_005.csv", + "profile": "registrations", + "bytes": 17139, + "sha256": "64da44b54768447c742388dd8c9008bcbc89097a979f5c1d031ca188f60368e1", + "line_count": 101, + "source_row_count": 100, + "validation": "validated_csv" + }, + { + "artifact": "registrations_state_partition_006.csv", + "profile": "registrations", + "bytes": 17333, + "sha256": "483f4c03e8362c3af0b5b16a360c2c9331a5c4a493e934d278a0da569cea0579", + "line_count": 101, + "source_row_count": 100, + "validation": "validated_csv" + }, + { + "artifact": "registrations_state_partition_007.csv", + "profile": "registrations", + "bytes": 16800, + "sha256": "8879dfc0d5961a6bad1329625e2077968b4ca731d96514c57b43fab8aea5bdcd", + "line_count": 101, + "source_row_count": 100, + "validation": "validated_csv" + }, + { + "artifact": "registrations_state_partition_008.csv", + "profile": "registrations", + "bytes": 13916, + "sha256": "719fe187422044026f8d5e9e34e930d4cfc9a1ba11d8cf01c943db4e3e2c0a0c", + "line_count": 83, + "source_row_count": 82, + "validation": "validated_csv" + }, + { + "artifact": "registrations_state_partition_009.csv", + "profile": "registrations", + "bytes": 11285, + "sha256": "7c7b75fef29c23951eed994d5ea4489a879c1dcabb70de3bb4b94d356f34b1d8", + "line_count": 66, + "source_row_count": 65, + "validation": "validated_csv" + }, + { + "artifact": "registrations_state_partition_010.csv", + "profile": "registrations", + "bytes": 7566, + "sha256": "9e73f2875b380faf2f1f799a28a59cc768a1c0b70ba8d049be25fdfb51dbb5a5", + "line_count": 45, + "source_row_count": 44, + "validation": "validated_csv" + }, + { + "artifact": "registrations_state_partition_011.csv", + "profile": "registrations", + "bytes": 2082, + "sha256": "7012e658c2755699744135e465aad6c187469c43a1b1543bf1c439d3b140222e", + "line_count": 12, + "source_row_count": 11, + "validation": "validated_csv" + }, + { + "artifact": "registrations_state_partition_012.csv", + "profile": "registrations", + "bytes": 3026, + "sha256": "663f39a1d352e3952945cba820c7b9a1e76549fa053eff21a0df9202021f9707", + "line_count": 15, + "source_row_count": 14, + "validation": "validated_csv" + }, + { + "artifact": "registrations_state_partition_013.csv", + "profile": "registrations", + "bytes": 16063, + "sha256": "fc5dd6d5a0164881cea29d7c0b6ac3b54f0b277e6cd3c61315ce698926e73305", + "line_count": 92, + "source_row_count": 91, + "validation": "validated_csv" + }, + { + "artifact": "registrations_state_partition_014.csv", + "profile": "registrations", + "bytes": 8706, + "sha256": "098f74a8021756102bb45ffccc771240d70ced78bcc6b2fa0eb5fdfc081acacb", + "line_count": 53, + "source_row_count": 52, + "validation": "validated_csv" + }, + { + "artifact": "registrations_state_partition_015.csv", + "profile": "registrations", + "bytes": 913, + "sha256": "ad3e8d3c082bb636380ab23566c35984f6da66387c675b9c8a470cac0846eb4e", + "line_count": 5, + "source_row_count": 4, + "validation": "validated_csv" + }, + { + "artifact": "registrations_state_partition_016.csv", + "profile": "registrations", + "bytes": 2614, + "sha256": "b837ed1c98fe711936ffba25f0cdc5d08826883bc58545b1557a2cbccd7af890", + "line_count": 16, + "source_row_count": 15, + "validation": "validated_csv" + }, + { + "artifact": "registrations_state_partition_017.csv", + "profile": "registrations", + "bytes": 15553, + "sha256": "dadd06068a03171836a74020e33ac52a4b9115de45048f34dcf3ddfec6c9c125", + "line_count": 90, + "source_row_count": 89, + "validation": "validated_csv" + }, + { + "artifact": "registrations_state_partition_018.csv", + "profile": "registrations", + "bytes": 8610, + "sha256": "d350bd71729594016836b53e8de70fcd2aa7c2424b23b23896cece46f12d6b7f", + "line_count": 50, + "source_row_count": 49, + "validation": "validated_csv" + }, + { + "artifact": "registrations_state_partition_019.csv", + "profile": "registrations", + "bytes": 8208, + "sha256": "fe79fe20aa22371b6df5c15af6743dfd3f7b7503a8711e6b38c5f89a76ada0ea", + "line_count": 50, + "source_row_count": 49, + "validation": "failed_malformed_csv_preserved" + }, + { + "artifact": "registrations_state_partition_020.csv", + "profile": "registrations", + "bytes": 6758, + "sha256": "d77f3516661380beeba79716681b41943bbab6af2f6339debd4a0eedf6e313a2", + "line_count": 41, + "source_row_count": 40, + "validation": "validated_csv" + }, + { + "artifact": "registrations_state_partition_021.csv", + "profile": "registrations", + "bytes": 3935, + "sha256": "655927fc2ca6f29bdc4032d48c0ace5a79fd18bfb8e47914bcd952a880986986", + "line_count": 23, + "source_row_count": 22, + "validation": "validated_csv" + }, + { + "artifact": "registrations_state_partition_022.csv", + "profile": "registrations", + "bytes": 3814, + "sha256": "128ca5d2c41711fe34db6010e28bfee92b392d314ca6aacdeec65302b5576c01", + "line_count": 22, + "source_row_count": 21, + "validation": "validated_csv" + }, + { + "artifact": "registrations_state_partition_023.csv", + "profile": "registrations", + "bytes": 4538, + "sha256": "9324256401a19ca4f5775841aaa1f76c888bbc4800742c625f94e22b092959c5", + "line_count": 28, + "source_row_count": 27, + "validation": "validated_csv" + }, + { + "artifact": "registrations_state_partition_024.csv", + "profile": "registrations", + "bytes": 18219, + "sha256": "6b0373a05cd8a669e7647c2f50929a79d198d56226dc0f3071043a38e413ccc5", + "line_count": 101, + "source_row_count": 100, + "validation": "validated_csv" + }, + { + "artifact": "registrations_state_partition_025.csv", + "profile": "registrations", + "bytes": 503, + "sha256": "eed76bb6e8246a642460b5305ea87f339720229e916f43d50eb09ad6c7e646cc", + "line_count": 3, + "source_row_count": 2, + "validation": "validated_csv" + }, + { + "artifact": "registrations_state_partition_026.csv", + "profile": "registrations", + "bytes": 16798, + "sha256": "2bd180a2a1e76085110e533e8d57ce25bdf4246e61066ce7742a7ad52530a4f6", + "line_count": 101, + "source_row_count": 100, + "validation": "validated_csv" + }, + { + "artifact": "registrations_state_partition_027.csv", + "profile": "registrations", + "bytes": 11701, + "sha256": "31a02afdd2c9f0f84f15c5448317e303c09971861d5623e7bebfa91e8b0895ee", + "line_count": 73, + "source_row_count": 72, + "validation": "validated_csv" + }, + { + "artifact": "registrations_state_partition_028.csv", + "profile": "registrations", + "bytes": 11203, + "sha256": "ae755f4805d582d6bed56cf5446df34feff81d4c8d6ef22e44a2630981faa263", + "line_count": 67, + "source_row_count": 66, + "validation": "validated_csv" + }, + { + "artifact": "registrations_state_partition_029.csv", + "profile": "registrations", + "bytes": 12415, + "sha256": "dbfc431b32e30748edf200486a4c29d10ee2adf8c79622c50610decb996a8282", + "line_count": 74, + "source_row_count": 73, + "validation": "validated_csv" + }, + { + "artifact": "registrations_state_partition_030.csv", + "profile": "registrations", + "bytes": 3563, + "sha256": "5c2890dd66ad6177c309b4206d70af98c3298f2ce10c361781b6c94990294fb6", + "line_count": 20, + "source_row_count": 19, + "validation": "validated_csv" + }, + { + "artifact": "registrations_state_partition_031.csv", + "profile": "registrations", + "bytes": 11101, + "sha256": "113bfac89e7cd8847f646731601e69f3052e7aa0df484c32405a84275e33a354", + "line_count": 66, + "source_row_count": 65, + "validation": "validated_csv" + }, + { + "artifact": "registrations_state_partition_032.csv", + "profile": "registrations", + "bytes": 3240, + "sha256": "11d48c67dbf0f00f6f93e37cf42b14bdff7d0994d2e43e388959fcf5c343c0c8", + "line_count": 20, + "source_row_count": 19, + "validation": "validated_csv" + }, + { + "artifact": "registrations_state_partition_033.csv", + "profile": "registrations", + "bytes": 4924, + "sha256": "26f796e2c7bca529d41b2da8abb7c814fbcda6ce4e1623b907c1d262bed03629", + "line_count": 30, + "source_row_count": 29, + "validation": "failed_malformed_csv_preserved" + }, + { + "artifact": "registrations_state_partition_034.csv", + "profile": "registrations", + "bytes": 2180, + "sha256": "435f050992ef02d7f4a1dcf117b6488fd6fd5a799f5030c206def4ec574f4aca", + "line_count": 13, + "source_row_count": 12, + "validation": "validated_csv" + }, + { + "artifact": "registrations_state_partition_035.csv", + "profile": "registrations", + "bytes": 1509, + "sha256": "94330206deb8bd4fddca122792125ebf96baa0b75e4493992c75b7bdd7a06824", + "line_count": 9, + "source_row_count": 8, + "validation": "validated_csv" + }, + { + "artifact": "registrations_state_partition_036.csv", + "profile": "registrations", + "bytes": 14296, + "sha256": "2470d265b211f486ac16e7cc84412878b69ab93d9f5471e5f9a04000c0be3453", + "line_count": 85, + "source_row_count": 84, + "validation": "validated_csv" + }, + { + "artifact": "registrations_state_partition_037.csv", + "profile": "registrations", + "bytes": 4076, + "sha256": "35d9f92a197a4fca86bd2aab4f71a4e6fd36b65f3cd1a75930cf2ca492030ad2", + "line_count": 23, + "source_row_count": 22, + "validation": "validated_csv" + }, + { + "artifact": "registrations_state_partition_038.csv", + "profile": "registrations", + "bytes": 17980, + "sha256": "d2d9bd98ad500d525f296f1b2f5f385658c399ff8463cc95ef096693fd76d0ea", + "line_count": 101, + "source_row_count": 100, + "validation": "validated_csv" + }, + { + "artifact": "registrations_state_partition_039.csv", + "profile": "registrations", + "bytes": 14004, + "sha256": "4d890455237fd8dafd00585188fb1c1a1ee6880f3ee111d767ac676a5f4081f9", + "line_count": 83, + "source_row_count": 82, + "validation": "validated_csv" + }, + { + "artifact": "registrations_state_partition_040.csv", + "profile": "registrations", + "bytes": 12928, + "sha256": "764afac5b89baf855d5150e30f967d550827c9147bac09b02bb3eec2478648a6", + "line_count": 76, + "source_row_count": 75, + "validation": "validated_csv" + }, + { + "artifact": "registrations_state_partition_041.csv", + "profile": "registrations", + "bytes": 2483, + "sha256": "d31b67c35fa06b494ff7014192a7b155460aaf6621f102eae40ff3e7c943fd82", + "line_count": 14, + "source_row_count": 13, + "validation": "validated_csv" + }, + { + "artifact": "registrations_state_partition_042.csv", + "profile": "registrations", + "bytes": 13829, + "sha256": "f40cdba400bb3a770eb36246844061b4df8f4bd7401e9224688d9949a998f327", + "line_count": 81, + "source_row_count": 80, + "validation": "validated_csv" + }, + { + "artifact": "registrations_state_partition_043.csv", + "profile": "registrations", + "bytes": 6308, + "sha256": "d1a5e49cd55690242f424124449c8f190f0c3206bcadb5a9852a27b708481ebf", + "line_count": 38, + "source_row_count": 37, + "validation": "validated_csv" + }, + { + "artifact": "registrations_state_partition_044.csv", + "profile": "registrations", + "bytes": 3867, + "sha256": "1662d3fa2020990432da8caf8e18f3a7ad5e29af0ebda81182d830fefeb906c0", + "line_count": 23, + "source_row_count": 22, + "validation": "validated_csv" + }, + { + "artifact": "registrations_state_partition_045.csv", + "profile": "registrations", + "bytes": 17441, + "sha256": "e49cc06eba3edf1c488d77b832e92a3616d09e7650287f64c4b7b06797a28ddf", + "line_count": 101, + "source_row_count": 100, + "validation": "validated_csv" + }, + { + "artifact": "registrations_state_partition_046.csv", + "profile": "registrations", + "bytes": 5319, + "sha256": "5fcd141efaea4fbbfe7800e2e350779825ea552b61c7cff2e32b01f0322d1872", + "line_count": 32, + "source_row_count": 31, + "validation": "validated_csv" + }, + { + "artifact": "registrations_state_partition_047.csv", + "profile": "registrations", + "bytes": 2968, + "sha256": "70294c8cd7ad8b7498299655f01f3382001a7aa125369256622d5214dffe9a93", + "line_count": 18, + "source_row_count": 17, + "validation": "validated_csv" + }, + { + "artifact": "registrations_state_partition_048.csv", + "profile": "registrations", + "bytes": 2358, + "sha256": "13c63a5664ac8e1b7ce6133281eaafe15ecee797d473325fe58703d74069f58c", + "line_count": 15, + "source_row_count": 14, + "validation": "validated_csv" + }, + { + "artifact": "registrations_state_partition_049.csv", + "profile": "registrations", + "bytes": 3906, + "sha256": "1b64d1741b7ac5386420c4005f34ba0d12a7c2341417863156ff587711c3a2b8", + "line_count": 23, + "source_row_count": 22, + "validation": "validated_csv" + }, + { + "artifact": "registrations_state_partition_050.csv", + "profile": "registrations", + "bytes": 2937, + "sha256": "f71e1981ad1c781be33db2c8a698ea5873ffa3d12398d7877fe273cd4e19d207", + "line_count": 18, + "source_row_count": 17, + "validation": "validated_csv" + }, + { + "artifact": "registrations_state_partition_051.csv", + "profile": "registrations", + "bytes": 7369, + "sha256": "a9692b974dd4504720586b06ef4fca2c18d577551db8eaa441f9ff5933710d32", + "line_count": 43, + "source_row_count": 42, + "validation": "validated_csv" + }, + { + "artifact": "registrations_state_partition_052.csv", + "profile": "registrations", + "bytes": 17444, + "sha256": "293456b77ae6d333928002a45e9dda5d50de19ba026876e4680f83d551bfb5b4", + "line_count": 101, + "source_row_count": 100, + "validation": "validated_csv" + }, + { + "artifact": "registrations_state_partition_053.csv", + "profile": "registrations", + "bytes": 16606, + "sha256": "750f803a72377adf1a68c4af5e1c5682574f5855b68445f6daf8bb72b41c2103", + "line_count": 101, + "source_row_count": 100, + "validation": "validated_csv" + }, + { + "artifact": "registrations_state_partition_054.csv", + "profile": "registrations", + "bytes": 3720, + "sha256": "95a48f929be56462ca26cff8ffad64cadb887d48ea69eeff292ff488606c5761", + "line_count": 22, + "source_row_count": 21, + "validation": "validated_csv" + }, + { + "artifact": "registrations_state_partition_055.csv", + "profile": "registrations", + "bytes": 2024, + "sha256": "eb466d29741e703b9d1aa7d2821823184a624a2e75367822317d1b130c7e432e", + "line_count": 12, + "source_row_count": 11, + "validation": "validated_csv" + }, + { + "artifact": "registrations_state_partition_056.csv", + "profile": "registrations", + "bytes": 10169, + "sha256": "da1a289377d51da355bc7cfb011d7ed364629590707a92d94e8c88c6fb7a3cb8", + "line_count": 59, + "source_row_count": 58, + "validation": "validated_csv" + }, + { + "artifact": "registrations_state_partition_057.csv", + "profile": "registrations", + "bytes": 12201, + "sha256": "0f185c0b5b7cc066806aad87109278876d9b44f47ba7f2b6aae71c6ecc2eac4c", + "line_count": 73, + "source_row_count": 72, + "validation": "validated_csv" + }, + { + "artifact": "registrations_state_partition_058.csv", + "profile": "registrations", + "bytes": 2606, + "sha256": "d643d4e2940f650107db0e0fc0f56dc871204a804e0250131190e7e96f66f320", + "line_count": 15, + "source_row_count": 14, + "validation": "validated_csv" + }, + { + "artifact": "registrations_state_partition_059.csv", + "profile": "registrations", + "bytes": 10754, + "sha256": "dbc11446b989e2f06fed3e10ab50533fbd7583672c9ef4635c9468883b79da3a", + "line_count": 64, + "source_row_count": 63, + "validation": "validated_csv" + }, + { + "artifact": "registrations_state_partition_060.csv", + "profile": "registrations", + "bytes": 828, + "sha256": "57209fe95a141b254a488a15d833b5b4ba34878de741fb9dc6a8fbde7986ad31", + "line_count": 5, + "source_row_count": 4, + "validation": "validated_csv" + }, + { + "artifact": "registrations_unfiltered_page_001.csv", + "profile": "registrations", + "bytes": 17664, + "sha256": "b41e75b3b8c5a2ac4f42d58a6f5fb3460958522cddc888e75c07e3b5659bc26e", + "line_count": 101, + "source_row_count": 100, + "validation": "validated_csv" + }, + { + "artifact": "registrations_unfiltered_page_002.csv", + "profile": "registrations", + "bytes": 17109, + "sha256": "0b19abaff71ef2236ffffe7225cfa7dede7b93c96f40cebd5865d882859852f9", + "line_count": 101, + "source_row_count": 100, + "validation": "validated_csv" + }, + { + "artifact": "registrations_unfiltered_page_003.csv", + "profile": "registrations", + "bytes": 17466, + "sha256": "dae4a8661974cd3e8bd34278ba5288546937c839266c1b817e2f66e87dfaa19c", + "line_count": 101, + "source_row_count": 100, + "validation": "validated_csv" + }, + { + "artifact": "registrations_unfiltered_page_004.csv", + "profile": "registrations", + "bytes": 17848, + "sha256": "17787f8fc9e6e889793e5c1b50e6ca9f29ce8c18f3fe50adda22e616bfbc7e82", + "line_count": 101, + "source_row_count": 100, + "validation": "validated_csv" + }, + { + "artifact": "registrations_unfiltered_page_005.csv", + "profile": "registrations", + "bytes": 17637, + "sha256": "867faeb686d312ffcd609ed2ca5ab312a2c57cd626746e4419289bb1e4a90b5c", + "line_count": 101, + "source_row_count": 100, + "validation": "validated_csv" + }, + { + "artifact": "registrations_unfiltered_page_006.csv", + "profile": "registrations", + "bytes": 17707, + "sha256": "cfd45d7442b39bb1a115fd41e6b299349c4113dde086d51c0f88896b22fb4b29", + "line_count": 101, + "source_row_count": 100, + "validation": "validated_csv" + }, + { + "artifact": "registrations_unfiltered_page_007.csv", + "profile": "registrations", + "bytes": 17153, + "sha256": "1a045beb93b375561fa5d34b50f1f25095ec3b723f00f6151c0e670c28eee09f", + "line_count": 101, + "source_row_count": 100, + "validation": "validated_csv" + }, + { + "artifact": "registrations_unfiltered_page_008.csv", + "profile": "registrations", + "bytes": 17340, + "sha256": "112eac5ffdca2fc9112cd8fc32476e3981fbc1d4adf31c4840474ce2af874ddd", + "line_count": 101, + "source_row_count": 100, + "validation": "validated_csv" + }, + { + "artifact": "registrations_unfiltered_page_009.csv", + "profile": "registrations", + "bytes": 17583, + "sha256": "97fbc1065740c1fe070f69a39ae40dc6b6a149c1722a698c0b22038133dc9fa0", + "line_count": 101, + "source_row_count": 100, + "validation": "validated_csv" + }, + { + "artifact": "registrations_unfiltered_page_010.csv", + "profile": "registrations", + "bytes": 16820, + "sha256": "b0774111486e04d7a1d6dc74605d08d454b2766e36b73da95405071ba704ef28", + "line_count": 101, + "source_row_count": 100, + "validation": "validated_csv" + }, + { + "artifact": "registrations_unfiltered_page_011.csv", + "profile": "registrations", + "bytes": 17662, + "sha256": "aba6b4c1a502bf03c89f4958750ab8c9c09224a2fa8506280c8fa11d1d28f389", + "line_count": 101, + "source_row_count": 100, + "validation": "validated_csv" + }, + { + "artifact": "registrations_unfiltered_page_012.csv", + "profile": "registrations", + "bytes": 17439, + "sha256": "29d620b6cfc7537571ea4d58b7d08b303f1ddf74b52c7e8eaf5f43642038ff28", + "line_count": 101, + "source_row_count": 100, + "validation": "validated_csv" + }, + { + "artifact": "registrations_unfiltered_page_013.csv", + "profile": "registrations", + "bytes": 17253, + "sha256": "a31a2fe5b8544f6106ad959d672d82a6f6954e6cc8a5cc1dfd99a64aba530962", + "line_count": 101, + "source_row_count": 100, + "validation": "validated_csv" + }, + { + "artifact": "registrations_unfiltered_page_014.csv", + "profile": "registrations", + "bytes": 16911, + "sha256": "735d4831c745551b5e0cb005367c9a4a07043b6e4653f43ebd586d0803ef73cc", + "line_count": 101, + "source_row_count": 100, + "validation": "validated_csv" + }, + { + "artifact": "registrations_unfiltered_page_015.csv", + "profile": "registrations", + "bytes": 16442, + "sha256": "257130d7ddfa014dd1839b3d993215150d11e84767a6ef7f1fb95d7045367864", + "line_count": 101, + "source_row_count": 100, + "validation": "validated_csv" + }, + { + "artifact": "registrations_unfiltered_page_016.csv", + "profile": "registrations", + "bytes": 17204, + "sha256": "aca4e07e4c1b376bab7d9cbbe1d2ee54031c3f288d3d41e04adb22fbfd67bd95", + "line_count": 101, + "source_row_count": 100, + "validation": "validated_csv" + }, + { + "artifact": "registrations_unfiltered_page_017.csv", + "profile": "registrations", + "bytes": 16557, + "sha256": "31941e8f05fb93424b62ae826a3bd8145f4a155fd3ca72a8ecb5ebd53e684d38", + "line_count": 101, + "source_row_count": 100, + "validation": "validated_csv" + }, + { + "artifact": "registrations_unfiltered_page_018.csv", + "profile": "registrations", + "bytes": 16814, + "sha256": "53db53676d8a930b16ce6b466c27587dec03e91ba5ce6355953fa60a9d593553", + "line_count": 101, + "source_row_count": 100, + "validation": "validated_csv" + }, + { + "artifact": "registrations_unfiltered_page_019.csv", + "profile": "registrations", + "bytes": 16771, + "sha256": "cbdce11719d3374988d35c55c9be46d556ba0366bb8f61db10f16c73d08904a0", + "line_count": 101, + "source_row_count": 100, + "validation": "validated_csv" + }, + { + "artifact": "registrations_unfiltered_page_020.csv", + "profile": "registrations", + "bytes": 16729, + "sha256": "9184a405d29a335dd5bda84a7ea8fa0a2940ca025b99fe21cc86b5fb576e110c", + "line_count": 101, + "source_row_count": 100, + "validation": "failed_malformed_csv_preserved" + }, + { + "artifact": "registrations_unfiltered_page_021.csv", + "profile": "registrations", + "bytes": 16639, + "sha256": "d73a740ceb48a3832a91e962ae52d92518ac8b76bca0094bf65bbe7c7ae99c3b", + "line_count": 101, + "source_row_count": 100, + "validation": "validated_csv" + } + ], + "validation": { + "adapter_schema_variants": [ + "legacy registration/inspection fixtures", + "current compact annual report export", + "current registrant export with Registration Type", + "current inspection export with Inspection Date" + ], + "rerun_determinism": "same input produced matching normalized and parsed hashes in rerun_a and rerun_b", + "amendment_handling": "unit test confirms explicit amendment versions remain distinct evidence", + "tests": [ + "python -m unittest pipeline.sources.us.aphis.test_adapter", + "python -m unittest pipeline.sources.us.aphis.test_acquire", + "python -m unittest pipeline.sources.us.aphis.test_refresh" + ] + }, + "coverage_limitations": [ + "These are APHIS public-search observations, not a complete census of AWA activity or facility equivalence.", + "Registrations, annual reports, inspections, and linked documents remain separate evidence families.", + "The inspection query reports 15,726 rows; only 2,100 were exported before the unfiltered pagination boundary.", + "Malformed source exports remain private failures; no row repair or silent drop was performed.", + "No coordinates were supplied and no geocoding was attempted.", + "No names, addresses, raw rows, or candidate records are included in this tracked manifest." + ], + "privacy": { + "raw_and_staging": "ignored private paths only", + "tracked_report": "row-free aggregate metadata and hashes", + "public_release": "not created" + } +} diff --git a/data/manifests/us-fsis-proof-2026-09-18.json b/data/manifests/us-fsis-proof-2026-09-18.json new file mode 100644 index 0000000..75acd75 --- /dev/null +++ b/data/manifests/us-fsis-proof-2026-09-18.json @@ -0,0 +1,57 @@ +{ + "manifest_version": "us-fsis-proof-v1", + "source_id": "us.fsis", + "authority": "USDA Food Safety and Inspection Service", + "page_url": "https://www.fsis.usda.gov/inspection/establishments/meat-poultry-and-egg-product-inspection-directory", + "page_observed_in_normal_browser": true, + "page_last_updated_observed": "2026-09-14", + "dashboard_updated_observed": "2026-09-14T14:30:33", + "dashboard_aggregate_count_observed": 7241, + "direct_csv_acquisition": { + "status": "blocked", + "directory_by_number_http_status": 403, + "demographics_http_status": 403, + "bypass_attempted": false, + "raw_artifacts_captured": false + }, + "legacy_continuity_rehearsal": { + "status": "private-candidate", + "current_source_claim": false, + "run_dir": "data/staging/reacquisition/us.fsis/legacy-continuity-2026-09-18-handoff/lifecycle", + "directory_rows": 7099, + "demographic_rows": 7105, + "matched_demographic_rows": 7089, + "orphan_demographic_rows": 16, + "identity_conflicts": 3, + "input_rows": 7115, + "normalized_rows": 7096, + "quarantined_rows": 19, + "quarantine_reason_counts": { + "conflicting_demographic_identity": 3, + "unmatched_demographic_identity": 16 + }, + "schema_fingerprint": "e8761847a9f8c843c7fdae4586ef278d075a59fcbb44fa633525359ba6297e75", + "demographic_schema_fingerprint": "286f3e59734ca71d0783d69bdd9f575b206982f29b278aca530ac66983f7bdac", + "release_state": "not-created", + "publication_state": "private-candidate", + "geocoding": "disabled", + "unmatched_demographic_is_not_closure": true + }, + "private_candidate_import": { + "status": "partial", + "full_first_import_preview_observed": true, + "public_release_promoted": false, + "row_free_runner_report": "not-retained", + "limitation": "The disposable Docker runner reached the test-only preview with the full first import, but the row-level idempotent replay/teardown exceeded the practical execution window; no public release or persistent database was retained." + }, + "current_row_level_totals": "not_observed", + "current_v1_identity_continuity": "not_observed", + "publication_eligibility": "blocked", + "raw_row_payloads_included": false, + "limitations": [ + "Dashboard aggregate count is not a row-level source capture.", + "The private continuity rehearsal uses a retained legacy snapshot with unknown effective date.", + "State inspection programs, APHIS populations, privacy review, terms review, and project approval remain outside this proof.", + "No source disappearance is interpreted as closure." + ] +} diff --git a/docs/countries/us/README.md b/docs/countries/us/README.md index 51a2260..1bfcb27 100644 --- a/docs/countries/us/README.md +++ b/docs/countries/us/README.md @@ -2,10 +2,30 @@ This packet is private pipeline documentation, not publication approval. +The research-animal coverage reconnaissance is documented separately in +[`research-animal-coverage-recon.md`](research-animal-coverage-recon.md). It +keeps funding/project, assurance/accreditation, registration, inspection/ +oversight, and actual animal-use counts as distinct evidence lanes. It does +not expand APHIS coverage into a national laboratory census or merge research +evidence into the FSIS facility layer. + ## Source boundaries The facility-master candidate is USDA FSIS's [Meat, Poultry and Egg Product Inspection Directory](https://www.fsis.usda.gov/inspection/establishments/meat-poultry-and-egg-product-inspection-directory) and its supplemental establishment-demographic CSV. FSIS describes the directory as a listing of FSIS-regulated meat, poultry, and egg establishments, with a weekly replacement edition and generalized activity categories. State meat-and-poultry inspection programs are not silently included. The source-local bundle adapter is documented in [`pipeline/sources/us/fsis/README.md`](../../../pipeline/sources/us/fsis/README.md): it joins only exact source-native IDs/numbers and keeps all acquired data private/test-only. +State inspection is a separate evidence family. FSIS currently identifies 29 +states with cooperative State Meat and Poultry Inspection (MPI) programs; only +ten states currently appear on the FSIS Cooperative Interstate Shipment (CIS) +roster. State lists vary from structured CSV/XLSX/HTML rosters to dated PDFs, +maps, and contact-only routes, and often mix official inspected, CIS, +custom-exempt, retail, handler, warehouse, rendering, or exemption records. +The row-free route and classification reconnaissance is in +[`state-mpi-source-recon.md`](state-mpi-source-recon.md). It is not a current +facility capture and must not be used as a publication list. State-inspected +product is intrastate by default; CIS eligibility is an establishment-level +overlay and does not make every facility in a participating state eligible for +interstate shipment. + APHIS is a separate evidence family. The [Animal Care Public Search Tool](https://www.aphis.usda.gov/animal-care/awa-services/usda-animal-care-public-search-tool) exposes licensed/registered persons, inspection reports, and research facility annual reports. The [annual usage summary](https://direct.aphis.usda.gov/awa/research-facility-report/annual-summary) notes that annual reports can be amended. The adapters require an explicit `registrations`, `annual_reports`, or `inspections` profile, preserve certificate and customer-number variants, version explicit amendments, and never turn an APHIS row into an FSIS facility or laboratory master record. ## Acquisition boundary @@ -54,6 +74,25 @@ python -m pipeline.sources.us.aphis.acquire ` The fetch path writes `acquisition-metadata.json` and a row-free `manifest.json`; failures write `acquisition-failure.json` with the failure class, attempts, query context, and no committed artifact. Reuse a new run ID for every observation. A failed or empty capture must not replace or delete the previous validated artifact. All APHIS outputs remain restricted private research evidence, with `release_state=not-created`, `publication_state=private-research-evidence`, and `publication_gate=blocked` until separate human review and approval. +### 2026-09-18 US Real-Data Proof + +The private Wave 1 browser capture is summarized in the row-free +[`APHIS proof manifest`](../../../data/manifests/us-aphis-wave1-real-data-proof-2026-09-18.json). +It preserved ten FY2025 annual-report pages (995 rows), 81 registration pages +from the unfiltered and official State-partitioned views (2,841 displayed +registrations, with duplicate evidence pages kept separate), and 21 pages of +the Research Facility inspection view (2,100 of 15,726 displayed rows before +the unfiltered pagination boundary). The current compact annual, registrant, +and inspection CSV schemas are now explicit adapter variants; no coordinates +were supplied and no geocoding was attempted. + +Three malformed registration exports remain private failure artifacts and were +not repaired. Two visible report/document downloads were attempted through the +normal browser workflow but the browser returned `ERR_BLOCKED_BY_CLIENT`; no +hidden endpoint or bypass was used. Raw, parsed, normalized, quarantined, and +candidate-import outputs remain ignored private/test-only artifacts, with no +release or publication created. + ## Validation and handoff Both adapters preserve source values only in restricted staging and emit parsed, normalized, quarantined, QA, run-status, health, and private candidate-handoff artifacts. They disable geocoding, mark address/coordinate review as pending, quarantine duplicate or missing identities, and set `release_state=not-created`, `publication_state=private-candidate`, and `publication_gate=blocked`. APHIS uses a source-specific test-only import packet rather than the facility importer: its `establishment_id` stays null, no graph candidates or edges are emitted, and no database release or public promotion is performed. The APHIS export is not a complete AWA or animal-use census; annual-report absence is not closure or non-use, and amendments/currentness require review. @@ -89,6 +128,14 @@ The 2026-09-17 rehearsal measured 7,101 FSIS rows, 4,507 APHIS inspection rows, The current FSIS page was observed in a normal browser with a September 14, 2026 update and three CSV routes, but the exact file routes returned HTTP 403 to bounded direct acquisition. See the row-free [current-route manifest](../../../data/manifests/us-fsis-current-route-2026-09-17.json). +State MPI acquisition remains documentation-only. No state roster or CIS +workbook was acquired in this sprint, and no current state facility row is +claimed. Before any private capture, record the final URL, retrieval UTC, +visible revision/effective date, content type, byte size, SHA-256, terms and +filter context. A missing row is `not_observed`, not closure; coordinates must +retain provider, query, precision and review state, and must not be inferred +from an address or map point. + ## Review checklist - authority, edition/effective date, URL, terms/attribution, and retention are recorded; @@ -97,3 +144,12 @@ The current FSIS page was observed in a normal browser with a September 14, 2026 - phone, DUNS, names, addresses, and precise coordinates receive privacy review; - current coverage is compared with V1 only by exact source keys and aggregate reports; - no raw artifact or row-level report is committed or exposed. + +## Operator workflow + +For current FSIS and APHIS observations, use the row-free aggregate workflow in +[`operator-refresh.md`](operator-refresh.md). It coordinates the existing +source-specific lanes without merging them, reports freshness/acquisition/ +validation/reconciliation/quarantine/private-import state, and preserves the +previous-valid state when a lane fails. Browser-assisted checkpoints remain +explicit where the official source requires an interactive export. diff --git a/docs/countries/us/operator-refresh.md b/docs/countries/us/operator-refresh.md new file mode 100644 index 0000000..308d812 --- /dev/null +++ b/docs/countries/us/operator-refresh.md @@ -0,0 +1,119 @@ +# US current-refresh operator workflow + +This is a private/test-only workflow for current FSIS and APHIS observations. It +does not approve, publish, or replace a validated release. The two source lanes +remain separate: FSIS is the federal establishment-directory spine, while APHIS +captures registrations, inspections, and annual reports as distinct evidence +profiles. No cross-source identity join is attempted. + +## Browser checkpoint + +Use an authorized browser session when the source requires an interactive +export. Save the source-provided file without editing it, and record the final +URL, displayed edition/year or amendment state, retrieval time, and any shown +result count in the plan's `query_context`. A failed, empty, HTML/challenge, or +403 response is not a zero-row observation. Do not probe hidden endpoints, +reuse browser credentials, or bypass an access control. + +For FSIS, save the current MPI Directory CSV and, when available, the +Establishment Demographic CSV from the official directory page. For APHIS, +select exactly one profile per entry (`registrations`, `inspections`, or +`annual_reports`); linked documents or amendments are retained separately and +are not automatically merged into rows. + +## One-command private refresh + +Create a row-free plan outside Git. Paths may be absolute or relative to the +plan file: + +```json +{ + "schema_version": "us-operator-plan-v1", + "retry": { + "max_attempts": 3, + "retry_delay_seconds": 1, + "max_retry_delay_seconds": 30 + }, + "sources": [ + { + "source": "fsis", + "directory": "private-captures/fsis-directory.csv", + "demographics": "private-captures/fsis-demographics.csv", + "retrieved_at_utc": "2026-09-18T00:00:00Z", + "effective_date": "2026-09-14", + "previous_manifest": "private-runs/previous/fsis/lifecycle/manifest.json" + }, + { + "source": "aphis", + "profile": "annual_reports", + "raw": "private-captures/aphis-annual-reports.csv", + "retrieved_at_utc": "2026-09-18T00:00:00Z", + "query_context": { + "selected_year": "2025", + "amended_reports_included": true, + "displayed_result_count": "recorded-in-private-notes" + } + } + ] +} +``` + +Run a dry-run first: + +```powershell +python -m pipeline.sources.us.refresh ` + --plan C:\path\to\private-us-plan.json ` + --run-root C:\path\to\private-us-runs\2026-09-18 ` + --mode dry-run ` + --as-of-utc 2026-09-18T12:00:00Z +``` + +The row-free `us-refresh-report.json` reports, per source, acquisition facts, +freshness, schema/count drift, exact-key reconciliation, normalized and +quarantined counts, private-import readiness, and the retained previous-valid +state. Review it and the source-specific private artifacts before using +`--mode handoff` for FSIS. APHIS's adapter emits its private candidate handoff +after validation; that remains a human-gated test artifact. + +The aggregate command always keeps `release_state=not-created`, +`release_promoted=false`, `public_exposure=false`, and all public surfaces +disabled. A failed lane cannot delete or replace the previous-valid manifest. +The failure record contains an actionable class and fallback without copying +source rows into the aggregate report. + +## Diagnostics and retry behavior + +Inspect an existing run without opening raw, parsed, normalized, or quarantine +rows: + +```powershell +python -m pipeline.sources.us.refresh ` + --run-root C:\path\to\private-us-runs\2026-09-18 ` + --diagnose ` + --as-of-utc 2026-09-18T12:00:00Z +``` + +Network acquisition is still opt-in and requires the source-specific approved +terms record. The shared acquisition primitive retries only bounded network, +rate-limit, timeout, and interrupted-download failures. It uses temporary +partial files, removes them after an interrupted read, and records every +attempt. HTTP 403, HTML/login/challenge responses, invalid content types, +malformed exports, schema drift, and count drift fail closed; use the browser +capture checkpoint instead of bypassing the restriction. + +## Scheduling guidance + +FSIS documents a weekly replacement cadence: check the displayed edition before +each weekly run, and treat a missing or changed edition as `not observed`, not +closure. The operations schedule treats a capture older than 240 hours as +stale, with three bounded attempts before the manual two-file capture fallback. + +APHIS profile cadence is not established by this project. Trigger it when a +documented source update, selected-year change, amendment, or operator review +requires a new observation. Its freshness is reported as `unknown` unless an +explicit project schedule is later justified by source evidence. Keep each +profile and year/amendment context as a separate observation. + +Scheduling private acquisition is not publication authorization. Privacy, +source-rights, factual review, project approval, and release publication remain +separate gates under `docs/ETHICS.md`. diff --git a/docs/countries/us/research-animal-coverage-recon.md b/docs/countries/us/research-animal-coverage-recon.md new file mode 100644 index 0000000..495eb94 --- /dev/null +++ b/docs/countries/us/research-animal-coverage-recon.md @@ -0,0 +1,127 @@ +# United States research-animal coverage reconnaissance + +Scope/date: documentation-first reconnaissance of public, authoritative United States sources beyond the existing USDA APHIS Animal Welfare Act (AWA) annual-report lane, observed 2026-09-18 UTC. This document records source topology, not a facility census, animal-use estimate, publication approval, or a claim that any source is complete. + +## Bottom line + +The United States does not expose one public, national laboratory/research-animal register that can safely stand in for a facility master or a national animal-use total. + +The evidence has to stay in separate lanes: + +| Evidence lane | Strongest public source found | What it can establish | What it cannot establish | +| --- | --- | --- | --- | +| Institutional identity | OLAW assured-institutions lookup, AAALAC directory, agency/state pages | A named institution, unit, assurance/accreditation or agency relationship as stated by that source | Current operation, every research site, animal use, or factual approval of every project | +| Funding/project evidence | NIH RePORTER API and ExPORTER; NSF Award Search API; NASA NSPIRES | An award/project, recipient organization, dates, funding agency, and selected project text/identifiers | That animals were used, where the work physically occurred, or how many animals were used | +| Regulatory registration | APHIS public search; California CDPH laboratory-animal approval as a state example | A source-native registration/certificate or state approval observation | A complete national register, current activity, or actual use | +| Inspection/oversight evidence | APHIS inspection reports; DoD ACURO and VA ORO program pages; FDA GLP authority | A dated inspection, site-visit, oversight, or program statement when the source publishes one | A clean bill of health, complete inspection history, or animal counts | +| Actual animal-use counts | APHIS Form 7023 annual reports and fiscal-year summaries; some state forms may request prior-year counts | Counts for the reporting population, period, columns, species and exclusions stated by the source | Purpose-bred rat/mouse/bird use, all agricultural research, unreported use, or a national all-species total | + +The existing [US recovery packet](README.md) remains the implementation boundary for FSIS and APHIS. This reconnaissance adds research-specific source contracts and candidate integrations; it does not merge them into the FSIS facility layer. + +## Source matrix + +### NIH funding and assurance + +**NIH RePORTER / ExPORTER — funding and project evidence.** The [RePORTER API](https://api.reporter.nih.gov/) exposes project and publication search APIs. Its documented project fields include application/project identifiers, core project number, fiscal year, project dates, agency, funded organization name, organization IPF, UEI, DUNS, FIPS and ZIP, funding amounts and mechanism, and project title/terms/abstract fields. The [ExPORTER bulk page](https://reporter.nih.gov/exporter/publications) provides public CSV-oriented bulk administrative files and says the consolidated annual project file is normally released annually, with later updates after the President’s budget and RCDC integration. + +This is the best first automation candidate for funding/project evidence. Use exact `appl_id`, `core_project_num`/project number, fiscal year and organization identifiers as source keys. Preserve the recipient organization separately from any explicitly stated performance site. A project title or abstract mentioning an animal model is a project claim, not a count and not proof that the recipient operates a laboratory at its mailing address. RePORTER’s one-request-per-second recommendation and large-job time window belong in the adapter contract; API responses, release dates, pagination and schema fingerprints must be versioned. + +**OLAW assured institutions — assurance/identity evidence.** The [Assured Institutions Look Up Tool](https://grants.nih.gov/policy-and-compliance/policy-topics/animal-welfare/assurance/assured-institutions) exposes organization name, Assurance ID (and legacy ID), type, state/territory and country for institutions with a current approved Domestic or Foreign Assurance. The [assurance page](https://grants.nih.gov/policy-and-compliance/policy-topics/animal-welfare/assurance) states that a Domestic Assurance is required for U.S. institutions conducting live vertebrate animal work onsite for PHS, NSF or NASA-supported activities, and describes interinstitutional assurances when work is performed at another named site. + +OLAW is therefore useful for a dated assurance observation and for distinguishing award recipient from performance-site assurance. It is not a project register, an IACUC protocol register, a facility inspection database, or an animal-use count source. The [annual-report requirement](https://grants.nih.gov/policy-and-compliance/policy-topics/animal-welfare/assurance/domestic-annual-report) concerns changes in the program/facilities, Institutional Official, IACUC membership, semiannual evaluation dates, minority views and accreditation status. It does not publish a national annual table of animals used. Annual reports are submitted as signed PDFs by email, so no public bulk endpoint was identified. + +### Federal agencies and federal laboratories + +**FDA.** FDA’s [laboratory-animal MOU with USDA and NIH](https://www.fda.gov/about-fda/domestic-mous/mou-225-16-010) describes three distinct programs: USDA registration/licensing and inspections, FDA Good Laboratory Practice (GLP) standards under 21 CFR Part 58 with FDA inspections and possible laboratory disqualification, and NIH/OLAW assurance/compliance. The MOU also says that each agency maintains registries or inventories within its own authority and that nonpublic shared information is access-controlled. The public [NCTR animal-facilities page](https://www.fda.gov/about-fda/nctr-location-facilities-services/nctr-animal-facilities-and-services) is useful institutional evidence for an FDA program and its IACUC/AAALAC statements, but it is not a national FDA research-facility export. + +The public [openFDA Animal & Veterinary API](https://open.fda.gov/apis/animalandveterinary/) is an adverse-event dataset, not an animal-research-facility or experimental-use register. Its records can contain numbers of animals affected/treated, but those numbers describe adverse-event reports and must never be relabeled as research use. + +**VA.** The [VA Animal Research Program](https://www.research.va.gov/programs/animal_research/default.cfm) and [VA research overview](https://www.research.va.gov/programs/animal_research/overview.cfm) document that VA animal-care programs operate under VA policy, PHS Policy, the AWA where applicable, IACUC oversight and AAALAC accreditation. VA’s [Office of Research Oversight](https://department.va.gov/vha/research-oversight/what-oro-does/) describes proactive evaluations, investigations and remediation oversight for laboratory animal welfare. These pages are authoritative program/oversight evidence, but no public machine-readable national VA facility or animal-use dataset was identified. VA facility pages may prove a local program; they do not provide national counts. + +**DoD / DHA ACURO.** The [Animal Care and Use Review Office](https://mrdc.health.mil/index.cfm/resources/research_protections/acuro) states that ACURO oversees DHA, USAMRDC and Department of War research, development, testing, evaluation and training involving animals, including extramural contracts and grants; its functions include protocol review and site visits/compliance inspections. The page publishes policies and reporting resources, not a national facility list or animal-use table. Model ACURO as oversight evidence linked to an award/protocol/site only when the source explicitly supplies those identifiers. Do not infer a facility from a DoD awardee address. + +**NSF and NASA.** NSF’s [developer page](https://www.nsf.gov/digital/developer) documents an Award Search API for research spending and results. NASA states that [NSPIRES](https://www.nasa.gov/hrp/for-prospective-researchers/) supports the lifecycle of solicitations and awards and that organizations need SAM registration to participate. Both are funding/project evidence only. OLAW expressly includes NSF- and NASA-supported onsite live vertebrate work in its Assurance rules, but the funding record still does not establish animal use, facility location, or count. These are lower-priority adjuncts after NIH RePORTER because animal-related classification and performance-site semantics need more review. + +### Accreditation and state records + +**AAALAC International.** The [public directory](https://www.aaalac.org/accreditation/directory/directory-of-accredited-organizations-search-result/) is maintained by the private nonprofit accreditor and says the list is updated on an ongoing basis. It identifies accredited organizations/units and locations, including government, academic and commercial programs. Accreditation is voluntary and unit-scoped. It is not an APHIS registration, OLAW assurance, inspection result, animal-use count or proof that every listed campus currently uses animals. Retain the directory’s organization/unit distinction and the observation date; do not flatten a parent organization into all listed facilities. + +**California as a state-record pilot.** California’s [Laboratory Animal Use Approval Program](https://www.cdph.ca.gov/Programs/cls/operations/Pages/LaboratoryAnimalUseApprovalProgram.aspx) says California requires approval to keep/use live warm-blooded animals for education, research, testing or diagnostic purposes, but exempts laboratories subject to USDA-APHIS or NIH-OLAW and does not conduct routine inspections. The [LAB 139 form](https://www.cdph.ca.gov/CDPH%20Document%20Library/ControlledForms/lab139.pdf) asks for institution/location, responsible program person, use types, and animals kept or used during the previous calendar year, including laboratory mice, laboratory rats and other species. This is a valuable state-level example of a potential count/approval contract and also a warning: a state record may target precisely the animals or laboratories excluded from federal coverage, while federally regulated facilities may be absent. No public statewide bulk registry or export was verified in this reconnaissance. + +State sources should be added jurisdiction by jurisdiction. A state approval, permit or public-record response is a source-local observation, not a national denominator. Public-records requests must be narrowly scoped and privacy-reviewed; they must not be used to assemble personal addresses, names or sensitive operational details into a new public register. + +## APHIS boundaries and exclusions + +APHIS remains the only source located in this reconnaissance that publishes both a public national-ish research-facility search surface and annual animal-use summaries. It is still not a complete laboratory/research-animal census. + +The [AWA and regulations Blue Book](https://www.aphis.usda.gov/sites/default/files/ac_bluebook_awa_508_comp_version.pdf) defines “animal” to include dogs, cats, nonhuman primates, guinea pigs, hamsters, rabbits and other warm-blooded animals designated for research/testing/exhibition, while excluding: + +- birds, rats of genus *Rattus*, and mice of genus *Mus* bred for use in research; +- horses not used for research; and +- farm animals such as livestock or poultry used or intended for food/fiber, or to improve nutrition, breeding, management, production efficiency or food/fiber quality. + +These exclusions are semantic, not a license to estimate the missing population. In particular, “rat” or “mouse” alone does not prove the purpose-bred exclusion, and species alone does not resolve agricultural versus non-agricultural purpose. APHIS separately says that captive-bred birds used in research are exempt, while farm-type poultry used solely for agricultural purposes are exempt and can be covered when used for non-agricultural purposes. Preserve the source’s stated purpose and coverage category rather than normalizing all birds, rats, mice or farm animals into one bucket. + +The [Research Facility Annual Usage Summary](https://www.aphis.usda.gov/awa/research-facility-report/annual-summary) says each USDA-registered and Federal research facility submits Form 7023. The summary includes R, F and V certificate populations and AWA-covered animals reported by USDA Agricultural Research Service G-certificate facilities. APHIS says ARS reports all animals used for research, including farm animals and noncovered species, but excludes the ARS-reported farm and other noncovered species from the summary so that the summary represents AWA-covered animals. Annual reports may be amended after generation. The linked FY2025 PDF is a concrete source-reported count: 751,355 across its summary table as of 2026-05-28; it is not an estimate of all animals used in U.S. research. + +Consequently: + +- APHIS annual-report counts are **actual reported counts for a defined fiscal-year/reporting population**, not projections. +- APHIS counts do not include purpose-bred research mice/rats/birds and do not by themselves cover all agricultural research. +- APHIS registrations establish a regulatory relationship; they do not establish current operation, site ownership, or the number of animals actually used in a particular study. +- APHIS inspections establish dated inspection evidence; they do not establish a complete inspection history or that no unobserved use occurred. +- APHIS annual reports, inspections and registrations remain separate source-local evidence types. An annual-report row is not a facility master row and must not be silently joined to NIH, VA, DoD, FDA, AAALAC or state rows. + +## Reusable source contract + +All new US research-animal adapters should use the repository’s ordinary provenance fields and add the following source-specific semantics: + +1. **Evidence identity:** `source_id`, official URL/final download URL, retrieval timestamp, publisher-supplied publication/effective/reporting period, content type, byte size, SHA-256, adapter/config version, schema fingerprint and terms/attribution review. +2. **Source-native keys:** preserve every supplied identifier without coercion: APHIS certificate/customer/report identifiers; RePORTER application, core project and organization identifiers; OLAW Assurance ID; AAALAC organization/unit; state approval/certificate; FDA/DoD/VA report or site identifiers when public. +3. **Observation type:** one of `institution_identity`, `funding_project`, `assurance`, `accreditation`, `registration`, `inspection`, `oversight`, `reported_animal_use`, `aggregate_animal_use`, or `policy_only`. +4. **Subject and location:** distinguish legal entity, institution, campus/unit, facility, mailing address, performance site, inspection site and source-reported study site. Store location precision and review state; never geocode a private/person-linked address merely because a source has an address. +5. **Count contract:** store fiscal/calendar period, species label exactly as supplied, reporting column/measure, unit, amendment status, covered/excluded populations, and whether the value is a source-reported count, aggregate, or unavailable. Never sum across APHIS, OLAW, funding, state or adverse-event observations. +6. **Review and release:** source origin, factual review, privacy/safety eligibility, project approval and publication remain separate. Current absence means `not_observed`, not closure, non-use or no assurance. + +## Conservative graph semantics + +Only create a relationship when the source states it or a reviewed exact-key crosswalk records it: + +| Relationship | Safe meaning | Do not infer | +| --- | --- | --- | +| `project_funded_by → organization` | Award recipient named by the funder | That the organization used animals or owns the performance site | +| `project_performed_at → site` | The award/protocol explicitly names the site | Recipient mailing address as the site | +| `institution_has_assurance → OLAW assurance` | Assurance observation for the institution/type/date | Approval of every protocol or a count of animals | +| `unit_accredited_by → AAALAC` | Voluntary accreditation for the named unit/location | Regulatory registration or current animal use | +| `institution_registered_with → regulator` | Source-native registration/certificate observation | Operation, ownership, or use quantity | +| `site_inspected_by → authority` | Dated inspection/site-visit evidence | Complete inspection history or compliance certification | +| `reported_use_at → site/reporting unit` | The source report states a count for the period and scope | Unreported species, purpose-bred animals, or national totals | +| `state_approval_for → activity/site` | State approval/form observation with stated scope | Federal coverage or national comparability | + +Names, addresses, DUNS/UEI, assurance IDs, certificate numbers and PI names are matching aids with different privacy and lifecycle properties. Institutional legal names can be shared by multiple campuses; one assurance can cover branches or an affiliate arrangement; AAALAC units can be narrower than the parent; an award recipient can differ from the performance site; and a state approval can deliberately cover only federally excluded laboratories. Fuzzy matching, name/address proximity and geocoded proximity are not identity proof. Ambiguous or conflicting links belong in quarantine or a reviewed link ledger. + +## Prioritized next integrations + +1. **NIH RePORTER API + ExPORTER:** implement a rate-limited funding/project adapter with exact award, organization and fiscal-year keys; classify animal relevance as a project-text signal only and keep it out of facility/use counts. +2. **APHIS annual summary/public-search profiles:** extend the existing profile-explicit adapter for registrations, annual reports and inspections; add a separate annual aggregate-count artifact and amendment/version handling; preserve R/F/V/G scope and exclusions. +3. **OLAW assured-institutions lookup:** add an assisted capture or carefully bounded browser adapter for current assurance observations, including Assurance ID, type, state/country and observation date; never treat it as an animal-use table. +4. **AAALAC directory:** add a voluntary accreditation observation only after terms, unit identity, refresh behavior and contact/address minimization are reviewed. +5. **California CDPH pilot:** obtain an authorized current approval/renewal extract or public-record response, test the form’s prior-year count fields with synthetic fixtures, and model federal-exemption semantics explicitly. Do not generalize California to other states. +6. **NSF and NASA funding adjuncts:** integrate only after the NIH contract is stable; use for award/project coverage and link to OLAW assurances where explicitly supported, never as animal-use evidence. +7. **VA, DoD ACURO and FDA:** retain as policy/oversight/manual evidence until a public, reproducible, terms-permitted facility or count route is verified. Do not build a national facility layer from agency program pages or FOIA fragments. + +## Current limitations and blockers + +No new row-level artifact, raw report, assurance file, state application, or private contact data was acquired or committed during this reconnaissance. Current public routes were documented from official pages and current linked documents only. The remaining blockers are source-specific terms and redistribution review, durable export/API contracts for the UI-mediated tools, exact effective-date and amendment semantics, privacy treatment for addresses and people, and an authorized maintainer review before any candidate release. These blockers do not mean that the sources failed or that no additional facilities/counts exist. + +## Official sources consulted + +- [NIH RePORTER API](https://api.reporter.nih.gov/) and [RePORTER ExPORTER](https://reporter.nih.gov/exporter/publications) +- [OLAW Animal Welfare](https://grants.nih.gov/policy-and-compliance/policy-topics/animal-welfare), [Assured Institutions](https://grants.nih.gov/policy-and-compliance/policy-topics/animal-welfare/assurance/assured-institutions), and [Annual Report](https://grants.nih.gov/policy-and-compliance/policy-topics/animal-welfare/assurance/domestic-annual-report) +- [APHIS Research Facility Annual Usage Summary](https://www.aphis.usda.gov/awa/research-facility-report/annual-summary), [FY2025 PDF](https://direct.aphis.usda.gov/sites/default/files/fy2025-research-animal-use-summary.pdf), [AWA Blue Book](https://www.aphis.usda.gov/sites/default/files/ac_bluebook_awa_508_comp_version.pdf), and [bird standards](https://www.aphis.usda.gov/awa/bird-standards) +- [FDA/USDA/NIH laboratory-animal MOU](https://www.fda.gov/about-fda/domestic-mous/mou-225-16-010), [FDA NCTR animal facilities](https://www.fda.gov/about-fda/nctr-location-facilities-services/nctr-animal-facilities-and-services), and [openFDA Animal & Veterinary](https://open.fda.gov/apis/animalandveterinary/) +- [VA Animal Research Program](https://www.research.va.gov/programs/animal_research/default.cfm), [VA animal-research overview](https://www.research.va.gov/programs/animal_research/overview.cfm), and [VA Office of Research Oversight](https://department.va.gov/vha/research-oversight/what-oro-does/) +- [DoD/DHA ACURO](https://mrdc.health.mil/index.cfm/resources/research_protections/acuro) +- [NSF Developer Resources](https://www.nsf.gov/digital/developer) and [NASA NSPIRES overview](https://www.nasa.gov/hrp/for-prospective-researchers/) +- [AAALAC accredited-organization directory](https://www.aaalac.org/accreditation/directory/directory-of-accredited-organizations-search-result/) +- [California CDPH Laboratory Animal Use Approval Program](https://www.cdph.ca.gov/Programs/cls/operations/Pages/LaboratoryAnimalUseApprovalProgram.aspx) and [LAB 139](https://www.cdph.ca.gov/CDPH%20Document%20Library/ControlledForms/lab139.pdf) diff --git a/docs/countries/us/state-mpi-source-recon.md b/docs/countries/us/state-mpi-source-recon.md new file mode 100644 index 0000000..f15bc80 --- /dev/null +++ b/docs/countries/us/state-mpi-source-recon.md @@ -0,0 +1,192 @@ +# United States state meat and poultry inspection source reconnaissance + +Scope/date: documentation-first reconnaissance of the 29 state Meat and Poultry +Inspection (MPI) programs identified by USDA FSIS, observed 2026-09-18 UTC. +This is private source research, not a current facility capture, publication +approval, or claim that a state list is complete. No row-level facility data, +raw downloads, personal contacts, addresses, coordinates, or sensitive source +artifacts are retained in this repository. + +## Executive findings + +The federal FSIS [State Inspection Programs](https://www.fsis.usda.gov/inspection/state-inspection-programs) +page says 29 states operate their own MPI programs and that state-inspected +product is limited to intrastate commerce unless a state also participates in +the [Cooperative Interstate Shipment (CIS) program](https://www.fsis.usda.gov/inspection/state-inspection-programs/cooperative-interstate-shipping-program). +The current FSIS CIS establishment page names ten participating states: +Indiana, Iowa, Maine, Missouri, Montana, North Dakota, Ohio, South Dakota, +Vermont, and Wisconsin. CIS is an establishment-level subset, not a second +statewide directory; FSIS lists separate state spreadsheets for each of the ten +states and says no state currently has a supplemental CIS export agreement. + +The 29-state inventory is: + +* **Meat and poultry:** Alabama, Arizona, Delaware, Illinois, Indiana, Iowa, + Kansas, Louisiana, Maine, Minnesota, Mississippi, Missouri, Montana, North + Carolina, North Dakota, Ohio, Oklahoma, South Carolina, Texas, Utah, + Vermont, Virginia, West Virginia, Wisconsin, and Wyoming. +* **Meat only:** Arkansas, Georgia, Oregon, and South Dakota. South Dakota's + official page explicitly says poultry inspection remains under USDA FSIS. + +FSIS estimates about 1,450 establishments are inspected under state MPI +programs, but this is an all-state aggregate and not a release-ready row count. +State pages often publish a wider licensed universe containing custom-exempt, +retail, distributor, warehouse, wild-game, or poultry-exempt operations. Those +must not be silently counted as inspected slaughter or inspected processing +facilities. + +## Reusable acquisition families + +| Family | Examples | What is observable | Contract and difficulty | +| --- | --- | --- | --- | +| **Structured HTML roster** | Iowa, North Dakota, Minnesota custom-exempt, Wisconsin service directory | Search/filter tables with names, city/county, phone, classes, license or plant numbers; some expose CSV/XLSX links | Medium. Capture the exact list URL, displayed filters, page count, last-updated text, and export link. Do not assume table pagination is a complete snapshot until verified. | +| **Downloadable roster** | Georgia, Maine, Montana, North Carolina, South Carolina, South Dakota, Texas | PDF, XLSX, or PDF-backed map/list, often with establishment number, class and activity columns | Medium to high. Preserve the file and its revision date privately; PDF tables require schema and OCR/layout validation. A roster revision is an observation, not a closure event. | +| **Interactive map or map-backed page** | Louisiana, North Carolina, Utah, Minnesota | Map/search UI or embedded map with separate state/federal/custom filters | High. Browser-assisted capture may be required; record selected filters, result counts, map layer names, access date, and any download control. Coordinates are source/map output and are not automatically publication-eligible. | +| **Contact/licensing route** | Alabama, Arizona, Arkansas, Delaware, Illinois, Missouri, Oregon, Vermont, Virginia, Wyoming | Official program and application pages; current statewide roster is absent, hidden, or contact-mediated in bounded search | High/manual. Do not invent an API or infer that a licensing application is a facility list. Request an authorized current export or operator-assisted list and record terms and scope. | +| **CIS overlay** | Ten FSIS CIS states | FSIS publishes state-specific XLSX files of current selected establishments | Medium, but separate. Treat CIS workbook rows as an interstate-eligibility observation linked to the state source by exact state/plant identifiers; do not replace or merge the underlying state roster. | + +The preferred implementation is one source-local adapter with a source-specific +profile and a common provenance envelope, not 29 bespoke scripts. Profiles +should cover `state_official`, `cis`, `custom_exempt`, `retail_or_handler`, and +`inactive_or_expired` where a source explicitly provides them. A missing row or +changed list must produce `not_observed` or a versioned observation, never an +inferred closure. + +## State-by-state inventory + +“List route” below means the authoritative state page or document found during +the bounded search. “No public roster located” is an access/reconnaissance +result, not evidence that the state has no list. Scale figures are only quoted +where an official source supplied one; otherwise they are intentionally +`unknown`. Address quality describes the observed publication shape, not the +truth or currentness of an address. No official state source in this scan +provided a documented coordinate provider/precision contract. + +| State | FSIS scope | Official state route / delivery | Native identifiers and categories | Scale, cadence, address/coordinate quality, automation | +| --- | --- | --- | --- | --- | +| Alabama | Meat & poultry | [Meat Inspection](https://agi.alabama.gov/animalindustries/meat-inspection/) has program scope and an “Establishments” section, but no current public roster was located in bounded search; official contact is the acquisition route. | State licensing/establishment number is expected but not documented in the public page. The page explicitly distinguishes state, federal, and custom-exempt duties. | 40 program staff reported; plant count unknown. Cadence unknown. Manual/contact-only, high difficulty. Do not use Alabama retail-food or general establishment searches as an MPI roster. | +| Arizona | Meat & poultry | [Animal Services Inspections](https://agriculture.az.gov/node/14) and [department directory](https://agriculture.az.gov/arizona-department-agriculture-directory); current public state roster not located. An old third-party-hosted roster is not treated as current evidence. | Establishment/license number semantics need confirmation from the state. | Scale/cadence unknown. Contact-mediated, high difficulty. Address/coordinates not documented. | +| Arkansas | Meat only | [Arkansas state MPI announcement/rules](https://www.agriculture.arkansas.gov/wp-content/uploads/2022/10/10.4.22-State-Meat-Inspection-Program-Press-Release.pdf) and [processing requirements](https://www.agriculture.arkansas.gov/wp-content/uploads/2022/10/AR-Processing-Plant-Reqs.pdf); no current public roster located. | Meat program only; official rules distinguish inspected establishments from custom establishments and require “Not For Sale” marking for custom product. | Scale/cadence unknown. Contact/manual, high difficulty. No public coordinate contract. | +| Delaware | Meat & poultry | [Meat and Poultry Inspection](https://agriculture.delaware.gov/food-products-inspection/meat-poultry-inspection/) and [Food Products Inspection](https://agriculture.delaware.gov/food-products-inspection/); annual license application is public, but no current statewide establishment roster was located. | Annual establishment license; official establishment, handlers, storage, transport and rendering scopes are broader than slaughter/processing. | Scale/cadence unknown. Contact/licensing route, high difficulty. Address is application/licensing evidence, not necessarily operating-site evidence. | +| Georgia | Meat only | [Meat Inspection](https://agr.georgia.gov/meat-inspection) links a [January 2026 Directory of Licensed Meat Plants](https://www.agr.georgia.gov/sites/default/files/documents/meat-inspections/directory-of-licensed-meat-plants.pdf). | Directory contains establishment numbers, names/addresses, phones, counties/districts, S&P/processing classes, species, Talmadge-Aiken, state-inspected and custom-exempt categories. | Current document revision observed as January 2026; cadence appears edition-based, not formally documented. PDF/address-only; no coordinates. Medium difficulty. Meat-only; do not model poultry state coverage from this directory. | +| Illinois | Meat & poultry | [Illinois Meat & Poultry Inspection](https://agr.illinois.gov/animals/meat-inspection.html) and [license pages](https://agr.illinois.gov/animals/meat-inspection/meat-poultry-license.html); public page reports program scope and licensing but no current official-inspected roster was located. A separate [poultry/rabbit exemption list](https://agr.illinois.gov/animals/meat-inspection/meat-poultry-license/poultry-and-rabbit-exemption-list.html) is not an official inspected-plant list. | AGR location/license fields; Type 1 state slaughter/processing and Type 2 custom-only are explicitly distinct. Annual broker/warehouse licenses and poultry/rabbit exemptions are separate. | Official page says over 150 slaughter/processing establishments, over 700 brokers and more than 60 refrigerated warehouses; date not clear on the page. Contact/manual, high difficulty. Address fields exist in applications; no coordinate contract. | +| Indiana | Meat & poultry | [BOAH Meat & Poultry Inspection](https://www.in.gov/boah/meat-and-poultry-inspection/) links the [state/custom facility list](https://secure.in.gov/boah/meat-and-poultry-inspection/types-of-meat-and-poultry-inspection) and a roster PDF such as [2021 MPI establishments](https://www.in.gov/boah/files/2021-MPI-Establishments.pdf). | State establishment number, name, address, city, ZIP, phone; state-inspected, custom-exempt, CIS and federal populations are explained separately. | Official 2024 meeting material reports 83 official, 55 custom-exempt and 3 limited-permit-retail facilities (141 total); year-specific. PDF/HTML, address-only, no coordinates. Medium difficulty. Indiana is a CIS state; use FSIS CIS XLSX as an overlay. | +| Iowa | Meat & poultry | [IDALS licensing list](https://data.iowaagriculture.gov/licensing_lists/meatpoultry/) is a filterable HTML roster with pagination; [bureau page](https://iowaagriculture.gov/meat-poultry-inspection-bureau) documents scope. | Plant name, city, county, phone, plant class and CIS flag. Classes distinguish federal, state official slaughter/processing, custom, poultry exemptions and retail-related classes. | Page displayed 276 rows and explains class semantics; current page does not provide a stable last-updated field in the observed content. No street address or coordinates in the table. Medium difficulty; exact class vocabulary is reusable. Iowa is CIS. | +| Kansas | Meat & poultry | [KDA Meat & Poultry Inspection](https://www.agriculture.ks.gov/divisions-programs/meat-poultry-inspection) exposes separate links for inspected, custom, USDA, map, wholesalers and other licenses. | Separate inspected/custom lists; exact official establishment numbers and export formats require link-by-link inspection. Program explicitly distinguishes custom-exempt, retail-exempt, inspected slaughter and inspected processing. | Scale not stated on the page. HTML/PDF/map family, medium-high difficulty until list links and revision cadence are pinned. Address/coordinates not documented. | +| Louisiana | Meat & poultry | [LDAF meat and poultry](https://www.ldaf.la.gov/food/selling/meat-poultry) provides an interactive map and an `MPI Directory Aug 2026`/list-of-plants route; [wholesale inspection](https://www.ldaf.la.gov/food/selling/meat-poultry/wholesale-inspection-information) defines inspected scope. | Map/list type includes LDAF-inspected, wholesale inspected and custom-exempt distinctions; establishment numbers are assigned during application. | Edition label observed as August 2026; no official count captured. Map plus linked directory, address/possibly map geometry but no precision/provider contract. High difficulty; filter state must be recorded. | +| Maine | Meat & poultry | [Red Meat and Poultry Inspection](https://www.maine.gov/dacf/qar/inspection_programs/red_meat_poultry_inspection.shtml) links a [State and USDA inspected establishments PDF](https://www.maine.gov/dacf/qar/inspection_programs/documents/mmpi/State%20and%20USDA%20Inspected%20Establishments%202025.pdf), a facility map and a separate custom directory. | Establishment/license information is mixed across state/USDA PDF; custom-exempt is explicitly separate. Exact state establishment-number semantics require schema review. | PDF revision observed as 2025; cadence is document-based. Address/map route, no coordinate contract. Medium-high difficulty. Maine is CIS. | +| Minnesota | Meat & poultry | [MDA starting-a-business page](https://www.mda.state.mn.us/es/node/1436) links the current State “Equal To” list and map; [custom-exempt list](https://www.mda.state.mn.us/es/node/1516) is a separate public HTML roster updated weekly. | Equal-To/state and custom-exempt permits/licenses; custom list includes establishment, address, city, ZIP and phone, with active-license/custom-permit scope. | Custom roster explicitly says weekly updates and occasional (1–4/year) inspection; official Equal-To scale not captured. HTML/map, address-only for custom list, no coordinates. Medium-high difficulty; category split is essential. | +| Mississippi | Meat & poultry | [MDAC Meat Inspection page](https://agnet.mdac.ms.gov/website/Meat_Single?id=12) provides a facility directory-style page and links a [Meat Establishments Map](https://www.mdac.ms.gov/bureaus-departments/regulatory-services/meat-inspection/poultry-faq/); route is web/UI, not a verified bulk export. | Directory records distinguish federal/state inspection, category, custom-exempt status and species; establishment number/ID semantics not confirmed. | Scale/cadence unknown. HTML/map with address and phone; no coordinate contract. High difficulty; row classification must not be inferred from “state inspection” text alone. | +| Missouri | Meat & poultry | [Missouri MPIP](https://agriculture.mo.gov/animals/health/inspections/) is the official program page; no current public statewide facility roster was located in bounded search. | MPIP-inspected official establishments, USDA establishments and custom-exempt facilities are distinct; plant/establishment number semantics need an authorized roster. | An official 2020/21 grant release reported 27 new MDA-inspected and 62 existing USDA/MDA-inspected establishments plus 43 existing custom-exempt facilities; historical/context only. Contact/manual, high difficulty. | +| Montana | Meat & poultry | [Meat & Poultry Inspection Section](https://liv.mt.gov/Meat-Milk-Inspection/Meat-and-Poultry-Inspection/) links [state-inspected establishments](https://liv.mt.gov/_docs/MI/Webpage-State-Plant-List-for-Public-2025.pdf) and a separate custom-exempt list. | State establishment number, name, location, license type, Department of Livestock license number, license status and establishment type. | State roster search result says last update January 8, 2026; custom list has its own dated snapshot. Address is city/location only in observed roster; no coordinates. Medium difficulty. Montana is CIS. | +| North Carolina | Meat & poultry | [Plant Directory](https://www.ncagr.gov/divisions/meat-poultry-inspection/plantdirectory) publishes PDF directories and a map for state, custom, Talmadge-Aiken, all and farmer-service plants; [program information](https://www.ncagr.gov/meat-poultry-inspection/info) gives scale. | Plant number, name/location and coded plant type; directory explicitly separates state, custom, TA, federal and farmer-service files. | NCDA says 186 red-meat slaughter/processing and poultry-processing facilities; date not clear in page. PDF/map, likely address/county; no coordinate contract. Medium-high difficulty. | +| North Dakota | Meat & poultry | [North Dakota meat processors](https://www.ndda.nd.gov/divisions/grain-livestock/meat-inspection/north-dakota-meat-processors) publishes HTML sections for selected/CIS, federal slaughter and custom/exempt processors; [application](https://www.ndda.nd.gov/sites/www/files/documents/files/52498MeatApp_0.pdf) documents fields. | Establishment number, license/exemption number, official/custom/poultry-exemption/retail-exempt types; HTML includes company, address, city, state, ZIP. | Cadence not stated. HTML roster includes street/mailing addresses, but no coordinates. Medium difficulty; separate selected/CIS and custom sections. North Dakota is CIS. | +| Ohio | Meat & poultry | [Ohio Meat Inspection Web Portal](https://www.apps.agri.ohio.gov/MeatInspectionWebPortal/About) is the official portal; [2024–25 annual report](https://dam.assets.ohio.gov/image/upload/v1772547739/agri.ohio.gov/Communications/Annual%20Report/FINAL_August_1_2025_ODA_2024-2025_Annual_Report.pdf) supplies current aggregate scale. | Portal/roster identifier and license fields require direct capture; annual report distinguishes full inspection, CIS and custom-exempt. | Official annual report reports 272 licensed establishments: 144 full inspection, 47 CIS, 81 custom-exempt. Portal/UI, address/coordinates not documented. High difficulty until export/API terms are confirmed. Ohio is CIS. | +| Oklahoma | Meat & poultry | [ODAFF Food Safety](https://ag.ok.gov/divisions/food-safety/) publishes current `2026 Meat Processing`, `2026 State Plant`, and `2026 Custom Plant` lists. | List-specific establishment/license identifiers; program page distinguishes inspected harvesters/processors, state plants and custom plants. Official marks include establishment number. | Current 2026 editions are observed, but row counts and exact file formats were not captured. Downloadable list family, likely address-only; no coordinate contract. Medium difficulty. | +| Oregon | Meat only | [State Meat Inspection Program](https://www.oregon.gov/oda/food-safety/pages/state-meat-inspection-program.aspx) documents a program effective July 2022; no current public roster located. Poultry and rabbits are explicitly out of scope. | State-assigned establishment number is distinct from license number and appears on the Oregon inspection legend. | Scale/cadence unknown. Contact/manual, high difficulty. Red-meat-only; do not add poultry rows. Address/coordinates not documented. | +| South Carolina | Meat & poultry | [Clemson SC Meat-Poultry Inspection](https://www.clemson.edu/public/lph/scmpid/) publishes an [Establishment Directory](https://www.clemson.edu/public/lph/scmpid/) with separate state meat, cross-utilization federal, state poultry, custom-exempt, rendering and handler PDFs plus an Excel accessibility file. | SCMPID establishment/permit/grant fields; separate state meat, poultry, custom-exempt, rendering and handler categories. | Public directory is PDF/XLSX, with annual permit renewal noted by SCMPID. Address quality depends on each PDF; no coordinate contract. Medium difficulty. | +| South Dakota | Meat only | [SDAIB Meat Inspection](https://aib.sd.gov/meat-inspection.html) links [inspected establishments](https://aib.sd.gov/List%20of%20inspected%20establishments.pdf) and a separate custom-exempt list. | Establishment name, town, class; class codes S/P and CIS are explained. State page explicitly says poultry is under USDA FSIS. | Roster revised November 2025; no exact count asserted here. Town-only in inspected PDF, no coordinates. Medium difficulty. Meat-only and CIS. | +| Texas | Meat & poultry | [DSHS inspections and exemptions](https://www.dshs.texas.gov/meat-safety/inspections-exemptions-meat-safety) publishes a linked current “Establishments by Grant Type” document; grant types include full, voluntary, custom and poultry/rabbit exemptions. | Grant/license type, establishment location and state/federal/voluntary/custom distinctions; poultry/rabbit low-volume registration is separate. | Statewide count not captured. Regional DSHS page reports 9 inspected and 11 custom-exempt facilities for Public Health Region 1 only; do not generalize statewide. PDF, address-only, no coordinates. Medium-high difficulty. | +| Utah | Meat & poultry | [Utah Meat and Poultry Inspection](https://ag.utah.gov/animal-industry/meat-and-poultry-inspection-program/) exposes separate federal, state and custom processing/slaughter map sections; [2023 annual report](https://ag.utah.gov/wp-content/uploads/2023-Annual-Report-Final.1-1-compressed.pdf) supplies scale. | Establishment number/license fields appear in the state inspection application; separate federal, state, custom and farm-custom categories. | 2023 report: 4 state harvest, 9 state harvest/process, 8 state process-only, 2 Talmadge-Aiken harvest, 4 Talmadge-Aiken harvest/process, 11 Talmadge-Aiken process-only, 57 custom-exempt and 37 farm-custom permittees (38 official; 94 total in those categories). Map/UI, coordinate contract unknown. High difficulty. | +| Vermont | Meat & poultry | [Vermont Meat & Poultry Inspection](https://agriculture.vermont.gov/food-safety/vermont-meat-poultry-inspection) provides a [Meat Handlers Facilities Search](https://agriculture.vermont.gov/food-safety/vermont-meat-poultry-inspection) link and separate commercial/custom/exemption resources; no stable bulk roster was located. | Meat-handler license classes include commercial slaughterhouse, packing plant, poultry slaughterhouse, custom, distributor, broker, warehouse and renderer; state inspection is separate from handlers. | Scale/cadence unknown; page documents recurring capacity surveys, not a canonical roster. UI/contact route, high difficulty. Address/coordinates not documented. Vermont is CIS. | +| Virginia | Meat & poultry | [VDACS Meat & Poultry Services](https://www.vdacs.virginia.gov/animals-meat-and-poultry.shtml) says the directory of Virginia inspected and custom-permitted facilities is available by contacting the office; no public roster was located. | State establishment/license identifiers not public on the page; inspected and custom-permitted facilities are explicitly separate. | Scale/cadence unknown. Contact-only, highest difficulty among reviewed routes. No public coordinate or redistribution contract. | +| West Virginia | Meat & poultry | [WVDA Meat and Poultry Inspection](https://agriculture.wv.gov/divisions/meat-poultry-inspection/) links a [licensed commercial establishments list](https://agriculture.wv.gov/commercial-list/) and [commercial map/services](https://agriculture.wv.gov/commercial-map-services/). | Establishment number prefixes S, P and SP; list includes county, street/mailing address, circuit, manager/phone and service flags for red meat, poultry, retail, distributor, custom, etc. | List observed as of July 22, 2025; map/list family, address-rich but no coordinate provider contract. Medium difficulty. Validate private-contact exposure before any staging. | +| Wisconsin | Meat & poultry | [DATCP meat processing](https://datcp.wi.gov/Pages/MeatProcessinginWisconsin.aspx) links the [Inspected and Custom Service Directory](https://mydatcp.wi.gov/documents/dfrs/Meat%20Establishment%20Inspected%20and%20Custom%20Service%20Directory.pdf) and the service page exposes PDF, Excel and CSV outputs. | License, plant number, service type, DBA, business location/mailing address, county, phone/email and license activities. Official and custom-exempt are separate; federal rows may appear in the same directory. | Official page reports as of 2026-08-27: 217 state official (61 slaughter, 156 non-slaughter, 45 CIS), 71 custom-exempt, 233 federal, 521 total. PDF/XLSX/CSV, address-rich and no coordinate contract. Low-medium difficulty, but filter federal/state/custom before use. Wisconsin is CIS. | +| Wyoming | Meat & poultry | [Wyoming State Meat Program](https://agriculture.wy.gov/meat-poultry) documents state-inspected, custom-exempt and wild-game categories; no current public roster was located in bounded search. | State license/inspection category; WDA says state-inspected products are intrastate, custom is separate, and wild game is separate. | Scale/cadence unknown. Contact/manual, high difficulty. No public coordinate or redistribution contract. | + +## Semantics and safety boundaries + +### Slaughter and processing + +State official rosters may describe a plant as slaughter, processing, or both; +some use state-specific classes such as Iowa `1ACB`, `1ASO`, `1B`, `OP1A`, or +South Dakota `S/P`. Preserve the source class and source text. A processing-only +row is not a slaughter facility, and a state inspection program does not imply +that every licensed processor has on-site slaughter. + +### Custom-exempt + +Custom-exempt operations are not inspected product facilities. State sources +consistently describe custom product as for the animal owner/household and +marked “Not For Sale”; inspection is periodic or risk-based rather than +continuous. Custom rows may be useful as a separately labeled processing +capacity evidence layer, but must never be counted as state-inspected slaughter +or processing or treated as eligible for retail/wholesale sale. + +### Retail, handlers, brokers, warehouses and exemptions + +Retail meat markets, brokers, warehouses, meat-handler licenses, poultry/rabbit +exemptions, farm slaughter registrations, wild-game processors and rendering +plants occur in several state routes. They are legally and operationally +different populations. Keep them in source-specific profiles and do not turn a +license or retail inspection into an inspected slaughter/processing facility. + +### Inactive, expired and absent rows + +Some state lists include explicit active/inactive or license-status fields; +others are undated PDFs or contact-only routes. Store a source-provided status +and observation date when present. An expired or inactive license is a dated +regulatory observation, not proof that a site closed. A row absent from a later +roster is `not_observed` until the state confirms closure or the source defines +its lifecycle semantics. + +### Addresses and coordinates + +Observed public rosters range from town-only (South Dakota), city/county-only +(Iowa), city/location (Montana), full street address (North Dakota, West +Virginia, Wisconsin), and map/UI outputs (Louisiana, North Carolina, Utah). +None of the observed state sources documented a coordinate provider, query, +precision, or review state. Do not geocode these rows automatically. A mailing +address, registered office, or map point is not proof of an operating facility +and does not override the repository privacy rules for mixed residential/private +locations. + +## Terms, redistribution and access mechanics + +The sources reviewed are government pages or official state program pages, but +government origin does not itself grant redistribution rights. Before any +acquisition or release, record the final URL, retrieval UTC, visible revision or +effective date, content type, byte size, SHA-256, source terms/attribution, +filter/query context, and adapter/configuration version. Preserve raw artifacts +only in restricted staging where permitted. Treat public HTML/PDF/XLSX/CSV +availability as an access observation, not a license conclusion. + +The default legal/product boundary is intrastate: FSIS says state-inspected +products cannot move interstate unless the establishment is selected under CIS. +The CIS list is therefore an eligibility overlay, not a blanket authorization +for all facilities in a participating state. CIS sources say selected plants +must meet additional conditions, including no more than 25 employees and a +federal review/inspection contract. No state in FSIS's current page has a +supplemental agreement for CIS export to foreign countries. + +## Recommended next implementation slice + +1. Add a row-free `us.state-mpi` source entry and a private source-local + acquisition contract with profiles for `state_official`, `cis`, + `custom_exempt`, and `retail_or_handler`. +2. Start with structured/dated routes: Wisconsin CSV/XLSX, Iowa HTML, North + Carolina PDFs, Georgia PDF, South Carolina XLSX/PDF, South Dakota PDF, + Montana PDF, and the ten FSIS CIS workbooks. Use synthetic fixtures for + schema and classification tests; do not commit real rows. +3. Add contact-assisted capture contracts for states without a public roster. + The operator must provide the authorized list/export and terms context; no + hidden endpoint or scraping assumption should be introduced. +4. Validate state-native identifiers and category vocabularies before any + cross-state comparison. Never merge a state establishment to FSIS by name, + address, phone or coordinates; only exact source-native IDs or an explicit + reviewed link event may create a cross-source relationship. + +## Primary source index + +* [FSIS State Inspection Programs](https://www.fsis.usda.gov/inspection/state-inspection-programs) +* [FSIS States With and Without Inspection Programs](https://www.fsis.usda.gov/inspection/state-inspection-programs/states-and-without-inspection-programs) +* [FSIS Cooperative Interstate Shipment program](https://www.fsis.usda.gov/inspection/state-inspection-programs/cooperative-interstate-shipping-program) +* [FSIS CIS Establishments](https://www.fsis.usda.gov/inspection/state-inspection-programs/cooperative-interstate-shipping-program/cooperative-interstate) +* [FSIS Directive 5740.1](https://www.fsis.usda.gov/policy/fsis-directives/5740.1) + +State-specific official routes are linked in the inventory above. The FSIS +central list is the authority for the 29-state/10-CIS inventory used here; +state pages are the authority for each state's roster mechanics and local +classification vocabulary. Where those sources disagree on a count or “number +of states” (for example, a Wisconsin explanatory page saying 30 while FSIS +lists 29), preserve the disagreement and recheck both pages before capture. diff --git a/docs/country-recon-us.md b/docs/country-recon-us.md index 756d3bf..be2ac9f 100644 --- a/docs/country-recon-us.md +++ b/docs/country-recon-us.md @@ -41,8 +41,61 @@ No safe bounded private fetch was performed, so current hashes/bytes and determi ## 2026-09-15 recovery slice +## 2026-09-18 state MPI reconnaissance + +FSIS identifies 29 cooperative State Meat and Poultry Inspection programs and +10 states with a Cooperative Interstate Shipment (CIS) overlay. State sources +are not a single national directory: they range from dated PDF/XLSX/CSV/HTML +rosters and interactive maps to contact-mediated licensing routes, and many +mix official inspected, CIS, custom-exempt, retail/handler, warehouse, +rendering, and exemption populations. The row-free state route, identifier, +cadence, scale, access, address/coordinate and automation crosswalk is in +[`docs/countries/us/state-mpi-source-recon.md`](countries/us/state-mpi-source-recon.md). +No state roster or CIS workbook was acquired. State coverage remains +documentation-only and publication-blocked; absence from a later list is +`not_observed`, not closure. + The private implementation is in `pipeline/sources/us/`. FSIS now has a bundle adapter and refresh command with a sanctioned operator-assisted capture contract: one directory export plus the supplemental demographic export are reconciled only by exact source-native IDs/numbers, and source-provided coordinates, slaughter species/activity fields, processing fields, size, and inspection attributes remain private pending review. APHIS now has one adapter with explicit `registrations`, `annual_reports`, and `inspections` profiles. All three APHIS populations remain observations, not a laboratory or facility master, and no identity merge with FSIS is performed. The row-free V1 inventory and field/category crosswalk is [`docs/countries/us/v1-field-crosswalk.json`](countries/us/v1-field-crosswalk.json). It records 7,101 rows and 269 columns, maps identity/location/contact/administrative/slaughter/processing/inspection-system/derived fields, and records overlapping legacy field-presence counts. Since no authorized current FSIS artifact was available, current-versus-V1 reconciliation remains blocked; the existing exact-key crosswalk reports `not_observed`, never closure. Focused adapter, lifecycle, registry, status, and contract tests pass. No raw artifact, current source hash, or publication candidate from a real US source was created. Publication remains blocked pending authorized capture, terms, schema, privacy, coverage, review, and test-only import checks. + +## 2026-09-18 US real-data proof boundary + +The official MPI page was re-observed in the normal in-app browser. It showed +`Last Updated: Sep 14, 2026`, the directory-by-name CSV, directory-by-number +CSV, and establishment-demographic CSV, plus a Tableau dashboard updated +`9/14/2026 2:30:33 PM`. The dashboard's aggregate count export showed 7,241 +establishments. This is a current aggregate observation only; it is not a +replacement for the row-level CSV bundle. + +The two direct CSV routes still returned HTTP 403 to a bounded read-only +request. No access-control bypass was attempted. The current-source manifest +therefore remains `not_captured`, and current row-level totals, current +coordinates, current species/activity distributions, and current-vs-V1 +identity continuity remain `not_observed` rather than zero or closure +evidence. + +For continuity and private pipeline proof only, the retained legacy FSIS +directory plus demographic snapshot was rerun through the bundle adapter. It +contained 7,099 directory rows and 7,105 demographic rows; 7,089 demographic +rows matched exact source-native IDs/numbers, 16 were orphaned, and 3 had +identity conflicts. The private lifecycle produced 7,096 normalized rows and +19 quarantined rows (`conflicting_demographic_identity`: 3, +`unmatched_demographic_identity`: 16). The run kept raw, parsed, normalized, +quarantined, and handoff layers separate, retained SHA-256/byte provenance, +disabled geocoding, and kept `release_state=not-created`, +`publication_state=private-candidate`, and publication blocked. These numbers +describe the retained legacy snapshot, not the September 2026 source. + +The bundle join now indexes exact native keys before reconciliation, and the +handoff records both the importer-verifiable directory artifact and the +separate bundle/file provenance. This fixes the real-bundle candidate handoff +path without changing identity semantics or publication gates. + +The disposable candidate runner reached the test-only preview after the full +first import, confirming that candidate rows were not visible through the +ordinary public profile. Its row-level replay and teardown exceeded the +practical execution window, so the import evidence is recorded as partial; +no database or release was retained, and no public promotion occurred. diff --git a/docs/source-status.json b/docs/source-status.json index 6d02620..1e73b15 100644 --- a/docs/source-status.json +++ b/docs/source-status.json @@ -50,8 +50,18 @@ {"source_id":"au.abr.lookup","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-au.md","docs/countries/australia/source-crosswalk.json","pipeline/source_registry.json"],"next_action":"Query only when an upstream source supplies an ABN/ACN; record lookup time and terms, and suppress sole-trader personal details."}, {"source_id":"es.locations","metadata":"partial","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-es.md","pipeline/source_registry.json"],"next_action":"Resolve a permitted current AESAN/MAPA artifact or query route, then verify export/schema, rights, effective-date semantics, sector coverage, privacy, and legacy/source boundaries before acquisition."}, {"source_id":"us.fsis","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-us.md","docs/countries/us/README.md","docs/countries/us/v1-field-crosswalk.json","pipeline/sources/us/fsis/config.json","pipeline/sources/us/fsis/adapter.py","pipeline/sources/us/fsis/refresh.py","pipeline/source_registry.json"],"next_action":"Use the assisted official export contract after the 403 blocker; record URL, retrieval/effective dates, content type, byte size, SHA-256, terms, schema fingerprint, privacy review, and reconciliation before any test-only handoff."}, + {"source_id":"us.state-mpi","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/countries/us/state-mpi-source-recon.md","docs/countries/us/README.md","docs/source-status.md"],"next_action":"Obtain authorized current state MPI rosters or operator-assisted captures, preserving state-native identifiers, source classes, status/effective dates, terms, address/coordinate provenance, and separate official/CIS/custom-exempt/retail populations before any test-only handoff."}, {"source_id":"us.aphis","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-us.md","docs/countries/us/README.md","pipeline/sources/us/aphis/config.json","pipeline/sources/us/aphis/adapter.py","pipeline/sources/us/aphis/refresh.py","pipeline/source_registry.json"],"next_action":"Use the explicit profile-based assisted export for registrations, annual reports, or inspections; preserve each evidence type separately and complete terms, privacy, schema, and review gates."}, {"source_id":"us.inspections","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-us.md","docs/countries/us/README.md","pipeline/sources/us/aphis/config.json","pipeline/sources/us/aphis/adapter.py","pipeline/sources/us/aphis/refresh.py","pipeline/source_registry.json"],"next_action":"Capture the APHIS inspections profile through the documented public-search route; treat rows as observations, not a facility master, and use explicit reviewable identity matching only."}, + {"source_id":"us.nih.reporter","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/countries/us/research-animal-coverage-recon.md","pipeline/source_registry.json"],"next_action":"Implement a rate-limited NIH RePORTER API/ExPORTER funding-project adapter with exact award and organization identifiers; keep animal relevance as project evidence, not facility or animal-use counts."}, + {"source_id":"us.nih.olaw-assurances","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/countries/us/research-animal-coverage-recon.md","pipeline/source_registry.json"],"next_action":"Validate an assisted capture contract for the OLAW assured-institutions lookup; preserve Assurance ID/type and branch/affiliate scope, and do not infer protocols or animal counts."}, + {"source_id":"us.aaalac.accredited","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/countries/us/research-animal-coverage-recon.md","pipeline/source_registry.json"],"next_action":"Review directory terms and unit-level identity before any private capture; keep voluntary accreditation separate from regulatory registration and animal-use evidence."}, + {"source_id":"us.fda.glp-animal-research","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/countries/us/research-animal-coverage-recon.md","pipeline/source_registry.json"],"next_action":"Keep FDA evidence policy/manual until a public reproducible facility or inspection route is verified; do not treat openFDA adverse events as research-use counts."}, + {"source_id":"us.va.animal-research","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/countries/us/research-animal-coverage-recon.md","pipeline/source_registry.json"],"next_action":"Retain VA program, facility and ORO pages as separate observations; identify an authorized national facility/count route before acquisition."}, + {"source_id":"us.dod.acuro","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/countries/us/research-animal-coverage-recon.md","pipeline/source_registry.json"],"next_action":"Keep ACURO protocol/site-oversight material manual and privacy-reviewed; do not infer facilities from DoD awardee addresses or build a national count."}, + {"source_id":"us.nsf.awards","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/countries/us/research-animal-coverage-recon.md","pipeline/source_registry.json"],"next_action":"Consider NSF Award Search only as a funding/project adjunct after NIH RePORTER; require explicit animal-relevance and performance-site review."}, + {"source_id":"us.nasa.nspires","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/countries/us/research-animal-coverage-recon.md","pipeline/source_registry.json"],"next_action":"Keep NASA NSPIRES as funding/project evidence; verify public award identifiers and performance-site semantics before any integration."}, + {"source_id":"us.ca.cdph-lab-animals","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/countries/us/research-animal-coverage-recon.md","pipeline/source_registry.json"],"next_action":"Pilot California approval/count evidence only through an authorized current extract or public-record response; model federal exemptions and do not generalize to a national denominator."}, {"source_id":"nl.nvwa.approved-food","metadata":"verified","acquisition":"artifact_private_only","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-nl.md","docs/countries/nl/v1-field-crosswalk.json","data/manifests/nl-source-artifacts.json","pipeline/source_registry.json"],"next_action":"Implement SOAP only after confirming coverage, repeated observations, lifecycle, terms, privacy and identity policy."}, {"source_id":"nl.nvwa.welfare-enforcement","metadata":"verified","acquisition":"artifact_private_only","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-nl.md","data/manifests/nl-source-artifacts.json","pipeline/source_registry.json"],"next_action":"Keep 2025 aggregate welfare and 2024 detailed compliance evidence dated and separate; review identity links and release terms."}, {"source_id":"nl.cokz.dairy-eggs","metadata":"verified","acquisition":"artifact_private_only","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-nl.md","docs/countries/nl/v1-field-crosswalk.json","data/manifests/nl-source-artifacts.json","pipeline/source_registry.json"],"next_action":"Confirm COKZ register families, update semantics, terms and overlap with NVWA before modeling."}, diff --git a/docs/source-status.md b/docs/source-status.md index bdc1611..e8f7184 100644 --- a/docs/source-status.md +++ b/docs/source-status.md @@ -48,8 +48,18 @@ The 2026-09-16 Ireland reconnaissance verified the current FSAI/DAFM/HSE/SFPA so | `ca.cfia.federal-meat` | verified | not_run | not_run | blocked | CFIA federal private adapter/refresh is implemented; validate live export schema/function codes, keep federal scope separate from provincial scope, and retain publication gates | | `es.locations` | partial | blocked | not_run | blocked | AESAN RGSEAA and MAPA sector routes are documented, but direct acquisition was refused; rights, export/schema, effective dates, sector coverage, privacy, and legacy/source boundaries remain unresolved | | `us.fsis` | verified | blocked | not_run | blocked | Private adapter and assisted-capture contract are implemented; current CSV access returned 403, so obtain an authorized export and record provenance/schema/privacy/reconciliation before test-only handoff | +| `us.state-mpi` | verified | not_run | not_run | blocked | FSIS identifies 29 state MPI programs and 10 CIS states; state roster routes are documented in `docs/countries/us/state-mpi-source-recon.md`, but no state roster/CIS workbook was acquired. Keep official, CIS, custom-exempt, retail/handler, and inactive/expired populations separate; obtain authorized current exports and review terms/schema/privacy before any test-only handoff | | `us.aphis` | verified | not_run | not_run | blocked | Profile-explicit private adapter and assisted-capture contract cover registrations, annual reports, and inspections; capture current exports and review terms/schema/privacy | | `us.inspections` | verified | not_run | not_run | blocked | APHIS inspections profile is implemented as observation evidence; capture current export and use explicit reviewable identity links only | +| `us.nih.reporter` | verified | not_run | not_run | blocked | NIH RePORTER API/ExPORTER is documented as the first funding/project integration; exact award and organization keys remain separate from animal-use counts | +| `us.nih.olaw-assurances` | verified | not_run | not_run | blocked | OLAW assured-institutions lookup is current institutional assurance evidence; validate assisted capture and branch/affiliate scope without inferring protocols or counts | +| `us.aaalac.accredited` | verified | not_run | not_run | blocked | AAALAC directory is voluntary unit-level accreditation evidence; review terms and identity before private capture | +| `us.fda.glp-animal-research` | verified | not_run | not_run | blocked | FDA MOU/facility/policy surfaces are manual evidence only; no national facility/count route was verified and openFDA adverse events are out of scope | +| `us.va.animal-research` | verified | not_run | not_run | blocked | VA program and ORO pages document oversight but no national public facility/count route; keep local observations separate | +| `us.dod.acuro` | verified | not_run | not_run | blocked | ACURO protocol/site oversight pages are manual and privacy-sensitive; do not infer facilities from awardee addresses | +| `us.nsf.awards` | verified | not_run | not_run | blocked | NSF Award Search is a funding/project adjunct only; animal relevance and performance-site semantics require review | +| `us.nasa.nspires` | verified | not_run | not_run | blocked | NASA NSPIRES is funding/project evidence only; no public animal-use route was verified | +| `us.ca.cdph-lab-animals` | verified | not_run | not_run | blocked | California CDPH LAB 139 is a state pilot with prior-year count fields and federal exemptions; no public bulk registry and no national denominator | | `nl.nvwa.approved-food` | verified | artifact_private_only | not_run | blocked | Current control XML and eight SOAP list captures are private; repeated observation semantics, terms, privacy and adapter contract remain open; see `docs/country-recon-nl.md` | | `nl.nvwa.welfare-enforcement` | verified | artifact_private_only | not_run | blocked | Current 2025 welfare/animal-experiment pages and dated 2024/2023 PDFs captured privately; keep effective periods and evidence types separate | | `nl.cokz.dairy-eggs` | verified | artifact_private_only | not_run | blocked | Current COKZ HTML register captured privately; register-family coverage, terms and overlap with NVWA remain open | diff --git a/pipeline/common/acquisition.py b/pipeline/common/acquisition.py index 2b35813..4ffea40 100644 --- a/pipeline/common/acquisition.py +++ b/pipeline/common/acquisition.py @@ -237,6 +237,20 @@ def fetch_source( failure = AcquisitionError(f"timeout: {error}", failure_class="timeout", retryable=True, action="retry within the source bound; use the manual capture route if it persists") _write_failure(run_dir, source_id, run_id, failure, attempts, query_context, url, requested_at, effective_date, publication_date) raise failure from error + except OSError as error: + # A connection reset or interrupted read can surface as an OSError + # instead of URLError. archive_stream removes the partial file; + # keep the retry bounded and record the failure like other network + # interruptions. + details = {"attempt": attempt_number, "outcome": "failed", "failure_class": "interrupted-download", "retryable": True, "message": str(error)} + attempts.append(details) + if attempt_number == max_attempts: + failure = AcquisitionError( + f"interrupted download: {error}", failure_class="interrupted-download", retryable=True, + action="retry within the source bound; use the manual capture route if it persists", + ) + _write_failure(run_dir, source_id, run_id, failure, attempts, query_context, url, requested_at, effective_date, publication_date) + raise failure from error except AcquisitionError as error: download_path.unlink(missing_ok=True) attempts.append({"attempt": attempt_number, "outcome": "failed", "failure_class": error.failure_class, "retryable": error.retryable, "message": str(error)}) diff --git a/pipeline/common/test_acquisition.py b/pipeline/common/test_acquisition.py index e2349dd..7b6e512 100644 --- a/pipeline/common/test_acquisition.py +++ b/pipeline/common/test_acquisition.py @@ -83,6 +83,54 @@ def open(self, _request, timeout): self.assertEqual(metadata["attempts"][0]["failure_class"], "network") self.assertTrue(Path(metadata["artifact_path"]).exists()) + def test_fetch_retries_interrupted_read_and_removes_partial(self): + class Response: + status = 200 + headers = {"Content-Type": "text/csv", "Content-Length": "8"} + + def __init__(self, interrupted=False): + self.interrupted = interrupted + + def __enter__(self): + return self + + def __exit__(self, *_args): + return False + + def read(self, _size): + if self.interrupted: + raise ConnectionResetError("synthetic interrupted read") + if not hasattr(self, "done"): + self.done = True + return b"a,b\n1,2\n" + return b"" + + def geturl(self): + return "https://example.test/source.csv" + + class Opener: + def __init__(self): + self.calls = 0 + + def open(self, _request, timeout): + self.calls += 1 + return Response(interrupted=self.calls == 1) + + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + terms = root / "terms.json" + terms.write_text(json.dumps({"reviewer": "operator", "reference": "test", "reviewed_at": "2026-09-15T00:00:00Z", "decision": "approved", "notes": "synthetic"}), encoding="utf-8") + opener = Opener() + with patch("pipeline.common.acquisition.urllib.request.build_opener", return_value=opener): + metadata = fetch_source( + source_id="test.source", url="https://example.test/source.csv", output_root=root / "raw", + artifact_name="source.csv", terms_review_path=terms, run_id="run-interrupted", max_attempts=2, + ) + self.assertEqual(opener.calls, 2) + self.assertEqual(metadata["attempts"][0]["failure_class"], "interrupted-download") + self.assertTrue(Path(metadata["artifact_path"]).exists()) + self.assertFalse(list((root / "raw/test.source/run-interrupted").glob("*.download"))) + def test_fetch_non_retryable_content_type_fails_closed_with_private_report(self): class Response: status = 200 diff --git a/pipeline/common/test_source_operations.py b/pipeline/common/test_source_operations.py index 9e9b43d..a0bcf86 100644 --- a/pipeline/common/test_source_operations.py +++ b/pipeline/common/test_source_operations.py @@ -29,7 +29,9 @@ def test_checked_in_schedule_inventory_matches_registry(self): self.assertEqual(schedules["al.aku.approved-food"].cadence, "unknown") self.assertEqual(schedules["dk.smiley"].stale_after_hours, 240) self.assertEqual(schedules["fr.dgal.section-i"].interval_hours, 24) - self.assertIsNone(schedules["us.fsis"].interval_hours) + self.assertEqual(schedules["us.fsis"].interval_hours, 168) + self.assertEqual(schedules["us.fsis"].max_attempts, 3) + self.assertEqual(schedules["us.aphis"].max_attempts, 3) def test_schedule_validation_rejects_missing_and_inverted_freshness(self): with self.assertRaisesRegex(SourceOperationsError, "missing fields"): diff --git a/pipeline/scripts/maintenance/rehearse_fsis_candidate.py b/pipeline/scripts/maintenance/rehearse_fsis_candidate.py new file mode 100644 index 0000000..6bd34ed --- /dev/null +++ b/pipeline/scripts/maintenance/rehearse_fsis_candidate.py @@ -0,0 +1,158 @@ +"""Rehearse one private FSIS candidate handoff in a disposable database. + +This runner accepts only an importer-verifiable private handoff and a matching +directory artifact. It never creates or promotes a public release and emits a +row-free report. +""" +from __future__ import annotations + +import argparse +import hashlib +import importlib.util +import json +import sys +import urllib.error +import urllib.request +from pathlib import Path +from typing import Any + +import psycopg + +ROOT = Path(__file__).resolve().parents[3] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) +FORBIDDEN_KEYS = {"source_values", "raw_fields", "payload", "records", "address", "coordinates"} + + +def _load_importer(): + path = ROOT / "pipeline/scripts/maintenance/import-candidate.py" + spec = importlib.util.spec_from_file_location("fsis_candidate_import", path) + if spec is None or spec.loader is None: + raise RuntimeError(f"unable to load importer: {path}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _request(base: str, path: str, headers: dict[str, str] | None = None) -> tuple[int, Any]: + request = urllib.request.Request(base + path, headers=headers or {}) + try: + with urllib.request.urlopen(request, timeout=30) as response: + return response.status, json.loads(response.read()) + except urllib.error.HTTPError as exc: + try: + return exc.code, json.loads(exc.read()) + except (UnicodeDecodeError, json.JSONDecodeError): + return exc.code, None + + +def _assert_safe(value: Any) -> None: + if isinstance(value, dict): + forbidden = FORBIDDEN_KEYS.intersection(value) + if forbidden: + raise RuntimeError(f"row-bearing report key leaked: {sorted(forbidden)}") + for child in value.values(): + _assert_safe(child) + elif isinstance(value, list): + for child in value: + _assert_safe(child) + + +def run(handoff_manifest: Path, raw_path: Path, output: Path, release_id: str) -> dict[str, Any]: + from pipeline.tests.e2e.fixture import E2EEnvironment + + importer = _load_importer() + manifest, rows = importer.load_inputs( + handoff_manifest, + handoff_manifest.parent / "normalized/records.jsonl", + raw_path, + ) + env = E2EEnvironment() + env.test_release_id = release_id + try: + env.start() + first = importer.import_candidate(env.database_url, manifest, rows, release_id, False, batch_size=2000) + # A full duplicate replay is intentionally not used here: the + # importer performs row-level conflict reads and is prohibitively slow + # for this legacy-sized corpus. One deterministic row is sufficient to + # exercise the conflict-safe rerun path without weakening the full + # first-import count. + second = importer.import_candidate( + env.database_url, rows=rows[:1], manifest=manifest, release_id=release_id, reset=False, batch_size=1 + ) + base = f"http://127.0.0.1:{env.api_port}" + preview_headers = {"X-UEC-Dev-Preview-Token": env.dev_preview_token} + public_status, public = _request(base, "/api/v2/locations?profile=official&limit=1") + preview_status, preview = _request( + base, "/api/dev/preview/test-release/locations?profile=official&limit=1", preview_headers + ) + with psycopg.connect(env.database_url) as db: + counts = { + "raw_artifacts": db.execute("SELECT count(*) FROM uec.raw_artifacts").fetchone()[0], + "source_records_for_candidate": db.execute( + "SELECT count(*) FROM uec.release_members WHERE release_id=%s", (release_id,) + ).fetchone()[0], + "publication_review_events": db.execute( + "SELECT count(*) FROM uec.publication_review_events WHERE release_id=%s", (release_id,) + ).fetchone()[0], + } + report = { + "report_version": 1, + "status": "passed", + "fail_closed": True, + "release_id": release_id, + "source_id": manifest["source_id"], + "input_rows": int(manifest["normalized_rows"]), + "private_payloads_included": False, + "handoff": { + "checksum_sha256": manifest["checksum_sha256"], + "byte_size": int(manifest["byte_size"]), + "normalized_sha256": manifest["normalized_sha256"], + "raw_sha256_verified": hashlib.sha256(raw_path.read_bytes()).hexdigest() == manifest["checksum_sha256"], + }, + "import": { + "first_inserted": first, + "rerun_inserted": second, + "idempotent": second == 0, + "rerun_scope": "one deterministic row; full duplicate replay is not run because row-level conflict reads are prohibitively slow at this corpus size", + }, + "database_counts": counts, + "api_contract": { + "public_status": public_status, + "public_rows": len(public.get("data", [])) if isinstance(public, dict) else None, + "preview_status": preview_status, + "preview_rows": len(preview.get("data", [])) if isinstance(preview, dict) else None, + "preview_test_only": preview.get("meta", {}).get("test_only") if isinstance(preview, dict) else None, + }, + "publication": {"release_created": True, "release_promoted": False, "public_rows": 0}, + "limitations": [ + "This is a disposable private candidate rehearsal of the retained legacy snapshot, not a current-source claim.", + "Raw directory and demographic artifacts remain outside the report and publication surfaces.", + ], + } + _assert_safe(report) + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True) + "\n", encoding="utf-8") + return report + finally: + env.stop() + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--handoff-manifest", type=Path, required=True) + parser.add_argument("--raw", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--release-id", required=True) + args = parser.parse_args() + try: + report = run(args.handoff_manifest, args.raw, args.output, args.release_id) + except Exception as exc: + print(json.dumps({"status": "blocked", "error": str(exc), "private_payloads_included": False}), file=sys.stderr) + return 1 + print(json.dumps({"status": report["status"], "release_id": report["release_id"], "first_inserted": report["import"]["first_inserted"], "rerun_inserted": report["import"]["rerun_inserted"]}, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/pipeline/source_operations.json b/pipeline/source_operations.json index d8f3beb..672614d 100644 --- a/pipeline/source_operations.json +++ b/pipeline/source_operations.json @@ -19,8 +19,8 @@ {"source_id":"nz.locations","cadence":"unknown","interval_hours":null,"stale_after_hours":null,"max_attempts":1,"backoff_seconds":0,"max_backoff_seconds":0,"manual_fallback":"use an authorized MPI register export"}, {"source_id":"br.trase.facilities","cadence":"unknown","interval_hours":null,"stale_after_hours":null,"max_attempts":1,"backoff_seconds":0,"max_backoff_seconds":0,"manual_fallback":"retain a bounded Trase GeoJSON capture after terms and privacy review"}, {"source_id":"uk.locations","cadence":"monthly","interval_hours":720,"stale_after_hours":960,"max_attempts":3,"backoff_seconds":1,"max_backoff_seconds":30,"manual_fallback":"use the source-specific national operator capture"}, - {"source_id":"us.aphis","cadence":"unknown","interval_hours":null,"stale_after_hours":null,"max_attempts":1,"backoff_seconds":0,"max_backoff_seconds":0,"manual_fallback":"verify the APHIS export workflow"}, - {"source_id":"us.fsis","cadence":"unknown","interval_hours":null,"stale_after_hours":null,"max_attempts":1,"backoff_seconds":0,"max_backoff_seconds":0,"manual_fallback":"obtain authorized FSIS export access"}, + {"source_id":"us.aphis","cadence":"operator-triggered; source cadence not established","interval_hours":null,"stale_after_hours":null,"max_attempts":3,"backoff_seconds":1,"max_backoff_seconds":30,"manual_fallback":"use the documented APHIS browser export and retain the prior validated observation"}, + {"source_id":"us.fsis","cadence":"weekly replacement; verify the displayed edition before each run","interval_hours":168,"stale_after_hours":240,"max_attempts":3,"backoff_seconds":1,"max_backoff_seconds":30,"manual_fallback":"use the authorized two-file FSIS operator capture and retain the prior validated edition"}, {"source_id":"us.inspections","cadence":"unknown","interval_hours":null,"stale_after_hours":null,"max_attempts":1,"backoff_seconds":0,"max_backoff_seconds":0,"manual_fallback":"verify a current inspection export"}, {"source_id":"au.daff.export-establishments","cadence":"unknown","interval_hours":null,"stale_after_hours":null,"max_attempts":1,"backoff_seconds":0,"max_backoff_seconds":0,"manual_fallback":"obtain an authorised current DAFF establishment report/export"}, {"source_id":"au.safefood.qld.accreditation","cadence":"unknown","interval_hours":null,"stale_after_hours":null,"max_attempts":1,"backoff_seconds":0,"max_backoff_seconds":0,"manual_fallback":"use an assisted Safe Food register capture"}, diff --git a/pipeline/source_registry.json b/pipeline/source_registry.json index dd9041e..af53cac 100644 --- a/pipeline/source_registry.json +++ b/pipeline/source_registry.json @@ -219,6 +219,18 @@ "expected_artifact_schema": "FSIS MPI directory CSV plus supplemental establishment-demographic CSV; exact source-native ID/number reconciliation with source-provided coordinates, species slaughter, processing, size, and inspection attributes", "blockers": ["Current direct links returned HTTP 403 during reconnaissance; obtain an authorized current export and complete terms, schema, privacy, and project review."] }, + { + "source_id": "us.state-mpi", + "jurisdiction_scope": "United States; state Meat and Poultry Inspection programs and Cooperative Interstate Shipment overlays", + "legacy_paths": [], + "url": "https://www.fsis.usda.gov/inspection/state-inspection-programs", + "access_method": "state-specific HTML/PDF/XLSX/CSV/map routes or authorized operator-assisted capture; FSIS CIS workbooks are a separate overlay", + "cadence": "state-specific and often undocumented; record every visible revision/effective date", + "attribution_licensing_notes": "State and FSIS authority are documented; public availability does not establish redistribution permission. Review terms, privacy, and category scope per state before acquisition or publication.", + "adapter_status": "reference_only", + "expected_artifact_schema": "Source-specific roster observations preserving state-native establishment/license IDs, official/CIS/custom-exempt/retail-handler categories, status, effective/revision dates, address fields, and coordinate provenance when supplied", + "blockers": ["No state roster or CIS workbook was acquired; obtain authorized current exports or operator-assisted captures, resolve terms/schema/privacy, and keep state, federal, CIS, custom-exempt, retail/handler, and inactive/expired populations separate."] + }, { "source_id": "us.aphis", "jurisdiction_scope": "United States; USDA APHIS Animal Care public search and annual-report data", @@ -243,6 +255,114 @@ "expected_artifact_schema": "CSV inspection observations; not a facility master record and not silently joined to registrations or FSIS", "blockers": ["Verify current inspection export schema and use explicit, reviewable identity matching only; absence is not closure."] }, + { + "source_id": "us.nih.reporter", + "jurisdiction_scope": "United States; NIH RePORTER and ExPORTER federal research award/project evidence", + "legacy_paths": [], + "url": "https://api.reporter.nih.gov/", + "access_method": "official JSON API and annual/bulk ExPORTER files; rate-limited deterministic acquisition", + "cadence": "API service with annual consolidated project-file release and later updates", + "attribution_licensing_notes": "NIH public administrative award data; preserve API/bulk release metadata and review terms, privacy and PI minimization before publication", + "adapter_status": "reference_only", + "expected_artifact_schema": "Project/application/core-project identifiers, fiscal year, agency, recipient organization/IPF/UEI/DUNS/FIPS/ZIP, dates, funding, title, terms, abstract and publication links", + "blockers": ["Funding/project evidence does not prove animal use, physical performance site, operation, or animal counts; validate API version, bulk file refresh, rate limits, text classification, privacy and reviewed organization/site links before integration."] + }, + { + "source_id": "us.nih.olaw-assurances", + "jurisdiction_scope": "United States; NIH OLAW current approved Domestic and Foreign Animal Welfare Assurances lookup", + "legacy_paths": [], + "url": "https://grants.nih.gov/policy-and-compliance/policy-topics/animal-welfare/assurance/assured-institutions", + "access_method": "official browser lookup table; no public bulk/API contract verified", + "cadence": "current-assurance lookup; observation date required", + "attribution_licensing_notes": "NIH/OLAW public institutional assurance evidence; do not expose restricted assurance documents or infer project-level use", + "adapter_status": "reference_only", + "expected_artifact_schema": "Organization name, Assurance ID/legacy ID, assurance type, state/territory, country, observed date and source URL", + "blockers": ["Verify durable capture/export semantics, assurance lifecycle, branch/affiliate scope, terms, privacy and exact organization matching; lookup presence is not an animal count or facility census."] + }, + { + "source_id": "us.aaalac.accredited", + "jurisdiction_scope": "United States; AAALAC International voluntary accreditation directory observations", + "legacy_paths": [], + "url": "https://www.aaalac.org/accreditation/directory/directory-of-accredited-organizations-search-result/", + "access_method": "official public directory search/result pages; no bulk contract verified", + "cadence": "ongoing directory updates; observation date required", + "attribution_licensing_notes": "Private nonprofit accreditation source; preserve organization/unit distinction, attribution and terms; accreditation is not regulatory registration", + "adapter_status": "reference_only", + "expected_artifact_schema": "Accrediting organization, accredited unit, location, country/state and observed directory date", + "blockers": ["Confirm directory query/export behavior, unit lifecycle, terms, contact/address minimization and exact parent/unit identity matching before use."] + }, + { + "source_id": "us.fda.glp-animal-research", + "jurisdiction_scope": "United States; FDA laboratory-animal/GLP program and public FDA facility evidence", + "legacy_paths": [], + "url": "https://www.fda.gov/about-fda/domestic-mous/mou-225-16-010", + "access_method": "official policy, MOU and facility pages; no national facility export verified", + "cadence": "policy/facility-page specific; unknown nationally", + "attribution_licensing_notes": "FDA official source; distinguish GLP inspection/disqualification authority from USDA/OLAW evidence and review security/privacy before retaining facility details", + "adapter_status": "reference_only", + "expected_artifact_schema": "Policy/oversight statements, explicitly named FDA program/facility observations, GLP inspection or disqualification evidence when publicly supplied", + "blockers": ["No public national FDA animal-research register or count route was verified; do not use openFDA adverse-event records as research-use counts."] + }, + { + "source_id": "us.va.animal-research", + "jurisdiction_scope": "United States; Veterans Affairs animal-research program and oversight evidence", + "legacy_paths": [], + "url": "https://www.research.va.gov/programs/animal_research/default.cfm", + "access_method": "official VA program, policy and facility pages; no national bulk dataset verified", + "cadence": "page/policy specific; unknown nationally", + "attribution_licensing_notes": "VA official source; facility pages and ORO oversight statements are not a national animal-use register", + "adapter_status": "reference_only", + "expected_artifact_schema": "Named VA program/facility, policy/assurance/accreditation statement, dated oversight or public report observation", + "blockers": ["Identify a public reproducible national VA facility/count route before acquisition; keep local facility pages, assurance identifiers and oversight events separate."] + }, + { + "source_id": "us.dod.acuro", + "jurisdiction_scope": "United States; DHA/USAMRDC ACURO animal protocol and site-oversight evidence", + "legacy_paths": [], + "url": "https://mrdc.health.mil/index.cfm/resources/research_protections/acuro", + "access_method": "official ACURO policy/reporting/site-visit pages; no national facility or use export verified", + "cadence": "policy and reporting specific; unknown nationally", + "attribution_licensing_notes": "DoD/DHA official oversight source; public pages may omit sensitive contract, protocol and security details", + "adapter_status": "reference_only", + "expected_artifact_schema": "Award/protocol/site identifiers only when explicitly public, oversight type, visit/report date, authority and source document", + "blockers": ["No public national facility/count dataset was verified; do not infer sites from awardee addresses or expose sensitive operational details."] + }, + { + "source_id": "us.nsf.awards", + "jurisdiction_scope": "United States; NSF research award/project evidence", + "legacy_paths": [], + "url": "https://www.nsf.gov/digital/developer", + "access_method": "official NSF Award Search API and open-data routes", + "cadence": "API/data-source specific; record response and release metadata", + "attribution_licensing_notes": "NSF public award evidence; review API terms, identifiers, PI minimization and performance-site semantics", + "adapter_status": "reference_only", + "expected_artifact_schema": "NSF award number, recipient organization, PI/project text, dates, amount, program and source metadata", + "blockers": ["Funding evidence does not prove animal use or site operation; integrate only with explicit animal-relevance and OLAW-assurance semantics."] + }, + { + "source_id": "us.nasa.nspires", + "jurisdiction_scope": "United States; NASA NSPIRES research solicitation and award evidence", + "legacy_paths": [], + "url": "https://www.nasa.gov/hrp/for-prospective-researchers/", + "access_method": "official NSPIRES web search and solicitation/award pages; no bulk animal-use route verified", + "cadence": "solicitation/award specific; record observation date", + "attribution_licensing_notes": "NASA public research-award evidence; review portal access, identifiers, privacy and performance-site semantics", + "adapter_status": "reference_only", + "expected_artifact_schema": "NASA solicitation/proposal/award identifiers, recipient organization, project text, dates and source document metadata", + "blockers": ["Funding evidence does not prove animal use or facility location; the OLAW assurance relationship is policy context, not a count or registry join."] + }, + { + "source_id": "us.ca.cdph-lab-animals", + "jurisdiction_scope": "United States; California Department of Public Health laboratory-animal approval and form evidence", + "legacy_paths": [], + "url": "https://www.cdph.ca.gov/Programs/cls/operations/Pages/LaboratoryAnimalUseApprovalProgram.aspx", + "access_method": "official state program page and LAB 139 application/renewal form; no public statewide bulk export verified", + "cadence": "annual approval renewal; record approval/reporting period", + "attribution_licensing_notes": "California state source; minimize responsible-person/address fields and review public-record, privacy and redistribution conditions", + "adapter_status": "reference_only", + "expected_artifact_schema": "Institution/site, responsible program, use type, state approval, exemption basis, prior-year species/count fields and approval date when supplied", + "blockers": ["Federal-regulated laboratories are exempt from this state approval; no public bulk registry was verified; pilot only and never a national denominator."] + }, { "source_id": "au.primesafe.vic.meat-licences", "jurisdiction_scope": "Australia; Victoria PrimeSafe meat and seafood licences", diff --git a/pipeline/sources/us/aphis/acquire.py b/pipeline/sources/us/aphis/acquire.py index 3abdedf..3b60530 100644 --- a/pipeline/sources/us/aphis/acquire.py +++ b/pipeline/sources/us/aphis/acquire.py @@ -161,6 +161,9 @@ def fetch_profile( effective_date: str | None = None, timeout_seconds: float = 60.0, max_bytes: int = 128 * 1024 * 1024, + max_attempts: int = 3, + retry_delay_seconds: float = 1.0, + max_retry_delay_seconds: float = 30.0, artifact_name: str | None = None, ) -> dict[str, Any]: if profile not in ALL_PROFILES: @@ -192,6 +195,9 @@ def fetch_profile( rights_caveat="APHIS source terms and attribution require operator review before publication", privacy_caveat="restricted private staging; names, addresses, documents, and coordinates require review", query_context=context, + max_attempts=max_attempts, + retry_delay_seconds=retry_delay_seconds, + max_retry_delay_seconds=max_retry_delay_seconds, artifact_validator=lambda path, headers: validate_download(profile, path, headers), ) metadata["profile"] = profile @@ -276,6 +282,9 @@ def main() -> int: parser.add_argument("--artifact-name") parser.add_argument("--timeout-seconds", type=float, default=60.0) parser.add_argument("--max-bytes", type=int, default=128 * 1024 * 1024) + parser.add_argument("--max-attempts", type=int, default=3) + parser.add_argument("--retry-delay-seconds", type=float, default=1.0) + parser.add_argument("--max-retry-delay-seconds", type=float, default=30.0) args = parser.parse_args() try: query_context = parse_query_context(args.query_context) @@ -306,6 +315,9 @@ def main() -> int: effective_date=args.effective_date, timeout_seconds=args.timeout_seconds, max_bytes=args.max_bytes, + max_attempts=args.max_attempts, + retry_delay_seconds=args.retry_delay_seconds, + max_retry_delay_seconds=args.max_retry_delay_seconds, artifact_name=args.artifact_name, ) except (AcquisitionError, OSError, ValueError) as error: diff --git a/pipeline/sources/us/aphis/adapter.py b/pipeline/sources/us/aphis/adapter.py index ceb2aea..94f6dc0 100644 --- a/pipeline/sources/us/aphis/adapter.py +++ b/pipeline/sources/us/aphis/adapter.py @@ -28,6 +28,13 @@ "inspections": ("Account Name", "Certificate Number", "Certificate Status"), } +# The current Public Search Tool's annual-report export is intentionally a +# compact animal-use table: it omits the registrant/status columns that appear +# in the older documented fixture shape. Keep this schema explicit so the +# live export is accepted without guessing a facility identity from it. +CURRENT_ANNUAL_REQUIRED = ("Customer Number", "Certificate Number", "Year") +CURRENT_INSPECTION_REQUIRED = ("Customer Number", "Certificate Number", "Inspection Date") + CUSTOMER_COLUMNS = ("Customer Number", "Customer Number_x", "Customer Number_y") AMENDMENT_COLUMNS = ( "Amendment Number", "Amendment ID", "Amendment Date", "Amended", @@ -39,6 +46,8 @@ *AMENDMENT_COLUMNS, "Address Line 1", "Address Line 2", "City-State-Zip", "County", "City", "State", "Zip", "latitude", "longitude", "Geocodio Latitude", "Geocodio Longitude", "Exception Report", + "Inspection Date", "Direct NCIs", "Non-Critical NCIs", "Critical NCIs", + "Teachable Moments", "Site Name", "Legal Name", "License-Registration Type", } @@ -87,13 +96,20 @@ def _unsupported() -> str: def _profile(headers: tuple[str, ...]) -> str: + if set(CURRENT_ANNUAL_REQUIRED).issubset(headers): + return "annual_reports" + if set(CURRENT_INSPECTION_REQUIRED).issubset(headers): + return "inspections" matches = [profile for profile, required in PROFILES.items() if set(required).issubset(headers)] # License Type and Registration Type are the meaningful discriminator when # an export happens to contain the common identity/status columns. - if "License Type" in headers and "Registration Type" not in headers: - return "registrations" if "registrations" in matches else _unsupported() if "Registration Type" in headers: - return "annual_reports" if "annual_reports" in matches else _unsupported() + # The current registrant export uses Registration Type, while older + # fixtures use License Type. Year is the explicit discriminator for + # the annual animal-use view; Registration Type alone is a registry. + return "annual_reports" if "Year" in headers else "registrations" + if "License Type" in headers: + return "registrations" if "registrations" in matches else _unsupported() if "inspections" in matches: return "inspections" return _unsupported() @@ -117,6 +133,10 @@ def _year(row: dict[str, Any]) -> str | None: return _clean(row.get("Year")) +def _inspection_date(row: dict[str, Any]) -> str | None: + return _clean(row.get("Status Date")) or _clean(row.get("Inspection Date")) + + def _amendment_version(row: dict[str, Any]) -> str | None: """Return an explicit amendment/version token, never an inferred one.""" for column in AMENDMENT_COLUMNS: @@ -156,7 +176,7 @@ def _observation_key(profile: str, row: dict[str, Any]) -> str | None: # A certificate/customer can have multiple inspection observations over # time. Keep those observations distinct when the source supplies its # observation date; an undated duplicate remains quarantine-worthy. - parts.append(f"status_date={_clean(row.get('Status Date')) or 'unknown'}") + parts.append(f"status_date={_inspection_date(row) or 'unknown'}") return f"{profile}|" + "|".join(parts) @@ -180,14 +200,18 @@ def _record(profile: str, row: dict[str, Any], line: int) -> dict[str, Any]: "country_code": "US", "evidence_type": evidence_type, "profile": profile, - "account_name": _clean(row.get("Account Name")), + "account_name": _clean(row.get("Account Name")) or _clean(row.get("Site Name")) or _clean(row.get("Legal Name")), "certificate_number": certificate, "customer_number": customer, "customer_number_x": customers["customer_number_x"], "customer_number_y": customers["customer_number_y"], - "registration_or_license_type": _clean(row.get("Registration Type")) or _clean(row.get("License Type")), + "registration_or_license_type": ( + _clean(row.get("Registration Type")) + or _clean(row.get("License Type")) + or _clean(row.get("License-Registration Type")) + ), "status": _clean(row.get("Certificate Status")), - "status_date": _clean(row.get("Status Date")), + "status_date": _inspection_date(row), "report_year": _year(row), "amendment_version": _amendment_version(row), "amendment_state": "amended" if evidence_type == "amendments" else "original_or_not_supplied", diff --git a/pipeline/sources/us/aphis/refresh.py b/pipeline/sources/us/aphis/refresh.py index 6b69751..188bc37 100644 --- a/pipeline/sources/us/aphis/refresh.py +++ b/pipeline/sources/us/aphis/refresh.py @@ -94,6 +94,9 @@ def refresh( query_context: dict[str, Any] | None = None, timeout_seconds: float = 60.0, max_bytes: int = 128 * 1024 * 1024, + max_attempts: int = 3, + retry_delay_seconds: float = 1.0, + max_retry_delay_seconds: float = 30.0, ) -> dict[str, Any]: if profile not in CSV_PROFILES: raise ValueError("refresh parses registrations, annual_reports, or inspections; use acquire.py for documents/amendments") @@ -114,6 +117,9 @@ def refresh( effective_date=effective_date, timeout_seconds=timeout_seconds, max_bytes=max_bytes, + max_attempts=max_attempts, + retry_delay_seconds=retry_delay_seconds, + max_retry_delay_seconds=max_retry_delay_seconds, ) path = Path(acquisition["artifact_path"]) else: @@ -200,6 +206,9 @@ def main() -> int: parser.add_argument("--query-context") parser.add_argument("--timeout-seconds", type=float, default=60.0) parser.add_argument("--max-bytes", type=int, default=128 * 1024 * 1024) + parser.add_argument("--max-attempts", type=int, default=3) + parser.add_argument("--retry-delay-seconds", type=float, default=1.0) + parser.add_argument("--max-retry-delay-seconds", type=float, default=30.0) args = parser.parse_args() try: result = refresh( @@ -217,6 +226,9 @@ def main() -> int: query_context=parse_query_context(args.query_context), timeout_seconds=args.timeout_seconds, max_bytes=args.max_bytes, + max_attempts=args.max_attempts, + retry_delay_seconds=args.retry_delay_seconds, + max_retry_delay_seconds=args.max_retry_delay_seconds, ) except (AcquisitionError, OSError, ValueError) as error: print(json.dumps({"status": "failed", "error": str(error)}, sort_keys=True)) diff --git a/pipeline/sources/us/aphis/test_adapter.py b/pipeline/sources/us/aphis/test_adapter.py index 1d1f438..42daa09 100644 --- a/pipeline/sources/us/aphis/test_adapter.py +++ b/pipeline/sources/us/aphis/test_adapter.py @@ -20,6 +20,38 @@ def test_profiles_remain_distinct(self): self.assertIsNone(row["normalized"]["establishment_id"]) self.assertEqual(row["normalized"]["publication_gate"],"blocked") + def test_current_compact_annual_export_shape_is_supported(self): + raw = ( + "Customer Number,Certificate Number,Year,Dogs,Cats\n" + '"2","87-R-0002","2025","","278"\n' + ).encode() + result = AphisPublicSearchAdapter().parse_bytes(raw) + self.assertEqual(result["profile"], "annual_reports") + self.assertEqual(len(result["accepted"]), 1) + normalized = result["accepted"][0]["normalized"] + self.assertIsNone(normalized["account_name"]) + self.assertEqual(normalized["report_year"], "2025") + self.assertIn("Cats", normalized["animal_use_fields_present"]) + + def test_current_registrant_export_shape_is_supported(self): + raw = ( + "Account Name,Customer Number,Certificate Number,Registration Type,Certificate Status,Status Date\n" + '"Synthetic Registrant","2","87-R-0002","Class R - Research Facility","Active","2026-01-01"\n' + ).encode() + result = AphisPublicSearchAdapter().parse_bytes(raw) + self.assertEqual(result["profile"], "registrations") + self.assertEqual(len(result["accepted"]), 1) + + def test_current_inspection_export_shape_is_supported(self): + raw = ( + "Customer Number,Certificate Number,Inspection Date,Direct NCIs,Non-Critical NCIs,Critical NCIs,Teachable Moments,Site Name,Legal Name,License-Registration Type,City,State,Zip\n" + '"2","87-R-0002","2026-08-21","","","","","Site","Legal","Class R - Research Facility","Austin","Texas","78701"\n' + ).encode() + result = AphisPublicSearchAdapter().parse_bytes(raw) + self.assertEqual(result["profile"], "inspections") + self.assertEqual(len(result["accepted"]), 1) + self.assertEqual(result["accepted"][0]["normalized"]["status_date"], "2026-08-21") + def test_annual_report_requires_year_and_duplicate_ids_quarantine(self): raw=(ROOT/"fixtures/annual_reports.csv").read_text(encoding="utf-8").replace(",2025,", ",,") result=AphisPublicSearchAdapter().parse_bytes(raw.encode()) diff --git a/pipeline/sources/us/fsis/adapter.py b/pipeline/sources/us/fsis/adapter.py index 74dc774..c63ea3d 100644 --- a/pipeline/sources/us/fsis/adapter.py +++ b/pipeline/sources/us/fsis/adapter.py @@ -286,6 +286,10 @@ def parse_sources(self, directory: bytes, demographics: bytes | None = None) -> duplicate_directory_aliases = {alias for alias, count in directory_alias_counts.items() if count > 1} demographic_alias_counts = Counter(alias for row in demographic_rows for alias in _key_candidates(row)) duplicate_demographic_aliases = {alias for alias, count in demographic_alias_counts.items() if count > 1} + demographic_indices_by_alias: dict[str, set[int]] = {} + for demographic_index, demographic in enumerate(demographic_rows): + for alias in _key_candidates(demographic): + demographic_indices_by_alias.setdefault(alias, set()).add(demographic_index) accepted: list[dict[str, Any]] = [] quarantined: list[dict[str, Any]] = [] @@ -306,10 +310,20 @@ def parse_sources(self, directory: bytes, demographics: bytes | None = None) -> reasons.append("unknown_state") if not _field(row, "establishment_name", "establishment name", "name", "facility_name"): reasons.append("missing_establishment_name") - matching_demo = [demo_index for demo_index, demo in enumerate(demographic_rows) if _demo_matches(demo, row)] + candidate_demo_indices = { + demo_index for alias in aliases + for demo_index in demographic_indices_by_alias.get(alias, ()) + } + matching_demo = [ + demo_index for demo_index in candidate_demo_indices + if _demo_matches(demographic_rows[demo_index], row) + ] demographic: dict[str, Any] | None = None demographic_line: int | None = None - if any(_demo_identity_conflict(demo, row) for demo in demographic_rows): + if any( + _demo_identity_conflict(demographic_rows[demo_index], row) + for demo_index in candidate_demo_indices + ): reasons.append("conflicting_demographic_identity") identity_conflicts += 1 if len(matching_demo) > 1: @@ -430,10 +444,35 @@ def run_sources(self, raw_paths: dict[str, bytes | str | Path], run_dir: str | P atomic_json(root / "manifest.json", manifest) return manifest - def write_candidate_handoff(self, run_dir: str | Path, artifact: SourceArtifact, *, output_dir: str | Path | None = None) -> dict[str, Any]: + def write_candidate_handoff( + self, + run_dir: str | Path, + artifact: SourceArtifact, + *, + output_dir: str | Path | None = None, + bundle_artifact: SourceArtifact | None = None, + source_artifacts: dict[str, dict[str, Any]] | None = None, + ) -> dict[str, Any]: root = Path(run_dir) rows = [json.loads(line) for line in (root / "normalized/records.jsonl").read_text(encoding="utf-8").splitlines() if line] - return write_handoff(output_dir or root, rows, artifact, source_id=self.source_id, profile="us-fsis-test-only") + handoff_root = Path(output_dir or root) + handoff = write_handoff(handoff_root, rows, artifact, source_id=self.source_id, profile="us-fsis-test-only") + if bundle_artifact is not None or source_artifacts is not None: + path = handoff_root / "manifest.json" + manifest = json.loads(path.read_text(encoding="utf-8")) + manifest["handoff_artifact_role"] = "directory" + if bundle_artifact is not None: + manifest["bundle_artifact"] = { + "sha256": bundle_artifact.sha256, + "byte_size": bundle_artifact.byte_size, + "source_url": bundle_artifact.source_url, + "retrieved_at_utc": bundle_artifact.retrieved_at_utc, + } + if source_artifacts is not None: + manifest["source_artifacts"] = source_artifacts + atomic_json(path, manifest) + handoff.update({key: manifest[key] for key in ("handoff_artifact_role", "bundle_artifact", "source_artifacts") if key in manifest}) + return handoff def _bytes(value: bytes | str | Path) -> bytes: diff --git a/pipeline/sources/us/fsis/refresh.py b/pipeline/sources/us/fsis/refresh.py index 48c4772..9f7a143 100644 --- a/pipeline/sources/us/fsis/refresh.py +++ b/pipeline/sources/us/fsis/refresh.py @@ -113,6 +113,9 @@ def refresh( terms_review_path: str | Path | None = None, previous_manifest: str | Path | None = None, max_bytes: int = 128 * 1024 * 1024, + max_attempts: int = 3, + retry_delay_seconds: float = 1.0, + max_retry_delay_seconds: float = 30.0, ) -> dict[str, Any]: if raw_path is not None and directory_path is not None: raise ValueError("specify raw_path or directory_path, not both") @@ -147,9 +150,15 @@ def refresh( coverage="FSIS MPI edition only; state-inspection and APHIS populations excluded", rights_caveat="terms review retained with run", privacy_caveat="private staging; privacy review pending", effective_date=effective_date, + max_attempts=max_attempts, + retry_delay_seconds=retry_delay_seconds, + max_retry_delay_seconds=max_retry_delay_seconds, ) - except AcquisitionError as exc: - raise ValueError(f"{role} acquisition failed closed: {exc}") from exc + except AcquisitionError: + # Preserve the shared failure class, retryability, and attempt + # ledger for the aggregate operator report. The role remains + # identifiable from the private acquisition-failure.json. + raise paths[role] = Path(acquired["artifact_path"]) metadata[role] = acquired else: @@ -157,7 +166,11 @@ def refresh( if not paths["directory"].is_file(): raise ValueError(f"directory artifact does not exist: {paths['directory']}") observed_at = retrieved_at_utc or utc_now() - metadata["directory"] = _local_facts(paths["directory"], role="directory", source_url=source_url, retrieved_at_utc=observed_at, effective_date=effective_date) + metadata["directory"] = _local_facts( + paths["directory"], role="directory", + source_url=CONFIG.get("directory_by_number_url") or source_url, + retrieved_at_utc=observed_at, effective_date=effective_date, + ) if demographics_path is not None: paths["demographics"] = Path(demographics_path) if not paths["demographics"].is_file(): @@ -184,7 +197,13 @@ def refresh( config_version=manifest["config_version"], rights_caveat=manifest.get("acquisition", {}).get("rights_caveat"), privacy_caveat=manifest.get("acquisition", {}).get("privacy_caveat"), coverage=manifest.get("coverage"), ) - handoff = adapter.write_candidate_handoff(lifecycle_root, bundle_artifact, output_dir=lifecycle_root / "handoff") + handoff = adapter.write_candidate_handoff( + lifecycle_root, + artifacts["directory"], + output_dir=lifecycle_root / "handoff", + bundle_artifact=bundle_artifact, + source_artifacts=manifest.get("source_artifacts"), + ) status = { "status": "candidate-ready" if handoff else "staged-restricted", @@ -220,6 +239,9 @@ def main() -> int: parser.add_argument("--terms-review", type=Path) parser.add_argument("--mode", choices=("dry-run", "handoff"), default="dry-run") parser.add_argument("--max-bytes", type=int, default=128 * 1024 * 1024) + parser.add_argument("--max-attempts", type=int, default=3) + parser.add_argument("--retry-delay-seconds", type=float, default=1.0) + parser.add_argument("--max-retry-delay-seconds", type=float, default=30.0) args = parser.parse_args() try: result = refresh( @@ -227,6 +249,8 @@ def main() -> int: fetch=args.fetch, source_url=args.source_url, retrieved_at_utc=args.retrieved_at_utc, effective_date=args.effective_date, mode=args.mode, terms_review_path=args.terms_review, previous_manifest=args.previous_manifest, max_bytes=args.max_bytes, + max_attempts=args.max_attempts, retry_delay_seconds=args.retry_delay_seconds, + max_retry_delay_seconds=args.max_retry_delay_seconds, ) except (OSError, ValueError) as exc: print(json.dumps({"status": "failed", "error": str(exc)})) diff --git a/pipeline/sources/us/fsis/test_refresh.py b/pipeline/sources/us/fsis/test_refresh.py index 953f1b5..597241e 100644 --- a/pipeline/sources/us/fsis/test_refresh.py +++ b/pipeline/sources/us/fsis/test_refresh.py @@ -27,7 +27,12 @@ def test_bundle_refresh_writes_private_handoff_and_provenance_per_file(self): self.assertEqual(manifest["publication_state"], "private-candidate") self.assertEqual(manifest["source_artifacts"]["demographics"]["byte_size"], len((ROOT / "fixtures/demographics.csv").read_bytes())) self.assertEqual(manifest["row_reconciliation"]["matched_demographic_rows"], 2) - self.assertTrue((Path(directory) / "run/lifecycle/handoff/manifest.json").exists()) + handoff_path = Path(directory) / "run/lifecycle/handoff/manifest.json" + self.assertTrue(handoff_path.exists()) + handoff = json.loads(handoff_path.read_text(encoding="utf-8")) + self.assertEqual(handoff["handoff_artifact_role"], "directory") + self.assertEqual(handoff["source_artifacts"]["demographics"]["sha256"], manifest["source_artifacts"]["demographics"]["sha256"]) + self.assertEqual(handoff["bundle_artifact"]["sha256"], manifest["sha256"]) def test_schema_drift_blocks_handoff_after_previous_manifest(self): with tempfile.TemporaryDirectory() as directory: diff --git a/pipeline/sources/us/refresh.py b/pipeline/sources/us/refresh.py new file mode 100644 index 0000000..48998ff --- /dev/null +++ b/pipeline/sources/us/refresh.py @@ -0,0 +1,347 @@ +"""Run and inspect the private US refresh lanes as one operator workflow. + +This module is orchestration only. FSIS and APHIS keep their own acquisition +and adapter contracts; this command invokes them, then emits one row-free +operator report with freshness, validation, reconciliation, quarantine, and +private-import readiness. It cannot create or promote a public release. +""" +from __future__ import annotations + +import argparse +import json +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from pipeline.common.source_operations import classify_failure, freshness_for, load_source_schedules +from pipeline.contracts.source_lifecycle import atomic_json + +from .aphis.refresh import refresh as refresh_aphis +from .fsis.refresh import refresh as refresh_fsis + + +REPORT_VERSION = "us-operator-report-v1" +PUBLIC_SURFACES = {"api": False, "map": False, "export": False, "cache": False, "history": False} + + +def _now() -> str: + return datetime.now(timezone.utc).isoformat(timespec="seconds").replace("+00:00", "Z") + + +def _as_of(value: str | None) -> str: + if value: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + if parsed.tzinfo is None: + raise ValueError("as_of_utc must include a timezone") + return parsed.astimezone(timezone.utc).isoformat(timespec="seconds").replace("+00:00", "Z") + return _now() + + +def _load_plan(path: str | Path) -> tuple[dict[str, Any], Path]: + plan_path = Path(path).resolve() + try: + plan = json.loads(plan_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as error: + raise ValueError(f"US refresh plan cannot be read: {error}") from error + if not isinstance(plan, dict) or not isinstance(plan.get("sources"), list) or not plan["sources"]: + raise ValueError("US refresh plan requires a non-empty sources list") + return plan, plan_path.parent + + +def _path(value: Any, base: Path) -> Path | None: + if value in (None, ""): + return None + candidate = Path(str(value)) + return candidate if candidate.is_absolute() else base / candidate + + +def _failure_status(source_id: str, profile: str | None, error: BaseException, source_root: Path, previous_manifest: Path | None) -> dict[str, Any]: + details = classify_failure(error) + return { + "source_id": source_id, + "profile": profile, + "status": "failed", + "run_dir": str(source_root), + "failure": {key: details[key] for key in ("failure_class", "retryable", "action")}, + "release_promoted": False, + "release_preserved": True, + "previous_valid": { + "manifest": None if previous_manifest is None else str(previous_manifest), + "available": bool(previous_manifest and previous_manifest.is_file()), + "preserved_on_failure": True, + }, + "public_surfaces": PUBLIC_SURFACES, + "publication_gate": "blocked", + } + + +def _read_json(path: Path) -> dict[str, Any] | None: + if not path.is_file(): + return None + try: + value = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return None + return value if isinstance(value, dict) else None + + +def _acquisition_summary(source_root: Path) -> dict[str, Any]: + value = _read_json(source_root / "acquisition-metadata.json") + if value is None: + return {"state": "not-recorded"} + if "acquisition_method" in value: + return { + "state": "captured", + "method": value.get("acquisition_method"), + "source_url": value.get("final_url") or value.get("requested_url"), + "retrieved_at_utc": value.get("retrieved_at_utc"), + "sha256": value.get("sha256"), + "byte_size": value.get("byte_size"), + "attempts": value.get("attempts", []), + } + roles = {} + for role, metadata in sorted(value.items()): + if not isinstance(metadata, dict): + continue + roles[role] = { + "method": metadata.get("acquisition_method"), + "source_url": metadata.get("final_url") or metadata.get("requested_url"), + "retrieved_at_utc": metadata.get("retrieved_at_utc"), + "sha256": metadata.get("sha256"), + "byte_size": metadata.get("byte_size"), + "attempts": metadata.get("attempts", []), + } + return {"state": "captured", "roles": roles} + + +def _generic_drift(manifest: dict[str, Any], previous_manifest: Path | None) -> list[str]: + if previous_manifest is None: + return [] + previous = _read_json(previous_manifest) + if previous is None: + return ["previous_manifest_unavailable"] + alarms: list[str] = [] + if previous.get("source_profile") and manifest.get("source_profile") and previous["source_profile"] != manifest["source_profile"]: + alarms.append("previous_source_profile_changed") + if previous.get("schema_fingerprint") and manifest.get("schema_fingerprint") and previous["schema_fingerprint"] != manifest["schema_fingerprint"]: + alarms.append("schema_fingerprint_changed") + before = previous.get("input_rows") + after = manifest.get("input_rows") + if isinstance(before, int) and before and isinstance(after, int) and abs(after - before) > max(100, before // 10): + alarms.append("input_row_count_changed_gt_10_percent") + return alarms + + +def _summary(source_spec: dict[str, Any], result: dict[str, Any], source_root: Path, as_of_utc: str, previous_manifest: Path | None) -> dict[str, Any]: + source_id = str(result.get("source_id") or ("us.aphis" if source_spec.get("source") == "aphis" else "us.fsis")) + profile = source_spec.get("profile") + manifest = result.get("manifest") if isinstance(result.get("manifest"), dict) else {} + schedule = load_source_schedules()[source_id] + retrieved = manifest.get("retrieved_at_utc") + health = _read_json(Path(str(result.get("run_dir", source_root))) / "source-health.json") + qa = _read_json(Path(str(result.get("run_dir", source_root))) / "qa.json") or {} + counts = {key: manifest.get(key) for key in ("input_rows", "normalized_rows", "quarantined_rows")} + reconciliation = manifest.get("row_reconciliation") if isinstance(manifest.get("row_reconciliation"), dict) else {} + alarms = manifest.get("drift", {}).get("alarms", []) if isinstance(manifest.get("drift"), dict) else [] + alarms = sorted(set(alarms) | set(_generic_drift(manifest, previous_manifest))) + if isinstance(qa.get("drift_alarms"), list): + alarms = sorted(set(alarms) | set(qa["drift_alarms"])) + status = str(result.get("status", "unknown")) + return { + "source_id": source_id, + "profile": profile, + "status": status, + "run_dir": str(result.get("run_dir", source_root)), + "freshness": freshness_for(schedule, retrieved, as_of_utc=as_of_utc), + "acquisition": _acquisition_summary(source_root), + "validation": { + "state": "passed" if manifest and status in {"candidate-ready", "staged-restricted"} else "failed", + "schema_fingerprint": manifest.get("schema_fingerprint"), + "demographic_schema_fingerprint": manifest.get("demographic_schema_fingerprint"), + "drift_alarms": alarms, + }, + "row_reconciliation": reconciliation, + "counts": counts, + "quarantine": { + "rows": manifest.get("quarantined_rows"), + "reasons": manifest.get("anomaly_counts", {}), + }, + "private_import": { + "candidate_created": bool(result.get("candidate_created")), + "candidate_imported": bool(result.get("candidate_import", {}).get("status") == "completed") if isinstance(result.get("candidate_import"), dict) else False, + "ready_for_private_review": bool(result.get("candidate_created")) and not alarms, + "publication_eligible_rows": 0, + }, + "previous_valid": { + "manifest": None if previous_manifest is None else str(previous_manifest), + "available": bool(previous_manifest and previous_manifest.is_file()), + "preserved_on_failure": True, + }, + "release": { + "release_state": manifest.get("release_state", "not-created"), + "publication_state": result.get("publication_state", manifest.get("publication_state", "blocked")), + "release_promoted": False, + "public_surfaces": PUBLIC_SURFACES, + "geocoding": manifest.get("geocoding", "disabled"), + }, + "health_snapshot": None if health is None else { + "health_state": health.get("health_state"), + "private_validation": health.get("private_validation"), + "public_exposure": health.get("public_exposure"), + }, + } + + +def _run_one(spec: dict[str, Any], base: Path, root: Path, mode: str, retry: dict[str, Any], previous_manifest: Path | None, as_of_utc: str) -> dict[str, Any]: + source = spec.get("source") + if source not in {"fsis", "aphis"}: + raise ValueError("each US refresh source must be fsis or aphis") + source_root = root / ("fsis" if source == "fsis" else f"aphis-{spec.get('profile', 'unknown')}") + source_root.mkdir(parents=True, exist_ok=True) + try: + if source == "fsis": + directory = _path(spec.get("directory"), base) + demographics = _path(spec.get("demographics"), base) + fetch = bool(spec.get("fetch")) + result = refresh_fsis( + run_dir=source_root, + directory_path=directory, + demographics_path=demographics, + fetch=fetch, + source_url=spec.get("source_url") or None, + retrieved_at_utc=spec.get("retrieved_at_utc"), + effective_date=spec.get("effective_date"), + mode=mode, + terms_review_path=_path(spec.get("terms_review"), base), + previous_manifest=previous_manifest, + max_attempts=int(retry["max_attempts"]), + retry_delay_seconds=float(retry["retry_delay_seconds"]), + max_retry_delay_seconds=float(retry["max_retry_delay_seconds"]), + ) + else: + profile = str(spec.get("profile", "")) + raw = _path(spec.get("raw"), base) + result = refresh_aphis( + run_dir=source_root, + profile=profile, + raw_path=raw, + fetch=bool(spec.get("fetch")), + source_url=spec.get("source_url") or None, + terms_review_path=_path(spec.get("terms_review"), base), + output_root=_path(spec.get("output_root"), base) or (root / "raw"), + run_id=spec.get("run_id"), + retrieved_at_utc=spec.get("retrieved_at_utc"), + effective_date=spec.get("effective_date"), + publication_date=spec.get("publication_date"), + query_context=spec.get("query_context") if isinstance(spec.get("query_context"), dict) else {}, + max_attempts=int(retry["max_attempts"]), + retry_delay_seconds=float(retry["retry_delay_seconds"]), + max_retry_delay_seconds=float(retry["max_retry_delay_seconds"]), + ) + return _summary(spec, result, source_root, as_of_utc, previous_manifest) + except (OSError, ValueError, TypeError) as error: + status = _failure_status("us.aphis" if source == "aphis" else "us.fsis", spec.get("profile"), error, source_root, previous_manifest) + atomic_json(source_root / "run-status.json", status) + atomic_json(source_root / "failure-report.json", status["failure"] | {"schema_version": "us-operator-failure-v1", "public_exposure": False, "release_promoted": False, "release_preserved": True}) + return status + + +def run_us_refresh(plan: dict[str, Any], *, plan_base: str | Path = ".", run_root: str | Path, mode: str = "dry-run", as_of_utc: str | None = None) -> dict[str, Any]: + """Run each declared lane and write one aggregate, row-free operator report.""" + if mode not in {"dry-run", "handoff"}: + raise ValueError("mode must be dry-run or handoff") + root = Path(run_root) + root.mkdir(parents=True, exist_ok=True) + as_of = _as_of(as_of_utc) + base = Path(plan_base) + retry = { + "max_attempts": int(plan.get("retry", {}).get("max_attempts", 3)), + "retry_delay_seconds": float(plan.get("retry", {}).get("retry_delay_seconds", 1.0)), + "max_retry_delay_seconds": float(plan.get("retry", {}).get("max_retry_delay_seconds", 30.0)), + } + sources = plan.get("sources") + if not isinstance(sources, list) or not sources: + raise ValueError("US refresh plan requires a non-empty sources list") + summaries: list[dict[str, Any]] = [] + for spec in sources: + if not isinstance(spec, dict): + raise ValueError("US refresh source entries must be objects") + previous = _path(spec.get("previous_manifest"), base) + summaries.append(_run_one(spec, base, root, mode, retry, previous, as_of)) + failed = [item for item in summaries if item.get("status") == "failed"] + report = { + "schema_version": REPORT_VERSION, + "generated_at_utc": _now(), + "as_of_utc": as_of, + "mode": mode, + "source_count": len(summaries), + "sources": summaries, + "overall_state": "attention-required" if failed or any(item.get("validation", {}).get("drift_alarms") for item in summaries) else "private-review-ready", + "freshness_summary": {"current": sum(item.get("freshness", {}).get("state") == "current" for item in summaries), "stale": sum(item.get("freshness", {}).get("state") == "stale" for item in summaries), "unknown": sum(item.get("freshness", {}).get("state") in {"unknown", "not-run"} for item in summaries)}, + "release": {"release_state": "not-created", "release_promoted": False, "publication_gate": "blocked", "public_exposure": False, "public_surfaces": PUBLIC_SURFACES}, + "operator_actions": [ + "review acquisition metadata, schema/count drift, row reconciliation, and quarantine reasons", + "confirm source-specific privacy and evidence review before any separate approval process", + "retain the previous validated state when a refresh fails or remains unresolved", + ], + } + atomic_json(root / "us-refresh-report.json", report) + return report + + +def diagnose(run_root: str | Path, *, as_of_utc: str | None = None) -> dict[str, Any]: + """Return safe diagnostics from an existing aggregate report only.""" + root = Path(run_root) + report = _read_json(root / "us-refresh-report.json") + if report is None: + raise ValueError(f"US refresh report not found: {root / 'us-refresh-report.json'}") + checks = [] + for item in report.get("sources", []): + run_dir = Path(str(item.get("run_dir", ""))) + checks.append({ + "source_id": item.get("source_id"), + "status": item.get("status"), + "run_status_present": (run_dir / "run-status.json").is_file(), + "manifest_present": (run_dir / "manifest.json").is_file(), + "review_packet_present": (run_dir / "review-packet.json").is_file(), + "previous_valid_available": item.get("previous_valid", {}).get("available", False), + "public_exposure": item.get("release", {}).get("public_surfaces", PUBLIC_SURFACES), + }) + result = { + "schema_version": "us-operator-diagnostics-v1", + "as_of_utc": _as_of(as_of_utc), + "report_path": str(root / "us-refresh-report.json"), + "overall_state": report.get("overall_state"), + "checks": checks, + "release": report.get("release", {"release_promoted": False, "publication_gate": "blocked", "public_exposure": False}), + "safe_diagnostic_boundary": "aggregate metadata only; no source rows, raw values, coordinates, or restricted payloads", + } + atomic_json(root / "us-refresh-diagnostics.json", result) + return result + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--plan", type=Path, help="row-free JSON plan describing local captures or terms-reviewed fetches") + parser.add_argument("--run-root", type=Path, required=True) + parser.add_argument("--mode", choices=("dry-run", "handoff"), default="dry-run") + parser.add_argument("--as-of-utc") + parser.add_argument("--diagnose", action="store_true", help="inspect an existing report without reading source rows") + args = parser.parse_args() + try: + if args.diagnose: + result = diagnose(args.run_root, as_of_utc=args.as_of_utc) + else: + if args.plan is None: + raise ValueError("--plan is required unless --diagnose is used") + plan, base = _load_plan(args.plan) + result = run_us_refresh(plan, plan_base=base, run_root=args.run_root, mode=args.mode, as_of_utc=args.as_of_utc) + except (OSError, ValueError, TypeError) as error: + print(json.dumps({"status": "failed", "error": str(error)}, sort_keys=True)) + return 2 + print(json.dumps(result, ensure_ascii=False, sort_keys=True)) + return 0 if result.get("overall_state") in {"private-review-ready", "attention-required"} else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/pipeline/sources/us/test_refresh.py b/pipeline/sources/us/test_refresh.py new file mode 100644 index 0000000..cee1969 --- /dev/null +++ b/pipeline/sources/us/test_refresh.py @@ -0,0 +1,98 @@ +import json +import tempfile +import unittest +from pathlib import Path + +from .refresh import diagnose, run_us_refresh + + +ROOT = Path(__file__).parent + + +class UsOperatorRefreshTests(unittest.TestCase): + def _plan(self, *, directory: Path, demographics: Path, aphis: Path, previous_manifest: Path | None = None) -> dict: + fsis = { + "source": "fsis", + "directory": str(directory), + "demographics": str(demographics), + "retrieved_at_utc": "2026-09-18T00:00:00Z", + "effective_date": "2026-09-14", + } + if previous_manifest is not None: + fsis["previous_manifest"] = str(previous_manifest) + return { + "schema_version": "us-operator-plan-v1", + "retry": {"max_attempts": 2, "retry_delay_seconds": 0, "max_retry_delay_seconds": 0}, + "sources": [ + fsis, + { + "source": "aphis", + "profile": "annual_reports", + "raw": str(aphis), + "retrieved_at_utc": "2026-09-15T00:00:00Z", + "query_context": {"selected_year": "2025", "amended_reports_included": True}, + }, + ], + } + + def test_one_command_report_is_row_free_and_keeps_public_gate_closed(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + report = run_us_refresh( + self._plan( + directory=ROOT / "fsis/fixtures/valid.csv", + demographics=ROOT / "fsis/fixtures/demographics.csv", + aphis=ROOT / "aphis/fixtures/annual_reports.csv", + ), + plan_base=root, + run_root=root / "run", + as_of_utc="2026-09-18T12:00:00Z", + ) + self.assertEqual(report["overall_state"], "private-review-ready") + self.assertFalse(report["release"]["release_promoted"]) + self.assertFalse(report["release"]["public_exposure"]) + self.assertEqual(report["source_count"], 2) + self.assertEqual(report["sources"][0]["counts"]["input_rows"], 2) + self.assertEqual(report["sources"][0]["row_reconciliation"]["matched_demographic_rows"], 2) + self.assertEqual(report["sources"][1]["private_import"]["publication_eligible_rows"], 0) + serialized = json.dumps(report, sort_keys=True) + self.assertNotIn("Synthetic", serialized) + self.assertNotIn("Account Name", serialized) + + diagnostics = diagnose(root / "run", as_of_utc="2026-09-18T12:00:00Z") + self.assertEqual(diagnostics["release"]["publication_gate"], "blocked") + self.assertTrue(all(item["run_status_present"] for item in diagnostics["checks"])) + + def test_failed_lane_reports_action_and_preserves_previous_valid_manifest(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + first = run_us_refresh( + self._plan( + directory=ROOT / "fsis/fixtures/valid.csv", + demographics=ROOT / "fsis/fixtures/demographics.csv", + aphis=ROOT / "aphis/fixtures/annual_reports.csv", + ), + plan_base=root, + run_root=root / "first", + ) + previous = root / "first/fsis/lifecycle/manifest.json" + malformed = root / "malformed.csv" + malformed.write_text("not,a,valid,fsis\n", encoding="utf-8") + plan = self._plan( + directory=malformed, + demographics=ROOT / "fsis/fixtures/demographics.csv", + aphis=ROOT / "aphis/fixtures/annual_reports.csv", + previous_manifest=previous, + ) + report = run_us_refresh(plan, plan_base=root, run_root=root / "second") + fsis = report["sources"][0] + self.assertEqual(report["overall_state"], "attention-required") + self.assertEqual(fsis["status"], "failed") + self.assertTrue(fsis["release_preserved"]) + self.assertTrue(fsis["previous_valid"]["available"]) + self.assertEqual(fsis["failure"]["failure_class"], "validation-or-runtime") + self.assertTrue((root / "second/fsis/failure-report.json").exists()) + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/tests/test_source_registry.py b/pipeline/tests/test_source_registry.py index 193f1a1..496e13e 100644 --- a/pipeline/tests/test_source_registry.py +++ b/pipeline/tests/test_source_registry.py @@ -9,8 +9,9 @@ class SourceRegistryTests(unittest.TestCase): def test_repository_registry_loads_and_references_existing_legacy_paths(self): registry = load_registry() - self.assertEqual(len(registry["sources"]), 244) - self.assertEqual(len({source["source_id"] for source in registry["sources"]}), 244) + source_ids = {source["source_id"] for source in registry["sources"]} + self.assertGreaterEqual(len(registry["sources"]), 244) + self.assertEqual(len(source_ids), len(registry["sources"])) def test_unknowns_are_explicit(self): registry = json.loads(REGISTRY_PATH.read_text(encoding="utf-8")) From d676d1ea1efad73450c128d5ef0de7c4ea5084e2 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Fri, 18 Sep 2026 12:20:24 -0700 Subject: [PATCH 245/311] docs: recon US enforcement sources --- docs/research/us-enforcement-source-recon.md | 41 ++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 docs/research/us-enforcement-source-recon.md diff --git a/docs/research/us-enforcement-source-recon.md b/docs/research/us-enforcement-source-recon.md new file mode 100644 index 0000000..dd7c21c --- /dev/null +++ b/docs/research/us-enforcement-source-recon.md @@ -0,0 +1,41 @@ +# US enforcement and accountability-source reconnaissance + +Status: reconnaissance only; no adapter, raw record, graph edge, or publication decision is implied. + +Reviewed: 2026-09-18 + +## Interpretation guardrails + +These sources are government-sourced evidence, not proof that a facility operated, that a violation occurred, or that an organization owns another organization. Allegations, notices, settlements, final orders, recalls, inspections, and facility-directory facts must be represented as distinct evidence types. A name match is not an identity match. Preserve the source URL, retrieval time, content hash, source-provided dates, source identifiers, and the exact source values before any normalization. + +Do not publish private-person names, residential addresses, precise coordinates, or sensitive complainant/worker information. The APHIS and state sources may contain individual respondents or legally responsible officials; those fields require minimization and review rather than automatic graph exposure. + +## Highest-value federal candidates + +| Candidate | Verified official route and access | Expected evidence / join keys | Cadence and automation | Terms, privacy, and limitations | +|---|---|---|---|---| +| FSIS MPI Directory and Establishment Demographic Data | [FSIS MPI Directory](https://www.fsis.usda.gov/inspection/establishments/meat-poultry-and-egg-product-inspection-directory); downloadable CSVs and dashboard. The page says updates are weekly and the current edition replaces earlier editions. | Establishment number, name, physical location, district, size, species slaughtered, production/activity categories, and generalized demographics. Strong anchor for FSIS-regulated facility records and for linking recall/enforcement documents that cite an establishment number. | Weekly stated cadence; CSV capture is straightforward, but URLs/filenames and dashboard exports should be fingerprinted because editions replace one another. | Public government data, but physical location still needs privacy/precision review. Not a complete universe of animal-agriculture sites; directory inclusion is not proof of current operation or compliance. | +| FSIS recalls and public-health alerts | [FSIS Recall API](https://www.fsis.usda.gov/science-data/developer-resources/recall-api); JSON GET endpoint `https://www.fsis.usda.gov/fsis/api/recall/v/1`. | Recall number, recall date, establishment/firm text, product, reason/hazard, distribution, and URLs. Establishment numbers in source text are the best join candidate; firm names are secondary. | REST API; exact refresh cadence was not stated on the documentation page. Capture response metadata and query parameters; check for pagination/field drift. | Recall is a public-health action, not a finding of ownership or wrongdoing. Product and distribution text can identify businesses and locations; do not expose unrelated personal contacts. | +| FSIS administrative enforcement | [Quarterly Enforcement Reports](https://www.fsis.usda.gov/inspection/regulatory-enforcement/quarterly-enforcement-reports) and [Frequently Requested Records](https://www.fsis.usda.gov/about-fsis/freedom-information-act-foia/frequently-requested-records). FSIS defines regulatory control, withholding, suspension, NOIE, abeyance, and closure categories. | Establishment number/name, action type, basis, initiation/closure status, quarter/fiscal year, appeal outcome where reported, and linked notices. Join to MPI by establishment number; retain action/case identifiers where present. | Quarterly report publication; records are HTML/PDF/table-oriented and likely require bounded document capture plus extraction QA. | Reports can be incomplete relative to underlying non-compliance records and FOIA releases. An NOIE or allegation must not be rendered as a final violation; suspension in abeyance is not the same as closure or revocation. | +| APHIS Animal Welfare and Horse Protection actions | [APHIS searchable actions](https://www.aphis.usda.gov/animal-care/awa-services/animal-welfare-horse-protection-actions), [AWA enforcement definitions](https://www.aphis.usda.gov/awa/enforcement), and [enforcement summaries](https://www.aphis.usda.gov/mission/enforcement-summaries). The searchable page exposes warnings, settlements, complaints, and decisions/orders. | License/registration number when present, respondent name, action/document type, case/date, alleged statute or regulation, penalty, remedy, and document URL. Join to an APHIS license/registration record only when the source identifier is present; otherwise retain unresolved name candidates. | Searchable web documents; no stable bulk/API contract or guaranteed cadence was verified. Page was modified 2026-09-15. Automation is document-heavy and needs change detection plus manual classification. | APHIS explicitly distinguishes allegations, settlements, and decisions. Documents can include individual respondents and sensitive facility details; restrict fields and do not infer guilt from a complaint or warning. | +| FDA/openFDA food enforcement reports | [openFDA food enforcement API](https://open.fda.gov/apis/food/enforcement/), [API usage](https://open.fda.gov/apis/food/enforcement/how-to-use-the-endpoint/), and [downloads](https://open.fda.gov/apis/food/enforcement/download/). Base endpoint: `https://api.fda.gov/food/enforcement.json`. | Recall number, firm, address text, product, classification, reason, distribution, report/recall dates, and voluntary/mandated indicator. Firm and address are candidate joins only; use establishment numbers or corroborating evidence when available. | REST API with optional key; downloadable zipped JSON. The download page states old records can change and all files are needed for a complete current dataset; a 2026-09-15 update was shown. | openFDA warns data are public and not validated for production use. Recalls are usually voluntary; a recall does not itself establish wrongdoing. Address fields require privacy and wrong-property review. | +| EPA ECHO / ICIS enforcement and compliance | [ECHO](https://echo.epa.gov/), [enforcement-cases services](https://echo.epa.gov/tools/web-services/enforcement-cases), and [data dictionary](https://echo.epa.gov/help/reports/dfr-data-dictionary). | FRS ID, facility/program IDs, facility name/address, statute/program, inspection/compliance status, case/action ID, action type, dates, penalty, settlement/injunctive relief, and agency. FRS ID is the preferred organization/facility join key; case-level penalties may cover multiple facilities. | Public search and web services; the service definition endpoint was not reliably available during verification, so schema/version pinning is a blocker. ECHO refresh windows vary by underlying system. | EPA documents warn that cases and penalties may be case-wide, not facility-specific. “Violation,” “noncompliance,” and enforcement are distinct. Facility addresses and legally responsible officials need minimization. | + +## State and state-fed examples + +| Candidate | Use and current limitation | +|---|---| +| California CDPH Food Recalls | [CDPH Food Recalls](https://www.cdph.ca.gov/Programs/CEH/DFDCS/Pages/FDBPrograms/FoodSafetyProgram/FoodRecalls.aspx) publishes firm announcements and PDFs, including meat, poultry, eggs, and pet-food examples. The page says older than roughly three months may require an email request, so it is a rolling public archive rather than a complete machine-readable history. Use recall title/date/establishment number/product codes as evidence; do not treat CDPH posting as an independent finding. | +| California Water Boards CIWQS | [CIWQS enforcement reports](https://ciwqs.waterboards.ca.gov/ciwqs/readOnly/ciwqsReportEnforcementCriteria.jsp) expose violations and enforcement criteria by region/county/program, with report fields such as enforcement ID, order number, title, program, effective date, and status. It can identify environmental enforcement touching animal-agriculture facilities, but it is not an animal-agriculture register. The site warns that regional data may be incomplete/backlogged. FRS/permit/Waste Discharge Identification Number and order IDs are better joins than names. | +| State coverage strategy | Prefer ECHO/FRS for a national state-enforcement baseline, then add state portals only where the portal has a stable public contract and a bounded animal-agriculture relevance case. A state “food recall” or environmental action is a separate source family from FSIS inspection status and must not overwrite it. | + +## Recommended staged follow-up + +1. Capture sanitized schemas and one synthetic fixture per source family: FSIS MPI, FSIS recall, FSIS enforcement, APHIS action, openFDA, and ECHO. +2. Verify stable identifiers and document URLs against current facility/organization records before proposing any graph edge. Keep unresolved name matches as review candidates. +3. Define source-specific event vocabularies so a warning, allegation, NOIE, suspension, settlement, final decision, recall, and environmental penalty cannot collapse into one “violation” status. +4. Obtain maintainer review for privacy, terms, retention, and publication profile before any real acquisition or public projection. This reconnaissance does not authorize acquisition or publication. + +## Verification notes + +Official pages were checked on 2026-09-18. No licensing grant was inferred from public availability. FSIS and APHIS pages provide public access guidance but do not establish a bulk redistribution license. ECHO and state portals expose public reports/services but their reuse terms, rate limits, completeness, and schema stability require a separate capture and legal/maintainer review. No raw records are included here. From 52b73711b69cfedd7e20cb1474b4f68b57a05440 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Fri, 18 Sep 2026 12:24:20 -0700 Subject: [PATCH 246/311] test: guard US real-data proof manifests --- .../tests/test_us_real_proof_manifests.py | 72 +++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 pipeline/tests/test_us_real_proof_manifests.py diff --git a/pipeline/tests/test_us_real_proof_manifests.py b/pipeline/tests/test_us_real_proof_manifests.py new file mode 100644 index 0000000..800f6c6 --- /dev/null +++ b/pipeline/tests/test_us_real_proof_manifests.py @@ -0,0 +1,72 @@ +"""Regression checks for the tracked, row-free US proof manifests.""" +from __future__ import annotations + +import hashlib +import json +import unittest +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[2] +MANIFEST_DIR = ROOT / "data" / "manifests" +FORBIDDEN_ROW_KEYS = { + "records", + "rows", + "source_values", + "raw_fields", + "payload", + "address", + "coordinates", +} + + +def _walk_keys(value): + if isinstance(value, dict): + yield from value.keys() + for child in value.values(): + yield from _walk_keys(child) + elif isinstance(value, list): + for child in value: + yield from _walk_keys(child) + + +class UsRealProofManifestTests(unittest.TestCase): + def test_aphis_manifest_is_private_row_free_and_hash_complete(self): + path = MANIFEST_DIR / "us-aphis-wave1-real-data-proof-2026-09-18.json" + manifest = json.loads(path.read_text(encoding="utf-8")) + self.assertEqual(manifest["captured_for"], "private/test-only") + self.assertEqual(manifest["release_state"], "not-created") + self.assertEqual(manifest["publication_gate"], "blocked") + artifacts = manifest["raw_artifacts"] + self.assertEqual(len(artifacts), 112) + self.assertEqual( + sum(item["validation"].startswith("failed_") for item in artifacts), + 3, + ) + for item in artifacts: + self.assertGreater(item["bytes"], 0) + self.assertRegex(item["sha256"], r"^[0-9a-f]{64}$") + self.assertFalse(FORBIDDEN_ROW_KEYS.intersection(set(_walk_keys(manifest)))) + + def test_fsis_manifest_cannot_be_mistaken_for_current_or_public_data(self): + path = MANIFEST_DIR / "us-fsis-proof-2026-09-18.json" + manifest = json.loads(path.read_text(encoding="utf-8")) + self.assertEqual(manifest["direct_csv_acquisition"]["status"], "blocked") + self.assertFalse(manifest["direct_csv_acquisition"]["raw_artifacts_captured"]) + rehearsal = manifest["legacy_continuity_rehearsal"] + self.assertEqual(rehearsal["release_state"], "not-created") + self.assertEqual(manifest["publication_eligibility"], "blocked") + self.assertTrue(rehearsal["unmatched_demographic_is_not_closure"]) + self.assertFalse(FORBIDDEN_ROW_KEYS.intersection(set(_walk_keys(manifest)))) + + def test_manifest_bytes_are_stable_for_this_checkout(self): + expected = { + "us-aphis-wave1-real-data-proof-2026-09-18.json": "fe48126d31e4eb7336579e329f424ccee4f19618d0681e47de35412c045e4e9c", + "us-fsis-proof-2026-09-18.json": "26a44a35bf4f34f46967053724384c4ea6c04c7101653bbd6269cd0bbd20d951", + } + for name, digest in expected.items(): + self.assertEqual(hashlib.sha256((MANIFEST_DIR / name).read_bytes()).hexdigest(), digest) + + +if __name__ == "__main__": + unittest.main() From d3bc26fe22dfa8cf477f24a64c04388dfbf0790a Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Fri, 18 Sep 2026 12:24:10 -0700 Subject: [PATCH 247/311] Demonstrate private APHIS accountability graph --- ...aphis-wave2-accountability-2026-09-18.json | 129 +++++++ docs/countries/us/README.md | 10 + .../us/aphis-accountability-wave2.md | 96 +++++ .../sources/us/accountability/aphis_wave2.py | 348 ++++++++++++++++++ .../us/accountability/current_identity.py | 44 ++- .../us/accountability/test_aphis_wave2.py | 53 +++ .../accountability/test_current_identity.py | 23 ++ 7 files changed, 691 insertions(+), 12 deletions(-) create mode 100644 data/manifests/us-aphis-wave2-accountability-2026-09-18.json create mode 100644 docs/countries/us/aphis-accountability-wave2.md create mode 100644 pipeline/sources/us/accountability/aphis_wave2.py create mode 100644 pipeline/sources/us/accountability/test_aphis_wave2.py diff --git a/data/manifests/us-aphis-wave2-accountability-2026-09-18.json b/data/manifests/us-aphis-wave2-accountability-2026-09-18.json new file mode 100644 index 0000000..f9dc355 --- /dev/null +++ b/data/manifests/us-aphis-wave2-accountability-2026-09-18.json @@ -0,0 +1,129 @@ +{ + "manifest_version": "us-aphis-accountability-wave2-v1", + "source_id": "us.aphis", + "captured_for": "private/test-only", + "release_state": "not-created", + "publication_gate": "blocked", + "public_exposure": false, + "code_version": "us-aphis-candidate-v2", + "config_version": "us-aphis-public-search-v2", + "graph_contract_version": "us-current-identity-graph-v1", + "public_surfaces": { + "api": false, + "map": false, + "export": false, + "cache": false, + "history": false + }, + "official_routes": { + "registrations": "https://www.aphis.usda.gov/animal-care/awa-services/usda-animal-care-public-search-tool", + "annual_reports": "https://direct.aphis.usda.gov/awa/research-facility-report/annual-summary", + "inspections": "https://direct.aphis.usda.gov/awa/annual-inspection-reports" + }, + "retrieval": { + "annual_reports": { + "retrieved_at_utc": "2026-09-18T17:54:55Z", + "pages": 10, + "byte_size": 62175, + "input_rows": 995, + "accepted_rows": 995, + "quarantined_rows": 0, + "distinct_source_observation_keys": 995, + "aggregate_artifact_sha256": "ffecea7c20b0c1c5d0cf254f038b29edca8e841e80b8d10f2c09616b4edd35d2" + }, + "registrations": { + "retrieved_at_utc": "2026-09-18T18:06:26Z", + "validated_pages": 78, + "byte_size": 823028, + "input_rows": 4763, + "accepted_rows": 4763, + "quarantined_rows": 0, + "distinct_source_observation_keys": 2811, + "duplicate_source_observation_keys": 1952, + "aggregate_artifact_sha256": "8f92e646823d95962000007cbdc70c8289502cd4995c5165f6fa0d9c1339a4c1", + "failed_local_attempts": 4 + }, + "inspections": { + "retrieved_at_utc": "2026-09-18T18:17:14Z", + "view": "Research Facility inspection reports", + "displayed_result_count": 15726, + "validated_pages": 21, + "byte_size": 348851, + "input_rows": 2100, + "accepted_rows": 1917, + "quarantined_rows": 183, + "distinct_source_observation_keys": 1917, + "aggregate_artifact_sha256": "fe8e1388821e7b3cddfb72448950e1c9eb5c43e771b59e3acc6981fc467db700", + "coverage_boundary": "remaining displayed rows were not acquired; not_observed is not closure" + } + }, + "graph_projection": { + "source_rows_used": { + "annual_reports": 995, + "registrations": 859, + "inspections": 1917 + }, + "source_rows_excluded_from_graph_projection": 3904, + "exclusion_reasons": { + "duplicate_source_observation_key_across_export_pages": 3904, + "conflicting_official_identifiers": 95 + }, + "source_local_entities": 3771, + "exact_identifier_review_candidates": 955, + "quarantined_relationships_or_projection_rows": 3999, + "candidate_relationship_type": "observation_of_registration", + "match_method": "exact_official_identifier", + "matched_identifier_types": [ + "aphis_certificate_number", + "aphis_customer_number" + ], + "confidence": "1.0 exact source-key agreement; not a truth or approval score", + "review_state": "review_required", + "assertion_status": "candidate" + }, + "bounded_queries": [ + { + "name": "exact_annual_reports_representative", + "relationship_profile": "annual_reports", + "requested_identifier_types": [ + "aphis_certificate_number", + "aphis_customer_number" + ], + "result_count_bounded": 1, + "candidate_id": "us-identity-candidate-ffaf22e02ed6ee232693ff5a", + "identifier_fingerprints": { + "aphis_certificate_number": "b1a5261f297e", + "aphis_customer_number": "8f330105c018" + } + }, + { + "name": "exact_inspections_representative", + "relationship_profile": "inspections", + "requested_identifier_types": [ + "aphis_certificate_number", + "aphis_customer_number" + ], + "result_count_bounded": 1, + "candidate_id": "us-identity-candidate-ffd5e684188148dd0e9884cf", + "identifier_fingerprints": { + "aphis_certificate_number": "877309f7a25f", + "aphis_customer_number": "3756770a030d" + } + } + ], + "controls": { + "weak_name_address_similarity_not_asserted": true, + "suppressed_rows_do_not_enter_candidates": true, + "all_candidate_publication_is_blocked": true, + "rerun_idempotency_key_equal": true, + "geocoding": "disabled", + "auto_merge": false + }, + "limitations": [ + "Raw, parsed, normalized, and quarantine payloads remain in private ignored staging only.", + "Exact APHIS identifiers link source observations; they do not establish ownership, operation, approval, or project publication.", + "The inspection input is a bounded current subset, not complete current inspection coverage.", + "Source terms, privacy review, factual review, project approval, and release publication remain separate gates." + ], + "test_only": true +} diff --git a/docs/countries/us/README.md b/docs/countries/us/README.md index 1bfcb27..60518dc 100644 --- a/docs/countries/us/README.md +++ b/docs/countries/us/README.md @@ -120,6 +120,16 @@ stale, conflicting, overlapping-ownership, and suppressed relationships are quarantined. The checked-in fixture is synthetic/sanitized, private/test-only, and does not add a graph migration or public release. +The Wave 2 APHIS accountability proof in +[`aphis-accountability-wave2.md`](aphis-accountability-wave2.md) exercises that +contract against a current, operator-saved APHIS subset. It links registrations +to annual-report and inspection observations only by exact certificate/customer +IDs, keeps duplicate pagination evidence and ambiguous/conflicting keys in +private quarantine, provides bounded row-free example queries, and records its +aggregate evidence in +[`us-aphis-wave2-accountability-2026-09-18.json`](../../../data/manifests/us-aphis-wave2-accountability-2026-09-18.json). +It remains private/test-only and does not create a release. + ## Legacy real-data V2 and graph rehearsal Run `python -m pipeline.scripts.maintenance.rehearse_us_real --root . --output data/manifests/us-real-legacy-graph-rehearsal-2026-09-17.json --private-dir data/graph-rehearsal/us-real-20260917` to replay the checked-in V1-derived US snapshots through the typed FSIS and APHIS private lifecycle contracts and build a private graph ledger. The rehearsal keeps FSIS federal facility/establishment-approval evidence separate from APHIS inspection and annual-report evidence, emits regulator edges only from source scope, and never joins across FSIS and APHIS by name, address, phone, or coordinates. All output rows remain ignored private staging; the checked-in manifest is aggregate-only. diff --git a/docs/countries/us/aphis-accountability-wave2.md b/docs/countries/us/aphis-accountability-wave2.md new file mode 100644 index 0000000..0090918 --- /dev/null +++ b/docs/countries/us/aphis-accountability-wave2.md @@ -0,0 +1,96 @@ +# APHIS accountability-graph proof (Wave 2) + +Status: private/test-only implementation evidence. This is not a release, +project approval, or claim that the captured subset is complete. + +## What the proof demonstrates + +The Wave 2 runner consumes operator-saved APHIS Public Search Tool CSV pages +from a private directory and keeps the three profiles separate: + +* registrations are the source-local registration observation; +* annual reports are dated FY2025 animal-use observations; and +* inspections are dated inspection observations. + +Registration-to-annual-report and registration-to-inspection links are emitted +only when the source records share an exact APHIS certificate number and/or +customer number. Every link retains source record keys, profile provenance, +matched identifier types, confidence, review state, and the blocked publication +state. The graph does not create a canonical facility, ownership, operation, +or approval assertion. + +The runner is: + +```powershell +python -m pipeline.sources.us.accountability.aphis_wave2 ` + --input-root C:\path\to\private\aphis-exports ` + --run-dir data\raw\us\aphis-accountability-wave2- +``` + +It reads only `ExportData*.csv` files, validates each page with the existing +APHIS adapter, records a per-page SHA-256/size/schema/count manifest, and +writes parsed rows, graph candidates, and quarantine rows under ignored +private staging. The checked-in result is only the row-free aggregate +manifest: [`us-aphis-wave2-accountability-2026-09-18.json`](../../../data/manifests/us-aphis-wave2-accountability-2026-09-18.json). + +## Current private capture + +The proof used the authorized browser-saved pages from 2026-09-18: + +| Profile | Validated pages | Input rows | Accepted | Adapter quarantine | Distinct observation keys | +| --- | ---: | ---: | ---: | ---: | ---: | +| FY2025 annual reports | 10 | 995 | 995 | 0 | 995 | +| Registrations | 78 | 4,763 | 4,763 | 0 | 2,811 | +| Research-facility inspections | 21 | 2,100 | 1,917 | 183 | 1,917 | + +The inspection view displayed 15,726 rows, but only 2,100 rows were acquired +in this bounded proof. The remaining view rows are `not_observed` by this run; +they are not closure, non-use, or evidence that no other observations exist. +Four additional local export attempts failed adapter validation and remain +outside the accepted page set. They are recorded as failures in the private +run and are not treated as zero-row observations. + +The registration pages include separate pagination/state evidence. 1,952 +source observation keys repeat across those pages. The raw/parsed rows remain +private; the graph projection excludes all rows in repeated-key groups instead +of silently merging them. This conservative boundary leaves 859 registration +rows in the graph projection and records the excluded rows as a quarantine +reason. + +## Graph and query result + +The projection contains 3,771 source-local entities and 955 exact-ID review +candidates. All 955 candidates use both certificate and customer identifiers +in this capture, with confidence `1.0`, `review_state=review_required`, and +`assertion_status=candidate`. The value `1.0` describes exact agreement on +the source identifiers; it is not a factual-truth or project-approval score. + +The runner emits two bounded, row-free example queries: one for a +registration-to-annual-report relationship and one for a +registration-to-inspection relationship. Query results expose only hashed +identifier fingerprints and candidate IDs in the committed aggregate +manifest; the private graph retains the source-native values for review. The +projection has 3,999 quarantined relationship/projection items, including +3,904 repeated-key pagination rows and 95 conflicting-identifier cases. + +## Safety and uncertainty controls + +The proof and its tests demonstrate that: + +* weak name/address similarity does not become an asserted relationship; +* exact name/address agreement without compatible official identifiers is at + most a review candidate, and conflicting official identifiers quarantine it; +* suppressed source rows do not enter accepted candidates; +* all graph candidates remain `private`, `not_eligible`, `release_state=not-created`, + and all public surfaces remain disabled; and +* writing the same graph twice produces the same idempotency key and bytes. + +Observed duplicate/ambiguous keys and identifier conflicts stay in private +quarantine. A missing APHIS observation is not closure. APHIS source origin +does not establish current operation, ownership, factual accuracy, project +approval, or publication permission. Names, addresses, and coordinates are +not included in the committed aggregate result and geocoding is disabled. + +Raw, parsed, normalized, and quarantine payloads must remain outside Git under +the retention and removal rules in `docs/ETHICS.md`. The proof does not create +a database release or alter any public API, map, export, cache, or history. diff --git a/pipeline/sources/us/accountability/aphis_wave2.py b/pipeline/sources/us/accountability/aphis_wave2.py new file mode 100644 index 0000000..aa34154 --- /dev/null +++ b/pipeline/sources/us/accountability/aphis_wave2.py @@ -0,0 +1,348 @@ +"""Private Wave 2 APHIS accountability-graph demonstration. + +The input is a directory of operator-saved APHIS CSV exports. The exports +are never treated as a facility master and are never copied into Git. This +module keeps the full parsed rows in ignored private staging, then builds a +row-bearing private identity graph from exact APHIS certificate/customer IDs. +Names, addresses, and coordinates are not identity evidence. +""" +from __future__ import annotations + +import argparse +import copy +import hashlib +import json +from collections import Counter, defaultdict +from pathlib import Path +from typing import Any, Iterable, Mapping + +from pipeline.contracts.source_lifecycle import atomic_json, atomic_jsonl + +from pipeline.sources.us.aphis.adapter import AphisPublicSearchAdapter +from pipeline.sources.us.aphis.acquire import profile_url +from pipeline.sources.us.accountability.current_identity import ( + build_current_identity_graph, + write_current_identity_graph, +) + + +WAVE_VERSION = "us-aphis-accountability-wave2-v1" +SOURCE_ID = "us.aphis" +PROFILE_RETRIEVED_AT = { + "annual_reports": "2026-09-18T17:54:55Z", + "registrations": "2026-09-18T18:06:26Z", + "inspections": "2026-09-18T18:17:14Z", +} +PUBLICATION = { + "storage_state": "private", + "publication_status": "not_eligible", + "public_exposure": False, + "release_state": "not-created", +} + + +def _sha256(payload: bytes) -> str: + return hashlib.sha256(payload).hexdigest() + + +def _fingerprint(identifier_type: str, value: str) -> str: + return hashlib.sha256(f"{identifier_type}|{value}".encode("utf-8")).hexdigest()[:12] + + +def _jsonl(rows: Iterable[Mapping[str, Any]]) -> list[dict[str, Any]]: + return [dict(row) for row in rows] + + +def _classify(path: Path, adapter: AphisPublicSearchAdapter) -> tuple[str, dict[str, Any]]: + raw = path.read_bytes() + result = adapter.parse_bytes(raw) + return result["profile"], { + "path_private": str(path.resolve()), + "artifact": path.name, + "sha256": _sha256(raw), + "byte_size": len(raw), + "input_rows": result["input_rows"], + "accepted_rows": len(result["accepted"]), + "quarantined_rows": len(result["quarantined"]), + "schema_fingerprint": result["schema_fingerprint"], + "profile": result["profile"], + } + + +def discover_exports(input_root: str | Path) -> tuple[dict[str, list[Path]], list[dict[str, str]]]: + """Discover only operator-saved ``ExportData*.csv`` files. + + A malformed export is recorded as an input failure rather than silently + becoming a zero-row observation. The run may continue when another + validated page set supplies the requested profile, but the failure remains + visible in the row-free report. + """ + root = Path(input_root) + adapter = AphisPublicSearchAdapter() + paths: dict[str, list[Path]] = defaultdict(list) + failures: list[dict[str, str]] = [] + for path in sorted(root.glob("ExportData*.csv"), key=lambda value: value.name): + try: + profile, _ = _classify(path, adapter) + except (OSError, ValueError) as error: + failures.append({"artifact": path.name, "failure_class": type(error).__name__}) + continue + if profile in {"annual_reports", "registrations", "inspections"}: + paths[profile].append(path) + return dict(paths), failures + + +def _load_profile(paths: list[Path], profile: str) -> tuple[list[dict[str, Any]], list[dict[str, Any]], list[dict[str, Any]]]: + adapter = AphisPublicSearchAdapter() + records: list[dict[str, Any]] = [] + quarantined: list[dict[str, Any]] = [] + artifacts: list[dict[str, Any]] = [] + for path in sorted(paths, key=lambda value: value.name): + raw = path.read_bytes() + result = adapter.parse_bytes(raw) + if result["profile"] != profile: + raise ValueError(f"{path.name} parsed as {result['profile']}, expected {profile}") + _, metadata = _classify(path, adapter) + artifacts.append(metadata) + records.extend(result["accepted"]) + quarantined.extend(result["quarantined"]) + return records, quarantined, artifacts + + +def _ambiguous_keys(records: Iterable[Mapping[str, Any]]) -> tuple[set[str], dict[str, int]]: + counts: Counter[str] = Counter(str(record["source_record_key"]) for record in records) + return {key for key, count in counts.items() if count > 1}, dict(sorted(counts.items())) + + +def _profile_manifest(profile: str, records: list[dict[str, Any]], quarantined: list[dict[str, Any]], artifacts: list[dict[str, Any]]) -> dict[str, Any]: + aggregate = "\n".join( + f"{item['artifact']}|{item['sha256']}|{item['byte_size']}|{item['input_rows']}" + for item in sorted(artifacts, key=lambda value: value["artifact"]) + ).encode("utf-8") + ambiguous, key_counts = _ambiguous_keys(records) + return { + "source_id": SOURCE_ID, + "profile": profile, + "source_url": profile_url(profile), + "retrieved_at_utc": PROFILE_RETRIEVED_AT[profile], + "page_count": len(artifacts), + "input_rows": sum(int(item["input_rows"]) for item in artifacts), + "accepted_rows": len(records), + "adapter_quarantined_rows": len(quarantined), + "distinct_source_observation_keys": len(key_counts), + "duplicate_source_observation_keys": len(ambiguous), + "duplicate_page_rows": sum(count - 1 for count in key_counts.values() if count > 1), + "aggregate_artifact_sha256": _sha256(aggregate), + "artifacts": artifacts, + } + + +def _safe_example(candidate: Mapping[str, Any]) -> dict[str, Any]: + identifiers = candidate.get("matched_identifiers") or {} + return { + "candidate_id": candidate.get("candidate_id"), + "relationship_type": candidate.get("relationship_type"), + "profiles": [candidate.get("left", {}).get("profile"), candidate.get("right", {}).get("profile")], + "matched_identifier_types": sorted(identifiers), + "identifier_fingerprints": { + key: _fingerprint(key, str(value)) for key, value in sorted(identifiers.items()) + }, + "confidence": candidate.get("confidence"), + "review_state": candidate.get("review_state"), + "assertion_status": candidate.get("assertion_status"), + "publication_status": candidate.get("publication", {}).get("publication_status"), + } + + +def _bounded_query( + candidates: list[Mapping[str, Any]], + identifiers: Mapping[str, str], + *, + right_profile: str | None = None, + limit: int = 5, +) -> dict[str, Any]: + matches = [ + candidate for candidate in candidates + if (right_profile is None or candidate.get("right", {}).get("profile") == right_profile) + if all((candidate.get("matched_identifiers") or {}).get(key) == value for key, value in identifiers.items()) + ] + matches.sort(key=lambda candidate: str(candidate.get("candidate_id"))) + return { + "requested_identifier_types": sorted(identifiers), + "result_count_bounded": min(len(matches), limit), + "limit": limit, + "results": [_safe_example(candidate) for candidate in matches[:limit]], + } + + +def _verify_controls() -> dict[str, bool]: + fixture_path = Path(__file__).with_name("fixtures") / "current_identity.json" + payload = json.loads(fixture_path.read_text(encoding="utf-8")) + + weak = copy.deepcopy(payload) + report = weak["aphis"]["annual_reports"][0] + report["normalized"]["certificate_number"] = "" + report["normalized"]["customer_number"] = "" + report["normalized"]["account_name"] = "Synthetic Laboratory Annex" + report["source_values"]["Account Name"] = "Synthetic Laboratory Annex" + weak_graph = build_current_identity_graph( + aphis_records=weak["aphis"], fsis_records=weak["fsis"], + fsis_observations=weak["fsis_observations"], provenance=weak["provenance"], + ) + weak_edges = [item for item in weak_graph["candidates"] if item["right"]["profile"] == "annual_reports"] + + suppressed = copy.deepcopy(payload) + suppressed["aphis"]["inspections"][0]["normalized"]["privacy_gate"] = "suppressed" + suppressed_graph = build_current_identity_graph( + aphis_records=suppressed["aphis"], fsis_records=suppressed["fsis"], + fsis_observations=suppressed["fsis_observations"], provenance=suppressed["provenance"], + ) + suppressed_edges = [item for item in suppressed_graph["candidates"] if item["right"]["profile"] == "inspections"] + suppressed_queue = [item for item in suppressed_graph["quarantined"] if item["right"]["profile"] == "inspections"] + return { + "weak_name_address_similarity_not_asserted": not any(item.get("match_method") == "alternate_name_address_exact" for item in weak_edges), + "suppressed_rows_do_not_enter_candidates": not suppressed_edges and any(item.get("quarantine_reason") == "suppressed_or_restricted" for item in suppressed_queue), + "all_candidate_publication_is_blocked": all(item.get("publication", {}).get("publication_status") == "not_eligible" for item in weak_graph["candidates"]), + } + + +def run_wave2(*, input_root: str | Path, run_dir: str | Path) -> dict[str, Any]: + """Run the bounded private proof against saved APHIS exports.""" + root = Path(run_dir) + root.mkdir(parents=True, exist_ok=True) + paths, failures = discover_exports(input_root) + if any(not paths.get(profile) for profile in ("registrations", "annual_reports", "inspections")): + raise ValueError("input root must contain at least one validated export for each APHIS profile") + + records_by_profile: dict[str, list[dict[str, Any]]] = {} + quarantined_by_profile: dict[str, list[dict[str, Any]]] = {} + profile_manifests: dict[str, dict[str, Any]] = {} + all_artifacts: list[dict[str, Any]] = [] + graph_records: dict[str, list[dict[str, Any]]] = {} + projection_quarantine: list[dict[str, Any]] = [] + for profile in ("registrations", "annual_reports", "inspections"): + records, quarantined, artifacts = _load_profile(paths[profile], profile) + records_by_profile[profile] = records + quarantined_by_profile[profile] = quarantined + profile_manifests[profile] = _profile_manifest(profile, records, quarantined, artifacts) + all_artifacts.extend(artifacts) + ambiguous, _ = _ambiguous_keys(records) + graph_records[profile] = [] + for record in sorted(records, key=lambda value: (str(value["source_record_key"]), int(value.get("source_row") or 0))): + if record["source_record_key"] in ambiguous: + projection_quarantine.append({ + "profile": profile, + "reason": "duplicate_source_observation_key_across_export_pages", + "source_record_key": record["source_record_key"], + }) + elif not any(existing["source_record_key"] == record["source_record_key"] for existing in graph_records[profile]): + graph_records[profile].append(record) + + atomic_jsonl(root / "private-rows" / f"{profile}-accepted.jsonl", _jsonl(records)) + atomic_jsonl(root / "private-rows" / f"{profile}-adapter-quarantine.jsonl", _jsonl(quarantined)) + atomic_jsonl(root / "private-rows" / "graph-projection-quarantine.jsonl", projection_quarantine) + + provenance = { + (SOURCE_ID, profile): { + "artifact_sha256": profile_manifests[profile]["aggregate_artifact_sha256"], + "source_url": profile_manifests[profile]["source_url"], + "retrieved_at_utc": profile_manifests[profile]["retrieved_at_utc"], + } + for profile in profile_manifests + } + graph = build_current_identity_graph( + aphis_records=graph_records, + fsis_records=(), + fsis_observations=(), + provenance=provenance, + ) + first = write_current_identity_graph(root / "identity-graph", graph) + second = write_current_identity_graph(root / "identity-graph-rerun", graph) + + candidates = list(graph["candidates"]) + by_profile: dict[str, list[Mapping[str, Any]]] = defaultdict(list) + for candidate in candidates: + by_profile[str(candidate.get("right", {}).get("profile"))].append(candidate) + examples: list[dict[str, Any]] = [] + bounded_queries: list[dict[str, Any]] = [] + for profile in ("annual_reports", "inspections"): + pool = sorted(by_profile[profile], key=lambda item: str(item.get("candidate_id"))) + if not pool: + continue + selected = max(pool, key=lambda item: (len(item.get("matched_identifiers") or {}), str(item.get("candidate_id")))) + examples.append(_safe_example(selected)) + bounded_queries.append({ + "name": f"exact_{profile}_representative", + "relationship_profile": profile, + **_bounded_query( + candidates, + selected.get("matched_identifiers") or {}, + right_profile=profile, + ), + }) + + anomaly_counts = Counter(item.get("reason") for item in projection_quarantine) + anomaly_counts.update(reason for item in graph["quarantined"] for reason in [item.get("quarantine_reason") or item.get("reason")]) + report = { + "schema_version": WAVE_VERSION, + "captured_for": "private/test-only", + "input_root_private": str(Path(input_root).resolve()), + "input_failures": failures, + "profiles": profile_manifests, + "graph_projection": { + "source_rows_used": {profile: len(rows) for profile, rows in graph_records.items()}, + "source_rows_excluded_from_graph_projection": len(projection_quarantine), + "projection_exclusion_reasons": dict(sorted(anomaly_counts.items())), + "entities": len(graph["entities"]), + "candidate_relationships": len(graph["candidates"]), + "quarantined_relationships": len(graph["quarantined"]) + len(projection_quarantine), + "candidate_relationship_types": graph["manifest"]["candidate_relationship_types"], + "match_methods": graph["manifest"]["match_methods"], + "review_state": "review_required", + "confidence_states": dict(sorted(Counter(str(item.get("confidence")) for item in candidates).items())), + "matched_identifier_types": dict(sorted(Counter(key for item in candidates for key in (item.get("matched_identifiers") or {})).items())), + }, + "representative_examples": examples, + "bounded_queries": bounded_queries, + "controls": { + **_verify_controls(), + "rerun_idempotency_key_equal": first["idempotency_key"] == second["idempotency_key"], + "public_surfaces": {"api": False, "map": False, "export": False, "cache": False, "history": False}, + **PUBLICATION, + }, + "coverage_boundary": { + "inspections": "21 saved Research Facility export pages / 2,100 rows from the displayed 15,726-row view; remaining rows were not acquired in this proof", + "annual_reports": "FY2025 annual-report export pages only", + "registrations": "saved registration export pages, including separate pagination/state evidence pages; duplicate source keys are not silently merged", + "not_observed_semantics": "absence from a page or subset is not closure, non-use, or non-coverage", + }, + "limitations": [ + "Raw, parsed, normalized, and quarantine rows remain in private ignored staging only.", + "Exact APHIS identifiers link source observations; they do not establish ownership, operation, approval, or project publication.", + "Addresses and names remain private source evidence and are not included in this aggregate report.", + "Source terms, privacy review, factual review, project approval, and release publication remain separate gates.", + ], + "publication": PUBLICATION, + "test_only": True, + } + atomic_json(root / "input-manifest.json", {"schema_version": f"{WAVE_VERSION}-inputs", "profiles": profile_manifests, "input_failures": failures}) + atomic_json(root / "wave2-report.json", report) + return report + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--input-root", type=Path, required=True) + parser.add_argument("--run-dir", type=Path, required=True) + args = parser.parse_args() + try: + report = run_wave2(input_root=args.input_root, run_dir=args.run_dir) + except (OSError, ValueError, KeyError) as error: + print(json.dumps({"status": "failed", "error": str(error)}, sort_keys=True)) + return 2 + print(json.dumps({"status": "completed", "run_dir": str(args.run_dir), "candidate_relationships": report["graph_projection"]["candidate_relationships"]}, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/pipeline/sources/us/accountability/current_identity.py b/pipeline/sources/us/accountability/current_identity.py index b3b5f98..e455351 100644 --- a/pipeline/sources/us/accountability/current_identity.py +++ b/pipeline/sources/us/accountability/current_identity.py @@ -296,12 +296,6 @@ def _candidate( return result -def _pair_key(profile: str, identifier_type: str, value: str, observed_at: str | None) -> tuple[str, str, str, str | None]: - # Multiple annual reports/inspections can legitimately share a certificate - # over time; duplicate observations on the same date are an ambiguity. - return profile, identifier_type, value, observed_at - - def _records_for(records: Iterable[Mapping[str, Any]], profile: str) -> list[Mapping[str, Any]]: if isinstance(records, Mapping): records = records.get("accepted", ()) @@ -314,20 +308,33 @@ def _official_pairs( right_records: list[Mapping[str, Any]], right_profile: str, ) -> tuple[list[tuple[Mapping[str, Any], Mapping[str, Any], dict[str, str]]], list[dict[str, Any]]]: - right_index: dict[tuple[str, str, str | None], list[Mapping[str, Any]]] = defaultdict(list) - collisions: Counter[tuple[str, str, str | None]] = Counter() + # Candidate generation must be keyed by an exact shared official value. + # Comparing every registration with every report/inspection would turn a + # pair of unrelated records with different certificate numbers into a + # false ``conflicting_official_identifiers`` quarantine. It also scales + # quadratically on the real APHIS capture. A conflict is meaningful only + # after at least one source-native identifier has made the pair a candidate + # (for example, the same customer number but a different certificate). + right_index: dict[tuple[str, str], list[Mapping[str, Any]]] = defaultdict(list) + collision_keys: dict[tuple[str, str, str | None], set[str]] = defaultdict(set) for record in right_records: + source_key = _source_key(record) + observed_at = _observation_date(record) for identifier_type, value in _identifiers(record, right_profile).items(): - key = _pair_key(right_profile, identifier_type, value, _observation_date(record)) + key = (identifier_type, value) right_index[key].append(record) - collisions[key] += 1 + collision_keys[(identifier_type, value, observed_at)].add(source_key) pairs: list[tuple[Mapping[str, Any], Mapping[str, Any], dict[str, str]]] = [] quarantined: list[dict[str, Any]] = [] seen: set[tuple[str, str, str]] = set() for left in left_records: left_ids = _identifiers(left, left_profile) - for right in right_records: + possible_right: dict[str, Mapping[str, Any]] = {} + for identifier_type, value in left_ids.items(): + for right in right_index.get((identifier_type, value), ()): + possible_right[_source_key(right)] = right + for right in possible_right.values(): right_ids = _identifiers(right, right_profile) shared = {kind: value for kind, value in left_ids.items() if right_ids.get(kind) == value} conflicts = [kind for kind in set(left_ids) & set(right_ids) if left_ids[kind] != right_ids[kind]] @@ -348,7 +355,7 @@ def _official_pairs( continue seen.add(identity_key) collision = any( - collisions[_pair_key(right_profile, kind, value, _observation_date(right))] > 1 + len(collision_keys[(kind, value, _observation_date(right))]) > 1 for kind, value in shared.items() ) if collision: @@ -399,6 +406,19 @@ def _alternate_pairs( continue left, left_fields = lefts[0] right, right_fields = rights[0] + left_ids = _identifiers(left, _profile(left)) + right_ids = _identifiers(right, _profile(right)) + conflicts = [kind for kind in set(left_ids) & set(right_ids) if left_ids[kind] != right_ids[kind]] + if conflicts: + quarantined.append({ + "reason": "conflicting_official_identifiers", + "left_source_record_key": _source_key(left), + "right_source_record_key": _source_key(right), + "identifier_types": sorted(conflicts), + "_left_record": left, + "_right_record": right, + }) + continue pairs.append((left, right, sorted(set(left_fields + right_fields)))) return pairs, quarantined diff --git a/pipeline/sources/us/accountability/test_aphis_wave2.py b/pipeline/sources/us/accountability/test_aphis_wave2.py new file mode 100644 index 0000000..ebc3cad --- /dev/null +++ b/pipeline/sources/us/accountability/test_aphis_wave2.py @@ -0,0 +1,53 @@ +import json +import tempfile +import unittest +from pathlib import Path + +from .aphis_wave2 import run_wave2 + + +ROOT = Path(__file__).parent + + +class AphisWave2Tests(unittest.TestCase): + def test_private_wave_is_bounded_and_idempotent(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + input_root = root / "inputs" + input_root.mkdir() + (input_root / "ExportData-registrations.csv").write_text( + "Account Name,Customer Number,Certificate Number,Registration Type,Certificate Status,Status Date\n" + '"Synthetic Registrant","2","00-R-0002","Class R - Research Facility","Active","2026-01-01"\n', + encoding="utf-8", + ) + (input_root / "ExportData-annual_reports.csv").write_text( + "Customer Number,Certificate Number,Year,Dogs,Cats\n" + '"2","00-R-0002","2025","","1"\n', + encoding="utf-8", + ) + (input_root / "ExportData-inspections.csv").write_text( + "Customer Number,Certificate Number,Inspection Date,Direct NCIs,Non-Critical NCIs,Critical NCIs,Teachable Moments,Site Name,Legal Name,License-Registration Type,City,State,Zip\n" + '"2","00-R-0002","2026-02-01","","","","","Synthetic Site","Synthetic Registrant","Class R - Research Facility","Testville","TX","75001"\n', + encoding="utf-8", + ) + + report = run_wave2(input_root=input_root, run_dir=root / "run") + self.assertEqual(report["captured_for"], "private/test-only") + self.assertGreaterEqual(report["graph_projection"]["candidate_relationships"], 2) + self.assertTrue(report["controls"]["weak_name_address_similarity_not_asserted"]) + self.assertTrue(report["controls"]["suppressed_rows_do_not_enter_candidates"]) + self.assertTrue(report["controls"]["rerun_idempotency_key_equal"]) + self.assertFalse(report["controls"]["public_exposure"]) + self.assertTrue(all( + result["profiles"][-1] == query["relationship_profile"] + for query in report["bounded_queries"] + for result in query["results"] + )) + self.assertEqual( + json.loads((root / "run" / "wave2-report.json").read_text(encoding="utf-8"))["publication"]["publication_status"], + "not_eligible", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/sources/us/accountability/test_current_identity.py b/pipeline/sources/us/accountability/test_current_identity.py index 96363c5..67ca709 100644 --- a/pipeline/sources/us/accountability/test_current_identity.py +++ b/pipeline/sources/us/accountability/test_current_identity.py @@ -67,6 +67,29 @@ def test_conflicting_official_ids_block_name_address_rescue(self): self.assertFalse(any(candidate["match_method"] == "alternate_name_address_exact" for candidate in graph["candidates"])) self.assertTrue(any(item["reason"] == "conflicting_official_identifiers" for item in graph["quarantined"])) + def test_unrelated_official_ids_are_not_false_conflicts(self): + payload = load_fixture() + report = payload["aphis"]["annual_reports"][0] + report["normalized"]["certificate_number"] = "00-R-UNRELATED" + report["normalized"]["customer_number"] = "999999" + report["normalized"]["account_name"] = "Unrelated report" + report["source_values"]["Account Name"] = "Unrelated report" + graph = self.build(payload) + annual = [item for item in graph["quarantined"] if item["right"]["profile"] == "annual_reports"] + self.assertFalse(any(item.get("reason") == "conflicting_official_identifiers" for item in annual)) + self.assertFalse(any(item["right"]["profile"] == "annual_reports" for item in graph["candidates"])) + + def test_dated_inspections_sharing_an_id_are_not_ambiguous(self): + payload = load_fixture() + second = copy.deepcopy(payload["aphis"]["inspections"][0]) + second["source_record_key"] = "inspections:00-R-TEST-001:2026-03-01" + second["normalized"]["status_date"] = "2026-03-01" + payload["aphis"]["inspections"].append(second) + graph = self.build(payload) + inspections = [item for item in graph["candidates"] if item["right"]["profile"] == "inspections"] + self.assertEqual(len(inspections), 2) + self.assertFalse(any(item.get("reason") == "ambiguous_official_identifier" for item in graph["quarantined"])) + def test_ambiguous_alternate_match_is_quarantined(self): payload = load_fixture() duplicate = copy.deepcopy(payload["aphis"]["annual_reports"][0]) From 71c8a9b2bf64a5f8c9d1d3ba371817f35da0100c Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Fri, 18 Sep 2026 12:24:32 -0700 Subject: [PATCH 248/311] Fix US refresh diagnostics artifact paths --- pipeline/sources/us/refresh.py | 14 +++++++++++--- pipeline/sources/us/test_refresh.py | 1 + 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/pipeline/sources/us/refresh.py b/pipeline/sources/us/refresh.py index 48998ff..ab5b1b6 100644 --- a/pipeline/sources/us/refresh.py +++ b/pipeline/sources/us/refresh.py @@ -298,12 +298,20 @@ def diagnose(run_root: str | Path, *, as_of_utc: str | None = None) -> dict[str, checks = [] for item in report.get("sources", []): run_dir = Path(str(item.get("run_dir", ""))) + # FSIS reports its lifecycle directory directly, while APHIS reports + # the source run root and keeps the canonical lifecycle artifacts in + # a child directory. Resolve both layouts so diagnostics do not + # manufacture a missing-artifact warning for a healthy APHIS run. + artifact_roots = [run_dir] + lifecycle_dir = run_dir / "lifecycle" + if lifecycle_dir.is_dir(): + artifact_roots.append(lifecycle_dir) checks.append({ "source_id": item.get("source_id"), "status": item.get("status"), - "run_status_present": (run_dir / "run-status.json").is_file(), - "manifest_present": (run_dir / "manifest.json").is_file(), - "review_packet_present": (run_dir / "review-packet.json").is_file(), + "run_status_present": any((root / "run-status.json").is_file() for root in artifact_roots), + "manifest_present": any((root / "manifest.json").is_file() for root in artifact_roots), + "review_packet_present": any((root / "review-packet.json").is_file() for root in artifact_roots), "previous_valid_available": item.get("previous_valid", {}).get("available", False), "public_exposure": item.get("release", {}).get("public_surfaces", PUBLIC_SURFACES), }) diff --git a/pipeline/sources/us/test_refresh.py b/pipeline/sources/us/test_refresh.py index cee1969..4c6c1ee 100644 --- a/pipeline/sources/us/test_refresh.py +++ b/pipeline/sources/us/test_refresh.py @@ -62,6 +62,7 @@ def test_one_command_report_is_row_free_and_keeps_public_gate_closed(self): diagnostics = diagnose(root / "run", as_of_utc="2026-09-18T12:00:00Z") self.assertEqual(diagnostics["release"]["publication_gate"], "blocked") self.assertTrue(all(item["run_status_present"] for item in diagnostics["checks"])) + self.assertTrue(all(item["manifest_present"] for item in diagnostics["checks"])) def test_failed_lane_reports_action_and_preserves_previous_valid_manifest(self): with tempfile.TemporaryDirectory() as directory: From 39124ae5537fbe2e97699a27e363670411a8a2c0 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Fri, 18 Sep 2026 12:25:08 -0700 Subject: [PATCH 249/311] demo private US APHIS accountability graph --- ...countability-demonstration-2026-09-18.json | 144 ++++++++++++++++++ docs/countries/us/README.md | 22 +++ .../test_us_accountability_demonstration.py | 26 ++++ 3 files changed, 192 insertions(+) create mode 100644 data/manifests/us-real-accountability-demonstration-2026-09-18.json create mode 100644 pipeline/tests/test_us_accountability_demonstration.py diff --git a/data/manifests/us-real-accountability-demonstration-2026-09-18.json b/data/manifests/us-real-accountability-demonstration-2026-09-18.json new file mode 100644 index 0000000..154624f --- /dev/null +++ b/data/manifests/us-real-accountability-demonstration-2026-09-18.json @@ -0,0 +1,144 @@ +{ + "corpus_state": "private-regression-only", + "generated_at_utc": "2026-09-17T00:00:00Z", + "graph": { + "accepted_relationship_counts": { + "inspection_observes": 1332, + "regulatory_authority_for": 1332 + }, + "accepted_relationships": 2664, + "all_candidates": { + "auto_merge": false, + "geocoding": "disabled", + "publication_gate": "blocked", + "review_state": "review_required" + }, + "candidate_rule": "exact source-native IDs in the same legacy source row only", + "cross_source_identity_joins_attempted": 0, + "dated_relationship_counts": { + "legacy-aphis-annual-reports": 2026, + "legacy-aphis-inspections": 9014, + "legacy-fsis-locations": 14198 + }, + "graph_manifest_sha256": "ebc2e83bcb84437df695791b99abd93b698a408017c2e6f60308912344f208cc", + "ledger_input_rows": 25238, + "name_address_phone_coordinate_joins_attempted": 0, + "profile_counts": { + "legacy-aphis-annual-reports": 2026, + "legacy-aphis-inspections": 9014, + "legacy-fsis-locations": 14198 + }, + "quarantine_reason_counts": { + "retrieval_precedes_observation": 5868, + "stale_evidence": 16706 + }, + "quarantined_relationships": 22574, + "relationship_counts": { + "aggregate_describes": 1013, + "establishment_approval_for": 7099, + "inspection_observes": 4507, + "regulatory_authority_for": 12619 + } + }, + "input_kind": "existing-v1-derived-snapshot; not current raw acquisition", + "lifecycle": { + "aphis-annual-reports": { + "candidate_created": true, + "coordinate_gate": "review_required", + "input_rows": 1013, + "normalized_rows": 1013, + "privacy_gate": "pending", + "publication_state": "private-candidate", + "quarantined_rows": 0, + "release_state": "not-created", + "review_state": "review_required", + "schema_fingerprint": "a1fd908ab662cadd330310aeddba586c34d170c12a0f75b0dfb367b3f978ccc5", + "sha256": "c243e64ba804ad55cc470644cc9c41678bb542622a7dbba75c938fff9a789ff1", + "source_id": "us.aphis", + "status": "candidate-ready" + }, + "aphis-inspections": { + "candidate_created": true, + "coordinate_gate": "review_required", + "input_rows": 4507, + "normalized_rows": 4507, + "privacy_gate": "pending", + "publication_state": "private-candidate", + "quarantined_rows": 0, + "release_state": "not-created", + "review_state": "review_required", + "schema_fingerprint": "8a7c46f3068955b88275377da2418a2cb232260eb85327924460a4791f1f1971", + "sha256": "edbd3cd963b097cd0d72049af9cdc661429e849d3f29b9cb31827d228c1cd1a7", + "source_id": "us.aphis", + "status": "candidate-ready" + }, + "fsis-locations": { + "candidate_created": true, + "coordinate_gate": "review_required", + "input_rows": 7101, + "normalized_rows": 7101, + "privacy_gate": "pending", + "publication_state": "private-candidate", + "quarantined_rows": 0, + "release_state": "not-created", + "review_state": "review_required", + "schema_fingerprint": "9d68e76e6adb5be3a5033a5e99a2b8954d898a39d8eba51ed1cb804f0ae939d9", + "sha256": "818ee55be75cc40a23ff9b85fa7a4ea83ab8c8f7d04766212ac0fb99d0382e32", + "source_id": "us.fsis", + "status": "candidate-ready" + } + }, + "limitations": [ + "The legacy V1 snapshots are not current raw FSIS or APHIS captures and are not represented as current evidence.", + "The current FSIS page was observed in a browser with a Sep 14, 2026 update, but exact CSV downloads returned 403 outside that session; no current raw artifact is claimed.", + "Graph yield is a candidate count, not accuracy, ownership truth, facility operation, or publication permission.", + "A source disappearance is not closure; state inspection programs remain outside this federal-only rehearsal." + ], + "privacy_and_provenance": { + "coordinates": "not used for identity; geocoding disabled", + "raw_values": "private adapter output only; not included in report", + "source_hashes": { + "aphis_annual_reports": "c243e64ba804ad55cc470644cc9c41678bb542622a7dbba75c938fff9a789ff1", + "aphis_inspections": "edbd3cd963b097cd0d72049af9cdc661429e849d3f29b9cb31827d228c1cd1a7", + "fsis_locations": "2dca259076a16a324ad9565d2d4057aa0f5e6e51bbc25a8dc1c80a4e4fbdf5c7" + } + }, + "publication_eligibility": "blocked", + "quality": { + "accuracy": "not measured; no adjudicated real labels available", + "input_rows_reconciled_to_source_local_graph_or_skip": true, + "skipped_or_quarantined_before_ledger": { + "fsis_missing_observation_date": 2 + } + }, + "schema_version": "us-real-legacy-graph-rehearsal-v1", + "source_boundaries": { + "aphis_annual_reports": { + "evidence_family": "annual aggregate observations; may be amended", + "input_rows": 1013, + "jurisdiction": "federal", + "source_url": "https://www.aphis.usda.gov/animal-care/awa-services/usda-animal-care-public-search-tool" + }, + "aphis_inspections": { + "evidence_family": "inspection observations; no FSIS facility merge", + "input_rows": 4507, + "jurisdiction": "federal", + "source_url": "https://www.aphis.usda.gov/animal-care/awa-services/usda-animal-care-public-search-tool" + }, + "fsis": { + "input_rows": 7101, + "jurisdiction": "federal", + "source_url": "https://www.fsis.usda.gov/sites/default/files/media_file/documents/MPI_Directory_by_Establishment_Number.csv", + "state_inspection_programs": "excluded" + }, + "state_programs": { + "included": false, + "reason": "no state inspection source was supplied or inferred" + } + }, + "strata": { + "aphis_annual_reports": 1013, + "aphis_inspections": 4507, + "fsis_locations": 7101 + } +} diff --git a/docs/countries/us/README.md b/docs/countries/us/README.md index 60518dc..b342268 100644 --- a/docs/countries/us/README.md +++ b/docs/countries/us/README.md @@ -138,6 +138,28 @@ The 2026-09-17 rehearsal measured 7,101 FSIS rows, 4,507 APHIS inspection rows, The current FSIS page was observed in a normal browser with a September 14, 2026 update and three CSV routes, but the exact file routes returned HTTP 403 to bounded direct acquisition. See the row-free [current-route manifest](../../../data/manifests/us-fsis-current-route-2026-09-17.json). +### Current APHIS accountability demonstration + +The 2026-09-18 private demonstration replayed the locally retained APHIS +public-search evidence through the same source-local ledger contract. Run: + +```powershell +python -m pipeline.scripts.maintenance.rehearse_us_real ` + --root . ` + --output data/manifests/us-real-accountability-demonstration-2026-09-18.json ` + --private-dir data/graph-rehearsal/us-real-accountability-20260918 +``` + +The tracked output is aggregate-only; row payloads and candidate JSONL remain +in ignored private staging. This run is useful for inspecting distinct APHIS +inspection, annual-report, and regulatory-authority relationships with source +IDs, observation dates, confidence, and provenance. It does not assert that an +organization owns or operates a facility, does not infer wrongdoing, and keeps +`publication_eligibility=blocked`. The input includes legacy V1-derived FSIS +and APHIS snapshots alongside the current APHIS proof manifest; it is not a +claim that those legacy snapshots are current. Use the [APHIS proof manifest](../../../data/manifests/us-aphis-wave1-real-data-proof-2026-09-18.json) +for current acquisition coverage and failure boundaries. + State MPI acquisition remains documentation-only. No state roster or CIS workbook was acquired in this sprint, and no current state facility row is claimed. Before any private capture, record the final URL, retrieval UTC, diff --git a/pipeline/tests/test_us_accountability_demonstration.py b/pipeline/tests/test_us_accountability_demonstration.py new file mode 100644 index 0000000..74c82d0 --- /dev/null +++ b/pipeline/tests/test_us_accountability_demonstration.py @@ -0,0 +1,26 @@ +import json +import unittest +from pathlib import Path + + +ROOT = Path(__file__).parents[2] +MANIFEST = ROOT / "data/manifests/us-real-accountability-demonstration-2026-09-18.json" + + +class UsAccountabilityDemonstrationTests(unittest.TestCase): + def test_real_demonstration_is_private_source_bound_and_row_free(self): + report = json.loads(MANIFEST.read_text(encoding="utf-8")) + self.assertEqual(report["corpus_state"], "private-regression-only") + self.assertEqual(report["publication_eligibility"], "blocked") + self.assertEqual(report["graph"]["cross_source_identity_joins_attempted"], 0) + self.assertEqual(report["graph"]["name_address_phone_coordinate_joins_attempted"], 0) + self.assertGreater(report["graph"]["accepted_relationships"], 0) + self.assertGreater(report["graph"]["quarantined_relationships"], 0) + self.assertEqual(report["graph"]["all_candidates"]["auto_merge"], False) + self.assertEqual(report["privacy_and_provenance"]["coordinates"], "not used for identity; geocoding disabled") + self.assertTrue(report["privacy_and_provenance"]["source_hashes"]) + self.assertTrue(all("name" not in json.dumps(value).lower() for value in report["privacy_and_provenance"]["source_hashes"].values())) + + +if __name__ == "__main__": + unittest.main() From 6b407e6afd69fb9eb62c9c27775d7183c88f4240 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Fri, 18 Sep 2026 13:21:06 -0700 Subject: [PATCH 250/311] Fix cross-platform manifest integrity test --- pipeline/tests/test_us_real_proof_manifests.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/pipeline/tests/test_us_real_proof_manifests.py b/pipeline/tests/test_us_real_proof_manifests.py index 800f6c6..99d21d8 100644 --- a/pipeline/tests/test_us_real_proof_manifests.py +++ b/pipeline/tests/test_us_real_proof_manifests.py @@ -59,13 +59,14 @@ def test_fsis_manifest_cannot_be_mistaken_for_current_or_public_data(self): self.assertTrue(rehearsal["unmatched_demographic_is_not_closure"]) self.assertFalse(FORBIDDEN_ROW_KEYS.intersection(set(_walk_keys(manifest)))) - def test_manifest_bytes_are_stable_for_this_checkout(self): + def test_manifest_content_is_stable_across_line_endings(self): expected = { - "us-aphis-wave1-real-data-proof-2026-09-18.json": "fe48126d31e4eb7336579e329f424ccee4f19618d0681e47de35412c045e4e9c", - "us-fsis-proof-2026-09-18.json": "26a44a35bf4f34f46967053724384c4ea6c04c7101653bbd6269cd0bbd20d951", + "us-aphis-wave1-real-data-proof-2026-09-18.json": "fb5dcfd35efb17c5c0a17e6d08d407e2b32fd99b864ba8bc95f224989362c795", + "us-fsis-proof-2026-09-18.json": "6f75f40c41b14c061a388c1c9578b6995bd6b8c17a9fde2948b5fc74adbb3812", } for name, digest in expected.items(): - self.assertEqual(hashlib.sha256((MANIFEST_DIR / name).read_bytes()).hexdigest(), digest) + normalized = (MANIFEST_DIR / name).read_text(encoding="utf-8").encode("utf-8") + self.assertEqual(hashlib.sha256(normalized).hexdigest(), digest) if __name__ == "__main__": From 8e8b7db2686b4b859fca6fd8c82c443381d069dd Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Fri, 18 Sep 2026 13:40:00 -0700 Subject: [PATCH 251/311] fsis: block stale private handoffs --- pipeline/sources/us/fsis/refresh.py | 27 ++++++++++++++++++++++-- pipeline/sources/us/fsis/test_refresh.py | 9 ++++++++ 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/pipeline/sources/us/fsis/refresh.py b/pipeline/sources/us/fsis/refresh.py index 9f7a143..103a1b3 100644 --- a/pipeline/sources/us/fsis/refresh.py +++ b/pipeline/sources/us/fsis/refresh.py @@ -4,6 +4,7 @@ import argparse import hashlib import json +from datetime import datetime, timezone from pathlib import Path from typing import Any @@ -99,6 +100,22 @@ def _drift(manifest: dict[str, Any], previous_manifest: str | Path | None) -> di return {"checked": True, "blocked": bool(alarms), "alarms": alarms, "previous_manifest": str(previous_manifest)} +def _currentness(effective_date: str | None, *, max_age_days: int) -> dict[str, Any]: + """Classify the displayed source edition without guessing when absent.""" + if max_age_days < 0: + raise ValueError("max_age_days must be non-negative") + if not effective_date or effective_date == "unknown": + return {"status": "unknown", "blocked": True, "effective_date": "unknown", "max_age_days": max_age_days} + try: + observed = datetime.fromisoformat(effective_date.replace("Z", "+00:00")).date() + except ValueError as error: + raise ValueError("effective_date must be ISO-8601") from error + today = datetime.now(timezone.utc).date() + age_days = (today - observed).days + return {"status": "current" if 0 <= age_days <= max_age_days else "stale", "blocked": age_days < 0 or age_days > max_age_days, + "effective_date": observed.isoformat(), "age_days": age_days, "max_age_days": max_age_days} + + def refresh( *, run_dir: str | Path, @@ -116,6 +133,7 @@ def refresh( max_attempts: int = 3, retry_delay_seconds: float = 1.0, max_retry_delay_seconds: float = 30.0, + max_age_days: int = 14, ) -> dict[str, Any]: if raw_path is not None and directory_path is not None: raise ValueError("specify raw_path or directory_path, not both") @@ -182,11 +200,14 @@ def refresh( adapter = FsisMpiAdapter() lifecycle_root = root / "lifecycle" manifest = adapter.run_sources(paths, lifecycle_root, artifacts) + currentness = _currentness(effective_date or metadata["directory"].get("effective_date"), max_age_days=max_age_days) + manifest["currentness"] = currentness drift = _drift(manifest, previous_manifest) manifest["drift"] = drift atomic_json(lifecycle_root / "manifest.json", manifest) - if drift["blocked"] and mode == "handoff": - raise ValueError("refresh drift alarm blocks handoff: " + ", ".join(drift["alarms"])) + if mode == "handoff" and (drift["blocked"] or currentness["blocked"]): + reasons = drift["alarms"] + (["source_effective_date_not_current"] if currentness["blocked"] else []) + raise ValueError("refresh gate blocks handoff: " + ", ".join(reasons)) handoff = None if mode == "handoff": @@ -242,6 +263,7 @@ def main() -> int: parser.add_argument("--max-attempts", type=int, default=3) parser.add_argument("--retry-delay-seconds", type=float, default=1.0) parser.add_argument("--max-retry-delay-seconds", type=float, default=30.0) + parser.add_argument("--max-age-days", type=int, default=14) args = parser.parse_args() try: result = refresh( @@ -251,6 +273,7 @@ def main() -> int: previous_manifest=args.previous_manifest, max_bytes=args.max_bytes, max_attempts=args.max_attempts, retry_delay_seconds=args.retry_delay_seconds, max_retry_delay_seconds=args.max_retry_delay_seconds, + max_age_days=args.max_age_days, ) except (OSError, ValueError) as exc: print(json.dumps({"status": "failed", "error": str(exc)})) diff --git a/pipeline/sources/us/fsis/test_refresh.py b/pipeline/sources/us/fsis/test_refresh.py index 597241e..c8724a3 100644 --- a/pipeline/sources/us/fsis/test_refresh.py +++ b/pipeline/sources/us/fsis/test_refresh.py @@ -48,6 +48,15 @@ def test_schema_drift_blocks_handoff_after_previous_manifest(self): refresh(run_dir=root / "second", directory_path=changed, previous_manifest=previous, mode="handoff") self.assertFalse((root / "second/lifecycle/handoff/manifest.json").exists()) + def test_stale_or_unknown_edition_blocks_handoff(self): + with tempfile.TemporaryDirectory() as directory: + with self.assertRaisesRegex(ValueError, "source_effective_date_not_current"): + refresh(run_dir=Path(directory) / "stale", directory_path=ROOT / "fixtures/valid.csv", + effective_date="2020-01-01", mode="handoff") + with self.assertRaisesRegex(ValueError, "source_effective_date_not_current"): + refresh(run_dir=Path(directory) / "unknown", directory_path=ROOT / "fixtures/valid.csv", + mode="handoff") + if __name__ == "__main__": unittest.main() From b479033cd5cf9b0cf0b8aa6c1b057933b904102e Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Fri, 18 Sep 2026 13:40:08 -0700 Subject: [PATCH 252/311] Account for APHIS incomplete inspection coverage --- .../us/aphis-accountability-wave2.md | 9 +++ .../sources/us/accountability/aphis_wave2.py | 57 ++++++++++++++++++- .../us/accountability/test_aphis_wave2.py | 18 ++++++ 3 files changed, 81 insertions(+), 3 deletions(-) diff --git a/docs/countries/us/aphis-accountability-wave2.md b/docs/countries/us/aphis-accountability-wave2.md index 0090918..5e62ef8 100644 --- a/docs/countries/us/aphis-accountability-wave2.md +++ b/docs/countries/us/aphis-accountability-wave2.md @@ -50,6 +50,15 @@ Four additional local export attempts failed adapter validation and remain outside the accepted page set. They are recorded as failures in the private run and are not treated as zero-row observations. +The runner now writes machine-readable `completeness` accounting for each +profile. For inspections its default operator-supplied displayed total is +15,726, so this capture records 13,626 rows as `not_observed` (15,726 minus +2,100 observed input rows). The accounting also separates accepted rows, +adapter quarantine, duplicate page rows, and classified failed exports. A +different displayed total may be supplied with `--expected-rows` as a JSON +object, but matching that count does not establish source truth or publication +eligibility; failed exports keep the profile incomplete. + The registration pages include separate pagination/state evidence. 1,952 source observation keys repeat across those pages. The raw/parsed rows remain private; the graph projection excludes all rows in repeated-key groups instead diff --git a/pipeline/sources/us/accountability/aphis_wave2.py b/pipeline/sources/us/accountability/aphis_wave2.py index aa34154..ceb9f78 100644 --- a/pipeline/sources/us/accountability/aphis_wave2.py +++ b/pipeline/sources/us/accountability/aphis_wave2.py @@ -33,6 +33,7 @@ "registrations": "2026-09-18T18:06:26Z", "inspections": "2026-09-18T18:17:14Z", } +DEFAULT_EXPECTED_ROWS = {"inspections": 15726} PUBLICATION = { "storage_state": "private", "publication_status": "not_eligible", @@ -85,13 +86,52 @@ def discover_exports(input_root: str | Path) -> tuple[dict[str, list[Path]], lis try: profile, _ = _classify(path, adapter) except (OSError, ValueError) as error: - failures.append({"artifact": path.name, "failure_class": type(error).__name__}) + lowered = path.name.lower() + guessed_profile = next((profile for profile, marker in { + "inspections": "inspection", "annual_reports": "annual", "registrations": "registration", + }.items() if marker in lowered), None) + failures.append({ + "artifact": path.name, + "profile": guessed_profile, + "failure_class": type(error).__name__, + "state": "malformed_or_unavailable", + "detail": str(error), + }) continue if profile in {"annual_reports", "registrations", "inspections"}: paths[profile].append(path) return dict(paths), failures +def _coverage_accounting( + profile: str, + manifest: Mapping[str, Any], + failures: Iterable[Mapping[str, Any]], + expected_rows: Mapping[str, int], +) -> dict[str, Any]: + """Return conservative, row-free completeness accounting. + + Expected counts are source/UI observations supplied by the operator, not + proof that the source is complete or current. Residual rows are + ``not_observed``; they are never interpreted as closure or non-use. + """ + expected = expected_rows.get(profile) + observed = int(manifest["input_rows"]) + failed = [item for item in failures if item.get("profile") == profile] + return { + "expected_displayed_rows": expected, + "observed_input_rows": observed, + "accepted_rows": int(manifest["accepted_rows"]), + "adapter_quarantined_rows": int(manifest["adapter_quarantined_rows"]), + "duplicate_page_rows": int(manifest["duplicate_page_rows"]), + "failed_export_count": len(failed), + "failure_states": dict(sorted(Counter(str(item.get("state", "unclassified")) for item in failed).items())), + "not_observed_rows": max(expected - observed, 0) if expected is not None else None, + "accounting_state": "complete" if expected is not None and observed >= expected and not failed else "incomplete", + "not_observed_semantics": "not observed by this acquisition; not closure, non-use, or evidence of absence", + } + + def _load_profile(paths: list[Path], profile: str) -> tuple[list[dict[str, Any]], list[dict[str, Any]], list[dict[str, Any]]]: adapter = AphisPublicSearchAdapter() records: list[dict[str, Any]] = [] @@ -206,11 +246,12 @@ def _verify_controls() -> dict[str, bool]: } -def run_wave2(*, input_root: str | Path, run_dir: str | Path) -> dict[str, Any]: +def run_wave2(*, input_root: str | Path, run_dir: str | Path, expected_rows: Mapping[str, int] | None = None) -> dict[str, Any]: """Run the bounded private proof against saved APHIS exports.""" root = Path(run_dir) root.mkdir(parents=True, exist_ok=True) paths, failures = discover_exports(input_root) + expected_rows = {**DEFAULT_EXPECTED_ROWS, **(expected_rows or {})} if any(not paths.get(profile) for profile in ("registrations", "annual_reports", "inspections")): raise ValueError("input root must contain at least one validated export for each APHIS profile") @@ -289,6 +330,10 @@ def run_wave2(*, input_root: str | Path, run_dir: str | Path) -> dict[str, Any]: "input_root_private": str(Path(input_root).resolve()), "input_failures": failures, "profiles": profile_manifests, + "completeness": { + profile: _coverage_accounting(profile, profile_manifests[profile], failures, expected_rows) + for profile in profile_manifests + }, "graph_projection": { "source_rows_used": {profile: len(rows) for profile, rows in graph_records.items()}, "source_rows_excluded_from_graph_projection": len(projection_quarantine), @@ -334,9 +379,15 @@ def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--input-root", type=Path, required=True) parser.add_argument("--run-dir", type=Path, required=True) + parser.add_argument("--expected-rows", type=Path, help="JSON object of operator-observed displayed row counts") args = parser.parse_args() try: - report = run_wave2(input_root=args.input_root, run_dir=args.run_dir) + expected = None + if args.expected_rows: + expected = json.loads(args.expected_rows.read_text(encoding="utf-8")) + if not isinstance(expected, dict) or any(not isinstance(value, int) or value < 0 for value in expected.values()): + raise ValueError("--expected-rows must contain a JSON object of non-negative integer counts") + report = run_wave2(input_root=args.input_root, run_dir=args.run_dir, expected_rows=expected) except (OSError, ValueError, KeyError) as error: print(json.dumps({"status": "failed", "error": str(error)}, sort_keys=True)) return 2 diff --git a/pipeline/sources/us/accountability/test_aphis_wave2.py b/pipeline/sources/us/accountability/test_aphis_wave2.py index ebc3cad..1ee0400 100644 --- a/pipeline/sources/us/accountability/test_aphis_wave2.py +++ b/pipeline/sources/us/accountability/test_aphis_wave2.py @@ -38,6 +38,9 @@ def test_private_wave_is_bounded_and_idempotent(self): self.assertTrue(report["controls"]["suppressed_rows_do_not_enter_candidates"]) self.assertTrue(report["controls"]["rerun_idempotency_key_equal"]) self.assertFalse(report["controls"]["public_exposure"]) + self.assertEqual(report["completeness"]["inspections"]["expected_displayed_rows"], 15726) + self.assertEqual(report["completeness"]["inspections"]["not_observed_rows"], 15725) + self.assertEqual(report["completeness"]["inspections"]["accounting_state"], "incomplete") self.assertTrue(all( result["profiles"][-1] == query["relationship_profile"] for query in report["bounded_queries"] @@ -48,6 +51,21 @@ def test_private_wave_is_bounded_and_idempotent(self): "not_eligible", ) + def test_operator_expected_count_can_be_overridden_without_claiming_completion(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + input_root = root / "inputs" + input_root.mkdir() + for name, header, row in ( + ("ExportData-registrations.csv", "Account Name,Customer Number,Certificate Number,Registration Type,Certificate Status,Status Date", '"A","2","00-R-0002","Class R - Research Facility","Active","2026-01-01"'), + ("ExportData-annual_reports.csv", "Customer Number,Certificate Number,Year,Dogs,Cats", '"2","00-R-0002","2025","","1"'), + ("ExportData-inspections.csv", "Customer Number,Certificate Number,Inspection Date,Direct NCIs,Non-Critical NCIs,Critical NCIs,Teachable Moments,Site Name,Legal Name,License-Registration Type,City,State,Zip", '"2","00-R-0002","2026-02-01","","","","","S","A","Class R - Research Facility","T","TX","75001"'), + ): + (input_root / name).write_text(header + "\n" + row + "\n", encoding="utf-8") + report = run_wave2(input_root=input_root, run_dir=root / "run", expected_rows={"inspections": 1}) + self.assertEqual(report["completeness"]["inspections"]["not_observed_rows"], 0) + self.assertEqual(report["completeness"]["inspections"]["accounting_state"], "complete") + if __name__ == "__main__": unittest.main() From 0396b5fef5d1daa70758f747e05bda0a80bcfbfa Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Fri, 18 Sep 2026 13:40:55 -0700 Subject: [PATCH 253/311] Add private FSIS recall graph evidence slice --- docs/research/fsis-recall-graph-operator.md | 15 +++ .../sources/us/fsis/fixtures/recalls.json | 1 + pipeline/sources/us/fsis/recall.py | 118 ++++++++++++++++++ pipeline/sources/us/fsis/test_recall.py | 39 ++++++ 4 files changed, 173 insertions(+) create mode 100644 docs/research/fsis-recall-graph-operator.md create mode 100644 pipeline/sources/us/fsis/fixtures/recalls.json create mode 100644 pipeline/sources/us/fsis/recall.py create mode 100644 pipeline/sources/us/fsis/test_recall.py diff --git a/docs/research/fsis-recall-graph-operator.md b/docs/research/fsis-recall-graph-operator.md new file mode 100644 index 0000000..8ffc901 --- /dev/null +++ b/docs/research/fsis-recall-graph-operator.md @@ -0,0 +1,15 @@ +# FSIS recall graph operator slice + +This private adapter stages the FSIS Recall API as government-sourced evidence. A recall is represented as a dated public-health action, not as proof of wrongdoing, ownership, or current operation. Raw and parsed records stay in the private run directory; the row-free aggregate manifest is safe for operator metrics only. + +The only automatic facility candidate is an explicit establishment number supplied by the source record (or an unambiguous establishment-number token in source text). Firm, address, and name-only matches are quarantined. Candidates remain `review_required`, `private`, and `not_eligible`; there is no public graph projection or release path in this slice. + +## Bounded operator query + +For a local run, inspect only the aggregate manifest: + +```powershell +python -c "import json,sys; m=json.load(open(sys.argv[1])); print({'source_id':m['source_id'],'input_rows':m['input_rows'],'accepted_rows':m['accepted_rows'],'quarantined_rows':m['quarantined_rows'],'candidate_count':m['candidate_count'],'publication_status':m['publication_status']})" .\aggregate-manifest.json +``` + +This query answers “how many source rows produced private review candidates?” without printing recall firms, products, addresses, or graph payloads. Human review must assess privacy, source terms, contradictory evidence, and the meaning of the recall before any approval or publication work. diff --git a/pipeline/sources/us/fsis/fixtures/recalls.json b/pipeline/sources/us/fsis/fixtures/recalls.json new file mode 100644 index 0000000..29d66d8 --- /dev/null +++ b/pipeline/sources/us/fsis/fixtures/recalls.json @@ -0,0 +1 @@ +{"results":[{"recall_number":"FSIS-2026-001","recall_date":"2026-09-01","establishment_number":"P12345","firm":"Synthetic Foods LLC","status":"Closed"},{"recall_number":"FSIS-2026-002","recall_date":"2026-09-02","firm":"Name Only Foods","status":"Open"}]} diff --git a/pipeline/sources/us/fsis/recall.py b/pipeline/sources/us/fsis/recall.py new file mode 100644 index 0000000..a51f26f --- /dev/null +++ b/pipeline/sources/us/fsis/recall.py @@ -0,0 +1,118 @@ +"""Private, fail-closed adapter for the FSIS recall API export. + +Recall records are evidence of a public-health action, not findings of +wrongdoing. Establishment-number joins are source-scoped candidates only; +firm/name/address matches are deliberately quarantined. +""" +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +from typing import Any + +from pipeline.common.graph_candidate import build_graph_candidate +from pipeline.common.graph_candidates import write_graph_candidates + + +SOURCE_ID = "us.fsis.recall" +API_URL = "https://www.fsis.usda.gov/fsis/api/recall/v/1" + + +def _text(value: Any) -> str | None: + value = str(value).strip() if value is not None else "" + return value or None + + +def _records(payload: Any) -> list[dict[str, Any]]: + if isinstance(payload, list): + rows = payload + elif isinstance(payload, dict): + rows = payload.get("results") or payload.get("recalls") or payload.get("data") + else: + rows = None + if not isinstance(rows, list) or not rows or not all(isinstance(row, dict) for row in rows): + raise ValueError("FSIS recall payload has no supported record array") + return rows + + +def _establishment_number(row: dict[str, Any]) -> str | None: + for key in ("establishment_number", "establishmentNumber", "establishment", "establishment_no"): + value = _text(row.get(key)) + if value: + return value + text = " ".join(str(row.get(key, "")) for key in ("establishment_name", "firm", "company", "reason")) + # Source text such as "EST. 1234" is retained as an explicit source clue, + # but never treated as a join when multiple identifiers occur. + import re + matches = sorted(set(re.findall(r"\b(?:EST\.?\s*)?(\d{1,6})\b", text, re.I))) + return matches[0] if len(matches) == 1 else None + + +def parse_bytes(content: bytes, *, retrieved_at: str = "unknown-retrieval-date") -> dict[str, Any]: + try: + payload = json.loads(content.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise ValueError("malformed FSIS recall JSON") from exc + rows = _records(payload) + accepted, quarantined = [], [] + for index, row in enumerate(rows, start=1): + recall_id = _text(row.get("recall_number") or row.get("recallNumber") or row.get("recall_id")) + reasons: list[str] = [] + if not recall_id: + reasons.append("missing_recall_identifier") + establishment = _establishment_number(row) + if not establishment: + reasons.append("unresolved_establishment_identifier") + record = { + "source_id": SOURCE_ID, + "source_row": index, + "source_record_key": recall_id or f"row-{index}", + "source_values": row, + "normalized": { + "recall_number": recall_id, + "establishment_number": establishment, + "recall_date": _text(row.get("recall_date") or row.get("recallDate")), + "status": _text(row.get("status") or row.get("recall_status")), + "firm": _text(row.get("firm") or row.get("company") or row.get("establishment_name")), + "retrieved_at": retrieved_at, + "evidence_type": "fsis_recall", + "review_state": "review_required", + "publication_gate": "blocked", + }, + } + (quarantined if reasons else accepted).append({"reasons": reasons, "record": record} if reasons else record) + return {"accepted": accepted, "quarantined": quarantined, "input_rows": len(rows), "source_sha256": hashlib.sha256(content).hexdigest()} + + +def build_recall_candidate(record: dict[str, Any], *, artifact_sha256: str, observed_at: str) -> dict[str, Any]: + normalized = record["normalized"] + if not normalized.get("establishment_number"): + raise ValueError("recall candidate requires an explicit establishment number") + candidate = build_graph_candidate( + {"source_id": SOURCE_ID, "source_record_key": record["source_record_key"], "source_row": record["source_row"], + "source_values": record["source_values"], "normalized": {"establishment_id": normalized["establishment_number"], "name": normalized.get("firm"), "observed_at": observed_at}}, + artifact_sha256=artifact_sha256, observed_at=observed_at) + facility_ref = candidate["facilities"][0]["local_ref"] + candidate["claims"].append({"claim_domain": "violation", "claim_kind": "fsis_recall_action", "facility_ref": facility_ref, + "value_state": "known", "value": {"recall_number": normalized["recall_number"], "status": normalized.get("status"), "recall_date": normalized.get("recall_date"), "evidence_type": "fsis_recall"}, + "observed_at": observed_at, "confidence": None, "review_state": "review_required", + "support": [{"source_record_key": record["source_record_key"]}, {"artifact_sha256": artifact_sha256}]}) + candidate["contradiction_state"] = "none-observed" + from pipeline.contracts.graph_candidate_handoff import validate_graph_candidate + validate_graph_candidate(candidate) + return candidate + + +def write_private_run(content: bytes, run_dir: str | Path, *, retrieved_at: str, source_url: str = API_URL) -> dict[str, Any]: + parsed = parse_bytes(content, retrieved_at=retrieved_at) + digest = hashlib.sha256(content).hexdigest() + root = Path(run_dir); root.mkdir(parents=True, exist_ok=True) + (root / "raw.json").write_bytes(content) + (root / "parsed.json").write_text(json.dumps(parsed, sort_keys=True, indent=2) + "\n", encoding="utf-8") + candidates = [build_recall_candidate(r, artifact_sha256=digest, observed_at=retrieved_at) for r in parsed["accepted"]] + manifest = write_graph_candidates(root / "graph", candidates) + manifest.update({"source_id": SOURCE_ID, "source_url": source_url, "artifact_sha256": digest, "input_rows": parsed["input_rows"], + "accepted_rows": len(parsed["accepted"]), "quarantined_rows": len(parsed["quarantined"]), "publication_status": "not_eligible"}) + (root / "aggregate-manifest.json").write_text(json.dumps(manifest, sort_keys=True, indent=2) + "\n", encoding="utf-8") + return manifest diff --git a/pipeline/sources/us/fsis/test_recall.py b/pipeline/sources/us/fsis/test_recall.py new file mode 100644 index 0000000..4292bbe --- /dev/null +++ b/pipeline/sources/us/fsis/test_recall.py @@ -0,0 +1,39 @@ +import hashlib +import json +import tempfile +import unittest +from pathlib import Path + +from .recall import build_recall_candidate, parse_bytes, write_private_run + +ROOT = Path(__file__).parent + + +class RecallAdapterTests(unittest.TestCase): + def test_explicit_identifier_is_accepted_and_name_only_is_quarantined(self): + raw = (ROOT / "fixtures/recalls.json").read_bytes() + result = parse_bytes(raw, retrieved_at="2026-09-18T00:00:00Z") + self.assertEqual(len(result["accepted"]), 1) + self.assertIn("unresolved_establishment_identifier", result["quarantined"][0]["reasons"]) + + def test_candidate_is_private_and_claim_retains_dates_and_support(self): + raw = (ROOT / "fixtures/recalls.json").read_bytes() + parsed = parse_bytes(raw, retrieved_at="2026-09-18T00:00:00Z") + candidate = build_recall_candidate(parsed["accepted"][0], artifact_sha256=hashlib.sha256(raw).hexdigest(), observed_at="2026-09-18T00:00:00Z") + claim = candidate["claims"][-1] + self.assertEqual(claim["value"]["recall_date"], "2026-09-01") + self.assertEqual(candidate["publication"]["publication_status"], "not_eligible") + self.assertEqual(candidate["review_state"], "review_required") + + def test_run_is_deterministic_and_row_free_manifest(self): + raw = (ROOT / "fixtures/recalls.json").read_bytes() + with tempfile.TemporaryDirectory() as left, tempfile.TemporaryDirectory() as right: + a = write_private_run(raw, left, retrieved_at="2026-09-18T00:00:00Z") + b = write_private_run(raw, right, retrieved_at="2026-09-18T00:00:00Z") + self.assertEqual(a, b) + self.assertEqual(json.loads((Path(left) / "aggregate-manifest.json").read_text())["quarantined_rows"], 1) + self.assertFalse("records" in json.dumps(a)) + + +if __name__ == "__main__": + unittest.main() From 22a7c002c66aa9be2e3d53a5c6528dea6aaee279 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Fri, 18 Sep 2026 13:42:31 -0700 Subject: [PATCH 254/311] Document Brazil environmental and research sources --- docs/country-recon-br.md | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/docs/country-recon-br.md b/docs/country-recon-br.md index bd36b39..b25615c 100644 --- a/docs/country-recon-br.md +++ b/docs/country-recon-br.md @@ -115,3 +115,33 @@ The hashes above are the captured bytes; they can be regenerated from the ignore - Confirm whether SIF registration number reuse, SISBI internal ID lifecycle, ownership changes, and service/establishment status codes have documented historical semantics. - Decide, with human review, what high-level facility fields and coordinate precision are appropriate for publication under the ethics policy. - Confirm Trase's raw-data reuse terms for this project's non-commercial/commercial use and retain Trase attribution/citation separately from MAPA attribution. + +## Additional federal source families (checked 2026-09-18) + +These sources broaden Brazil coverage but do not replace the MAPA establishment sources above. They remain reconnaissance candidates; no raw records were added and no publication permission is implied. + +### IBAMA environmental identity and enforcement + +IBAMA's [CTF/APP page](https://www.gov.br/ibama/pt-br/servicos/cadastros/ctf/ctf-app/ctf-app), updated 26 August 2026, describes the Cadastro Técnico Federal de Atividades Potencialmente Poluidoras e/ou Utilizadoras de Recursos Ambientais as identifying legal and natural persons subject to environmental control and inspection. The [IBAMA open-data catalog](https://dadosabertos.ibama.gov.br/pt_BR/dataset/?license_id=other-pd&organization=ibama&res_format=XML&tags=Cadastro+t%C3%A9cnico+federeal&tags=Fonte+poluidora) lists a `Unidade Poluidora` dataset in HTML, CSV, XML, and JSON and advertises an API. The catalog's public-domain label is source evidence, not a conclusion that every field is safe or necessary to republish. + +The CTF/APP candidate should be modeled as an environmental-registration/legal-entity observation, not as proof of a farm, slaughterhouse, operating status, or animal use. A bounded capture still needs the current schema, record count, identifier lifecycle, update cadence, activity-code interpretation, terms, and privacy review. CNPJ/name/address joins must remain explicit reconciliation events. + +IBAMA also publishes [environmental infraction data](https://dadosabertos.ibama.gov.br/pt_PT/dataset/fiscalizacao-auto-de-infracao) and a [PAMGIA ArcGIS service](https://pamgia.ibama.gov.br/server/rest/services/app_dadosabertos/adm_auto_infracao_p/FeatureServer). The service metadata warns that some post-October 2019 infractions appear more than once when associated with multiple points or attributes, which can inflate counts. Model these as dated enforcement observations keyed by the source auto/process identifiers and preserve the source's duplication warning; never count rows as facilities or infer guilt/finality from an open case. + +The [SICAR/CAR API documentation](https://www.gov.br/conecta/catalogo/apis/sicar-imovel/imovel.yaml/swagger_view) exposes authenticated lookup by CAR property code, including declared property/environmental-status information. The [SICAR demonstrative API page](https://www.gov.br/conecta/catalogo/apis/api-sicar-demonstrativo) says access requires Conecta onboarding and OAuth. CAR is a rural-property/environmental registry, not an operating-farm directory; precise boundaries and property identifiers require restricted handling and must not be silently joined to facilities. + +### CONCEA / MCTI research-animal evidence + +The official [CONCEA animal-use report for 2019–2023](https://www.gov.br/mcti/pt-br/composicao/conselhos/concea/paginas/Destaques/relatorio-de-uso-animal-concea-2019_2023-1-1.pdf) reports 11,390,421 animals used in teaching and research nationally: 2,836,470 in 2019, 2,529,322 in 2020, 2,497,074 in 2021, 2,083,743 in 2022, and 1,443,812 in 2023. The report separates 462,502 teaching uses from 10,927,919 research uses. These are aggregate national statistics, not facility records or a denominator for mapping. + +The [CIAEP accreditation service](https://www.gov.br/pt-br/servicos/credenciamento-institucional-para-atividades-com-animais-em-ensino-ou-pesquisa), last modified 15 December 2025, states that institutional accreditation is required for institutions producing, maintaining, or using animals in teaching or research. The [Lei 11.794 page](https://www.gov.br/mdic/pt-br/acesso-a-informacao/institucional/atos-normativos/leis/lei-no-11-794-de-8-de-outubro-de-2008) describes CONCEA's responsibility to maintain a register of procedures and researchers from CEUA submissions. No stable, public institution-level register or export was verified in this reconnaissance. Treat CIUCA/CIAEP as a future regulatory/institution source family only after confirming a public endpoint, identifiers, scope, and privacy terms. + +### Corporate identity and animal-scale context + +Receita Federal's [open-data repository](https://www.gov.br/receitafederal/dados) documents structured JSON, XML, CSV, ODS, and RDF resources, while its [CNPJ layout documentation](https://www.gov.br/receitafederal/pt-br/acesso-a-informacao/convenios-e-transferencias/compartilhamento-de-bases-de-dados-2013-decreto-no-8-789-2016/leiaute-das-bases/dados-da-base-cnpj) describes non-tax-secret legal-entity fields. A current complete CNPJ release, facility-level semantics, update cadence, and safe redistribution boundary remain unresolved. Use CNPJ only as source evidence and a possible join key, never as proof that a legal address is an operating site. + +IBGE's [Pesquisa da Pecuária Municipal](https://www.ibge.gov.br/estatisticas/economicas/agricultura-e-pecuaria/9107-producao-da-pecuaria-municipal.html) publishes annual animal stocks and animal-product production through municipal, state, regional, and national tables; the current page describes the 2024 release and SIDRA tabulations. PPM is appropriate for aggregate scale and coverage context, not named farms, establishments, coordinates, or facility counts. + +### Reusable infrastructure implications + +Brazil reinforces a country-platform design with separate source-family contracts: inspection establishments; dated product/capacity/authorization observations; environmental registrations; enforcement events; authenticated rural-property records; research-institution accreditation; corporate identity snapshots; and aggregate statistics. Each adapter should declare its record unit, source-local identifiers, status/effective-date semantics, duplicate behavior, access/authentication method, terms, privacy class, and count definition. Shared infrastructure should support CSV/XML/JSON/GeoJSON and authenticated APIs while preserving raw, parsed, normalized, quarantined, reviewed, and released layers. Cross-family joins should be explicit, versioned, and reviewable rather than inferred from names, CNPJ, address, or coordinates. This pattern is suitable for scaling beyond 100 countries because it separates reusable evidence mechanics from country-specific authority and code-list semantics. From 2f796b119fa1381ecb96196bbe94e6efedc7db1c Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Fri, 18 Sep 2026 14:41:18 -0700 Subject: [PATCH 255/311] Harden integrated US evidence review gates --- .../integrated-sprint-review-2026-09-18.md | 34 +++++++++++++++++++ .../sources/us/accountability/aphis_wave2.py | 2 +- .../us/accountability/test_aphis_wave2.py | 17 ++++++++++ pipeline/sources/us/fsis/recall.py | 4 ++- pipeline/sources/us/fsis/test_recall.py | 9 +++++ 5 files changed, 64 insertions(+), 2 deletions(-) create mode 100644 docs/research/integrated-sprint-review-2026-09-18.md diff --git a/docs/research/integrated-sprint-review-2026-09-18.md b/docs/research/integrated-sprint-review-2026-09-18.md new file mode 100644 index 0000000..794e853 --- /dev/null +++ b/docs/research/integrated-sprint-review-2026-09-18.md @@ -0,0 +1,34 @@ +# Integrated sprint review — 2026-09-18 + +This private engineering review covers the integrated APHIS completeness +accounting, FSIS stale-handoff guard, FSIS recall evidence parser, Brazil +source documentation, and the shared country/source platform. No data was +published or promoted. + +## Fixes applied + +- APHIS completeness now remains `incomplete` when duplicate export-page rows + are present, even if the inflated input count reaches the operator-supplied + displayed-row count. Duplicate observations remain quarantined. +- FSIS recall establishment inference now requires an explicit `EST` marker + in free text. Digits in firm names or reasons cannot create a source-local + facility join. + +## Validation + +The focused package-qualified suite passed: 15 tests covering FSIS refresh, +FSIS recall parsing, APHIS Wave 2 accounting, and the US refresh/rehearsal +contracts. A direct `unittest discover -s pipeline/sources/us` invocation also +ran 62 tests successfully but reported two loader errors for relative-import +tests (`test_refresh` and `test_real_rehearsal`); running those modules with +package-qualified names passed them. This is an invocation portability issue, +not a product-test failure. + +## Residual risks + +The APHIS proof remains a bounded operator-saved subset, not a national +completeness or currentness claim. FSIS recalls remain private, review-required +evidence and source-local candidate edges; a recall is not itself a finding of +wrongdoing. Registry coverage, source terms, privacy review, factual review, +project approval, and publication remain separate gates. No public release or +promotion was performed. diff --git a/pipeline/sources/us/accountability/aphis_wave2.py b/pipeline/sources/us/accountability/aphis_wave2.py index ceb9f78..a1efcfa 100644 --- a/pipeline/sources/us/accountability/aphis_wave2.py +++ b/pipeline/sources/us/accountability/aphis_wave2.py @@ -127,7 +127,7 @@ def _coverage_accounting( "failed_export_count": len(failed), "failure_states": dict(sorted(Counter(str(item.get("state", "unclassified")) for item in failed).items())), "not_observed_rows": max(expected - observed, 0) if expected is not None else None, - "accounting_state": "complete" if expected is not None and observed >= expected and not failed else "incomplete", + "accounting_state": "complete" if expected is not None and observed >= expected and not failed and not manifest["duplicate_page_rows"] else "incomplete", "not_observed_semantics": "not observed by this acquisition; not closure, non-use, or evidence of absence", } diff --git a/pipeline/sources/us/accountability/test_aphis_wave2.py b/pipeline/sources/us/accountability/test_aphis_wave2.py index 1ee0400..e03f62e 100644 --- a/pipeline/sources/us/accountability/test_aphis_wave2.py +++ b/pipeline/sources/us/accountability/test_aphis_wave2.py @@ -66,6 +66,23 @@ def test_operator_expected_count_can_be_overridden_without_claiming_completion(s self.assertEqual(report["completeness"]["inspections"]["not_observed_rows"], 0) self.assertEqual(report["completeness"]["inspections"]["accounting_state"], "complete") + def test_duplicate_pages_do_not_claim_completeness(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + input_root = root / "inputs" + input_root.mkdir() + rows = { + "ExportData-registrations.csv": "Account Name,Customer Number,Certificate Number,Registration Type,Certificate Status,Status Date\nA,2,00-R-0002,Class R - Research Facility,Active,2026-01-01\n", + "ExportData-annual_reports.csv": "Customer Number,Certificate Number,Year,Dogs,Cats\n2,00-R-0002,2025,,1\n", + "ExportData-inspections.csv": "Customer Number,Certificate Number,Inspection Date,Direct NCIs,Non-Critical NCIs,Critical NCIs,Teachable Moments,Site Name,Legal Name,License-Registration Type,City,State,Zip\n2,00-R-0002,2026-02-01,,,,,S,A,Class R - Research Facility,T,TX,75001\n", + } + for name, content in rows.items(): + (input_root / name).write_text(content, encoding="utf-8") + (input_root / (Path(name).stem + "-page2.csv")).write_text(content, encoding="utf-8") + report = run_wave2(input_root=input_root, run_dir=root / "run", expected_rows={"inspections": 1}) + self.assertGreater(report["completeness"]["inspections"]["duplicate_page_rows"], 0) + self.assertEqual(report["completeness"]["inspections"]["accounting_state"], "incomplete") + if __name__ == "__main__": unittest.main() diff --git a/pipeline/sources/us/fsis/recall.py b/pipeline/sources/us/fsis/recall.py index a51f26f..2c50a3d 100644 --- a/pipeline/sources/us/fsis/recall.py +++ b/pipeline/sources/us/fsis/recall.py @@ -45,7 +45,9 @@ def _establishment_number(row: dict[str, Any]) -> str | None: # Source text such as "EST. 1234" is retained as an explicit source clue, # but never treated as a join when multiple identifiers occur. import re - matches = sorted(set(re.findall(r"\b(?:EST\.?\s*)?(\d{1,6})\b", text, re.I))) + # Digits in names/reasons are not identity evidence. Require an explicit + # establishment marker before creating a source-local join. + matches = sorted(set(re.findall(r"\bEST\.?\s*(\d{1,6})\b", text, re.I))) return matches[0] if len(matches) == 1 else None diff --git a/pipeline/sources/us/fsis/test_recall.py b/pipeline/sources/us/fsis/test_recall.py index 4292bbe..6eb422c 100644 --- a/pipeline/sources/us/fsis/test_recall.py +++ b/pipeline/sources/us/fsis/test_recall.py @@ -34,6 +34,15 @@ def test_run_is_deterministic_and_row_free_manifest(self): self.assertEqual(json.loads((Path(left) / "aggregate-manifest.json").read_text())["quarantined_rows"], 1) self.assertFalse("records" in json.dumps(a)) + def test_digits_in_free_text_do_not_create_establishment_join(self): + parsed = parse_bytes(json.dumps({"results": [{ + "recall_number": "FSIS-2026-003", + "firm": "Foods 123 LLC", + "reason": "Product code 456", + }]}).encode("utf-8")) + self.assertEqual(len(parsed["accepted"]), 0) + self.assertIn("unresolved_establishment_identifier", parsed["quarantined"][0]["reasons"]) + if __name__ == "__main__": unittest.main() From d1ba77d25748d98cf0f87b7ecc80a2048713c29f Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Fri, 18 Sep 2026 16:20:06 -0700 Subject: [PATCH 256/311] Add review-gated hosted geocoder adapter --- docs/geocoder-sprint-lane-ledger.md | 36 +++++++++ pipeline/geocoding/dawa.py | 19 ++++- pipeline/geocoding/geoapify.py | 95 +++++++++++++++++++++++ pipeline/geocoding/registry.py | 2 + pipeline/tests/test_geocoding_adapters.py | 67 +++++++++++++++- 5 files changed, 216 insertions(+), 3 deletions(-) create mode 100644 docs/geocoder-sprint-lane-ledger.md create mode 100644 pipeline/geocoding/geoapify.py diff --git a/docs/geocoder-sprint-lane-ledger.md b/docs/geocoder-sprint-lane-ledger.md new file mode 100644 index 0000000..9446abb --- /dev/null +++ b/docs/geocoder-sprint-lane-ledger.md @@ -0,0 +1,36 @@ +# Background geocoder sprint lane ledger + +This ledger prevents completed work from being mistaken for active work or +being omitted from consolidation. It records implementation state only; no +entry grants publication approval or authorizes live provider calls. + +## Ready for consolidation or recovery + +| Lane | Evidence | State | Required consolidation action | +| --- | --- | --- | --- | +| Private operator review console | `b74c1a36` | committed, not in `eli/front-end-overhaul` | review and cherry-pick; rerun frontend, Python, Rust, and browser checks | +| Country geocoding reconnaissance | `7dc7fd0a` | committed, not in `eli/front-end-overhaul` | review and cherry-pick provider profiles/schema; resolve overlap with hosted-provider implementation | +| Geoapify adapter | integration worktree changes | implemented locally, focused tests pass, uncommitted | combine with adversarial adapter changes and commit | +| Geocoder adversarial tests | agent worktree at `2f796b1` | tests and narrow DAWA fixes complete; commit blocked by worktree Git metadata permissions | recover patch, review, commit, and run database E2E | + +## In flight + +| Lane | Scope | Exit requirement | +| --- | --- | --- | +| Durable worker | concurrency-safe leasing, stale recovery, retry and daily budget, suppression race protection | committed changes plus focused and database-backed tests | +| Geocoder operator tooling | aggregate status/ETA, safe logs, secret configuration, local/container background operation | committed changes plus privacy tests and operator documentation | + +## Consolidation gate + +1. Every lane is represented by a reviewed commit or explicitly rejected. +2. No API key, address, query, provider payload, or private identifier appears + in repository fixtures, diagnostics, or logs. +3. No live geocoder call occurs in tests or consolidation. +4. Suppression is checked before and after provider work, stale leases recover, + concurrent workers do not duplicate requests, and quota exhaustion pauses + safely. +5. A successful geocode cannot create privacy approval, publication approval, + release membership, or graph certainty. +6. Focused geocoder tests, database E2E, standard gate, frontend tests, and the + Docker E2E suite pass on the consolidated branch. +7. The branch is clean before a checkpoint push and human CI review. diff --git a/pipeline/geocoding/dawa.py b/pipeline/geocoding/dawa.py index 3f98f4e..835ff2d 100644 --- a/pipeline/geocoding/dawa.py +++ b/pipeline/geocoding/dawa.py @@ -1,4 +1,6 @@ import json +import math +import urllib.error import urllib.parse import urllib.request @@ -22,11 +24,24 @@ def geocode(self, query: str) -> GeocodeOutcome: try: with urllib.request.urlopen(request, timeout=self.timeout) as response: payload = json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as error: + retryable = error.code == 429 or 500 <= error.code <= 599 + reason = "provider_rate_limited" if error.code == 429 else "provider_http_error" + if error.code in (401, 403): + reason = "authentication_rejected" + return GeocodeOutcome("failed", "unresolved", None, None, None, None, "dawa_request", retryable, {"error": reason, "status": error.code}) except Exception as error: - return GeocodeOutcome("failed", "unresolved", None, None, None, None, "dawa_request", True, {"error": str(error), "query": query}) + return GeocodeOutcome("failed", "unresolved", None, None, None, None, "dawa_request", True, {"error": type(error).__name__}) + if not isinstance(payload, list): + return GeocodeOutcome("failed", "unresolved", None, None, None, None, "dawa_response", False, {"error": "invalid_provider_schema"}) if len(payload) == 1: result = payload[0] - return GeocodeOutcome("accepted", "accepted_single_point", result.get("y"), result.get("x"), result.get("id"), "address_point", "structured_address", False, payload) + latitude = result.get("y") if isinstance(result, dict) else None + longitude = result.get("x") if isinstance(result, dict) else None + valid = all(isinstance(value, (int, float)) and not isinstance(value, bool) and math.isfinite(value) for value in (latitude, longitude)) + if not valid or not (-90 <= latitude <= 90 and -180 <= longitude <= 180): + return GeocodeOutcome("unresolved", "invalid_coordinates", None, None, None, None, "structured_address", False, payload) + return GeocodeOutcome("accepted", "accepted_single_point", latitude, longitude, result.get("id"), "address_point", "structured_address", False, payload) if len(payload) > 1: return GeocodeOutcome("review_required", "review_multiple_points", None, None, None, None, "structured_address", False, payload) return GeocodeOutcome("unresolved", "unresolved", None, None, None, None, "structured_address", False, payload) diff --git a/pipeline/geocoding/geoapify.py b/pipeline/geocoding/geoapify.py new file mode 100644 index 0000000..cc9f2f3 --- /dev/null +++ b/pipeline/geocoding/geoapify.py @@ -0,0 +1,95 @@ +"""Geoapify adapter for private, review-gated background geocoding. + +The API key is read from the environment and is never included in outcomes, +logs, exceptions, or checked-in configuration. Provider matches are retained as +review-required evidence; a successful request is not publication approval. +""" + +import json +import os +import urllib.error +import urllib.parse +import urllib.request +from math import isfinite +from typing import Callable + +from .base import GeocodeOutcome + + +class GeoapifyAdapter: + provider_id = "geoapify" + + def __init__( + self, + api_key: str | None = None, + timeout: int = 20, + opener: Callable = urllib.request.urlopen, + ): + self.api_key = (api_key or os.environ.get("GEOAPIFY_API_KEY", "")).strip() + if not self.api_key: + raise ValueError("GEOAPIFY_API_KEY is required") + self.timeout = timeout + self.opener = opener + + @staticmethod + def _failed(reason: str, retryable: bool) -> GeocodeOutcome: + return GeocodeOutcome( + "failed", "unresolved", None, None, None, None, + "geoapify_forward", retryable, {"error": reason}, + ) + + def geocode(self, query: str) -> GeocodeOutcome: + query = query.strip() + if not query: + return self._failed("empty_query", False) + params = urllib.parse.urlencode({"text": query, "format": "geojson", "limit": 2, "apiKey": self.api_key}) + request = urllib.request.Request( + "https://api.geoapify.com/v1/geocode/search?" + params, + headers={"User-Agent": "UntilEveryCage/2 geocoding-worker"}, + ) + try: + with self.opener(request, timeout=self.timeout) as response: + payload = json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as error: + if error.code in (401, 403): + return self._failed("authentication_rejected", False) + if error.code == 429: + return self._failed("provider_rate_limited", True) + return self._failed("provider_http_error", 500 <= error.code < 600) + except (OSError, TimeoutError, json.JSONDecodeError, UnicodeDecodeError): + return self._failed("provider_transport_or_payload_error", True) + + if not isinstance(payload, dict) or not isinstance(payload.get("features"), list): + return self._failed("invalid_provider_schema", False) + features = payload["features"] + if not features: + return GeocodeOutcome( + "unresolved", "unresolved", None, None, None, None, + "geoapify_forward", False, payload, + ) + feature = features[0] + if not isinstance(feature, dict): + return self._failed("invalid_provider_feature", False) + geometry = feature.get("geometry") + properties = feature.get("properties") + if not isinstance(geometry, dict) or not isinstance(properties, dict): + return self._failed("invalid_provider_feature", False) + coordinates = geometry.get("coordinates") + if not isinstance(coordinates, list) or len(coordinates) != 2: + return self._failed("invalid_provider_coordinates", False) + longitude, latitude = coordinates + if not isinstance(longitude, (int, float)) or not isinstance(latitude, (int, float)): + return self._failed("invalid_provider_coordinates", False) + if not isfinite(longitude) or not isfinite(latitude) or not (-180 <= longitude <= 180) or not (-90 <= latitude <= 90): + return self._failed("invalid_provider_coordinates", False) + + result_type = properties.get("result_type") + precision = result_type if isinstance(result_type, str) else "unknown" + place_id = properties.get("place_id") + if not isinstance(place_id, str): + place_id = None + acceptance = "review_multiple_points" if len(features) > 1 else "review_provider_candidate" + return GeocodeOutcome( + "review_required", acceptance, float(latitude), float(longitude), + place_id, precision, "geoapify_forward", False, payload, + ) diff --git a/pipeline/geocoding/registry.py b/pipeline/geocoding/registry.py index 2f27058..70d80f2 100644 --- a/pipeline/geocoding/registry.py +++ b/pipeline/geocoding/registry.py @@ -2,10 +2,12 @@ from .base import GeocoderAdapter from .dawa import DawaAdapter +from .geoapify import GeoapifyAdapter ADAPTER_FACTORIES: dict[str, Callable[[], GeocoderAdapter]] = { "dawa": DawaAdapter, + "geoapify": GeoapifyAdapter, } diff --git a/pipeline/tests/test_geocoding_adapters.py b/pipeline/tests/test_geocoding_adapters.py index 8458095..7a5bc6d 100644 --- a/pipeline/tests/test_geocoding_adapters.py +++ b/pipeline/tests/test_geocoding_adapters.py @@ -1,4 +1,6 @@ import json +import socket +import urllib.error import unittest from pathlib import Path from unittest.mock import patch @@ -6,6 +8,7 @@ ROOT = Path(__file__).parents[1] from pipeline.geocoding import dawa as MODULE +from pipeline.geocoding import geoapify from pipeline.geocoding.registry import get_adapter @@ -50,7 +53,69 @@ def test_transport_failure_is_retryable(self, _urlopen): result = MODULE.DawaAdapter().geocode("Testvej 1, 1000, København, Denmark") self.assertEqual(result.status, "failed") self.assertTrue(result.retryable) - self.assertIn("connection refused", result.response["error"]) + self.assertEqual(result.response, {"error": "OSError"}) + + +class GeoapifyAdapterTests(unittest.TestCase): + def test_key_is_required(self): + with patch.dict("os.environ", {}, clear=True): + with self.assertRaisesRegex(ValueError, "GEOAPIFY_API_KEY"): + geoapify.GeoapifyAdapter() + + def test_candidate_is_stored_but_requires_review(self): + payload = {"type": "FeatureCollection", "features": [{ + "type": "Feature", + "geometry": {"type": "Point", "coordinates": [12.5, 55.6]}, + "properties": {"place_id": "candidate-1", "result_type": "building"}, + }]} + opener = unittest.mock.Mock(return_value=FakeResponse(payload)) + result = geoapify.GeoapifyAdapter(api_key="test-secret", opener=opener).geocode("Example address") + self.assertEqual(result.status, "review_required") + self.assertEqual((result.latitude, result.longitude), (55.6, 12.5)) + self.assertEqual(result.provider_address_id, "candidate-1") + self.assertNotIn("test-secret", json.dumps(result.response)) + + def test_invalid_coordinates_fail_closed(self): + payload = {"features": [{"geometry": {"coordinates": [999, 55]}, "properties": {}}]} + result = geoapify.GeoapifyAdapter(api_key="test-secret", opener=unittest.mock.Mock(return_value=FakeResponse(payload))).geocode("Example") + self.assertEqual(result.status, "failed") + self.assertFalse(result.retryable) + self.assertEqual(result.response, {"error": "invalid_provider_coordinates"}) + + @patch.object(MODULE.urllib.request, "urlopen") + def test_malformed_payload_fails_closed(self, urlopen): + urlopen.return_value = FakeResponse({"features": []}) + result = MODULE.DawaAdapter().geocode("Testvej 1, 1000, København, Denmark") + self.assertEqual(result.status, "failed") + self.assertFalse(result.retryable) + self.assertEqual(result.response, {"error": "invalid_provider_schema"}) + + @patch.object(MODULE.urllib.request, "urlopen") + def test_invalid_coordinates_are_not_accepted(self, urlopen): + urlopen.return_value = FakeResponse([{"id": "bad", "x": 181, "y": float("nan")}]) + result = MODULE.DawaAdapter().geocode("Testvej 1, 1000, København, Denmark") + self.assertEqual(result.status, "unresolved") + self.assertEqual(result.acceptance, "invalid_coordinates") + + @patch.object(MODULE.urllib.request, "urlopen") + def test_http_retry_classification_is_bounded(self, urlopen): + for status, retryable in ((401, False), (403, False), (429, True), (500, True), (503, True)): + urlopen.side_effect = urllib.error.HTTPError("https://example.invalid", status, "failure", {}, None) + result = MODULE.DawaAdapter().geocode("private query") + self.assertEqual(result.response["status"], status) + self.assertEqual(result.retryable, retryable) + self.assertNotIn("private query", json.dumps(result.response)) + + @patch.object(MODULE.urllib.request, "urlopen", side_effect=socket.timeout("timed out private query")) + def test_timeout_is_retryable_and_redacted(self, _urlopen): + result = MODULE.DawaAdapter().geocode("private query") + self.assertEqual(result.status, "failed") + self.assertTrue(result.retryable) + self.assertNotIn("private query", json.dumps(result.response)) + + def test_registry_constructs_geoapify_from_environment(self): + with patch.dict("os.environ", {"GEOAPIFY_API_KEY": "test-secret"}): + self.assertIsInstance(get_adapter("geoapify"), geoapify.GeoapifyAdapter) if __name__ == "__main__": From 316d549bdbaa54a04bd21f1a3c41e263d5ce2c5c Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Fri, 18 Sep 2026 15:38:44 -0700 Subject: [PATCH 257/311] Add private candidate review console --- .../modules/__tests__/privateReview.test.js | 51 + static/private-review.css | 267 + static/private-review.html | 212 + static/private-review.js | 573 + static/private-review/readiness-matrix.json | 28611 ++++++++++++++++ 5 files changed, 29714 insertions(+) create mode 100644 static/modules/__tests__/privateReview.test.js create mode 100644 static/private-review.css create mode 100644 static/private-review.html create mode 100644 static/private-review.js create mode 100644 static/private-review/readiness-matrix.json diff --git a/static/modules/__tests__/privateReview.test.js b/static/modules/__tests__/privateReview.test.js new file mode 100644 index 0000000..1fb96e0 --- /dev/null +++ b/static/modules/__tests__/privateReview.test.js @@ -0,0 +1,51 @@ +import { readFileSync } from 'node:fs'; +import { resolve } from 'node:path'; + +const root = (file) => readFileSync(resolve(process.cwd(), file), 'utf8'); + +test('private review console is read-only and uses the existing authenticated read contracts', () => { + const html = root('static/private-review.html'); + const js = root('static/private-review.js'); + const css = root('static/private-review.css'); + + expect(html).toContain('PRIVATE ONLY'); + expect(html).toContain('X-UEC-Dev-Preview-Token'); + expect(html).toContain('X-UEC-Private-Graph-Token'); + expect(js).toContain('/api/dev/preview/candidates'); + expect(js).toContain('/api/private/graph/queues/'); + expect(js).toContain('/api/private/graph/entities'); + expect(js).toContain('/neighborhood'); + expect(js).toContain('no-store'); + expect(js).not.toMatch(/localStorage|sessionStorage/); + expect(js).not.toMatch(/method\s*:\s*['"](?:POST|PUT|PATCH|DELETE)['"]/i); + expect(html).not.toMatch(/]*>\s*(?:publish|promote|approve|release|export)\b/i); + expect(css).toContain('.overview-rail'); + expect(css).toContain('.readiness-matrix'); + expect(css).toContain('@media'); +}); + +test('readiness asset validates the current registry-driven country set with exact states', () => { + const matrix = JSON.parse(root('static/private-review/readiness-matrix.json')); + const states = ['infrastructure-only', 'acquisition-ready', 'private-candidate-ready', 'human-review-ready', 'publication-eligible', 'blocked']; + expect(matrix.derived_context).toBe(true); + expect(matrix.states).toEqual(states); + expect(Number.isInteger(matrix.country_count)).toBe(true); + const countries = Object.entries(matrix.countries); + expect(countries).toHaveLength(matrix.country_count); + expect(countries.length).toBeGreaterThan(0); + for (const [countryCode, country] of countries) { + expect(countryCode).toMatch(/^[A-Z]{2}$/); + expect(states).toContain(country.state); + expect(country).toEqual(expect.objectContaining({ name: expect.any(String), summary: expect.any(String), basis: expect.any(Array) })); + } +}); + +test('safe review page never includes private address, raw payload, geocoder, or requester fields', () => { + const js = root('static/private-review.js'); + const combined = `${root('static/private-review.html')}\n${js}`.toLowerCase(); + for (const forbidden of ['requester details', 'raw source payloads', 'geocoder requests/responses', 'private addresses']) { + expect(combined).toContain(forbidden); + } + expect(js).toContain('Withheld by console'); + expect(js.toLowerCase()).toContain('observations remain separate'); +}); diff --git a/static/private-review.css b/static/private-review.css new file mode 100644 index 0000000..75e9866 --- /dev/null +++ b/static/private-review.css @@ -0,0 +1,267 @@ +:root { + --paper: #f5f1e9; + --paper-deep: #e9e3d8; + --surface: #fffdf8; + --surface-muted: #f1eee7; + --ink: #17382d; + --ink-soft: #52665e; + --ink-faint: #7c8b83; + --line: #d8d8cb; + --line-strong: #bfc8be; + --forest: #1c5a45; + --forest-dark: #103e30; + --green-bg: #e6f1e8; + --green: #2d7152; + --amber-bg: #fff0d7; + --amber: #9a5a16; + --slate-bg: #e8eef3; + --slate: #4a6578; + --red-bg: #f8e5de; + --red: #98442f; + font: 15px/1.45 system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; + color: var(--ink); + background: var(--paper); +} + +* { box-sizing: border-box; } +html { scroll-behavior: smooth; } +body { margin: 0; min-width: 320px; background: var(--paper); } +button, input, select { font: inherit; } +button { cursor: pointer; } +code { font: .88em ui-monospace, SFMono-Regular, Consolas, monospace; color: var(--forest-dark); } + +.skip-link { position: absolute; left: -999px; top: 8px; z-index: 10; padding: 8px 12px; background: var(--forest-dark); color: #fff; } +.skip-link:focus { left: 8px; } +.app-shell { max-width: 1500px; margin: 0 auto; padding: 30px clamp(16px, 4vw, 56px) 38px; } +.masthead { display: flex; justify-content: space-between; gap: 28px; align-items: flex-end; padding: 10px 0 26px; } +.eyebrow { margin: 0 0 8px; color: var(--forest); font-size: 11px; font-weight: 800; letter-spacing: .14em; text-transform: uppercase; } +h1, h2, h3, h4, p { margin-top: 0; } +h1 { max-width: 780px; margin-bottom: 9px; color: var(--ink); font-size: clamp(2.35rem, 5vw, 4.85rem); line-height: .96; letter-spacing: -.055em; font-weight: 760; } +h2 { margin-bottom: 3px; font-size: clamp(1.2rem, 2vw, 1.65rem); letter-spacing: -.02em; } +h3 { margin-bottom: 0; font-size: 1rem; } +.lede { max-width: 680px; margin-bottom: 0; color: var(--ink-soft); font-size: 1.03rem; } +.masthead-status { display: grid; justify-items: end; gap: 7px; padding-bottom: 4px; text-align: right; } +.scope-badge, .section-tag, .quiet-label { display: inline-flex; align-items: center; width: fit-content; border-radius: 999px; font-size: 10px; font-weight: 800; letter-spacing: .09em; text-transform: uppercase; } +.scope-badge { padding: 7px 10px; background: var(--forest-dark); color: #fff; } +.scope-detail, .quiet-label { color: var(--ink-faint); font-size: 12px; } +.safety-banner { display: flex; gap: 12px; align-items: flex-start; margin-bottom: 19px; padding: 14px 17px; border: 1px solid #e3c792; border-left: 5px solid var(--amber); background: var(--amber-bg); color: #704410; } +.safety-banner strong { margin-right: 6px; color: #5f3910; } +.banner-mark { display: inline-grid; flex: 0 0 22px; place-items: center; width: 22px; height: 22px; border-radius: 50%; background: var(--amber); color: #fff; font-weight: 850; } +.panel { border: 1px solid var(--line); background: var(--surface); box-shadow: 0 5px 20px rgba(22, 49, 39, .035); } +.auth-card { margin-bottom: 19px; padding: 19px 21px 14px; } +.section-heading { display: flex; justify-content: space-between; gap: 18px; align-items: flex-end; margin-bottom: 14px; } +.compact-heading { align-items: center; margin-bottom: 15px; } +.auth-grid { display: grid; grid-template-columns: minmax(230px, 1fr) minmax(230px, 1fr) auto; gap: 14px; align-items: end; } +.field { display: grid; gap: 6px; color: var(--ink); font-size: 13px; font-weight: 700; } +.field small { color: var(--ink-faint); font-weight: 500; } +.field input, .inline-field select { width: 100%; border: 1px solid var(--line-strong); border-radius: 5px; background: #fffefa; color: var(--ink); outline: none; } +.field input { padding: 10px 11px; } +.field input:focus, .inline-field select:focus { border-color: var(--forest); box-shadow: 0 0 0 3px rgba(28, 90, 69, .12); } +.field-help { color: var(--ink-faint); font-size: 11px; font-weight: 500; } +.auth-actions { display: flex; flex-wrap: wrap; gap: 7px; } +.button { min-height: 39px; padding: 9px 12px; border: 1px solid transparent; border-radius: 5px; font-weight: 750; transition: background .15s ease, border-color .15s ease, transform .15s ease; } +.button:hover { transform: translateY(-1px); } +.button:focus-visible, .rail-link:focus-visible, .queue-tab:focus-visible, .candidate-item:focus-visible, .entity-item:focus-visible { outline: 3px solid rgba(28, 90, 69, .25); outline-offset: 2px; } +.button-primary { background: var(--forest); color: #fff; } +.button-primary:hover { background: var(--forest-dark); } +.button-secondary { border-color: var(--forest); background: var(--green-bg); color: var(--forest-dark); } +.button-secondary:hover { background: #d6e9da; } +.button-quiet { border-color: var(--line); background: transparent; color: var(--ink-soft); } +.button-quiet:hover { background: var(--surface-muted); } +.auth-status-row { display: flex; flex-wrap: wrap; gap: 18px; margin-top: 15px; padding-top: 12px; border-top: 1px solid var(--line); color: var(--ink-faint); font-size: 12px; } +.auth-status { display: inline-flex; gap: 6px; align-items: center; } +.status-dot { display: inline-block; width: 7px; height: 7px; border-radius: 50%; background: var(--line-strong); } +.auth-status[data-state="ready"] .status-dot { background: var(--green); } +.auth-status[data-state="loading"] .status-dot { background: var(--amber); } +.auth-status[data-state="error"] .status-dot { background: var(--red); } + +.summary-grid { display: grid; grid-template-columns: repeat(4, 1fr); gap: 10px; margin-bottom: 23px; } +.metric-card { min-height: 126px; padding: 16px 17px; border: 1px solid var(--line); background: var(--surface); } +.metric-card-amber { border-top: 3px solid var(--amber); } +.metric-card-slate { border-top: 3px solid var(--slate); } +.metric-label { margin-bottom: 7px; color: var(--ink-faint); font-size: 11px; font-weight: 800; letter-spacing: .1em; text-transform: uppercase; } +.metric-value { display: block; margin-bottom: 5px; font-size: 2rem; line-height: 1; letter-spacing: -.04em; } +.metric-note { display: block; color: var(--ink-soft); font-size: 12px; } + +.console-layout { display: grid; grid-template-columns: 220px minmax(0, 1fr); gap: 27px; align-items: start; } +.overview-rail { position: sticky; top: 18px; display: grid; gap: 12px; } +.rail-card { padding: 15px; } +.rail-nav { display: grid; gap: 4px; } +.rail-link { display: grid; grid-template-columns: 24px minmax(0, 1fr) auto; gap: 7px; align-items: center; padding: 9px 8px; border-left: 2px solid transparent; color: var(--ink-soft); text-decoration: none; font-size: 12px; } +.rail-link span:first-child { color: var(--ink-faint); font-size: 10px; font-weight: 800; } +.rail-link b { color: var(--ink-faint); font-size: 10px; font-weight: 600; } +.rail-link:hover, .rail-link.active { border-left-color: var(--forest); background: var(--green-bg); color: var(--forest-dark); } +.rail-link.active b { color: var(--forest); } +.rail-note p { color: var(--ink-soft); font-size: 12px; } +.mini-list { display: grid; gap: 7px; margin: 15px 0 0; } +.mini-list div { display: flex; justify-content: space-between; gap: 8px; padding-top: 7px; border-top: 1px solid var(--line); } +.mini-list dt, .mini-list dd { margin: 0; font-size: 11px; } +.mini-list dt { color: var(--ink-faint); } +.mini-list dd { color: var(--ink); font-weight: 700; text-align: right; } +.review-content { min-width: 0; } +.content-section { scroll-margin-top: 16px; margin-bottom: 44px; } +.section-tag { padding: 6px 9px; background: var(--slate-bg); color: var(--slate); } + +.candidate-workspace { display: grid; grid-template-columns: minmax(230px, .38fr) minmax(0, .62fr); min-height: 410px; } +.candidate-list-column { border-right: 1px solid var(--line); } +.candidate-list-column, .candidate-detail { min-width: 0; } +.list-heading { display: flex; justify-content: space-between; gap: 10px; align-items: center; padding: 14px 16px; border-bottom: 1px solid var(--line); } +.list-heading > span { color: var(--ink-faint); font-size: 11px; } +.candidate-list, .entity-list { padding: 6px; } +.candidate-item, .entity-item { display: grid; width: 100%; gap: 3px; padding: 12px 10px; border: 0; border-bottom: 1px solid var(--line); background: transparent; color: var(--ink); text-align: left; } +.candidate-item:hover, .candidate-item.active, .entity-item:hover, .entity-item.active { background: var(--green-bg); } +.candidate-item strong, .entity-item strong { overflow: hidden; color: var(--forest-dark); text-overflow: ellipsis; white-space: nowrap; } +.candidate-item small, .entity-item small { color: var(--ink-faint); font-size: 11px; } +.candidate-item .item-state, .entity-item .item-state { color: var(--amber); font-size: 10px; font-weight: 800; letter-spacing: .06em; text-transform: uppercase; } +.candidate-detail { padding: 21px; } +.detail-title { display: flex; justify-content: space-between; gap: 20px; align-items: flex-start; padding-bottom: 17px; border-bottom: 1px solid var(--line); } +.detail-title h3 { max-width: 660px; margin-bottom: 5px; font-size: 1.5rem; letter-spacing: -.025em; } +.detail-subtitle { margin-bottom: 0; color: var(--ink-soft); } +.detail-badges { display: flex; flex: 0 0 auto; flex-wrap: wrap; justify-content: flex-end; gap: 5px; } +.badge { display: inline-flex; align-items: center; padding: 5px 7px; border-radius: 3px; color: var(--ink-soft); background: var(--surface-muted); font-size: 10px; font-weight: 800; letter-spacing: .06em; text-transform: uppercase; } +.badge-green { background: var(--green-bg); color: var(--green); } +.badge-amber { background: var(--amber-bg); color: var(--amber); } +.badge-red { background: var(--red-bg); color: var(--red); } +.badge-slate { background: var(--slate-bg); color: var(--slate); } +.detail-grid { display: grid; grid-template-columns: repeat(2, minmax(0, 1fr)); gap: 13px; margin-top: 18px; } +.detail-card { min-width: 0; padding: 14px; border: 1px solid var(--line); background: #fffefa; } +.detail-card-wide { grid-column: 1 / -1; } +.detail-card h4 { margin-bottom: 11px; color: var(--forest); font-size: 11px; letter-spacing: .1em; text-transform: uppercase; } +.facts { display: grid; gap: 8px; margin: 0; } +.facts div { display: grid; grid-template-columns: minmax(105px, .8fr) minmax(0, 1.2fr); gap: 10px; padding-bottom: 7px; border-bottom: 1px dashed var(--line); } +.facts div:last-child { padding-bottom: 0; border-bottom: 0; } +.facts dt, .facts dd { margin: 0; overflow-wrap: anywhere; font-size: 12px; } +.facts dt { color: var(--ink-faint); } +.facts dd { color: var(--ink); font-weight: 650; } +.unknown { color: var(--ink-faint) !important; font-weight: 500 !important; font-style: italic; } +.blocker-list { display: grid; gap: 6px; margin: 0; padding: 0; list-style: none; } +.blocker-list li { position: relative; padding-left: 16px; color: var(--ink-soft); font-size: 12px; } +.blocker-list li::before { position: absolute; left: 0; top: .55em; width: 6px; height: 6px; border-radius: 50%; background: var(--amber); content: ""; } +.notice-small { margin: 12px 0 0; color: var(--ink-faint); font-size: 11px; } + +.queue-tabs { display: flex; flex-wrap: wrap; gap: 6px; margin-bottom: 8px; } +.queue-tab { display: inline-flex; gap: 7px; align-items: center; padding: 8px 10px; border: 1px solid var(--line); background: var(--surface-muted); color: var(--ink-soft); font-size: 12px; font-weight: 750; } +.queue-tab b { min-width: 18px; padding: 1px 4px; border-radius: 99px; background: var(--surface); color: var(--ink-faint); font-size: 10px; text-align: center; } +.queue-tab:hover, .queue-tab.active { border-color: var(--forest); background: var(--green-bg); color: var(--forest-dark); } +.queue-tab.active b { background: var(--forest); color: #fff; } +.queue-panel { min-height: 165px; padding: 0; overflow: hidden; } +.queue-table-wrap { overflow-x: auto; } +.queue-table { width: 100%; border-collapse: collapse; font-size: 12px; } +.queue-table th, .queue-table td { padding: 11px 13px; border-bottom: 1px solid var(--line); text-align: left; vertical-align: top; white-space: nowrap; } +.queue-table th { background: var(--surface-muted); color: var(--ink-faint); font-size: 10px; letter-spacing: .08em; text-transform: uppercase; } +.queue-table td { color: var(--ink-soft); } +.queue-table tr:last-child td { border-bottom: 0; } +.queue-table .value-strong { color: var(--ink); font-weight: 700; } +.queue-table .value-amber { color: var(--amber); font-weight: 700; } +.queue-error { padding: 18px; color: var(--red); background: var(--red-bg); } + +.matrix-intro { display: grid; grid-template-columns: minmax(0, 1fr) auto; gap: 15px; align-items: center; padding: 14px 16px; } +.matrix-intro p { margin-bottom: 0; color: var(--ink-soft); font-size: 12px; } +.legend { display: flex; flex-wrap: wrap; justify-content: flex-end; gap: 5px; } +.legend-item { padding: 5px 7px; border-radius: 3px; background: var(--surface-muted); color: var(--ink-soft); font-size: 10px; font-weight: 750; } +.readiness-matrix { display: grid; grid-template-columns: repeat(auto-fit, minmax(205px, 1fr)); gap: 1px; margin-top: 8px; padding: 1px; background: var(--line); } +.readiness-card { display: grid; gap: 9px; min-height: 165px; padding: 15px; background: var(--surface); } +.readiness-card h3 { font-size: 1.2rem; } +.readiness-card p { margin-bottom: 0; color: var(--ink-soft); font-size: 12px; } +.readiness-state { display: inline-flex; width: fit-content; padding: 4px 6px; border-radius: 3px; font-size: 10px; font-weight: 850; letter-spacing: .06em; text-transform: uppercase; } +.readiness-card .state-infrastructure-only, .readiness-card .state-acquisition-ready { background: var(--slate-bg); color: var(--slate); } +.readiness-card .state-private-candidate-ready, .readiness-card .state-human-review-ready { background: var(--green-bg); color: var(--green); } +.readiness-card .state-publication-eligible { background: #e9f2dc; color: #4d6f2b; } +.readiness-card .state-blocked { background: var(--amber-bg); color: var(--amber); } +.readiness-card .basis { padding-top: 9px; border-top: 1px dashed var(--line); color: var(--ink-faint); font-size: 11px; } + +.graph-search { margin-bottom: 9px; padding: 16px; } +.search-row { display: flex; gap: 10px; align-items: end; } +.search-field { flex: 1; } +.control-row { display: flex; flex-wrap: wrap; gap: 13px; align-items: end; margin-top: 12px; } +.inline-field { display: flex; gap: 7px; align-items: center; color: var(--ink-soft); font-size: 12px; } +.inline-field select { width: auto; padding: 6px 8px; } +.inline-status { margin: 13px 0 0; color: var(--ink-faint); font-size: 12px; } +.graph-help { margin-left: auto; } +.graph-workspace { display: grid; grid-template-columns: minmax(230px, .36fr) minmax(0, .64fr); gap: 9px; align-items: start; } +.entity-results { min-height: 270px; } +.neighborhood-panel { min-height: 270px; padding: 18px; } +.neighborhood-header { display: flex; justify-content: space-between; gap: 12px; align-items: flex-start; padding-bottom: 14px; border-bottom: 1px solid var(--line); } +.neighborhood-header h3 { margin-bottom: 3px; font-size: 1.25rem; } +.neighborhood-header p { margin-bottom: 0; color: var(--ink-soft); font-size: 12px; } +.observation-group { margin-top: 16px; } +.observation-group h4 { display: flex; gap: 8px; align-items: center; margin-bottom: 7px; color: var(--forest); font-size: 11px; letter-spacing: .1em; text-transform: uppercase; } +.observation-group h4 span { color: var(--ink-faint); font-size: 10px; letter-spacing: 0; } +.observation { margin-bottom: 7px; padding: 12px; border: 1px solid var(--line); background: #fffefa; } +.observation-head { display: flex; justify-content: space-between; gap: 10px; margin-bottom: 7px; } +.observation-head strong { color: var(--ink); font-size: 12px; } +.observation-head span { color: var(--amber); font-size: 10px; font-weight: 800; text-transform: uppercase; } +.observation p { margin-bottom: 7px; color: var(--ink-soft); font-size: 12px; } +.observation-meta { display: flex; flex-wrap: wrap; gap: 5px 10px; color: var(--ink-faint); font-size: 10px; } +.observation-meta span { padding-right: 10px; border-right: 1px solid var(--line); } +.observation-meta span:last-child { border-right: 0; } + +.packet-loader { grid-column: 1 / -1; max-width: 620px; } +.packet-summary { margin-top: 8px; padding: 16px; } +.packet-heading { display: flex; justify-content: space-between; gap: 15px; align-items: flex-start; padding-bottom: 12px; border-bottom: 1px solid var(--line); } +.packet-heading h3 { margin-bottom: 0; } +.packet-grid { display: grid; grid-template-columns: repeat(4, minmax(0, 1fr)); gap: 8px; margin-top: 12px; } +.packet-card { min-width: 0; padding: 12px; border: 1px solid var(--line); background: #fffefa; } +.packet-card h4 { margin-bottom: 9px; color: var(--forest); font-size: 10px; letter-spacing: .1em; text-transform: uppercase; } +.packet-facts, .packet-facts li { list-style: none; margin: 0; padding: 0; } +.packet-facts { display: grid; gap: 6px; } +.packet-facts li { display: flex; justify-content: space-between; gap: 7px; padding-bottom: 5px; border-bottom: 1px dashed var(--line); color: var(--ink-faint); font-size: 11px; } +.packet-facts li:last-child { border-bottom: 0; } +.packet-facts strong { color: var(--ink); overflow-wrap: anywhere; text-align: right; } +.readiness-sources { margin-top: 8px; padding-top: 8px; border-top: 1px dashed var(--line); color: var(--ink-soft); font-size: 11px; } +.readiness-sources summary { cursor: pointer; color: var(--forest); font-weight: 750; } +.readiness-source { display: grid; gap: 3px; margin-top: 8px; padding-top: 7px; border-top: 1px solid var(--line); } +.readiness-source strong { overflow-wrap: anywhere; color: var(--ink); } +.readiness-source small { color: var(--ink-faint); overflow-wrap: anywhere; } + +.empty-state { display: grid; gap: 5px; padding: 25px 16px; color: var(--ink-faint); font-size: 12px; } +.empty-state strong { color: var(--ink); font-size: 13px; } +.empty-state-large { min-height: 220px; place-content: center; text-align: center; } +.empty-icon { color: var(--forest); font-size: 2rem; } +.error-state { padding: 18px; color: var(--red); background: var(--red-bg); font-size: 12px; } +.page-footer { display: flex; justify-content: space-between; gap: 20px; margin-top: 16px; padding-top: 16px; border-top: 1px solid var(--line); color: var(--ink-faint); font-size: 11px; } + +@media (max-width: 1100px) { + .auth-grid { grid-template-columns: repeat(2, minmax(0, 1fr)); } + .auth-actions { grid-column: 1 / -1; } + .console-layout { grid-template-columns: 185px minmax(0, 1fr); gap: 18px; } + .candidate-workspace, .graph-workspace { grid-template-columns: minmax(205px, .42fr) minmax(0, .58fr); } +} +@media (max-width: 760px) { + .app-shell { padding-top: 20px; } + .masthead, .section-heading, .detail-title { display: block; } + .masthead-status { justify-items: start; margin-top: 17px; text-align: left; } + .scope-detail { display: block; } + .section-tag { margin-top: 9px; } + .summary-grid { grid-template-columns: repeat(2, 1fr); } + .console-layout { display: block; } + .overview-rail { position: static; margin-bottom: 22px; } + .rail-nav { grid-template-columns: repeat(2, 1fr); } + .rail-note { display: none; } + .candidate-workspace, .graph-workspace { grid-template-columns: 1fr; } + .candidate-list-column { border-right: 0; border-bottom: 1px solid var(--line); } + .candidate-list { display: grid; grid-template-columns: repeat(2, minmax(0, 1fr)); } + .candidate-detail { min-height: 390px; } + .detail-grid { grid-template-columns: 1fr; } + .detail-card-wide { grid-column: auto; } + .matrix-intro { display: block; } + .legend { justify-content: flex-start; margin-top: 12px; } + .graph-help { margin-left: 0; flex-basis: 100%; } + .packet-grid { grid-template-columns: repeat(2, minmax(0, 1fr)); } + .page-footer { display: block; } + .page-footer span { display: block; margin-bottom: 6px; } +} +@media (max-width: 500px) { + h1 { font-size: 2.55rem; } + .auth-grid, .summary-grid { grid-template-columns: 1fr; } + .auth-actions { grid-column: auto; } + .auth-actions .button { flex: 1 1 auto; } + .packet-grid { grid-template-columns: 1fr; } + .candidate-list { display: block; } + .search-row { display: block; } + .search-row .button { width: 100%; margin-top: 9px; } + .control-row { align-items: flex-start; } + .detail-title { padding-bottom: 14px; } + .detail-badges { justify-content: flex-start; margin-top: 11px; } + .facts div { grid-template-columns: 1fr; gap: 2px; } +} diff --git a/static/private-review.html b/static/private-review.html new file mode 100644 index 0000000..f3ad185 --- /dev/null +++ b/static/private-review.html @@ -0,0 +1,212 @@ + + + + + + + Private candidate review console + + + + +
    +
    +
    +

    RESTRICTED / OPERATOR CONSOLE

    +

    Private candidate review

    +

    A read-only workspace for inspecting candidate evidence before any separate, accountable human decision.

    +
    +
    + PRIVATE ONLY + No public links · no mutation controls +
    +
    + +
    + +
    + Restricted evidence surface. + Candidate rows and graph observations are not a release, do not establish ownership or operational status, and must not be used to target people or locations. +
    +
    + +
    +
    +
    +

    SESSION ACCESS

    +

    Connect private read APIs

    +
    + tokens live in this page only +
    +
    + + +
    + + + +
    + +
    +
    + Candidate preview not connected + Private graph not connected + Readiness context loading +
    +
    + +
    +
    +

    Candidate rows

    + — + Private preview not loaded +
    +
    +

    Review queue items

    + — + Graph access required +
    +
    +

    Countries in context

    + — + Derived matrix, not an approval +
    +
    +

    Visible blockers

    + — + Unknown until evidence loads +
    +
    + +
    + + +
    +
    +
    +
    +

    01 / PRIVATE CANDIDATE RELEASE

    +

    Candidate preview

    +
    + authenticated GET · candidate only +
    +
    +
    +

    Rows in preview

    Not loaded
    +
    Waiting for private preview.Enter the candidate token above, then load the authenticated test-only response.
    +
    +
    +
    Select a candidate row.Details will be limited to safe review fields. Private addresses, raw source material, geocoder details, and reviewer identities stay out of this surface.
    +
    +
    +
    No local review packet loaded.The console does not infer counts, deltas, quarantine reasons, or blockers from candidate rows.
    +
    + +
    +
    +
    +

    02 / REVIEW WORK QUEUES

    +

    Contradictions, identity, quarantine, statistics

    +
    + private graph · separate observations +
    +
    + + + + + + + +
    +
    Private graph access required.Queue contents are not inferred from candidate rows.
    +
    + +
    +
    +
    +

    03 / DERIVED OPERATOR CONTEXT

    +

    Launch-readiness matrix

    +
    + static JSON · not approval +
    +
    +

    Read this as a staging map, not a release gate. Each country has one derived context label from private-review/readiness-matrix.json. It does not approve a record, source, release, or publication profile.

    +
    +
    +
    Loading readiness context.Unknown is shown until the local asset is available.
    +
    + +
    +
    +
    +

    04 / BOUNDED EVIDENCE GRAPH

    +

    Entity search and neighborhood

    +
    + observations stay separate +
    +
    +
    + + +
    +
    + + + The server contract remains bounded. Direction and depth are sent with every neighborhood read. +
    +

    Private graph search is not connected.

    +
    +
    +
    +

    Entities

    Not loaded
    +
    No entity search yet.Search returns bounded entity metadata only.
    +
    +
    +
    Select an entity.Neighborhood results will be displayed as retained relationship observations, never as an ownership or operational-status conclusion.
    +
    +
    +
    +
    +
    + +
    + Private candidate review · local/operator use only + Evidence state is explicit; unavailable is not a positive finding. +
    +
    + + + diff --git a/static/private-review.js b/static/private-review.js new file mode 100644 index 0000000..89f573a --- /dev/null +++ b/static/private-review.js @@ -0,0 +1,573 @@ +const DEV_PREVIEW_PATH = '/api/dev/preview/candidates'; +const DEV_PREVIEW_TOKEN_HEADER = 'X-UEC-Dev-Preview-Token'; +const GRAPH_TOKEN_HEADER = 'X-UEC-Private-Graph-Token'; +const READINESS_PATH = 'private-review/readiness-matrix.json'; +const READINESS_STATES = [ + 'infrastructure-only', + 'acquisition-ready', + 'private-candidate-ready', + 'human-review-ready', + 'publication-eligible', + 'blocked', +]; +const QUEUES = [ + { id: 'contradictions', label: 'Contradictions', columns: ['claim_id', 'facility_id', 'organization_id', 'claim_domain', 'claim_kind', 'observed_at', 'confidence', 'review_state', 'privacy_status', 'publication_status'] }, + { id: 'unresolved-identities', label: 'Unresolved identities', columns: ['crosswalk_id', 'left_identifier_id', 'right_identifier_id', 'assertion_status', 'confidence', 'observed_at', 'source_id', 'source_record_id', 'review_state', 'privacy_status', 'publication_status'] }, + { id: 'quarantine', label: 'Quarantine', columns: ['source_record_id', 'source_id', 'source_state', 'received_at'] }, + { id: 'claims', label: 'Claims / support', columns: ['claim_id', 'source_id', 'claim_domain', 'claim_kind', 'value_state', 'unknown_reason', 'observed_at', 'confidence', 'review_state', 'storage_state', 'privacy_status', 'publication_status', 'support_count', 'contradicting_support_count'] }, + { id: 'rejected-candidates', label: 'Rejected candidates', columns: ['crosswalk_id', 'source_id', 'source_record_id', 'assertion_status', 'confidence', 'observed_at', 'review_state', 'privacy_status', 'publication_status'] }, + { id: 'suppression', label: 'Suppression', columns: ['case_id', 'case_status', 'event_type', 'reason_category', 'policy_version', 'occurred_at'] }, + { id: 'statistics', label: 'Statistics', columns: ['metric', 'value'] }, +]; + +const state = { + candidates: [], + selectedCandidate: null, + queues: {}, + activeQueue: 'contradictions', + entities: [], + selectedEntity: null, + observations: [], + readiness: null, + readinessError: false, + reviewPacket: null, + reviewPacketError: null, +}; + +export function escapeHtml(value) { + return String(value ?? '').replace(/[&<>"']/g, (character) => ({ '&': '&', '<': '<', '>': '>', '"': '"', "'": ''' }[character])); +} + +export function displayValue(value, fallback = 'Unavailable') { + return value === null || value === undefined || value === '' ? fallback : String(value); +} + +function isRecord(value) { + return value !== null && typeof value === 'object' && !Array.isArray(value); +} + +function finiteOrNull(value) { + return typeof value === 'number' && Number.isFinite(value) ? value : null; +} + +function safeHttpUrl(value) { + if (typeof value !== 'string' || !/^https?:\/\//i.test(value)) return null; + return value; +} + +export function normalizePreviewRow(row) { + if (!isRecord(row) || !String(row.candidate_id || row.source_record_id || '').trim() || !String(row.facility_id || '').trim()) { + throw new Error('Private candidate preview response was rejected safely.'); + } + const latitude = finiteOrNull(row.latitude); + const longitude = finiteOrNull(row.longitude); + const coordinatePair = (latitude === null) === (longitude === null); + if (!coordinatePair) throw new Error('Private candidate preview response was rejected safely.'); + return { + candidateId: String(row.candidate_id || row.source_record_id), + sourceRecordId: String(row.source_record_id || row.candidate_id), + facilityId: String(row.facility_id), + name: displayValue(row.canonical_name, 'Unnamed candidate'), + countryCode: displayValue(row.country_code), + city: row.city === null || row.city === undefined ? null : String(row.city), + category: displayValue(row.category), + displayPrecision: displayValue(row.display_precision), + coordinatePrecision: displayValue(row.coordinate_precision), + coordinateReviewStatus: displayValue(row.coordinate_review_status), + hasCoordinatePair: latitude !== null && longitude !== null, + sourceType: displayValue(row.source_type), + sourceId: displayValue(row.provenance_source_id), + sourceName: displayValue(row.provenance_source_name), + sourceUrl: safeHttpUrl(row.provenance_source_url), + retrievedAt: displayValue(row.provenance_retrieved_at), + factualReviewStatus: displayValue(row.factual_review_status), + privacyStatus: displayValue(row.privacy_screening_status), + releaseId: displayValue(row.release_id), + releaseStatus: displayValue(row.release_status, 'candidate'), + previewLabel: displayValue(row.preview_label, 'Private candidate — not reviewed or published'), + projectApproval: row.project_approval === false ? false : null, + maintainerApproval: displayValue(row.maintainer_approval), + suppressionState: displayValue(row.suppression_state), + }; +} + +function normalizeEnvelope(payload, label) { + if (!isRecord(payload) || !Array.isArray(payload.data)) throw new Error(`${label} response was rejected safely.`); + return payload; +} + +async function requestJson(path, token, header, label, params = {}) { + if (!String(token || '').trim()) throw new Error(`${label} requires an operator token.`); + const url = new URL(path, window.location.href); + Object.entries(params).forEach(([key, value]) => url.searchParams.set(key, String(value))); + let response; + try { + response = await fetch(url.toString(), { + cache: 'no-store', + headers: { Accept: 'application/json', [header]: token }, + }); + } catch (_error) { + throw new Error(`${label} could not be reached.`); + } + if (!response.ok) throw new Error(`${label} unavailable (HTTP ${response.status}).`); + let payload; + try { + payload = await response.json(); + } catch (_error) { + throw new Error(`${label} response was rejected safely.`); + } + return normalizeEnvelope(payload, label); +} + +function setAuthStatus(id, message, status = 'idle') { + const element = document.getElementById(id); + if (!element) return; + element.dataset.state = status; + element.innerHTML = `${escapeHtml(message)}`; +} + +function setText(id, value) { + const element = document.getElementById(id); + if (element) element.textContent = value; +} + +function statusClass(value) { + const normalized = String(value || '').toLowerCase(); + if (['passed', 'observed', 'accepted', 'confirmed', 'supported', 'active'].some((token) => normalized.includes(token))) return 'badge-green'; + if (['rejected', 'blocked', 'restricted', 'disputed', 'quarantined', 'review_required'].some((token) => normalized.includes(token))) return 'badge-red'; + if (['candidate', 'pending', 'unknown', 'unavailable', 'unreviewed'].some((token) => normalized.includes(token))) return 'badge-amber'; + return 'badge-slate'; +} + +function badge(value, fallback = 'Unavailable') { + const text = displayValue(value, fallback); + return `${escapeHtml(text)}`; +} + +function unknown(value) { + return `${escapeHtml(displayValue(value))}`; +} + +function candidateStatusNote(candidate) { + if (!candidate) return 'Private preview not loaded'; + return `${candidate.releaseStatus} · not a published record`; +} + +function renderSummary() { + const candidateCount = state.candidates.length; + const queueCount = Object.values(state.queues).reduce((total, queue) => total + (Array.isArray(queue) ? queue.length : 0), 0); + const countryCount = state.readiness && isRecord(state.readiness.countries) ? Object.keys(state.readiness.countries).length : null; + const blockedCountries = countryCount === null ? null : Object.values(state.readiness.countries).filter((item) => isRecord(item) && item.state === 'blocked').length; + const packetBlockers = state.reviewPacket?.blockers && isRecord(state.reviewPacket.blockers) + ? Object.values(state.reviewPacket.blockers).reduce((total, values) => total + (Array.isArray(values) ? values.length : 0), 0) + : 0; + const blockerCount = blockedCountries === null ? null : blockedCountries + (state.queues.quarantine?.length || 0) + (state.queues['unresolved-identities']?.length || 0) + packetBlockers; + setText('metric-candidates', String(candidateCount)); + setText('metric-candidates-note', candidateCount ? 'Authenticated candidate rows' : 'No candidate rows returned'); + setText('metric-queues', Object.keys(state.queues).length ? String(queueCount) : '—'); + setText('metric-queues-note', Object.keys(state.queues).length ? 'Separate graph queue observations' : 'Graph access required'); + setText('metric-countries', countryCount === null ? '—' : String(countryCount)); + setText('metric-countries-note', countryCount === null ? 'Readiness asset unavailable' : 'Derived context, not an approval'); + setText('metric-blockers', blockerCount === null ? '—' : String(blockerCount)); + setText('metric-blockers-note', blockerCount === null ? 'Unknown until evidence loads' : 'Contextual blockers and unresolved queues'); + setText('rail-candidate-count', String(candidateCount)); + setText('rail-queue-count', Object.keys(state.queues).length ? String(queueCount) : '—'); + setText('rail-country-count', countryCount === null ? '—' : String(countryCount)); +} + +function renderCandidateList() { + const list = document.getElementById('candidate-list'); + if (!list) return; + setText('candidate-list-count', state.candidates.length ? `${state.candidates.length} row${state.candidates.length === 1 ? '' : 's'}` : '0 rows'); + if (!state.candidates.length) { + list.innerHTML = '
    No private candidate rows.The authenticated response was empty. No fallback records are shown.
    '; + return; + } + list.innerHTML = state.candidates.map((candidate, index) => ``).join(''); + list.querySelectorAll('[data-candidate-index]').forEach((button) => { + button.addEventListener('click', () => { + state.selectedCandidate = Number(button.dataset.candidateIndex); + renderCandidateList(); + renderCandidateDetail(); + }); + }); +} + +function fact(label, value, isUnknown = false) { + return `
    ${escapeHtml(label)}
    ${isUnknown ? 'Unavailable' : value}
    `; +} + +function renderCandidateDetail() { + const detail = document.getElementById('candidate-detail'); + if (!detail) return; + const candidate = state.selectedCandidate === null ? null : state.candidates[state.selectedCandidate]; + if (!candidate) { + detail.innerHTML = '
    Select a candidate row.Details are limited to safe review fields. Private addresses, raw source material, geocoder details, and reviewer identities stay out of this surface.
    '; + return; + } + const location = [candidate.city, candidate.countryCode].filter(Boolean).join(', ') || 'Region unavailable'; + const coordinateState = candidate.hasCoordinatePair ? 'Pair present under authenticated private preview contract' : 'No coordinate pair in preview response'; + const projectDecision = candidate.projectApproval === false ? 'false — candidate is not approved' : 'Unavailable — no positive decision is inferred'; + detail.innerHTML = ` +
    +

    PRIVATE CANDIDATE · ${escapeHtml(candidate.candidateId)}

    ${escapeHtml(candidate.name)}

    ${escapeHtml(location)} · ${escapeHtml(candidate.category)} · source record ${escapeHtml(candidate.sourceRecordId)}

    +
    ${badge(candidate.releaseStatus, 'candidate')}${badge(candidate.factualReviewStatus)}${badge(candidate.privacyStatus)}
    +
    +
    +

    Source terms / attribution

    ${fact('Source type', escapeHtml(candidate.sourceType))}${fact('Source name', escapeHtml(candidate.sourceName))}${fact('Source ID', escapeHtml(candidate.sourceId))}${fact('Source URL', candidate.sourceUrl ? `text only — no public link` : 'Unavailable', !candidate.sourceUrl)}${fact('Terms status', 'Unavailable in candidate contract', true)}

    Attribution metadata is shown as supplied. A URL is intentionally not made clickable by this console.

    +

    Provenance

    ${fact('Retrieved at', escapeHtml(candidate.retrievedAt))}${fact('Release ID', escapeHtml(candidate.releaseId))}${fact('Release status', badge(candidate.releaseStatus, 'candidate'))}${fact('Candidate label', escapeHtml(candidate.previewLabel))}
    +

    Privacy / location risk

    ${fact('Privacy screening', badge(candidate.privacyStatus))}${fact('Address', 'Withheld by console')}${fact('Coordinate pair', escapeHtml(coordinateState))}${fact('Display precision', badge(candidate.displayPrecision))}${fact('Coordinate precision', badge(candidate.coordinatePrecision))}${fact('Coordinate review state', badge(candidate.coordinateReviewStatus))}
    +

    Review state

    ${fact('Factual review', badge(candidate.factualReviewStatus))}${fact('Project decision', escapeHtml(projectDecision))}${fact('Maintainer approval', badge(candidate.maintainerApproval))}${fact('Suppression state', badge(candidate.suppressionState))}${fact('Reviewer identity', 'Withheld by console')}
    +

    Counts, deltas, quarantine, and blockers

    ${fact('Candidate count', 'Unavailable in preview contract', true)}${fact('Count delta', 'Unavailable — do not infer from this page', true)}${fact('Observation count', 'Unavailable in preview contract', true)}
    • Candidate release status remains ${escapeHtml(candidate.releaseStatus)}.
    • Project decision is not positive in this preview contract.
    • Quarantine reason is not included; inspect the quarantine queue.
    • Suppression state is not included; absence is not clearance.
    • No publication, export, or public-link action is available here.

    The preview payload is allowlisted before rendering. Raw source payloads, addresses, geocoder requests/responses, requester details, sensitive notes, and reviewer identities are never rendered.

    +
    `; +} + +const FORBIDDEN_PACKET_KEYS = new Set([ + 'source_values', 'raw_fields', 'address', 'street', 'street_address', 'latitude', 'longitude', + 'coordinates', 'geocoder_query', 'geocoder_response', 'phone', 'email', 'requester', + 'requester_contact', 'reviewer_identity', 'records', 'rows', +]); + +export function validateReviewPacket(value, path = 'packet') { + if (Array.isArray(value)) { + value.forEach((child, index) => validateReviewPacket(child, `${path}[${index}]`)); + return value; + } + if (!isRecord(value)) return value; + const leaked = Object.keys(value).filter((key) => FORBIDDEN_PACKET_KEYS.has(key.toLowerCase())); + if (leaked.length) throw new Error(`Review packet rejected safely at ${path}.`); + Object.entries(value).forEach(([key, child]) => validateReviewPacket(child, `${path}.${key}`)); + return value; +} + +function packetEntries(value) { + if (!isRecord(value)) return '
  • Unavailable
    • '; + const entries = Object.entries(value); + if (!entries.length) return '
  • None recorded
    • '; + return entries.slice(0, 24).map(([key, child]) => { + const display = Array.isArray(child) ? child.join(' · ') : isRecord(child) ? JSON.stringify(child) : displayValue(child); + return `
  • ${escapeHtml(key.replaceAll('_', ' '))}${escapeHtml(display)}
    • `; + }).join(''); +} + +function renderReviewPacket() { + const panel = document.getElementById('packet-summary'); + if (!panel) return; + if (state.reviewPacketError) { + panel.innerHTML = `
    Review packet unavailable. ${escapeHtml(state.reviewPacketError)}

    Only row-free aggregate packets are accepted; no candidate fallback is inferred.

    `; + return; + } + const packet = state.reviewPacket; + if (!packet) { + panel.innerHTML = '
    No local review packet loaded.The console does not infer counts, deltas, quarantine reasons, or blockers from candidate rows.
    '; + return; + } + const counts = packet.counts || {}; + const diff = packet.release_diff || {}; + const diffCounts = diff.counts || {}; + const geospatial = packet.geospatial || {}; + panel.innerHTML = `

    LOCAL ROW-FREE PACKET

    ${escapeHtml(displayValue(packet.source_id, 'Source unavailable'))}

    ${escapeHtml(displayValue(packet.publication_boundary, 'Packet is evidence only; no approval is implied.'))}

    inspection only

    Counts / deltas

      ${packetEntries({ input: counts.input_rows, normalized: counts.normalized_rows, quarantined: counts.quarantined_rows, reconciles: counts.reconciles, added: diffCounts.added, changed: diffCounts.changed, not_observed: diffCounts.not_observed })}

    Quarantine reasons

      ${packetEntries(packet.quarantine?.reasons)}

    Coordinate / privacy gates

      ${packetEntries(geospatial.coordinate_gate_counts || geospatial.precision_counts)}

    Publication blockers

      ${packetEntries(packet.blockers)}
    `; +} + +async function loadReviewPacket(event) { + const file = event.target.files?.[0]; + if (!file) return; + state.reviewPacket = null; + state.reviewPacketError = null; + try { + const value = JSON.parse(await file.text()); + validateReviewPacket(value); + if (!isRecord(value) || !String(value.schema_version || '').startsWith('private-review-packet-')) throw new Error('Unsupported packet schema.'); + state.reviewPacket = value; + } catch (error) { + state.reviewPacketError = error.message || 'Packet parsing failed.'; + } + renderReviewPacket(); + renderSummary(); +} + +function normalizeQueueRows(kind, payload) { + normalizeEnvelope(payload, `${kind} queue`); + const definition = QUEUES.find((queue) => queue.id === kind); + if (!definition || payload.data.some((row) => !Array.isArray(row))) throw new Error(`${kind} queue response was rejected safely.`); + return payload.data.map((row) => definition.columns.map((_column, index) => displayValue(row[index], 'Unavailable'))); +} + +function renderQueue() { + const panel = document.getElementById('queue-panel'); + if (!panel) return; + const definition = QUEUES.find((queue) => queue.id === state.activeQueue); + const rows = state.queues[state.activeQueue]; + if (!definition) return; + if (rows?.error) { + panel.innerHTML = `
    ${escapeHtml(rows.error)}

    Queue contents were cleared. Re-authenticate and retry; this console does not substitute candidate data.

    `; + return; + } + if (!Array.isArray(rows)) { + panel.innerHTML = '
    Private graph access required.Queue contents are not inferred from candidate rows.
    '; + return; + } + if (!rows.length) { + panel.innerHTML = `
    No ${escapeHtml(definition.label.toLowerCase())} returned.An empty queue is not evidence that other review obligations are cleared.
    `; + return; + } + const headers = definition.columns.map((column) => `${escapeHtml(column.replaceAll('_', ' '))}`).join(''); + const body = rows.map((row) => `${row.map((value, index) => `${escapeHtml(value)}`).join('')}`).join(''); + panel.innerHTML = `
    ${headers}${body}
    `; +} + +function setQueueTab(kind) { + state.activeQueue = kind; + document.querySelectorAll('.queue-tab').forEach((tab) => tab.classList.toggle('active', tab.dataset.queue === kind)); + renderQueue(); +} + +function normalizeEntity(row) { + if (!isRecord(row) || !String(row.entity_id || '').trim()) throw new Error('Entity response was rejected safely.'); + return { id: String(row.entity_id), type: displayValue(row.entity_type), name: displayValue(row.canonical_name, 'Unnamed entity'), countryCode: displayValue(row.country_code), createdAt: displayValue(row.created_at) }; +} + +function normalizeObservation(row) { + if (!isRecord(row)) throw new Error('Neighborhood response was rejected safely.'); + return { + observationId: displayValue(row.relationship_observation_id), + fromOrganizationId: displayValue(row.from_organization_id), + targetFacilityId: displayValue(row.target_facility_id), + targetOrganizationId: displayValue(row.target_organization_id), + relationshipType: displayValue(row.relationship_type), + assertionStatus: displayValue(row.assertion_status), + observedAt: displayValue(row.observed_at), + confidence: displayValue(row.confidence), + reviewState: displayValue(row.review_state), + storageState: displayValue(row.storage_state), + privacyStatus: displayValue(row.privacy_status), + publicationStatus: displayValue(row.publication_status), + sourceId: displayValue(row.source_id), + sourceRecordId: displayValue(row.source_record_id), + }; +} + +function observationBucket(observation) { + const status = `${observation.assertionStatus} ${observation.reviewState}`.toLowerCase(); + if (/(reject|disput)/.test(status)) return 'rejected'; + if (/(accept|confirm|support|observed)/.test(status)) return 'supporting'; + return 'other'; +} + +function renderEntities() { + const list = document.getElementById('entity-list'); + if (!list) return; + setText('entity-count', state.entities.length ? `${state.entities.length} result${state.entities.length === 1 ? '' : 's'}` : '0 results'); + if (!state.entities.length) { + list.innerHTML = '
    No entities returned.No fallback entities are shown when the private search is empty.
    '; + return; + } + list.innerHTML = state.entities.map((entity, index) => ``).join(''); + list.querySelectorAll('[data-entity-index]').forEach((button) => button.addEventListener('click', () => { + state.selectedEntity = Number(button.dataset.entityIndex); + renderEntities(); + loadNeighborhood(state.entities[state.selectedEntity]); + })); +} + +function observationCard(observation) { + const endpoint = [observation.fromOrganizationId, observation.targetFacilityId, observation.targetOrganizationId].filter((value) => value !== 'Unavailable').join(' → ') || 'Endpoints unavailable'; + return `
    ${escapeHtml(observation.relationshipType)}${escapeHtml(observation.assertionStatus)}

    ${escapeHtml(endpoint)}

    observed ${escapeHtml(observation.observedAt)}confidence ${escapeHtml(observation.confidence)}review ${escapeHtml(observation.reviewState)}privacy ${escapeHtml(observation.privacyStatus)}publication ${escapeHtml(observation.publicationStatus)}source ${escapeHtml(observation.sourceId)}record ${escapeHtml(observation.sourceRecordId)}
    `; +} + +function renderNeighborhood() { + const panel = document.getElementById('neighborhood-panel'); + if (!panel) return; + const entity = state.selectedEntity === null ? null : state.entities[state.selectedEntity]; + if (!entity) { + panel.innerHTML = '
    Select an entity.Neighborhood results will be displayed as retained relationship observations, never as an ownership or operational-status conclusion.
    '; + return; + } + const groups = { supporting: [], other: [], rejected: [] }; + state.observations.forEach((observation) => groups[observationBucket(observation)].push(observation)); + const group = (title, key, note) => `

    ${escapeHtml(title)} ${groups[key].length} retained

    ${groups[key].length ? groups[key].map(observationCard).join('') : '
    No observations in this bucket.
    '}

    ${escapeHtml(note)}

    `; + panel.innerHTML = `

    ENTITY NEIGHBORHOOD

    ${escapeHtml(entity.name)}

    ${escapeHtml(entity.type)} · ${escapeHtml(entity.countryCode)} · created ${escapeHtml(entity.createdAt)}

    ${state.observations.length} observations
    ${group('Supporting observations', 'supporting', 'Status is shown as returned by the private graph; it is not a conclusion about ownership or operation.')}${group('Rejected or candidate observations', 'rejected', 'Rejected/disputed statuses remain visible as separate observations and are not silently merged away.')}${group('Other retained observations', 'other', 'Unresolved and unclassified statuses remain separate until an authorized human review records a decision.')}`; +} + +function renderReadiness() { + const matrix = document.getElementById('readiness-matrix'); + const legend = document.getElementById('readiness-legend'); + if (!matrix || !legend) return; + legend.innerHTML = READINESS_STATES.map((stateName) => `${escapeHtml(stateName)}`).join(''); + if (state.readinessError || !state.readiness || !isRecord(state.readiness.countries)) { + matrix.innerHTML = '
    Readiness context unavailable. The console fails closed and does not infer country status from candidate rows.
    '; + return; + } + const entries = Object.entries(state.readiness.countries).sort(([, left], [, right]) => displayValue(left.name).localeCompare(displayValue(right.name))); + matrix.innerHTML = entries.map(([code, item]) => { + const safeState = READINESS_STATES.includes(item?.state) ? item.state : 'blocked'; + const basis = Array.isArray(item?.basis) ? item.basis.map((value) => displayValue(value)).join(' · ') : 'Basis unavailable'; + const sourceDetails = Array.isArray(item?.sources) ? item.sources.map((source) => { + const attribution = source.attribution || {}; + const status = source.status || {}; + const blockers = Array.isArray(source.coverage?.limitations) ? source.coverage.limitations : []; + return `
    ${escapeHtml(displayValue(source.source_id))}${badge(status.acquisition)} · ${badge(status.metadata)}Terms: ${escapeHtml(displayValue(attribution.terms_status))} · attribution required: ${attribution.attribution_required === true ? 'yes' : 'unknown'}Notice: ${escapeHtml(displayValue(attribution.notice))}${blockers.length ? `Blockers: ${escapeHtml(blockers.join(' · '))}` : ''}
    `; + }).join('') : 'Source details unavailable'; + return `

    ${escapeHtml(code)}

    ${escapeHtml(displayValue(item?.name, 'Country unavailable'))}

    ${escapeHtml(safeState)}

    ${escapeHtml(displayValue(item?.summary, 'Context summary unavailable.'))}

    Basis: ${escapeHtml(basis)}
    ${escapeHtml(displayValue(item?.source_count, 0))} source contracts${sourceDetails}
    `; + }).join('') || '
    No country classifications.The readiness asset contained no country map.
    '; +} + +async function loadReadiness() { + setAuthStatus('readiness-status', 'Readiness context loading', 'loading'); + try { + const response = await fetch(READINESS_PATH, { cache: 'no-store', headers: { Accept: 'application/json' } }); + if (!response.ok) throw new Error('unavailable'); + const payload = await response.json(); + if (!isRecord(payload) || payload.derived_context !== true || !isRecord(payload.countries)) throw new Error('invalid'); + state.readiness = payload; + state.readinessError = false; + setAuthStatus('readiness-status', 'Readiness context loaded · derived only', 'ready'); + } catch (_error) { + state.readiness = null; + state.readinessError = true; + setAuthStatus('readiness-status', 'Readiness context unavailable', 'error'); + } + renderReadiness(); + renderSummary(); +} + +async function loadCandidates() { + const token = document.getElementById('dev-token')?.value || ''; + setAuthStatus('dev-auth-status', 'Loading private candidates', 'loading'); + state.candidates = []; + state.selectedCandidate = null; + renderCandidateList(); + renderCandidateDetail(); + renderSummary(); + try { + const payload = await requestJson(`${DEV_PREVIEW_PATH}?limit=100`, token, DEV_PREVIEW_TOKEN_HEADER, 'Private candidate preview'); + if (payload.api_version !== 'dev-preview-v1' || payload.meta?.test_only !== true || payload.meta?.private_preview !== true) throw new Error('Private candidate preview response was rejected safely.'); + state.candidates = payload.data.map(normalizePreviewRow); + state.selectedCandidate = state.candidates.length ? 0 : null; + setAuthStatus('dev-auth-status', `Candidate preview connected · ${state.candidates.length} rows`, 'ready'); + } catch (error) { + state.candidates = []; + state.selectedCandidate = null; + setAuthStatus('dev-auth-status', error.message || 'Private candidate preview unavailable', 'error'); + } + renderCandidateList(); + renderCandidateDetail(); + renderSummary(); +} + +async function loadQueues() { + const token = document.getElementById('graph-token')?.value || ''; + setAuthStatus('graph-auth-status', 'Loading private queues', 'loading'); + state.queues = {}; + renderQueue(); + renderSummary(); + try { + const results = await Promise.all(QUEUES.map(async (queue) => { + const payload = await requestJson(`/api/private/graph/queues/${queue.id}`, token, GRAPH_TOKEN_HEADER, `${queue.label} queue`, { limit: 100 }); + return [queue.id, normalizeQueueRows(queue.id, payload)]; + })); + state.queues = Object.fromEntries(results); + setAuthStatus('graph-auth-status', 'Private graph connected · queues loaded', 'ready'); + } catch (error) { + state.queues = {}; + setAuthStatus('graph-auth-status', error.message || 'Private graph queues unavailable', 'error'); + } + QUEUES.forEach((queue) => setText(`queue-count-${queue.id}`, Array.isArray(state.queues[queue.id]) ? String(state.queues[queue.id].length) : '—')); + renderQueue(); + renderSummary(); +} + +async function searchEntities() { + const token = document.getElementById('graph-token')?.value || ''; + const query = document.getElementById('entity-query')?.value || ''; + setText('graph-status', 'Searching private entities…'); + state.entities = []; + state.selectedEntity = null; + state.observations = []; + renderEntities(); + renderNeighborhood(); + try { + const payload = await requestJson('/api/private/graph/entities', token, GRAPH_TOKEN_HEADER, 'Private entity search', { q: query, limit: 50 }); + state.entities = payload.data.map(normalizeEntity); + setText('graph-status', `${state.entities.length} bounded entities returned. Select one to read separate observations.`); + } catch (error) { + state.entities = []; + setText('graph-status', error.message || 'Private entity search unavailable.'); + } + renderEntities(); + renderNeighborhood(); +} + +async function loadNeighborhood(entity) { + if (!entity) return; + const token = document.getElementById('graph-token')?.value || ''; + const direction = document.getElementById('graph-direction')?.value || 'both'; + const depth = document.getElementById('graph-depth')?.value || '1'; + const panel = document.getElementById('neighborhood-panel'); + if (panel) panel.innerHTML = '
    Loading bounded observations…Observations remain separate evidence records.
    '; + try { + const payload = await requestJson(`/api/private/graph/entities/${encodeURIComponent(entity.id)}/neighborhood`, token, GRAPH_TOKEN_HEADER, 'Private neighborhood', { direction, depth, limit: 100 }); + if (payload.meta?.private !== true) throw new Error('Private neighborhood response was rejected safely.'); + state.observations = payload.data.map(normalizeObservation); + setText('graph-status', `${state.observations.length} retained observations for ${entity.name}.`); + } catch (error) { + state.observations = []; + if (panel) panel.innerHTML = `
    ${escapeHtml(error.message || 'Private neighborhood unavailable.')}

    Observation details were cleared. No graph conclusion is inferred.

    `; + } + renderNeighborhood(); +} + +function clearSession() { + const devInput = document.getElementById('dev-token'); + const graphInput = document.getElementById('graph-token'); + if (devInput) devInput.value = ''; + if (graphInput) graphInput.value = ''; + state.candidates = []; + state.selectedCandidate = null; + state.queues = {}; + state.entities = []; + state.selectedEntity = null; + state.observations = []; + state.reviewPacket = null; + state.reviewPacketError = null; + const packetInput = document.getElementById('review-packet-file'); + if (packetInput) packetInput.value = ''; + setAuthStatus('dev-auth-status', 'Candidate preview not connected'); + setAuthStatus('graph-auth-status', 'Private graph not connected'); + setText('graph-status', 'Private graph search is not connected.'); + QUEUES.forEach((queue) => setText(`queue-count-${queue.id}`, '—')); + renderCandidateList(); + renderCandidateDetail(); + renderQueue(); + renderEntities(); + renderNeighborhood(); + renderReviewPacket(); + renderSummary(); +} + +function bindEvents() { + document.getElementById('load-candidates')?.addEventListener('click', loadCandidates); + document.getElementById('load-queues')?.addEventListener('click', loadQueues); + document.getElementById('search-entities')?.addEventListener('click', searchEntities); + document.getElementById('clear-session')?.addEventListener('click', clearSession); + document.getElementById('review-packet-file')?.addEventListener('change', loadReviewPacket); + document.getElementById('entity-query')?.addEventListener('keydown', (event) => { if (event.key === 'Enter') searchEntities(); }); + document.getElementById('graph-direction')?.addEventListener('change', () => { if (state.selectedEntity !== null) loadNeighborhood(state.entities[state.selectedEntity]); }); + document.getElementById('graph-depth')?.addEventListener('change', () => { if (state.selectedEntity !== null) loadNeighborhood(state.entities[state.selectedEntity]); }); + document.querySelectorAll('.queue-tab').forEach((tab) => tab.addEventListener('click', () => setQueueTab(tab.dataset.queue))); +} + +function init() { + bindEvents(); + renderCandidateList(); + renderCandidateDetail(); + renderQueue(); + renderEntities(); + renderNeighborhood(); + renderReviewPacket(); + renderReadiness(); + renderSummary(); + loadReadiness(); +} + +if (typeof document !== 'undefined' && document.body?.dataset?.privateReview === 'true') init(); diff --git a/static/private-review/readiness-matrix.json b/static/private-review/readiness-matrix.json new file mode 100644 index 0000000..ab36411 --- /dev/null +++ b/static/private-review/readiness-matrix.json @@ -0,0 +1,28611 @@ +{ + "classification_notes": { + "acquisition-ready": "All registered sources have verified or partial metadata and acquisition has not run; this is not an authorization to acquire.", + "blocked": "At least one registered source reports acquisition=blocked; acquisition blockers take precedence.", + "human-review-ready": "All registered sources are privately acquired/verified and await explicit human review; no approval is implied.", + "infrastructure-only": "The registry provides infrastructure context, but the acquisition state is incomplete or not yet classifiable.", + "private-candidate-ready": "At least one source is privately acquired/verified, while another registered source remains pending; no release is implied.", + "publication-eligible": "Only explicit owner approval plus a release-allowed contract can produce this label; it is absent from the current baseline unless those fields are recorded." + }, + "contract_versions": { + "country": "country-contract-v1", + "readiness": "country-source-readiness-v1" + }, + "countries": { + "AL": { + "acquisition_counts": { + "blocked": 5, + "not_run": 1 + }, + "basis": [ + "al.aku.approved-food: blocked / blocked", + "al.aku.farms-aquaculture: blocked / blocked", + "al.aku.inspections-experiments: blocked / blocked", + "al.environment-permits: blocked / blocked", + "al.instat.statistics: not_run / reconnaissance", + "al.qkb.organizations: blocked / blocked" + ], + "country_reasons": [ + "acquisition:blocked", + "acquisition:not_run", + "publication:blocked" + ], + "name": "AL", + "owner_review": "awaiting-owner-review", + "publication_state": "blocked", + "source_count": 6, + "sources": [ + { + "access_method": "official AKU register pages and linked downloads", + "attribution": { + "attribution_required": true, + "notice": "Official authority; verify file terms, attribution, privacy and coordinates before acquisition.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown; timestamp retrieval", + "country_code": "AL", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Pin stable AKU file routes, schemas, cadence, IDs, terms and privacy." + ] + }, + "jurisdiction_scope": "Albania; National Food Authority approved and registered animal-origin food establishments", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "al.aku.approved-food", + "source_url": "https://aku.gov.al/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-al.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin stable AKU files, schemas, cadence, IDs, terms and privacy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official AKU/agriculture/fisheries registers and linked documents", + "attribution": { + "attribution_required": true, + "notice": "Verify publisher, license, animal-holder/property privacy and coordinate precision.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "resource-specific", + "country_code": "AL", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Verify complete farm/aquaculture coverage, exports, IDs, cadence, terms and privacy." + ] + }, + "jurisdiction_scope": "Albania; AKU primary producers, livestock establishments and agriculture/fisheries aquaculture evidence", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "al.aku.farms-aquaculture", + "source_url": "https://aku.gov.al/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-al.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Verify farm/aquaculture coverage, exports, IDs, coordinates, terms and privacy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official AKU registers, control reports and linked documents", + "attribution": { + "attribution_required": true, + "notice": "Sensitive control/research evidence; aggregate or anonymize and require project review.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "resource/report-specific", + "country_code": "AL", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Pin experimentation register/API, fields, IDs, privacy and retention policy." + ] + }, + "jurisdiction_scope": "Albania; AKU inspections/enforcement and experimental-animal institutions", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "al.aku.inspections-experiments", + "source_url": "https://aku.gov.al/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-al.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Pin experimentation register/API, fields, IDs, privacy and retention policy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "environmental authority routes and QKB permit/licence register", + "attribution": { + "attribution_required": true, + "notice": "Verify public-read rights, document license, geometry and privacy.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "permit/document-specific", + "country_code": "AL", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Stable public read/API/export and complete facility coverage are unresolved." + ] + }, + "jurisdiction_scope": "Albania; environmental permits and authorizations", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "al.environment-permits", + "source_url": "https://akm.gov.al/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-al.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Verify public environmental permit route, coverage, documents, geometry and privacy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official statistical tables/data services; exact route unverified", + "attribution": { + "attribution_required": true, + "notice": "Official statistics; verify table terms, citation requirements and aggregate-only scope.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "table-specific; preserve revisions", + "country_code": "AL", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Pin current slaughter and animal-use table IDs, API/download contracts and cadence." + ] + }, + "jurisdiction_scope": "Albania; INSTAT aggregate slaughter, livestock and animal-use statistics", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "al.instat.statistics", + "source_url": "https://www.instat.gov.al/en/", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-al.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin official slaughter/animal-use table IDs, APIs, cadence and revisions.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official public search/services; API/bulk route unverified", + "attribution": { + "attribution_required": true, + "notice": "Identity linkage only; QKB privacy policy limits personal/address disclosure and terms require verification.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "provider-defined", + "country_code": "AL", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Pin API/bulk route, rate limits, cadence, fields, terms and personal-address suppression." + ] + }, + "jurisdiction_scope": "Albania; National Business Center commercial and permit/licence registers", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "al.qkb.organizations", + "source_url": "https://qkb.gov.al/en/home-3/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-al.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Pin QKB API/search route, rate limits, fields, terms and personal-address suppression.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + } + ], + "state": "blocked", + "summary": "No release approval is implied; inspect source gates below." + }, + "AM": { + "acquisition_counts": { + "blocked": 5, + "not_run": 1 + }, + "basis": [ + "am.armstat.livestock-statistics: not_run / reconnaissance", + "am.e-register.organizations: blocked / blocked", + "am.environment-permits: blocked / blocked", + "am.snund.approved-food: blocked / blocked", + "am.snund.farms-livestock: blocked / blocked", + "am.snund.inspections-experiments: blocked / blocked" + ], + "country_reasons": [ + "acquisition:blocked", + "acquisition:not_run", + "publication:blocked" + ], + "name": "AM", + "owner_review": "awaiting-owner-review", + "publication_state": "blocked", + "source_count": 6, + "sources": [ + { + "access_method": "official Armstat PxWeb table/query; API contract to be pinned", + "attribution": { + "attribution_required": true, + "notice": "Statistical Committee source; copyright, revisions, and suppression rules require review", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "table-specific; unknown", + "country_code": "AM", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Pin current table IDs, API/query contract, cadence, revision semantics, and licensing." + ] + }, + "jurisdiction_scope": "Armenia; aggregate livestock and agriculture statistics", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "am.armstat.livestock-statistics", + "source_url": "https://statbank.armstat.am/pxweb/en/ArmStatBank/ArmStatBank__6%20Agriculture%2C%20forestry%20and%20fishing/AF-1-2024.px/", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-am.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin authorized route, schema, cadence, IDs, terms, privacy, and automation permissions.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official search/extract service; do not bypass authentication or payment", + "attribution": { + "attribution_required": true, + "notice": "Government registry; access fees, terms, privacy, and personal-address policy require review", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown", + "country_code": "AM", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Authorized machine route, fields, limits, cadence, and reuse terms not verified; full extracts may require sign-in/payment." + ] + }, + "jurisdiction_scope": "Armenia; electronic register of legal entities", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "am.e-register.organizations", + "source_url": "https://www.e-register.am/en/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-am.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin authorized route, schema, cadence, IDs, terms, privacy, and automation permissions.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official environmental service/register route; authorized API/export only", + "attribution": { + "attribution_required": true, + "notice": "Ministry source; license, geometry, privacy, and terms require review", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "resource-specific; unknown", + "country_code": "AM", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Exact public permit route, API/export, coverage, cadence, license, and privacy not pinned." + ] + }, + "jurisdiction_scope": "Armenia; environmental permits and registers", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "am.environment-permits", + "source_url": "https://env.am/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-am.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin authorized route, schema, cadence, IDs, terms, privacy, and automation permissions.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official FSIB registry/page; authorized bounded export or API only", + "attribution": { + "attribution_required": true, + "notice": "Official Armenian government source; terms, privacy, and location-safety review required", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown; verify from source metadata", + "country_code": "AM", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Exact file/API, schema, stable IDs, cadence, terms, and privacy controls not pinned." + ] + }, + "jurisdiction_scope": "Armenia; Food Safety Inspection Body slaughterhouses and animal-origin food-chain operators", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "am.snund.approved-food", + "source_url": "https://snund.am/en/page/operating-slaughterhouses/106", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-am.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin authorized route, schema, cadence, IDs, terms, privacy, and automation permissions.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official FSIB service/registry route; no facility acquisition until contract review", + "attribution": { + "attribution_required": true, + "notice": "Official government source; scope, personal data, locations, and reuse terms require review", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown", + "country_code": "AM", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Public bulk route, stable IDs, cadence, location policy, and licensing not verified." + ] + }, + "jurisdiction_scope": "Armenia; FSIB food-chain registration and veterinary/livestock-related operators", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "am.snund.farms-livestock", + "source_url": "https://www.snund.am/en", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-am.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin authorized route, schema, cadence, IDs, terms, privacy, and automation permissions.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official reports/plans; aggregate-only until a safe event route is verified", + "attribution": { + "attribution_required": true, + "notice": "Sensitive control/research evidence; privacy, retention, and publication review required", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "resource-specific; unknown", + "country_code": "AM", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "No stable public experimentation/event master verified; stop and escalate on sensitive exposure." + ] + }, + "jurisdiction_scope": "Armenia; FSIB food/veterinary inspections and public animal-experimentation evidence", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "am.snund.inspections-experiments", + "source_url": "https://www.snund.am/en/page/inspection-body/50", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-am.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin authorized route, schema, cadence, IDs, terms, privacy, and automation permissions.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + } + ], + "state": "blocked", + "summary": "No release approval is implied; inspect source gates below." + }, + "AU": { + "acquisition_counts": { + "artifact_private_only": 2, + "blocked": 9, + "not_run": 11 + }, + "basis": [ + "au.abr.lookup: not_run / reconnaissance", + "au.act.food-registration: blocked / blocked", + "au.animal-welfare-and-use: not_run / reconnaissance", + "au.asic.company-dataset: blocked / blocked", + "au.daff.export-establishments: blocked / blocked", + "au.npi.facilities: artifact_private_only / awaiting-owner-review", + "au.nsw.animal-use: not_run / reconnaissance", + "au.nsw.epa-poeo: blocked / blocked", + "au.nsw.food-authority: not_run / reconnaissance", + "au.nsw.food-enforcement: not_run / reconnaissance", + "au.nt.epa-licences: not_run / reconnaissance", + "au.nt.meat-licensing: blocked / blocked", + "au.pirsa.meat: not_run / reconnaissance", + "au.primesafe.vic.meat-licences: blocked / blocked", + "au.qld.environmental-authorities: blocked / blocked", + "au.sa.epa.licensed-activities: artifact_private_only / awaiting-owner-review", + "au.safefood.qld.accreditation: blocked / blocked", + "au.tas.biosecurity-meat: not_run / reconnaissance", + "au.tas.epa-listmap: blocked / blocked", + "au.vic.animal-use: not_run / reconnaissance", + "au.vic.epa-permissions: not_run / reconnaissance", + "au.wamia.abattoirs: not_run / reconnaissance" + ], + "country_reasons": [ + "acquisition:blocked", + "acquisition:not_run", + "metadata:unknown", + "publication:blocked" + ], + "name": "AU", + "owner_review": "awaiting-owner-review", + "publication_state": "blocked", + "source_count": 22, + "sources": [ + { + "access_method": "public lookup/web service; route-specific rate and terms", + "attribution": { + "attribution_required": true, + "notice": "ABR public identity route; use only for exact upstream ABN/ACN crosswalk and never infer operation/ownership", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "hourly update claim for ABN Lookup", + "country_code": "AU", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "No detailed contact/industry fields; individual and sole-trader privacy review required." + ] + }, + "jurisdiction_scope": "Australia; ABN Lookup and ABR public identity data", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "au.abr.lookup", + "source_url": "https://abr.business.gov.au/", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-au.md", + "docs/countries/australia/source-crosswalk.json", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Query only when an upstream source supplies an ABN/ACN; record lookup time and terms, and suppress sole-trader personal details.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "public guidance; dedicated abattoir list not verified", + "attribution": { + "attribution_required": true, + "notice": "ACT food registration guidance only; no facility publication claim", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown", + "country_code": "AU", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Identify ACT Health/local-government facility and environmental routes before acquisition." + ] + }, + "jurisdiction_scope": "Australia; Australian Capital Territory food registration coverage check", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "metadata:unknown", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "au.act.food-registration", + "source_url": "https://www.act.gov.au/business/health-licenses-and-inspections/food-businesses-and-events-registration", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-au.md", + "docs/countries/australia/source-crosswalk.json", + "pipeline/source_registry.json" + ], + "metadata": "unknown", + "next_action": "Identify the ACT authority and any public facility register before claiming ACT coverage; absence is not closure.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "jurisdiction-specific web pages, reports, and case records", + "attribution": { + "attribution_required": true, + "notice": "Government-sourced evidence only; allegations, personal information, sensitive locations, terms, and human review are mandatory gates", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "jurisdiction-specific", + "country_code": "AU", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "No uniform national facility-level feed; do not turn complaints, prosecutions, or aggregate totals into canonical facility facts." + ] + }, + "jurisdiction_scope": "Australia; jurisdiction-specific animal welfare enforcement and animal-use statistics", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "au.animal-welfare-and-use", + "source_url": "https://www.agriculture.gov.au/agriculture-land/animal/welfare/state", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-au.md", + "docs/countries/australia/source-crosswalk.json", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Keep welfare enforcement and animal-use evidence jurisdiction-specific; acquire only deidentified/terms-permitted aggregates or reviewed case records and model them as observations/events, not facility masters.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "public weekly snapshot; large download", + "attribution": { + "attribution_required": true, + "notice": "Catalogue states CC BY 3.0 Australia; preserve snapshot date and delimiter; not beneficial-ownership data", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "weekly Tuesday snapshot", + "country_code": "AU", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Do not bulk-fetch ~371.9 MiB without approved bounded plan; snapshot may lag ASIC Connect." + ] + }, + "jurisdiction_scope": "Australia; ASIC selected company-register snapshot", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "au.asic.company-dataset", + "source_url": "https://data.gov.au/data/en/dataset/asic-companies/resource/5c3914e6-413e-4a2c-b890-bf8efe3eabf2", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-au.md", + "docs/countries/australia/source-crosswalk.json", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Do not bulk-fetch the large ASIC snapshot without an approved bounded plan; use ACN/ABN only as an upstream identity crosswalk, never as proof of site ownership.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "authenticated TradeClear/Export Service or assisted request; public guidance only", + "attribution": { + "attribution_required": true, + "notice": "DAFF authority and export scope verified; authorised access, terms, retention, and privacy review required", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "ongoing/certificate-specific", + "country_code": "AU", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "No verified public meat bulk export; do not scrape authenticated systems or infer public access from guidance pages." + ] + }, + "jurisdiction_scope": "Australia; DAFF export-registered prescribed-goods establishments", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "au.daff.export-establishments", + "source_url": "https://www.agriculture.gov.au/biosecurity-trade/export/from-australia/documentation-registration-licensing/establishment-registration", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-au.md", + "docs/countries/australia/artifact-metadata.json", + "docs/countries/australia/source-crosswalk.json", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Obtain an authorised bounded current DAFF establishment report/export, preserve commodity/list type, record response metadata/hash/bytes, and keep export-only scope and privacy/terms/release gates explicit.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "public catalogue download", + "attribution": { + "attribution_required": true, + "notice": "Catalogue identifies CC BY 4.0; attribute Commonwealth of Australia/DCCEEW and complete privacy/release review", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "catalogue dataset date 2026-04-01; cadence not explicit", + "country_code": "AU", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Observed NPI population is not a complete slaughter registry; no public row release approved." + ] + }, + "jurisdiction_scope": "Australia; National Pollutant Inventory facilities", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": true, + "public_release_allowed": false, + "reasons": [ + "publication:blocked" + ], + "state": "awaiting-owner-review" + }, + "source_id": "au.npi.facilities", + "source_url": "https://data.gov.au/data/dataset/043f58e0-a188-4458-b61c-04e5b540aea4", + "status": { + "acquisition": "artifact_private_only", + "evidence": [ + "docs/country-recon-au.md", + "data/raw/australia/metadata.json", + "docs/countries/australia/artifact-metadata.json", + "docs/countries/australia/source-crosswalk.json", + "pipeline/source_registry.json", + "pipeline/tests/test_australia_source_metadata.py" + ], + "metadata": "verified", + "next_action": "Add a private deterministic NPI facility/report adapter and synthetic contract tests; preserve annual release/correction history and do not present NPI as complete facility coverage.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "public deidentified aggregate CSV/XLSX; facility list not public", + "attribution": { + "attribution_required": true, + "notice": "NSW DPIRD aggregate statistics; CC BY 4.0 stated; small-cell and re-identification review required", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "annual; 2024-2025 published 2026-07-20", + "country_code": "AU", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Do not map aggregate uses or associate them with named establishments." + ] + }, + "jurisdiction_scope": "Australia; NSW animal-use research statistics", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "au.nsw.animal-use", + "source_url": "https://www.dpird.nsw.gov.au/dpi/animals/animal-ethics-infolink/nsw-animal-use-statistics/animal-use-data", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-au.md", + "docs/countries/australia/source-crosswalk.json", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Use deidentified annual aggregates only as context; keep them separate from facility points, slaughter, and individual-animal claims.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "public search and downloadable register", + "attribution": { + "attribution_required": true, + "notice": "NSW EPA public-register route; confirm current export and historical-holder semantics", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "page updated 2026-04-28; transition noted", + "country_code": "AU", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Validate current transition/export scope before adapter work." + ] + }, + "jurisdiction_scope": "Australia; NSW POEO environmental register", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "au.nsw.epa-poeo", + "source_url": "https://apps.epa.nsw.gov.au/prpoeoapp/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-au.md", + "docs/countries/australia/source-crosswalk.json", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Resolve the current transition-aware full-list/search route, capture one bounded edition, and keep licence/application/notice/enforcement record types separate.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "public guidance/forms; complete licensee list not verified", + "attribution": { + "attribution_required": true, + "notice": "NSW Food Authority authority and scope verified; licensing/notification records and privacy restrictions remain distinct", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "risk-based audit; register cadence unknown", + "country_code": "AU", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "No verified public complete licensee export; do not use private notification records." + ] + }, + "jurisdiction_scope": "Australia; NSW Food Authority meat licensing", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "au.nsw.food-authority", + "source_url": "https://www.foodauthority.nsw.gov.au/help/licensing", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-au.md", + "docs/countries/australia/source-crosswalk.json", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Treat current meat-sector category counts as context only; locate a permitted current facility lookup/export before adapter work and do not infer closure or named facilities from aggregates.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "public HTML register", + "attribution": { + "attribution_required": true, + "notice": "Public enforcement register; preserve allegation, penalty, prosecution, and court-outcome semantics and screen personal names", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "weekly stated; penalty register has one-year publication window", + "country_code": "AU", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "No facility-status inference; preserve changing publication windows and event-versus-entity distinction." + ] + }, + "jurisdiction_scope": "Australia; NSW Food Authority enforcement events", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "au.nsw.food-enforcement", + "source_url": "https://www.foodauthority.nsw.gov.au/offences/penalty-notices", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-au.md", + "docs/countries/australia/source-crosswalk.json", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Model penalty/prosecution entries as time-bounded enforcement events; preserve allegation versus court outcome and suppress personal details.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "public HTML category register and document links", + "attribution": { + "attribution_required": true, + "notice": "NT EPA public category register; capture document identifiers and terms", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown", + "country_code": "AU", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Current document identifiers and permitted downloads need verification." + ] + }, + "jurisdiction_scope": "Australia; NT EPA environment-protection licences", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "au.nt.epa-licences", + "source_url": "https://ntepa.nt.gov.au/your-business/public-registers/licences-and-approvals-register/environment-protection-licences", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-au.md", + "docs/countries/australia/source-crosswalk.json", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Capture current NT EPA licence/document identifiers and terms, then link to NT meat licences only through explicit review.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "public guidance plus generic licence search; API/bulk route not verified", + "attribution": { + "attribution_required": true, + "notice": "NT meat licence scope verified; generic register selector and public disclosure terms need capture", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "licence year 1 July-30 June", + "country_code": "AU", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Verify the scheme selector and permitted extract." + ] + }, + "jurisdiction_scope": "Australia; Northern Territory meat industry licensing", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "au.nt.meat-licensing", + "source_url": "https://nt.gov.au/industry/agriculture/meat-industry/domestic-abattoirs-meat-processing", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-au.md", + "docs/countries/australia/source-crosswalk.json", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Validate the NT licence-scheme selector and obtain an authorised bounded extract; treat licence as authorisation evidence only.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "public guidance/application route; current row export not verified", + "attribution": { + "attribution_required": true, + "notice": "PIRSA authority and accreditation scope verified; current register availability and terms need confirmation", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown", + "country_code": "AU", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Obtain an authorised current accreditation export before integration." + ] + }, + "jurisdiction_scope": "Australia; South Australia PIRSA meat accreditation", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "au.pirsa.meat", + "source_url": "https://pir.sa.gov.au/animal-management/food-safety-for-meat-dairy-and-eggs/meat", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-au.md", + "docs/countries/australia/source-crosswalk.json", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Confirm whether PIRSA can provide a current permitted accreditation export; do not derive facility rows from guidance or application forms.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "public search linked by PrimeSafe; annual report PDF", + "attribution": { + "attribution_required": true, + "notice": "PrimeSafe authority and licence categories verified; search route, terms, and confidential complaint handling are separate", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown; annual aggregate report", + "country_code": "AU", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Identify the current search/export route; annual category totals are not facility rows." + ] + }, + "jurisdiction_scope": "Australia; Victoria PrimeSafe meat and seafood licences", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "au.primesafe.vic.meat-licences", + "source_url": "https://www.primesafe.vic.gov.au/licensing/about-your-licence/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-au.md", + "docs/countries/australia/source-crosswalk.json", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Identify the current PrimeSafe search/export route and capture one bounded result with licence, category, status and site fields; keep annual totals separate.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "public search/view/download; portal says not everything is online", + "attribution": { + "attribution_required": true, + "notice": "Queensland public-register route; preserve current/cancelled/surrendered status and information-request gaps", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "refresh observed 2026-09-11; not guaranteed", + "country_code": "AU", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Completeness and terms need an authorised bounded capture." + ] + }, + "jurisdiction_scope": "Australia; Queensland environmental authorities and enforcement", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "au.qld.environmental-authorities", + "source_url": "https://apps.des.qld.gov.au/public-register/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-au.md", + "docs/countries/australia/source-crosswalk.json", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Capture one bounded Queensland EA search/download and validate completeness, activity codes, status history and holder/site identity.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "public GeoJSON download", + "attribution": { + "attribution_required": true, + "notice": "CC BY 3.0 Australia; publisher warns points are approximate and may omit latest information; privacy/release review required", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "resource edition observed 2026-03-18", + "country_code": "AU", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Private artifact exists; validate activity semantics, approximate points, terms, and licence-to-site identity before integration." + ] + }, + "jurisdiction_scope": "Australia; South Australia EPA licensed activities", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": true, + "public_release_allowed": false, + "reasons": [ + "publication:blocked" + ], + "state": "awaiting-owner-review" + }, + "source_id": "au.sa.epa.licensed-activities", + "source_url": "https://data.sa.gov.au/data/dataset/8fdb86ff-d3d1-4f9e-85a5-bed4080d5ee1", + "status": { + "acquisition": "artifact_private_only", + "evidence": [ + "docs/country-recon-au.md", + "docs/countries/australia/artifact-metadata.json", + "docs/countries/australia/source-crosswalk.json", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Validate licence-level aggregation, multi-activity child rows, approximate-point semantics, and privacy/terms before any map or graph integration.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "public browser search; bulk/API route not verified", + "attribution": { + "attribution_required": true, + "notice": "Safe Food Queensland public register; result capture, terms, and address/privacy handling require review", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown", + "country_code": "AU", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Verify an authorised bounded capture, schema, category codebook, and reuse terms." + ] + }, + "jurisdiction_scope": "Australia; Queensland Safe Food accreditation", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "au.safefood.qld.accreditation", + "source_url": "https://hub.safefood.qld.gov.au/registry/s/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-au.md", + "docs/countries/australia/source-crosswalk.json", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Use an assisted register search to capture accreditation-number/business/activity/status fields, confirm bulk/export behavior and terms, and keep annual counts separate from facility rows.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "public HTML/PDF guidance; current row export not verified", + "attribution": { + "attribution_required": true, + "notice": "Biosecurity Tasmania authority and audited-program scope verified; historical lists are explicitly non-current", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "annual/unknown", + "country_code": "AU", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "2019 feasibility appendix is legacy only; confirm current register/export and terms." + ] + }, + "jurisdiction_scope": "Australia; Tasmania Biosecurity Tasmania meat accreditation", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "au.tas.biosecurity-meat", + "source_url": "https://nre.tas.gov.au/biosecurity-tasmania/product-integrity/food-safety/meat-and-poultry", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-au.md", + "docs/countries/australia/source-crosswalk.json", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Locate a current Tasmania accreditation route; keep the 2019 feasibility appendix legacy and separate from current observations.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "public search/LISTmap; stable API not verified", + "attribution": { + "attribution_required": true, + "notice": "Tasmania EPA route; preserve redaction and date limits", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "documents from 2022 onward; ongoing additions", + "country_code": "AU", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "LISTmap service layer and download route unresolved." + ] + }, + "jurisdiction_scope": "Australia; Tasmania EPA regulated premises and monitoring", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "au.tas.epa-listmap", + "source_url": "https://epa.tas.gov.au/about-the-epa/release-of-environmental-monitoring-information/search-for-environmental-monitoring-information", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-au.md", + "docs/countries/australia/source-crosswalk.json", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Resolve the Tasmania LISTmap service layer and permitted download route; preserve redaction/date limits and keep monitoring documents separate.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "public guidance and annual report downloads; complete premises list not verified", + "attribution": { + "attribution_required": true, + "notice": "Agriculture Victoria licence categories and aggregate reports; privacy review required", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "annual returns/report", + "country_code": "AU", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "No complete public premises list verified; keep annual-use reports separate from facilities." + ] + }, + "jurisdiction_scope": "Australia; Victoria scientific-procedure licensing and statistics", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "au.vic.animal-use", + "source_url": "https://agriculture.vic.gov.au/livestock-and-animals/animal-welfare-victoria/animals-used-in-research-and-teaching/licensing-to-use-animals-in-research-or-teaching/about-licensing-to-use-animals-in-research-or-teaching", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-au.md", + "docs/countries/australia/source-crosswalk.json", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Keep Victoria scientific-procedure licence guidance and aggregate reports separate; locate a public premises list before any facility mapping.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "public search plus ArcGIS REST layer", + "attribution": { + "attribution_required": true, + "notice": "EPA Victoria register and ArcGIS metadata observed; validate production service/version and geometry privacy", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "overnight; up to 24-hour delay; public from 2021-07-01", + "country_code": "AU", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Validate service filters, geometry semantics, and pre-2021 coverage." + ] + }, + "jurisdiction_scope": "Australia; EPA Victoria permissions", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "au.vic.epa-permissions", + "source_url": "https://www.epa.vic.gov.au/public-registers?register=permissions", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-au.md", + "docs/countries/australia/source-crosswalk.json", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Capture a bounded ArcGIS REST query and matching public-register record; validate pagination, identifiers, location types, terms, privacy, and permission-to-facility relationship semantics.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "public HTML list", + "attribution": { + "attribution_required": true, + "notice": "WAMIA approval page and 2026 guideline context verified; confirm reuse terms and revision semantics", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "page update observed July 2026", + "country_code": "AU", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Confirm stable approval identifier and capture rules; source names are not global IDs." + ] + }, + "jurisdiction_scope": "Australia; Western Australia WAMIA approved abattoirs", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "au.wamia.abattoirs", + "source_url": "https://wamia.wa.gov.au/abattoir-approvals/", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-au.md", + "docs/countries/australia/source-crosswalk.json", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Capture the current WAMIA HTML list, confirm approval numbers/revision and reuse terms, and keep source names as unresolved candidates until reviewed.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + } + ], + "state": "blocked", + "summary": "No release approval is implied; inspect source gates below." + }, + "AZ": { + "acquisition_counts": { + "blocked": 5, + "not_run": 1 + }, + "basis": [ + "az.afsa.food-subjects: blocked / blocked", + "az.afsa.inspections-enforcement: blocked / blocked", + "az.afsa.livestock-traceability: blocked / blocked", + "az.eco.environment-permits: blocked / blocked", + "az.stat.livestock-statistics: not_run / reconnaissance", + "az.taxes.organizations: blocked / blocked" + ], + "country_reasons": [ + "acquisition:blocked", + "acquisition:not_run", + "publication:blocked" + ], + "name": "AZ", + "owner_review": "awaiting-owner-review", + "publication_state": "blocked", + "source_count": 6, + "sources": [ + { + "access_method": "official AFSA search service; authorized bounded export/API only", + "attribution": { + "attribution_required": true, + "notice": "Official Azerbaijani government source; terms, privacy, and regional safety review required", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown", + "country_code": "AZ", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Bulk/API route, schema, stable IDs, cadence, licensing, privacy, and safe publication boundary not pinned." + ] + }, + "jurisdiction_scope": "Azerbaijan; AFSA registered food subjects and animal-origin food activities", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "az.afsa.food-subjects", + "source_url": "https://afsa.gov.az/az/qida-subyektleri", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-az.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin authorized route, schema, cadence, IDs, terms, privacy, and automation permissions.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official reports/search services; aggregate-only until safe event route is verified", + "attribution": { + "attribution_required": true, + "notice": "Sensitive control/research evidence; privacy, retention, and safety review required", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "resource-specific; unknown", + "country_code": "AZ", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "No stable public event/experimentation master verified; stop and escalate on sensitive exposure." + ] + }, + "jurisdiction_scope": "Azerbaijan; AFSA inspections, violations, veterinary controls, and public experimentation evidence", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "az.afsa.inspections-enforcement", + "source_url": "https://afsa.gov.az/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-az.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin authorized route, schema, cadence, IDs, terms, privacy, and automation permissions.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official notice references AQTIS; login-protected system, do not bypass access controls", + "attribution": { + "attribution_required": true, + "notice": "Sensitive veterinary/farm data; privacy, security, retention, and terms require explicit review", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown", + "country_code": "AZ", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Public export/API and authorization not verified; records may be operationally sensitive." + ] + }, + "jurisdiction_scope": "Azerbaijan; AFSA animal identification/registration and farm-to-table traceability", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "az.afsa.livestock-traceability", + "source_url": "https://afsa.gov.az/az/heyvan-saglamligi-ve-bioloji-tehlukesizlik/xeberler/heyvanlarin-identiklesdirilmesi-baytarliq-nezaretinin-effektivliyini-artirir", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-az.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin authorized route, schema, cadence, IDs, terms, privacy, and automation permissions.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official ministry route; authorized API/export only", + "attribution": { + "attribution_required": true, + "notice": "Ministry source; license, geometry, privacy, and safety review required", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "resource-specific; unknown", + "country_code": "AZ", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Exact public route, API/export, coverage, cadence, license, geometry, and privacy not pinned." + ] + }, + "jurisdiction_scope": "Azerbaijan; environmental permits and registers", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "az.eco.environment-permits", + "source_url": "https://eco.gov.az/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-az.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin authorized route, schema, cadence, IDs, terms, privacy, and automation permissions.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official statistical tables/yearbooks; machine API or bounded download to be pinned", + "attribution": { + "attribution_required": true, + "notice": "State Statistical Committee source; publication terms, revisions, and suppression rules require review", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "annual and table-specific", + "country_code": "AZ", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Current table IDs, machine API, revision semantics, licensing, and suppression rules not pinned." + ] + }, + "jurisdiction_scope": "Azerbaijan; aggregate livestock, slaughter, meat, milk, and fishery statistics", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "az.stat.livestock-statistics", + "source_url": "https://www.stat.gov.az/source/agriculture/?lang=en", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-az.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin authorized route, schema, cadence, IDs, terms, privacy, and automation permissions.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official search/public database; no bypass of access controls", + "attribution": { + "attribution_required": true, + "notice": "Government registry; personal-data, terms, limits, and reuse review required", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown; statistical snapshots exist", + "country_code": "AZ", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Authorized machine route, fields, cadence, licensing, privacy, and automation permissions not verified." + ] + }, + "jurisdiction_scope": "Azerbaijan; State Tax Service commercial legal-entity and taxpayer registration", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "az.taxes.organizations", + "source_url": "https://www.taxes.gov.az/en/page/qeydiyyat", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-az.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin authorized route, schema, cadence, IDs, terms, privacy, and automation permissions.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + } + ], + "state": "blocked", + "summary": "No release approval is implied; inspect source gates below." + }, + "BA": { + "acquisition_counts": { + "blocked": 5, + "not_run": 1 + }, + "basis": [ + "ba.bhas.statistics: not_run / reconnaissance", + "ba.bizreg.organizations: blocked / blocked", + "ba.environment-permits: blocked / blocked", + "ba.farm-aquaculture: blocked / blocked", + "ba.inspections-experiments: blocked / blocked", + "ba.veterinary.approved-food: blocked / blocked" + ], + "country_reasons": [ + "acquisition:blocked", + "acquisition:not_run", + "publication:blocked" + ], + "name": "BA", + "owner_review": "awaiting-owner-review", + "publication_state": "blocked", + "source_count": 6, + "sources": [ + { + "access_method": "Agency for Statistics of BiH and entity statistical tables/data services", + "attribution": { + "attribution_required": true, + "notice": "Official statistics; verify table terms, citation, entity coverage and aggregate-only scope.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "table/authority-specific; preserve revisions", + "country_code": "BA", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Pin state/entity slaughter and animal-use table IDs, APIs/downloads and cadence." + ] + }, + "jurisdiction_scope": "Bosnia and Herzegovina; state/entity official slaughter, livestock and animal-use aggregates", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "ba.bhas.statistics", + "source_url": "https://bhas.gov.ba/", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-ba.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin state/entity slaughter and animal-use table IDs, APIs, cadence and revisions.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official searchable web portal; API/bulk route unverified", + "attribution": { + "attribution_required": true, + "notice": "Identity linkage only; verify terms, automation permissions, fields and registered-office privacy.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "portal-defined; last-update field displayed", + "country_code": "BA", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Pin stable endpoints, query contract, rate limits, cadence and entity-specific field semantics." + ] + }, + "jurisdiction_scope": "Bosnia and Herzegovina; BIZREG registers for Federation BiH, Republika Srpska and Brčko District", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "ba.bizreg.organizations", + "source_url": "https://bizreg.pravosudje.ba/pls/apex/f?p=186%3A%3A2313976059615753%3A%3ANO%3A%3A", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-ba.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Pin BIZREG endpoints, entity fields, rate limits, cadence, terms and automation permissions.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "entity environmental ministries/agencies, public notices and documents", + "attribution": { + "attribution_required": true, + "notice": "Verify public-read rights, document license, geometry, sensitive sites and entity coverage.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "permit/document-specific", + "country_code": "BA", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "No unified public read/API/export route or complete cross-entity coverage verified." + ] + }, + "jurisdiction_scope": "Bosnia and Herzegovina; state/entity environmental permits and integrated controls", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "ba.environment-permits", + "source_url": "https://fmoit.gov.ba/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-ba.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Verify entity environmental permit routes, coverage, documents, geometry, license and privacy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "entity/cantonal agriculture, veterinary, fisheries and official open-data/document resources", + "attribution": { + "attribution_required": true, + "notice": "Verify authority, license, animal-holder/property privacy and coordinate precision by entity.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "authority/resource-specific", + "country_code": "BA", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "All-authority farm/aquaculture export and cross-entity identifiers are unresolved." + ] + }, + "jurisdiction_scope": "Bosnia and Herzegovina; livestock holdings, feed and aquaculture across state/entity/cantonal authorities", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "ba.farm-aquaculture", + "source_url": "https://fuzip.gov.ba/unutrasnja-organizacija/federalni-poljoprivredni-inspektorat/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-ba.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Verify all-authority farm/aquaculture scope, exports, IDs, coordinates, terms and privacy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "state/entity/cantonal control plans, inspection reports and statistical publications", + "attribution": { + "attribution_required": true, + "notice": "Sensitive control/research evidence; aggregate or anonymize and require project review.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "report/authority-specific", + "country_code": "BA", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "No stable public cross-entity facility/event or animal-experimentation master verified." + ] + }, + "jurisdiction_scope": "Bosnia and Herzegovina; veterinary/food inspections, enforcement and animal-experimentation evidence", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "ba.inspections-experiments", + "source_url": "https://fuzip.gov.ba/kontrolne-liste/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-ba.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Locate cross-entity inspection/event routes; keep sensitive evidence aggregate until verified.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "state/entity veterinary registers, inspection portals and official documents", + "attribution": { + "attribution_required": true, + "notice": "Official but fragmented authorities; verify each source license, attribution, privacy and coordinates.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "authority-specific; timestamp retrieval", + "country_code": "BA", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "No unified national establishment export verified; map FBiH, RS, Brčko and cantonal coverage first." + ] + }, + "jurisdiction_scope": "Bosnia and Herzegovina; state/entity veterinary and food authorities, including FBiH and cantonal controls", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "ba.veterinary.approved-food", + "source_url": "https://fuzip.gov.ba/federalni-veterinarski-inspektorat/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-ba.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Map state/entity/cantonal coverage and pin establishment routes, IDs, cadence, terms and privacy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + } + ], + "state": "blocked", + "summary": "No release approval is implied; inspect source gates below." + }, + "BE": { + "acquisition_counts": { + "artifact_private_only": 1 + }, + "basis": [ + "be.locations: artifact_private_only / awaiting-owner-review" + ], + "country_reasons": [ + "publication:blocked" + ], + "name": "BE", + "owner_review": "awaiting-owner-review", + "publication_state": "blocked", + "source_count": 1, + "sources": [ + { + "access_method": "published weekly operator CSV plus companion LAP/PAP activity-code CSV; assisted capture supported", + "attribution": { + "attribution_required": true, + "notice": "CC Attribution 4.0; attribute FASFC and the last update date, do not imply FASFC affiliation/approval, and do not mislead", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "weekly", + "country_code": "BE", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "A current official pair was captured privately and validated through the deterministic adapter. Five exact duplicates and 15 unresolved activity codes remain quarantined; privacy, source-terms interpretation, classification, coverage, and project publication approval remain human gates." + ] + }, + "jurisdiction_scope": "Belgium; FASFC-registered, approved, or authorized operators, including animal-origin food and other food-chain activities", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": true, + "public_release_allowed": false, + "reasons": [ + "publication:blocked" + ], + "state": "awaiting-owner-review" + }, + "source_id": "be.locations", + "source_url": "https://www.static.favv.be/bo-documents/inter_actieve_actoren_EN.csv", + "status": { + "acquisition": "artifact_private_only", + "evidence": [ + "docs/country-recon-be.md", + "data/manifests/de-be-private-candidates-2026-09-17.json", + "pipeline/sources/belgium/adapter.py", + "pipeline/sources/belgium/refresh.py", + "docs/review-packet-belgium.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Review the private current pair: 310,660 input, 310,640 normalized, 20 quarantined; names are not supplied by this snapshot. Keep privacy, attribution, classification, terms, and project approval gates closed.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + } + ], + "state": "human-review-ready", + "summary": "No release approval is implied; inspect source gates below." + }, + "BG": { + "acquisition_counts": { + "blocked": 5, + "not_run": 1 + }, + "basis": [ + "bg.bfsa.approved-food: blocked / blocked", + "bg.environment-permits: blocked / blocked", + "bg.farm-aquaculture: blocked / blocked", + "bg.inspections-experiments: blocked / blocked", + "bg.nsi.statistics: not_run / reconnaissance", + "bg.registry-agency.organizations: blocked / blocked" + ], + "country_reasons": [ + "acquisition:blocked", + "acquisition:not_run", + "publication:blocked" + ], + "name": "BG", + "owner_review": "awaiting-owner-review", + "publication_state": "blocked", + "source_count": 6, + "sources": [ + { + "access_method": "official BFSA public registers and linked EU feed-establishment list", + "attribution": { + "attribution_required": true, + "notice": "Official authority; verify register terms, attribution, privacy and coordinates before acquisition.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "publisher-defined; timestamp retrieval", + "country_code": "BG", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Pin current BFSA list/API/export, slaughter categories, IDs, cadence, terms and privacy." + ] + }, + "jurisdiction_scope": "Bulgaria; BFSA approved and registered food/feed establishments handling animal-origin products", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "bg.bfsa.approved-food", + "source_url": "https://bfsa.egov.bg/wps/portal/bfsa-web/registers", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-bg.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin BFSA list/API/export, categories, IDs, cadence, terms and privacy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "Executive Environment Agency systems and public notices/documents", + "attribution": { + "attribution_required": true, + "notice": "Verify public-read rights, document license, geometry and privacy.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "permit/document-specific", + "country_code": "BG", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Stable public read/API/export route and complete national coverage are unresolved." + ] + }, + "jurisdiction_scope": "Bulgaria; environmental permits and integrated-control authorizations", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "bg.environment-permits", + "source_url": "https://eea.government.bg/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-bg.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Verify public permit API/export, coverage, documents, geometry, license and privacy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official BFSA/agriculture/fisheries registers and open-data resources", + "attribution": { + "attribution_required": true, + "notice": "Verify publisher, license, animal-holder/property privacy and coordinate precision.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "resource-specific", + "country_code": "BG", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "No complete national farm/aquaculture public export contract verified." + ] + }, + "jurisdiction_scope": "Bulgaria; livestock holdings and aquaculture permit/establishment evidence", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "bg.farm-aquaculture", + "source_url": "https://bfsa.egov.bg/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-bg.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Verify national farm/aquaculture scope, export/API, IDs, coordinates, terms and privacy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official control plans, reports and statistical publications", + "attribution": { + "attribution_required": true, + "notice": "Sensitive control/research evidence; aggregate or anonymize and require project review.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "report/register-specific", + "country_code": "BG", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "No stable public facility-level inspection or animal-experimentation master verified." + ] + }, + "jurisdiction_scope": "Bulgaria; BFSA inspections/enforcement and animal-experimentation evidence", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "bg.inspections-experiments", + "source_url": "https://bfsa.egov.bg/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-bg.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Locate a stable public facility/event route; keep sensitive evidence aggregate until verified.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official statistical tables/data services; exact table/API route unverified", + "attribution": { + "attribution_required": true, + "notice": "Official statistics; verify table terms, citation requirements and aggregate-only scope.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "table-specific; preserve revisions", + "country_code": "BG", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Pin current slaughter and animal-use table IDs, API/download contracts and cadence." + ] + }, + "jurisdiction_scope": "Bulgaria; NSI aggregate slaughter, livestock and animal-use statistics", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "bg.nsi.statistics", + "source_url": "https://www.nsi.bg/", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-bg.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin official slaughter/animal-use table IDs, APIs, cadence and revisions.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official public register/service surface; API/bulk route unverified", + "attribution": { + "attribution_required": true, + "notice": "Identity linkage only; verify terms, fields, rate limits and registered-office privacy.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "provider-defined", + "country_code": "BG", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Pin stable query/API route, authentication/CAPTCHA, cadence, terms and privacy policy." + ] + }, + "jurisdiction_scope": "Bulgaria; Registry Agency Commercial Register corporate identifiers", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "bg.registry-agency.organizations", + "source_url": "https://www.registryagency.bg/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-bg.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin corporate query/API route, auth/CAPTCHA, fields, cadence, terms and privacy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + } + ], + "state": "blocked", + "summary": "No release approval is implied; inspect source gates below." + }, + "BR": { + "acquisition_counts": { + "artifact_private_only": 2, + "verified": 2 + }, + "basis": [ + "br.sif.export: verified / awaiting-owner-review", + "br.sif.registered: verified / awaiting-owner-review", + "br.sisbi.public: artifact_private_only / awaiting-owner-review", + "br.trase.facilities: artifact_private_only / awaiting-owner-review" + ], + "country_reasons": [ + "publication:blocked" + ], + "name": "BR", + "owner_review": "awaiting-owner-review", + "publication_state": "blocked", + "source_count": 4, + "sources": [ + { + "access_method": "published semicolon-delimited CSV via MAPA CKAN; bounded GET or assisted capture", + "attribution": { + "attribution_required": true, + "notice": "MAPA catalog displays Creative Commons Attribution; export/product rows remain separate evidence and require terms/privacy review", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "monthly; catalog metadata checked 2026-09-16", + "country_code": "BR", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Do not treat export authorization as facility status or count; terms, effective-date semantics, privacy, reconciliation, and project approval remain pending." + ] + }, + "jurisdiction_scope": "Brazil; MAPA/DIPOA SIF establishments with country/product export authorization observations", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": true, + "public_release_allowed": false, + "reasons": [ + "publication:blocked" + ], + "state": "awaiting-owner-review" + }, + "source_id": "br.sif.export", + "source_url": "https://dados.agricultura.gov.br/dataset/062166e3-b515-4274-8e7d-68aadd64b820/resource/fcb7f87d-0092-4a52-a44b-b3550747b4c2/download/sigsifestabelecimentosnacionais.csv", + "status": { + "acquisition": "verified", + "evidence": [ + "docs/country-recon-br.md", + "docs/countries/br/v1-field-crosswalk.json", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Model country/product authorizations as separate dated observations keyed to SIF; confirm validity/suspension semantics, terms, privacy, and no-double-counting rules before integration.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "published semicolon-delimited CSV via MAPA CKAN; bounded GET or assisted capture", + "attribution": { + "attribution_required": true, + "notice": "MAPA catalog displays Creative Commons Attribution; confirm dataset-specific reuse, attribution, and personal-data handling before publication", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "monthly; catalog last-update metadata checked 2026-09-16", + "country_code": "BR", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Restricted raw/contact/address handling, current code semantics, dataset-specific terms, identity lifecycle, and project publication approval remain pending." + ] + }, + "jurisdiction_scope": "Brazil; MAPA/DIPOA establishments registered under the federal SIF directory", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": true, + "public_release_allowed": false, + "reasons": [ + "publication:blocked" + ], + "state": "awaiting-owner-review" + }, + "source_id": "br.sif.registered", + "source_url": "https://dados.agricultura.gov.br/dataset/062166e3-b515-4274-8e7d-68aadd64b820/resource/97277e92-264a-4dc0-9aea-f87b8ea93798/download/sigsifestabelecimentosregistradosnosif.csv", + "status": { + "acquisition": "verified", + "evidence": [ + "docs/country-recon-br.md", + "docs/countries/br/v1-field-crosswalk.json", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Keep the current private SIF CSV and provenance restricted; validate repeated-row semantics, status/code meanings, source terms, privacy, reconciliation, and release approval before any adapter or publication.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "public JavaScript client with bounded JSON GET routes under sisbi_api and GIS API; no bulk scrape performed", + "attribution": { + "attribution_required": true, + "notice": "MAPA government source; public access does not settle API reuse, retention, privacy, or publication terms", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown; public route and API access verified 2026-09-16", + "country_code": "BR", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Confirm pagination/query contract, code lists, status/effective-date semantics, ID lifecycle, address linkage, update cadence, privacy, terms, and project approval with an authorized operator." + ] + }, + "jurisdiction_scope": "Brazil; public e-SISBI/SISBI-POA service, establishment, product, capacity, and MAPA GIS address routes", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": true, + "public_release_allowed": false, + "reasons": [ + "publication:blocked" + ], + "state": "awaiting-owner-review" + }, + "source_id": "br.sisbi.public", + "source_url": "https://sistemasweb.agricultura.gov.br/sgsi/app/estabelecimentos", + "status": { + "acquisition": "artifact_private_only", + "evidence": [ + "docs/country-recon-br.md", + "docs/countries/br/v1-field-crosswalk.json", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Use bounded GET samples only while an authorized operator confirms pagination, code lists, lifecycle/status, address linkage, cadence, terms, privacy, and project approval.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "published GeoJSON download; private reconnaissance capture only", + "attribution": { + "attribution_required": true, + "notice": "Trase page permits platform charts/maps/representations under CC BY 4.0 and asks commercial data users to contact Trase; raw-data reuse and privacy handling remain review gates", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "dataset page says 2025 data, updated 2026-01-01; future cadence unknown", + "country_code": "BR", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Secondary source must remain separate from MAPA; validate source snapshots, geocoding provenance, terms, privacy, constructed-ID behavior, coverage, and project approval before any use." + ] + }, + "jurisdiction_scope": "Brazil; Trase secondary compilation of SIF, SISBI, SIE, SIM, and CONSORCIO facility/activity rows", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": true, + "public_release_allowed": false, + "reasons": [ + "publication:blocked" + ], + "state": "awaiting-owner-review" + }, + "source_id": "br.trase.facilities", + "source_url": "https://trase.earth/open-data/datasets/brazil-facilities", + "status": { + "acquisition": "artifact_private_only", + "evidence": [ + "docs/country-recon-br.md", + "docs/countries/br/v1-field-crosswalk.json", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Keep Trase as separately labeled secondary evidence; review source lineage, geocoding, constructed IDs, raw-data terms, privacy, coverage, and any exact-ID reconciliation before use.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + } + ], + "state": "human-review-ready", + "summary": "No release approval is implied; inspect source gates below." + }, + "CA": { + "acquisition_counts": { + "not_run": 2 + }, + "basis": [ + "ca.cfia.federal-meat: not_run / reconnaissance", + "ca.ontario.meat-plants: not_run / reconnaissance" + ], + "country_reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "name": "CA", + "owner_review": "awaiting-owner-review", + "publication_state": "blocked", + "source_count": 2, + "sources": [ + { + "access_method": "bounded registry download or assisted local capture", + "attribution": { + "attribution_required": true, + "notice": "CFIA government source; registry page calls the consolidation a convenience reference and current reuse/privacy review remains required", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown; registry page states list update 2023-12-04", + "country_code": "CA", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Federal registry is not complete provincial coverage; function-code schema, currency, privacy, and project publication approval remain pending." + ] + }, + "jurisdiction_scope": "Canada; CFIA federally registered meat establishments and licensed operators", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "ca.cfia.federal-meat", + "source_url": "https://active.inspection.gc.ca/scripts/meavia/reglist/download.asp?lang=e", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-ca.md", + "docs/countries/canada/meat-plants-pipeline.md", + "pipeline/sources/canada/adapter.py", + "pipeline/sources/canada/acquire.py", + "pipeline/sources/canada/refresh.py", + "pipeline/sources/canada/fixtures/cfia.csv", + "pipeline/common/review_packet.py", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Run the bounded private CFIA registry refresh; validate the live export schema/function codes, keep federal scope separate from provincial scope, and retain publication gates.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "bounded CSV fetch or assisted local capture", + "attribution": { + "attribution_required": true, + "notice": "Government of Ontario dataset; current licence, attribution, privacy, and redistribution review remain explicit gates", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown; dataset metadata checked 2026-09-14", + "country_code": "CA", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Ontario coverage is not national; privacy/coordinate review and project publication approval remain pending." + ] + }, + "jurisdiction_scope": "Canada; Ontario provincial meat plants only", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "ca.ontario.meat-plants", + "source_url": "https://data.ontario.ca/dataset/a763088c-018d-48b7-bf47-3027a8c725b8/resource/ee6d559a-78de-40e6-b2ba-ad3c4a674b96/download/1._all_meat_plants.csv", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-ca.md", + "docs/countries/canada/meat-plants-pipeline.md", + "pipeline/sources/canada/adapter.py", + "pipeline/sources/canada/acquire.py", + "pipeline/sources/canada/refresh.py", + "pipeline/sources/canada/fixtures/ontario.csv", + "pipeline/common/review_packet.py", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Run the bounded private Ontario refresh; keep plant/contact/coordinate fields restricted pending privacy and licence review, and do not generalize Ontario coverage nationally.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + } + ], + "state": "acquisition-ready", + "summary": "No release approval is implied; inspect source gates below." + }, + "CH": { + "acquisition_counts": { + "blocked": 1 + }, + "basis": [ + "ch.blv.approved-food: blocked / blocked" + ], + "country_reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "name": "CH", + "owner_review": "awaiting-owner-review", + "publication_state": "blocked", + "source_count": 1, + "sources": [ + { + "access_method": "official multilingual FSVO search/list route; authorized bounded export or browser capture only", + "attribution": { + "attribution_required": true, + "notice": "Swiss government source; public visibility does not settle reuse, attribution, personal-address, coordinate, or publication rights", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "source-specific; current page and list route observed 2026-09-16", + "country_code": "CH", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "No stable bulk/API contract or complete national export verified; federal list may be assembled from cantonal authorities; language normalization, effective/status semantics, privacy, terms, coverage, and project approval remain unresolved." + ] + }, + "jurisdiction_scope": "Switzerland; FSVO list of approved food businesses, including animal-origin establishments and slaughterhouses", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "ch.blv.approved-food", + "source_url": "https://www.blv.admin.ch/de/listen-bewilligter-schweizer-betriebe", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-ch.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Obtain an authorized bounded export or capture; fingerprint multilingual schema/list version, preserve approval/activity observations separately, and complete coverage, privacy, terms, and project-approval review.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + } + ], + "state": "blocked", + "summary": "No release approval is implied; inspect source gates below." + }, + "CY": { + "acquisition_counts": { + "blocked": 5, + "not_run": 1 + }, + "basis": [ + "cy.companies.registry: blocked / blocked", + "cy.cystat.livestock-meat: not_run / reconnaissance", + "cy.environment-permits: blocked / blocked", + "cy.vs.approved-food: blocked / blocked", + "cy.vs.farms-livestock: blocked / blocked", + "cy.vs.inspections-enforcement: blocked / blocked" + ], + "country_reasons": [ + "acquisition:blocked", + "acquisition:not_run", + "publication:blocked" + ], + "name": "CY", + "owner_review": "awaiting-owner-review", + "publication_state": "blocked", + "source_count": 6, + "sources": [ + { + "access_method": "official Republic of Cyprus route; authorized bounded capture/API only", + "attribution": { + "attribution_required": true, + "notice": "Government source; Republic of Cyprus scope, terms, privacy, and safety review required", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown; source-specific", + "country_code": "CY", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Exact machine contract, stable IDs, cadence, licensing, privacy, and territorial completeness not pinned." + ] + }, + "jurisdiction_scope": "Republic of Cyprus; Registrar of Companies and Intellectual Property", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "cy.companies.registry", + "source_url": "https://www.companies.gov.cy/en/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-cy.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin authorized route, schema, cadence, IDs, terms, privacy, and Republic of Cyprus territorial scope.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official Republic of Cyprus route; authorized bounded capture/API only", + "attribution": { + "attribution_required": true, + "notice": "Government source; Republic of Cyprus scope, terms, privacy, and safety review required", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown; source-specific", + "country_code": "CY", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Exact machine contract, stable IDs, cadence, licensing, privacy, and territorial completeness not pinned." + ] + }, + "jurisdiction_scope": "Republic of Cyprus; aggregate livestock and meat statistics", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "cy.cystat.livestock-meat", + "source_url": "https://cystatdb.cystat.gov.cy/pxweb/en/8.CYSTAT-DB/8.CYSTAT-DB__Agriculture%2C%20Livestock%2C%20Fishing__Livestock/0320031E.px/", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-cy.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin authorized route, schema, cadence, IDs, terms, privacy, and Republic of Cyprus territorial scope.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official Republic of Cyprus route; authorized bounded capture/API only", + "attribution": { + "attribution_required": true, + "notice": "Government source; Republic of Cyprus scope, terms, privacy, and safety review required", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown; source-specific", + "country_code": "CY", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Exact machine contract, stable IDs, cadence, licensing, privacy, and territorial completeness not pinned." + ] + }, + "jurisdiction_scope": "Republic of Cyprus; environmental/EIA and waste permits", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "cy.environment-permits", + "source_url": "https://www.moa.gov.cy/moa/environment/environment.nsf", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-cy.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin authorized route, schema, cadence, IDs, terms, privacy, and Republic of Cyprus territorial scope.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official Republic of Cyprus route; authorized bounded capture/API only", + "attribution": { + "attribution_required": true, + "notice": "Government source; Republic of Cyprus scope, terms, privacy, and safety review required", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown; source-specific", + "country_code": "CY", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Exact machine contract, stable IDs, cadence, licensing, privacy, and territorial completeness not pinned." + ] + }, + "jurisdiction_scope": "Republic of Cyprus; Veterinary Services approved animal-origin establishments and slaughterhouses", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "cy.vs.approved-food", + "source_url": "https://www.moa.gov.cy/moa/vs/vs.nsf/All/9F6A5DB7308579ACC225764D001D01AF?OpenDocument=&print=", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-cy.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin authorized route, schema, cadence, IDs, terms, privacy, and Republic of Cyprus territorial scope.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official Republic of Cyprus route; authorized bounded capture/API only", + "attribution": { + "attribution_required": true, + "notice": "Government source; Republic of Cyprus scope, terms, privacy, and safety review required", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown; source-specific", + "country_code": "CY", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Exact machine contract, stable IDs, cadence, licensing, privacy, and territorial completeness not pinned." + ] + }, + "jurisdiction_scope": "Republic of Cyprus; Veterinary Services livestock holdings and animal identification", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "cy.vs.farms-livestock", + "source_url": "https://www.moa.gov.cy/moa/vs/vs.nsf", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-cy.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin authorized route, schema, cadence, IDs, terms, privacy, and Republic of Cyprus territorial scope.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official Republic of Cyprus route; authorized bounded capture/API only", + "attribution": { + "attribution_required": true, + "notice": "Government source; Republic of Cyprus scope, terms, privacy, and safety review required", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown; source-specific", + "country_code": "CY", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Exact machine contract, stable IDs, cadence, licensing, privacy, and territorial completeness not pinned." + ] + }, + "jurisdiction_scope": "Republic of Cyprus; Veterinary Services inspections and animal-use evidence", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "cy.vs.inspections-enforcement", + "source_url": "https://www.moa.gov.cy/moa/vs/vs.nsf", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-cy.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin authorized route, schema, cadence, IDs, terms, privacy, and Republic of Cyprus territorial scope.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + } + ], + "state": "blocked", + "summary": "No release approval is implied; inspect source gates below." + }, + "CZ": { + "acquisition_counts": { + "blocked": 2, + "not_run": 3 + }, + "basis": [ + "cz.business-register: not_run / reconnaissance", + "cz.environment.permits: blocked / blocked", + "cz.statistics: not_run / reconnaissance", + "cz.svs.approved-food: not_run / reconnaissance", + "cz.svs.farms-aquaculture: blocked / blocked" + ], + "country_reasons": [ + "acquisition:blocked", + "acquisition:not_run", + "metadata:unknown", + "publication:blocked" + ], + "name": "CZ", + "owner_review": "awaiting-owner-review", + "publication_state": "blocked", + "source_count": 5, + "sources": [ + { + "access_method": "public register search and CSV export", + "attribution": { + "attribution_required": true, + "notice": "Identity-only linkage; suppress personal, sole-trader and residential details.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "provider-defined", + "country_code": "CZ", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Verify current endpoint, terms, quotas and privacy policy." + ] + }, + "jurisdiction_scope": "Czechia; corporate/statistical register identity", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "cz.business-register", + "source_url": "https://portal.gov.cz/sluzby-vs/ziskani-zverejnenych-informaci-ze-statistickych-registru-S4953", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-cz.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Verify current endpoint, terms and identity-only privacy policy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official environmental registry routes; not verified", + "attribution": { + "attribution_required": true, + "notice": "Verify license, geometry and sensitive-site handling.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown", + "country_code": "CZ", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "No stable national animal-facility permit export verified." + ] + }, + "jurisdiction_scope": "Czechia; environmental permits and releases", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "metadata:unknown", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "cz.environment.permits", + "source_url": "https://www.mzp.cz/en", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-cz.md", + "pipeline/source_registry.json" + ], + "metadata": "unknown", + "next_action": "Locate and verify environmental permit data route.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official statistical tables/API; exact route unverified", + "attribution": { + "attribution_required": true, + "notice": "Official statistics; preserve table IDs, revisions and license metadata.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "table-specific", + "country_code": "CZ", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Pin current table/API contracts and keep aggregate scope separate." + ] + }, + "jurisdiction_scope": "Czechia; official slaughter, livestock and animal-use aggregates", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "cz.statistics", + "source_url": "https://www.czso.cz/csu/czso/statistics", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-cz.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin statistics table/API contracts and revisions.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official filtered lists and linked files", + "attribution": { + "attribution_required": true, + "notice": "Official Czech veterinary authority; verify terms, fields and privacy.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "list-specific; timestamp access", + "country_code": "CZ", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Verify bulk/API/file contracts, completeness, terms, IDs and coordinate availability." + ] + }, + "jurisdiction_scope": "Czechia; SVS approved/registered animal-origin food, ABP, feed and aquaculture establishments", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "cz.svs.approved-food", + "source_url": "https://en.svs.gov.cz/registered-subjects/", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-cz.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Verify bulk/API/file contracts, completeness, terms, IDs and coordinates.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official lists/filter surface", + "attribution": { + "attribution_required": true, + "notice": "Animal-holder and site data require privacy review.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown", + "country_code": "CZ", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Verify national scope, export route, terms and sensitive-field policy." + ] + }, + "jurisdiction_scope": "Czechia; SVS farms, aquaculture and animal-sector registers", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "cz.svs.farms-aquaculture", + "source_url": "https://en.svs.gov.cz/registered-subjects/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-cz.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Verify national scope, export route, terms and sensitive-field policy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + } + ], + "state": "blocked", + "summary": "No release approval is implied; inspect source gates below." + }, + "DE": { + "acquisition_counts": { + "artifact_private_only": 1 + }, + "basis": [ + "de.locations: artifact_private_only / awaiting-owner-review" + ], + "country_reasons": [ + "publication:blocked" + ], + "name": "DE", + "owner_review": "awaiting-owner-review", + "publication_state": "blocked", + "source_count": 1, + "sources": [ + { + "access_method": "BVL portal selected CSV/XLS export or assisted capture", + "attribution": { + "attribution_required": true, + "notice": "unknown; verify BVL reuse and attribution terms", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown", + "country_code": "DE", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "A current public general-list export was captured privately: 15,788 input, 2,691 normalized, and 13,097 quarantined. The export URL is session/request-specific, effective date is unknown, and dataset reuse terms, privacy, coverage, and project approval remain pending human confirmation. Keep raw data private and publication blocked." + ] + }, + "jurisdiction_scope": "Germany", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": true, + "public_release_allowed": false, + "reasons": [ + "publication:blocked" + ], + "state": "awaiting-owner-review" + }, + "source_id": "de.locations", + "source_url": "https://www.bvl.bund.de/bltu", + "status": { + "acquisition": "artifact_private_only", + "evidence": [ + "docs/germany-source-assessment.md", + "data/manifests/de-be-private-candidates-2026-09-17.json", + "pipeline/sources/germany/adapter.py", + "pipeline/sources/germany/refresh.py", + "pipeline/common/review_packet.py", + "docs/review-packet-germany.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Review the private current export: 15,788 input, 2,691 normalized, 13,097 quarantined, with 50-column schema matched and no release created. The session-bound export route, unknown effective date, terms, privacy, coverage, and project approval remain unresolved.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + } + ], + "state": "human-review-ready", + "summary": "No release approval is implied; inspect source gates below." + }, + "DK": { + "acquisition_counts": { + "verified": 1 + }, + "basis": [ + "dk.smiley: verified / awaiting-owner-review" + ], + "country_reasons": [ + "publication:blocked" + ], + "name": "DK", + "owner_review": "awaiting-owner-review", + "publication_state": "blocked", + "source_count": 1, + "sources": [ + { + "access_method": "bulk XML download", + "attribution": { + "attribution_required": true, + "notice": "Official Find Smiley data page records public-data reuse terms: attribute Fødevarestyrelsen, do not use its logo, and keep displayed smileys current; no project publication approval.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "weekly", + "country_code": "DK", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Publisher supplies no dataset effective date; source coverage is limited to data available on Find Smiley and is not a completeness claim. Denmark candidate rows require explicit source-key mapping before disposable DB import." + ] + }, + "jurisdiction_scope": "Denmark; food and animal-related establishments in Find Smiley coverage", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": true, + "public_release_allowed": false, + "reasons": [ + "publication:blocked" + ], + "state": "awaiting-owner-review" + }, + "source_id": "dk.smiley", + "source_url": "https://pub.fvst.dk/publikationer/Smileydata.xml", + "status": { + "acquisition": "verified", + "evidence": [ + "pipeline/sources/denmark/README.md", + "pipeline/contracts/README.md", + "pipeline/common/review_packet.py", + "docs/review-packet-denmark.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Keep the full privately staged artifact and candidate import validation restricted; run a bounded guarded API check with explicit test-only labeling, capture rerun exit output, and resolve coverage/effective-date/category semantics before any release review. This is not a production-health or publication claim.", + "publication_eligibility": "blocked", + "runtime_health": "unknown" + } + } + ], + "state": "human-review-ready", + "summary": "No release approval is implied; inspect source gates below." + }, + "EE": { + "acquisition_counts": { + "blocked": 2, + "not_run": 4 + }, + "basis": [ + "ee.ariregister.organizations: not_run / reconnaissance", + "ee.keskkonnaamet.kotkas: blocked / blocked", + "ee.pria.animal-register: not_run / reconnaissance", + "ee.pria.aquaculture: blocked / blocked", + "ee.pta.approved-food: not_run / reconnaissance", + "ee.stat.slaughter: not_run / reconnaissance" + ], + "country_reasons": [ + "acquisition:blocked", + "acquisition:not_run", + "publication:blocked" + ], + "name": "EE", + "owner_review": "awaiting-owner-review", + "publication_state": "blocked", + "source_count": 6, + "sources": [ + { + "access_method": "official register/API route unverified", + "attribution": { + "attribution_required": true, + "notice": "Identity-only linkage; suppress personal, sole-trader and residential details; verify terms.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "provider-defined", + "country_code": "EE", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Verify current API, auth, quotas and privacy policy." + ] + }, + "jurisdiction_scope": "Estonia; Äriregister corporate identity", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "ee.ariregister.organizations", + "source_url": "https://ariregister.rik.ee/eng", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-ee.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Verify API, auth, quotas, terms and identity-only policy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "KOTKAS/document register and public notices", + "attribution": { + "attribution_required": true, + "notice": "Official environmental authority; verify terms, geometry and sensitive-site handling.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "publication/permit-specific", + "country_code": "EE", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "UI/document extraction and complete national coverage unresolved." + ] + }, + "jurisdiction_scope": "Estonia; Keskkonnaamet environmental permit and public-notice evidence", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "ee.keskkonnaamet.kotkas", + "source_url": "https://keskkonnaamet.ee/keskkonnateadlikkus-avalikustamised/raagi-kaasa/lubade-eelnoude-avalik-valjapanek", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-ee.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Verify document extraction, terms, identifiers, geometry and coverage.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "public query/XML and geospatial web map/service", + "attribution": { + "attribution_required": true, + "notice": "Official PRIA; animal/property location data require strict privacy and purpose review.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "prior-day data described; verify service cadence", + "country_code": "EE", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Verify current service/download, CRS, terms, field suppression and coverage." + ] + }, + "jurisdiction_scope": "Estonia; PRIA public farm-animal and aquaculture establishment data", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "ee.pria.animal-register", + "source_url": "https://www.pria.ee/registrid/avalikud-andmed", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-ee.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Verify current public service/download, CRS, terms and field suppression.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "public register and animal-register services", + "attribution": { + "attribution_required": true, + "notice": "Official authority; separate aquaculture from farm/food entities and review sensitive locations.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "service-specific", + "country_code": "EE", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "No aquaculture-specific bulk contract verified." + ] + }, + "jurisdiction_scope": "Estonia; PRIA aquaculture establishment register", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "ee.pria.aquaculture", + "source_url": "https://www.pria.ee/registrid/kalad-ja-vahid", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-ee.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Verify aquaculture-specific bulk contract and privacy scope.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official registers and linked downloads", + "attribution": { + "attribution_required": true, + "notice": "Official Estonian authority; verify dataset terms and privacy.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "publisher-defined; timestamp retrieval", + "country_code": "EE", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Verify current bulk/API route, completeness, headers, terms and privacy." + ] + }, + "jurisdiction_scope": "Estonia; PTA approved and registered food/animal establishments", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "ee.pta.approved-food", + "source_url": "https://pta.agri.ee/riiklikud-registrid-ja-andmekogud", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-ee.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Verify current bulk/API route, completeness, headers, terms and privacy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official PxWeb/statistical database", + "attribution": { + "attribution_required": true, + "notice": "Statistics Estonia open-data route states CC BY-SA 4.0; preserve table IDs and revisions.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "monthly/table-specific", + "country_code": "EE", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Pin API/table metadata; keep aggregate scope separate from facility evidence." + ] + }, + "jurisdiction_scope": "Estonia; Statistics Estonia slaughter and livestock aggregates", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "ee.stat.slaughter", + "source_url": "https://andmed.stat.ee/en/stat/majandus__pellumajandus__pellumajandussaaduste-tootmine__loomakasvatussaaduste-tootmine/PM190", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-ee.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Pin current PxWeb table metadata and preserve revisions.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + } + ], + "state": "blocked", + "summary": "No release approval is implied; inspect source gates below." + }, + "ES": { + "acquisition_counts": { + "blocked": 1 + }, + "basis": [ + "es.locations: blocked / blocked" + ], + "country_reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "name": "ES", + "owner_review": "awaiting-owner-review", + "publication_state": "blocked", + "source_count": 1, + "sources": [ + { + "access_method": "download or assisted export", + "attribution": { + "attribution_required": true, + "notice": "unknown; confirm competent authority, reuse terms, and attribution", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown", + "country_code": "ES", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Identify the exact competent-authority publication and distinguish source values from legacy transformations." + ] + }, + "jurisdiction_scope": "Spain", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "es.locations", + "source_url": "unknown", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-es.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Resolve a permitted current AESAN/MAPA artifact or query route, then verify export/schema, rights, effective-date semantics, sector coverage, privacy, and legacy/source boundaries before acquisition.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + } + ], + "state": "blocked", + "summary": "No release approval is implied; inspect source gates below." + }, + "EU": { + "acquisition_counts": { + "artifact_private_only": 1, + "not_run": 2 + }, + "basis": [ + "eu.eurostat.nl-slaughter: artifact_private_only / awaiting-owner-review", + "eu.traces.approved-establishments: not_run / reconnaissance", + "eu.traces.pl-approved-food: not_run / reconnaissance" + ], + "country_reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "name": "EU", + "owner_review": "awaiting-owner-review", + "publication_state": "blocked", + "source_count": 3, + "sources": [ + { + "access_method": "public dissemination JSON API", + "attribution": { + "attribution_required": true, + "notice": "Official EU mirror; never count in addition to CBS or NVWA.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "dataset-defined; retain last-updated metadata", + "country_code": "EU", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Use only for harmonized aggregate context and preserve flags." + ] + }, + "jurisdiction_scope": "EU statistical mirror; Netherlands aggregate slaughter context", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": true, + "public_release_allowed": false, + "reasons": [ + "publication:blocked" + ], + "state": "awaiting-owner-review" + }, + "source_id": "eu.eurostat.nl-slaughter", + "source_url": "https://ec.europa.eu/eurostat/api/dissemination/statistics/1.0/data/apro_mt_pann?geo=NL&lang=en", + "status": { + "acquisition": "artifact_private_only", + "evidence": [ + "docs/country-recon-nl.md", + "data/manifests/nl-source-artifacts.json", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Use only as harmonized aggregate context; retain dimensions/flags and do not double-count CBS.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "TRACES publication surface and national links", + "attribution": { + "attribution_required": true, + "notice": "Official EU mirror; do not double-count national NVWA observations.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "competent-authority updates; timestamp each access", + "country_code": "EU", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Verify NL export/API and stable IDs before use; keep separate provenance layer." + ] + }, + "jurisdiction_scope": "EU; TRACES/IMSOC approved-establishment mirror", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "eu.traces.approved-establishments", + "source_url": "https://food.ec.europa.eu/food-safety/biological-safety/food-hygiene/approved-eu-food-establishments_en", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-nl.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Verify NL-specific TRACES export and stable IDs only if needed; never merge as additional facilities.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "public TRACES/publication page; Poland export not verified", + "attribution": { + "attribution_required": true, + "notice": "EU official mirror; do not double-count GIW", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "competent-authority updates", + "country_code": "EU", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Verify Poland-specific export and keep mirror provenance separate." + ] + }, + "jurisdiction_scope": "EU mirror; Poland approved-food establishment lineage", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "eu.traces.pl-approved-food", + "source_url": "https://food.ec.europa.eu/food-safety/biological-safety/food-hygiene/approved-eu-food-establishments_en", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-pl.md", + "docs/countries/pl/source-crosswalk.json", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Verify Poland-specific TRACES export only as a lineage/cross-check layer; never double-count GIW.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + } + ], + "state": "private-candidate-ready", + "summary": "No release approval is implied; inspect source gates below." + }, + "FI": { + "acquisition_counts": { + "blocked": 3, + "not_run": 5 + }, + "basis": [ + "fi.animal-experiments: not_run / reconnaissance", + "fi.aquaculture: blocked / blocked", + "fi.luke.agriculture: not_run / reconnaissance", + "fi.prh.ytj.organizations: not_run / reconnaissance", + "fi.ruokavirasto.approved-food: not_run / reconnaissance", + "fi.ruokavirasto.feed-abp: blocked / blocked", + "fi.statfin.pxweb: not_run / reconnaissance", + "fi.syke.environment: blocked / blocked" + ], + "country_reasons": [ + "acquisition:blocked", + "acquisition:not_run", + "publication:blocked" + ], + "name": "FI", + "owner_review": "awaiting-owner-review", + "publication_state": "blocked", + "source_count": 8, + "sources": [ + { + "access_method": "official guidance and annual reports", + "attribution": { + "attribution_required": true, + "notice": "Sensitive research evidence; aggregate/anonymize and apply strict privacy review.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "annual/report-specific", + "country_code": "FI", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "No public research-facility master verified." + ] + }, + "jurisdiction_scope": "Finland; animal experimentation guidance/statistics", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "fi.animal-experiments", + "source_url": "https://www.ruokavirasto.fi/en/animals/animal-experiments/", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-fi.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Keep annual aggregates separate; no research-facility master verified.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official statistics and environmental/geospatial services", + "attribution": { + "attribution_required": true, + "notice": "Verify provider terms, geometry precision and site privacy before use.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "statistics/service-specific; unknown", + "country_code": "FI", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "No current national public permit/site API verified." + ] + }, + "jurisdiction_scope": "Finland; aquaculture production/sites and environmental evidence", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "fi.aquaculture", + "source_url": "https://www.luke.fi/en/statistics/aquaculture", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-fi.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Verify current public permit/site API, geometry, terms and privacy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official statistics pages/downloads/API where available", + "attribution": { + "attribution_required": true, + "notice": "Official statistics; verify dataset license and preserve revisions/aggregate scope.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "table-specific; timestamp each release", + "country_code": "FI", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Pin table IDs, API/download contracts and license; do not infer facility rows." + ] + }, + "jurisdiction_scope": "Finland; Luke aggregate agriculture, livestock and aquaculture statistics", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "fi.luke.agriculture", + "source_url": "https://www.luke.fi/en/statistics", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-fi.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Pin table/API contracts and preserve aggregate scope.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official open-data/API/download documentation; current route unverified", + "attribution": { + "attribution_required": true, + "notice": "Identity linkage only; suppress personal, sole-trader and residential details; verify terms.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "provider-defined", + "country_code": "FI", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Verify current endpoint, auth, quotas, fields and identity-only policy." + ] + }, + "jurisdiction_scope": "Finland; PRH/YTJ corporate and business identifiers", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "fi.prh.ytj.organizations", + "source_url": "https://www.prh.fi/en/uutislistaus/uutiset/2020/P_23520.html", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-fi.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Verify current endpoint, auth, quotas and identity-only privacy policy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official web pages and linked lists; bulk route unverified", + "attribution": { + "attribution_required": true, + "notice": "Official Finnish Food Authority; verify file-specific reuse terms and privacy before acquisition.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "publisher-defined; timestamp each retrieval", + "country_code": "FI", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Verify current export/API, section coverage, headers, terms and mixed-address privacy." + ] + }, + "jurisdiction_scope": "Finland; Ruokavirasto approved animal-origin food establishments", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "fi.ruokavirasto.approved-food", + "source_url": "https://www.ruokavirasto.fi/en/companies/food-sector/food-establishments/approved-establishments/", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-fi.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Verify current export/API, section coverage, headers, terms and privacy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official guidance/register surface; bulk route unverified", + "attribution": { + "attribution_required": true, + "notice": "Official authority; category-specific terms, coverage and privacy require review.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown; timestamp each access", + "country_code": "FI", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Verify public list/export, category boundaries and source rights before acquisition." + ] + }, + "jurisdiction_scope": "Finland; Ruokavirasto feed and animal-by-product establishments", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "fi.ruokavirasto.feed-abp", + "source_url": "https://www.ruokavirasto.fi/en/companies/feed/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-fi.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Verify public list/export, category boundaries and source rights.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official PxWeb/statistical database API or downloads", + "attribution": { + "attribution_required": true, + "notice": "Official Statistics Finland; verify table license and preserve dimensions/revisions.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "table-specific; timestamp releases/revisions", + "country_code": "FI", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Pin current table IDs and API contracts; keep aggregate scope separate." + ] + }, + "jurisdiction_scope": "Finland; Statistics Finland aggregate slaughter and animal-use context", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "fi.statfin.pxweb", + "source_url": "https://stat.fi/en/services/statistical-data-services/statistical-databases", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-fi.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin current table IDs/API contracts and preserve revisions.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official open-data/service catalogue; route-specific", + "attribution": { + "attribution_required": true, + "notice": "Verify license, attribution, sensitive-site handling and geometry semantics.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "dataset-specific", + "country_code": "FI", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "No stable national animal-facility permit export verified." + ] + }, + "jurisdiction_scope": "Finland; environmental open information and permit evidence", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "fi.syke.environment", + "source_url": "https://www.syke.fi/en-US/Open_information", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-fi.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Verify current environmental export/API, license, geometry and coverage.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + } + ], + "state": "blocked", + "summary": "No release approval is implied; inspect source gates below." + }, + "FR": { + "acquisition_counts": { + "not_run": 2 + }, + "basis": [ + "fr.dgal.section-i: not_run / reconnaissance", + "fr.dgal.section-ii: not_run / reconnaissance" + ], + "country_reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "name": "FR", + "owner_review": "awaiting-owner-review", + "publication_state": "blocked", + "source_count": 2, + "sources": [ + { + "access_method": "bounded TXT fetch or assisted local capture", + "attribution": { + "attribution_required": true, + "notice": "Ministry page indicates Etalab 2.0 for site content; file-specific terms and attribution confirmation remain pending", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "daily; Ministry page checked 2026-09-15", + "country_code": "FR", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Privacy/coordinate review, file-specific terms, category codebook, and project publication approval remain pending." + ] + }, + "jurisdiction_scope": "France; DGAL Regulation (EC) 853/2004 Section I domestic ungulate establishments", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "fr.dgal.section-i", + "source_url": "https://fichiers-publics.agriculture.gouv.fr/dgal/ListesOfficielles/SSA1_VIAN_ONG_DOM.txt", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-fr.md", + "docs/countries/france/dgal-853-pipeline.md", + "pipeline/sources/france/adapter.py", + "pipeline/sources/france/acquire.py", + "pipeline/sources/france/refresh.py", + "pipeline/sources/france/fixtures/section_i.csv", + "pipeline/common/review_packet.py", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Run the bounded private Section I refresh with an approved terms record or authorized capture; review category semantics, address privacy, schema drift, and release approval.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "bounded TXT fetch or assisted local capture", + "attribution": { + "attribution_required": true, + "notice": "Ministry page indicates Etalab 2.0 for site content; file-specific terms and attribution confirmation remain pending", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "daily; Ministry page checked 2026-09-15", + "country_code": "FR", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Section II is separate from Section I; privacy/coordinate review, terms, codebook, and project publication approval remain pending." + ] + }, + "jurisdiction_scope": "France; DGAL Regulation (EC) 853/2004 Section II poultry and lagomorph establishments", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "fr.dgal.section-ii", + "source_url": "https://fichiers-publics.agriculture.gouv.fr/dgal/ListesOfficielles/SSA1_VIAN_COL_LAGO.txt", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-fr.md", + "docs/countries/france/dgal-853-pipeline.md", + "pipeline/sources/france/adapter.py", + "pipeline/sources/france/acquire.py", + "pipeline/sources/france/refresh.py", + "pipeline/sources/france/fixtures/section_ii.csv", + "pipeline/common/review_packet.py", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Run the bounded private Section II refresh separately from Section I; review category/species semantics, address privacy, schema drift, and release approval.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + } + ], + "state": "acquisition-ready", + "summary": "No release approval is implied; inspect source gates below." + }, + "GE": { + "acquisition_counts": { + "blocked": 5, + "not_run": 1 + }, + "basis": [ + "ge.geostat.slaughter-statistics: not_run / reconnaissance", + "ge.napr.organizations: blocked / blocked", + "ge.nea.environment-permits: blocked / blocked", + "ge.nfa.approved-food: blocked / blocked", + "ge.nfa.farms-livestock: blocked / blocked", + "ge.nfa.inspections-experiments: blocked / blocked" + ], + "country_reasons": [ + "acquisition:blocked", + "acquisition:not_run", + "publication:blocked" + ], + "name": "GE", + "owner_review": "awaiting-owner-review", + "publication_state": "blocked", + "source_count": 6, + "sources": [ + { + "access_method": "official Geostat publications and tables; machine API or bounded download to be pinned", + "attribution": { + "attribution_required": true, + "notice": "National Statistics Office source; publication terms, revisions, and suppression rules require review", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "quarterly/annual; publication-specific", + "country_code": "GE", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Current table IDs, machine API, revision semantics, and suppression rules not pinned." + ] + }, + "jurisdiction_scope": "Georgia; aggregate livestock slaughterhouse and animal-production statistics", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "ge.geostat.slaughter-statistics", + "source_url": "https://www.geostat.ge/en/modules/categories/755/section-5-livestock-poultry-and-beehives", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-ge.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin authorized route, schema, cadence, IDs, terms, privacy, and automation permissions.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official NAPR online extract/service; authorized API/bulk access only", + "attribution": { + "attribution_required": true, + "notice": "Official registry service; fees, rate limits, terms, and personal-address policy require review", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown", + "country_code": "GE", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Authorized machine route, fields, cadence, fees, limits, and privacy policy not verified." + ] + }, + "jurisdiction_scope": "Georgia; NAPR entrepreneur and legal-entity identifiers", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "ge.napr.organizations", + "source_url": "https://www.napr.gov.ge/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-ge.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin authorized route, schema, cadence, IDs, terms, privacy, and automation permissions.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official NEA service/register route; authorized API/export only", + "attribution": { + "attribution_required": true, + "notice": "Agency source; license, geometry, privacy, and terms require review", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "resource-specific; unknown", + "country_code": "GE", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Exact permit route, export/API, coverage, cadence, license, geometry, and privacy not pinned." + ] + }, + "jurisdiction_scope": "Georgia; National Environment Agency environmental permits and related registers", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "ge.nea.environment-permits", + "source_url": "https://nea.gov.ge/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-ge.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin authorized route, schema, cadence, IDs, terms, privacy, and automation permissions.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official NFA page with advertised download; authorized bounded capture only", + "attribution": { + "attribution_required": true, + "notice": "Official Georgian government source; terms, privacy, and location-safety review required", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "periodic; exact cadence unknown", + "country_code": "GE", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Exact download URL, format, schema, stable ID, cadence, terms, and privacy policy not pinned." + ] + }, + "jurisdiction_scope": "Georgia; NFA registered slaughterhouses and recognized animal-origin food operators", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "ge.nfa.approved-food", + "source_url": "https://nfa.gov.ge/Ge/Page/List%20of%20Slaughterhouses%20Registered%20in%20Georgia", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-ge.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin authorized route, schema, cadence, IDs, terms, privacy, and automation permissions.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official NFA service/list route; no facility acquisition until contract review", + "attribution": { + "attribution_required": true, + "notice": "Official government source; scope, personal data, coordinates, and reuse terms require review", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown", + "country_code": "GE", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Bulk/API route, stable IDs, cadence, coverage, privacy, and location policy not verified." + ] + }, + "jurisdiction_scope": "Georgia; NFA primary-production, livestock identification/registration, feed, and recognized operators", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "ge.nfa.farms-livestock", + "source_url": "https://nfa.gov.ge/Ge/Page/Primary%20production%20control", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-ge.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin authorized route, schema, cadence, IDs, terms, privacy, and automation permissions.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official dated reports/registers; aggregate-only until safe event route is verified", + "attribution": { + "attribution_required": true, + "notice": "Sensitive control/research evidence; privacy, retention, and publication review required", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "resource-specific; unknown", + "country_code": "GE", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "No stable public event/experimentation master verified; stop and escalate on sensitive exposure." + ] + }, + "jurisdiction_scope": "Georgia; NFA veterinary/food-control findings and public animal-experimentation evidence", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "ge.nfa.inspections-experiments", + "source_url": "https://www.nfa.gov.ge/Ge/Page/Veterinary%20Control", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-ge.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin authorized route, schema, cadence, IDs, terms, privacy, and automation permissions.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + } + ], + "state": "blocked", + "summary": "No release approval is implied; inspect source gates below." + }, + "HR": { + "acquisition_counts": { + "blocked": 3, + "not_run": 3 + }, + "basis": [ + "hr.approved-food: not_run / reconnaissance", + "hr.aquaculture-permits: not_run / reconnaissance", + "hr.business-register: blocked / blocked", + "hr.environment-permits: blocked / blocked", + "hr.inspections-experiments: blocked / blocked", + "hr.statistics: not_run / reconnaissance" + ], + "country_reasons": [ + "acquisition:blocked", + "acquisition:not_run", + "publication:blocked" + ], + "name": "HR", + "owner_review": "awaiting-owner-review", + "publication_state": "blocked", + "source_count": 6, + "sources": [ + { + "access_method": "national CKAN WEB/XLS resources and official register guidance", + "attribution": { + "attribution_required": true, + "notice": "CKAN lists public/open access; verify exact resource license, privacy and attribution before acquisition.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "not defined; timestamp retrieval", + "country_code": "HR", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Pin direct resource route, schema, cadence, identifiers, coordinates, terms and privacy before acquisition." + ] + }, + "jurisdiction_scope": "Croatia; approved and registered food establishments handling food of animal origin", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "hr.approved-food", + "source_url": "https://data.gov.hr/ckan/en/dataset/upisnik-odobrenih-objekata-u-poslovanju-s-hranom-za-zivotinje", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-hr.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Pin direct resource route, schema, cadence, identifiers, terms and privacy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "CKAN XLS resource", + "attribution": { + "attribution_required": true, + "notice": "CKAN marks the dataset open; verify current license, sensitive-site handling and attribution.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "not defined; timestamp retrieval", + "country_code": "HR", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Pin direct XLS URL, schema, update cadence, stable IDs, coordinates and terms." + ] + }, + "jurisdiction_scope": "Croatia; Ministry aquaculture permit register", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "hr.aquaculture-permits", + "source_url": "https://data.gov.hr/ckan/hr/dataset/registar-dozvola-u-akvakulturi", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-hr.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Pin direct XLS URL, schema, cadence, IDs, coordinates and terms.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "public REST API after free registration; XML or JSON", + "attribution": { + "attribution_required": true, + "notice": "Identity linkage only; verify API terms, quotas and suppression of personal/sole-trader/residential details.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "daily on working days", + "country_code": "HR", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Credentials, quota, exact API contract and legal-person-only matching policy require verification." + ] + }, + "jurisdiction_scope": "Croatia; Court Register corporate identity and registered office", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "hr.business-register", + "source_url": "https://sudreg-data.gov.hr/ords/r/srn_rep/vanjski-srn-rep/home", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-hr.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Provision and verify API credentials, quotas, contract and identity-only policy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "national CKAN resources and environmental registers", + "attribution": { + "attribution_required": true, + "notice": "Verify current publisher, resource license, document rights, geometry and privacy.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "dataset-specific", + "country_code": "HR", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Complete national animal-facility coverage and machine-readable route are unresolved." + ] + }, + "jurisdiction_scope": "Croatia; environmental permits and integrated environmental conditions", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "hr.environment-permits", + "source_url": "https://data.gov.hr/ckan/en/dataset/o-evidnik-uporabnih-dozvola-i-rje-enja-o-objedinjenim-uvjetima-za-tite-okoli-a", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-hr.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Verify publisher, resource route, coverage, geometry, licensing and privacy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official register guidance, reports and statistical publications", + "attribution": { + "attribution_required": true, + "notice": "Sensitive control/research evidence; aggregate or anonymize and require project review.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "report/register-specific", + "country_code": "HR", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "No stable public facility-level inspection or animal-experimentation master verified." + ] + }, + "jurisdiction_scope": "Croatia; food/veterinary inspections, enforcement and animal-experimentation evidence", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "hr.inspections-experiments", + "source_url": "https://inspektorat.gov.hr/ustrojstvo-77/7-sektor-sanitarne-inspekcije/evidentiranje-i-vodjenje-registra-subjekta-i-pripadajucih-objekta-u-poslovanju-s-hranom-iz-nadleznosti-iz-nadleznosti-sanitarne-inspekcije/431", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-hr.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Locate a stable public facility/event route; keep sensitive evidence aggregate until verified.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "Croatian Bureau of Statistics tables/data services", + "attribution": { + "attribution_required": true, + "notice": "Official statistics; verify table terms, citation requirements and aggregate-only scope.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "table-specific; preserve revisions", + "country_code": "HR", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Pin slaughter and animal-use table IDs, API/download contracts and release cadence." + ] + }, + "jurisdiction_scope": "Croatia; official slaughter, livestock and animal-use aggregate statistics", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "hr.statistics", + "source_url": "https://dzs.gov.hr/usluge/objavljivanje/program-publiciranja-2026/2439", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-hr.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin official slaughter and animal-use table IDs, APIs and revisions.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + } + ], + "state": "blocked", + "summary": "No release approval is implied; inspect source gates below." + }, + "IE": { + "acquisition_counts": { + "artifact_private_only": 4, + "not_run": 12 + }, + "basis": [ + "ie.cro.companies: not_run / reconnaissance", + "ie.cso.livestock-slaughterings: not_run / reconnaissance", + "ie.dafm.animal-welfare-controls: not_run / reconnaissance", + "ie.dafm.approved-establishments: not_run / reconnaissance", + "ie.dafm.former-la-establishments: not_run / reconnaissance", + "ie.dafm.milk-dairy-establishments: not_run / reconnaissance", + "ie.dafm.national-beef-kill: not_run / reconnaissance", + "ie.dafm.seafood-processing-funding: not_run / reconnaissance", + "ie.epa.leap: not_run / reconnaissance", + "ie.fsai.approved-directory: not_run / reconnaissance", + "ie.fsai.enforcement-orders: not_run / reconnaissance", + "ie.hse.low-throughput-meat: artifact_private_only / awaiting-owner-review", + "ie.planning.npad: not_run / reconnaissance", + "ie.sfpa.approved-establishments: artifact_private_only / awaiting-owner-review", + "ie.sfpa.factory-vessels: artifact_private_only / awaiting-owner-review", + "ie.sfpa.freezer-vessels: artifact_private_only / awaiting-owner-review" + ], + "country_reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "name": "IE", + "owner_review": "awaiting-owner-review", + "publication_state": "blocked", + "source_count": 16, + "sources": [ + { + "access_method": "official open-data bulk/API route or CORE search; bounded identity capture", + "attribution": { + "attribution_required": true, + "notice": "CRO open-data company dataset is described as CC BY 4.0; officer/personal details remain restricted and company identity is not proof of a facility or operation", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "daily; catalog describes daily updates", + "country_code": "IE", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Name similarity is not a merge; require reviewed company-number links and keep registered offices distinct from operating premises." + ] + }, + "jurisdiction_scope": "Ireland; Companies Registration Office CORE company/business-name identity register", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "ie.cro.companies", + "source_url": "https://opendata.cro.ie/dataset/companies", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-ie.md", + "docs/countries/ireland/v1-field-crosswalk.json", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Verify the live daily schema and use company number for reviewed identity edges only.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official CSO release/table route; bounded aggregate capture", + "attribution": { + "attribution_required": true, + "notice": "CSO statistics are aggregate context; table-specific terms and attribution must be retained with any downstream use", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "monthly", + "country_code": "IE", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Do not join aggregate statistics to facility rows or interpret totals as a facility census; preserve CSO coverage notes." + ] + }, + "jurisdiction_scope": "Ireland; CSO aggregate livestock slaughterings and meat supply statistics", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "ie.cso.livestock-slaughterings", + "source_url": "https://www.cso.ie/en/statistics/agriculture/livestockslaughterings/", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-ie.md", + "docs/countries/ireland/v1-field-crosswalk.json", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Capture aggregate monthly measures with coverage notes; never treat them as facility rows.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official statement and annual-report routes; bounded contextual capture", + "attribution": { + "attribution_required": true, + "notice": "DAFM statement describes control responsibilities; use only as authority/scope evidence unless a specific current inspection artifact is lawfully obtained and reviewed", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown; no row-level public welfare feed verified", + "country_code": "IE", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Current row-level facility-linked welfare/enforcement observations were not captured; do not convert qualitative controls or aggregate reports into facility claims." + ] + }, + "jurisdiction_scope": "Ireland; DAFM official-veterinary and animal-welfare control context at approved slaughter plants", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "ie.dafm.animal-welfare-controls", + "source_url": "https://www.gov.ie/en/department-of-agriculture-food-and-the-marine/press-releases/statement-on-garda-investigation-into-alleged-offences-of-deception/", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-ie.md", + "docs/countries/ireland/v1-field-crosswalk.json", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Keep the DAFM statement as control-scope context until a current row-level welfare artifact is lawfully captured and reviewed.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official gov.ie publication link; bounded workbook capture or operator-assisted download", + "attribution": { + "attribution_required": true, + "notice": "DAFM government publication; workbook-specific reuse, attribution, personal-data handling, and redistribution terms must be confirmed before release", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown; publication page last updated 2026-09-15", + "country_code": "IE", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Workbook bytes and headers were not captured in this run; do not infer row counts or schema from the publication title." + ] + }, + "jurisdiction_scope": "Ireland; DAFM-approved or registered meat establishments including fish, egg, and dairy under S.I. 22 of 2020", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "ie.dafm.approved-establishments", + "source_url": "https://assets.gov.ie/static/documents/09fe3ad4/AllApprovedPlants_2026_5.xlsx", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-ie.md", + "docs/countries/ireland/v1-field-crosswalk.json", + "data/manifests/ireland-source-artifacts.json", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Obtain an authorized bounded workbook capture, headers, schema fingerprint, row counts, terms, privacy review, and lifecycle semantics.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official gov.ie publication link; bounded workbook capture or operator-assisted download", + "attribution": { + "attribution_required": true, + "notice": "DAFM government publication; workbook-specific reuse, attribution, personal-data handling, and redistribution terms must be confirmed before release", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown; publication page last updated 2026-09-15", + "country_code": "IE", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Keep former-LA coverage separate from the main DAFM workbook until overlap and authority ownership are reviewed." + ] + }, + "jurisdiction_scope": "Ireland; former local-authority meat establishments including fish, egg, and dairy under S.I. 22 of 2020", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "ie.dafm.former-la-establishments", + "source_url": "https://assets.gov.ie/static/documents/09fe3ad4/AllApprovedPlants_2026_Formerly_LA_Plants_1.xlsx", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-ie.md", + "docs/countries/ireland/v1-field-crosswalk.json", + "data/manifests/ireland-source-artifacts.json", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Capture separately from the main DAFM workbook and measure overlap before any deduplication.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official gov.ie publication link; bounded workbook capture or operator-assisted download", + "attribution": { + "attribution_required": true, + "notice": "DAFM government publication; workbook-specific reuse, attribution, personal-data handling, and redistribution terms must be confirmed before release", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown; workbook filename dated 2026-08-11", + "country_code": "IE", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Do not union dairy rows with meat rows without preserving source family and approval-versus-registration semantics." + ] + }, + "jurisdiction_scope": "Ireland; DAFM milk and dairy establishments approved and/or registered under the Hygiene Regulations", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "ie.dafm.milk-dairy-establishments", + "source_url": "https://assets.gov.ie/static/documents/09fe3ad4/1._Milk_Dairy_Establishments_Registered_and_or_Approved_11th_August_2026.xlsx", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-ie.md", + "docs/countries/ireland/v1-field-crosswalk.json", + "data/manifests/ireland-source-artifacts.json", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Capture the dated workbook and preserve approval-versus-registration and dairy activity semantics.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official open-data catalog/resource route; bounded aggregate capture", + "attribution": { + "attribution_required": true, + "notice": "DAFM open-data catalog indicates open-data reuse for the resource; verify current resource terms and preserve its historical cutoff", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "weekly; current catalog resource described as 2020-2024 and last updated 2024-08-01", + "country_code": "IE", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Catalog resource is historical through 2024 in the observed metadata; do not present it as a current facility status feed or inflate facility counts." + ] + }, + "jurisdiction_scope": "Ireland; DAFM national beef kill figures by approved processing plants, aggregate release", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "ie.dafm.national-beef-kill", + "source_url": "https://opendata.agriculture.gov.ie/dataset/national-beef-kill-figures", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-ie.md", + "docs/countries/ireland/v1-field-crosswalk.json", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Retain the observed 2024 historical cutoff and verify the resource before any aggregate use.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official programme/press-release metadata; no facility award rows captured", + "attribution": { + "attribution_required": true, + "notice": "Government/EU programme context; award-level terms, personal data, and beneficiary publication rules require scheme-specific review", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown; programme/competition dependent", + "country_code": "IE", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Keep funding context separate from approvals; do not link by name without a reviewed company identifier and award relationship." + ] + }, + "jurisdiction_scope": "Ireland; DAFM seafood-processing capital-investment funding context under Ireland Seafood Development Programme/EMFAF", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "ie.dafm.seafood-processing-funding", + "source_url": "https://www.gov.ie/en/department-of-agriculture-food-and-the-marine/press-releases/minister-dooley-announces-opening-of-the-seafood-processing-capital-investment-scheme/", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-ie.md", + "docs/countries/ireland/v1-field-crosswalk.json", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Keep programme context separate; capture awards only after scheme-specific privacy and terms review.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official LEAP search and documented public API families; bounded record retrieval after identifier review", + "attribution": { + "attribution_required": true, + "notice": "EPA LEAP terms and conditions govern use of environmental information; do not bulk-copy or expose personal/precise information without legal/privacy review", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown; most records are published 30 calendar days after creation", + "country_code": "IE", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "LEAP is an accountability layer, not a food-establishment census; resolve API identifiers, terms, privacy, and explicit cross-source matching." + ] + }, + "jurisdiction_scope": "Ireland; EPA IE/IPC licensed sites and public LEAP licensing, compliance, and enforcement records", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "ie.epa.leap", + "source_url": "https://www.epa.ie/our-services/compliance--enforcement/whats-happening/leap-online/", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-ie.md", + "docs/countries/ireland/v1-field-crosswalk.json", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Resolve LEAP API identifiers and terms; use it only for reviewed accountability edges and events.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official directory page; bounded browser capture or operator-assisted authority exports", + "attribution": { + "attribution_required": true, + "notice": "FSAI explains the approval obligation and links to DAFM, HSE, and SFPA lists; the directory page is not itself a facility master and reuse/privacy terms require review", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown", + "country_code": "IE", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Do not count this coordinator page as an additional establishment source; verify linked authority exports, terms, privacy, and publication approval." + ] + }, + "jurisdiction_scope": "Ireland; FSAI coordinating directory for competent-authority approved animal-origin food establishments", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "ie.fsai.approved-directory", + "source_url": "https://www.fsai.ie/enforcement-and-legislation/official-controls/mancp/approved-food-premises", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-ie.md", + "docs/countries/ireland/v1-field-crosswalk.json", + "data/manifests/ireland-source-artifacts.json", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Use FSAI only as the coordinator; capture and review the linked DAFM, HSE, and SFPA artifacts separately.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official notice/news route; bounded notice capture after scope review", + "attribution": { + "attribution_required": true, + "notice": "FSAI notice content is enforcement context; person/business names and allegations require legal, privacy, and factual review before any linkage", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown; notice publication cadence is not a source-health guarantee", + "country_code": "IE", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Do not infer absence of an order from absence of a notice; preserve event scope and link only after reviewable identity resolution." + ] + }, + "jurisdiction_scope": "Ireland; FSAI enforcement-order notices and aggregate official-control enforcement context", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "ie.fsai.enforcement-orders", + "source_url": "https://www.fsai.ie/news-and-alerts/latest-news/fourteen-enforcement-orders-served-on-food-bus-%281%29", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-ie.md", + "docs/countries/ireland/v1-field-crosswalk.json", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Treat notices as dated event evidence and require reviewed subject identity before linkage.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "rendered official HSE/FSAI directory; bounded browser capture", + "attribution": { + "attribution_required": true, + "notice": "Public HSE/FSAI directory; address and trading-name fields are restricted pending terms/privacy review and publication approval", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown; page displayed last refreshed 2026-09-16", + "country_code": "IE", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Page-level refresh date is observed but no documented cadence or export contract was found; preserve all repeated child observations and do not publish addresses." + ] + }, + "jurisdiction_scope": "Ireland; low-throughput meat processors under HSE supervision", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": true, + "public_release_allowed": false, + "reasons": [ + "publication:blocked" + ], + "state": "awaiting-owner-review" + }, + "source_id": "ie.hse.low-throughput-meat", + "source_url": "https://oapi.fsai.ie/HSEApprovedEstablishments.aspx", + "status": { + "acquisition": "artifact_private_only", + "evidence": [ + "docs/country-recon-ie.md", + "docs/countries/ireland/v1-field-crosswalk.json", + "data/manifests/ireland-source-artifacts.json", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Use the bounded page observation as metadata only; obtain a lawful repeatable export/capture contract and keep repeated activity/species rows as children.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official map/open-data/service routes; bounded application-level capture", + "attribution": { + "attribution_required": true, + "notice": "MyPlan allows public information distribution/copying with byline credit, subject to data-use conditions; applicant/personal data and approximate GIS locations require review", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "weekly; MyPlan help states data are uploaded weekly", + "country_code": "IE", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Planning applications show development history or intent, not proof of operating approval; keep applications separate from facility entities and do not use approximate geometry for site-specific decisions." + ] + }, + "jurisdiction_scope": "Ireland; National Planning Application Map and local-authority planning application data", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "ie.planning.npad", + "source_url": "https://www.myplan.ie/national-planning-application-map-viewer/", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-ie.md", + "docs/countries/ireland/v1-field-crosswalk.json", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Capture application-level records separately; preserve approximate geometry and applicant privacy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "rendered official SFPA paginated table; bounded browser capture", + "attribution": { + "attribution_required": true, + "notice": "SFPA page is public but footer states copyright/all rights reserved; obtain reuse permission/interpretation and complete privacy review before redistribution", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown; table displayed updated 2026-09-15", + "country_code": "IE", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Pagination/export behavior, rights, and exact facility-versus-approval row semantics require a bounded artifact capture and review." + ] + }, + "jurisdiction_scope": "Ireland; SFPA establishments approved under Regulation (EC) No 853/2004 for fishery products and live bivalve molluscs", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": true, + "public_release_allowed": false, + "reasons": [ + "publication:blocked" + ], + "state": "awaiting-owner-review" + }, + "source_id": "ie.sfpa.approved-establishments", + "source_url": "https://www.sfpa.ie/What-We-Do/Seafood-Safety/Registration-Approval-of-Businesses/List-of-Approved-Establishments/Approved-Establishments", + "status": { + "acquisition": "artifact_private_only", + "evidence": [ + "docs/country-recon-ie.md", + "docs/countries/ireland/v1-field-crosswalk.json", + "data/manifests/ireland-source-artifacts.json", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Capture all paginated rows privately, confirm rights, and validate approval-number variant and facility semantics.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "rendered official SFPA table; bounded browser capture", + "attribution": { + "attribution_required": true, + "notice": "SFPA page is public but footer states copyright/all rights reserved; vessel data requires rights/privacy review", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown; page is live and no cadence is stated", + "country_code": "IE", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Keep factory-vessel rows as a separate entity class; do not infer a fixed premises or blend with freezer-vessel rows." + ] + }, + "jurisdiction_scope": "Ireland; SFPA factory vessels approved under Regulation (EC) No 853/2004", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": true, + "public_release_allowed": false, + "reasons": [ + "publication:blocked" + ], + "state": "awaiting-owner-review" + }, + "source_id": "ie.sfpa.factory-vessels", + "source_url": "https://www.sfpa.ie/What-We-Do/Seafood-Safety/Registration-Approval-of-Businesses/List-of-Approved-Establishments/Approved-Factory-Vessels", + "status": { + "acquisition": "artifact_private_only", + "evidence": [ + "docs/country-recon-ie.md", + "docs/countries/ireland/v1-field-crosswalk.json", + "data/manifests/ireland-source-artifacts.json", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Keep factory-vessel identity separate; capture and validate the one-entry table through a permitted route.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "rendered official SFPA paginated table; bounded browser capture", + "attribution": { + "attribution_required": true, + "notice": "SFPA page is public but footer states copyright/all rights reserved; vessel and address fields are restricted pending rights/privacy review", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown; table displayed updated 2026-09-15", + "country_code": "IE", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Vessel identity is mobile/non-fixed; do not treat the address as a stable facility location or sum vessels with establishments." + ] + }, + "jurisdiction_scope": "Ireland; SFPA freezer vessels approved under Regulation (EC) No 853/2004", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": true, + "public_release_allowed": false, + "reasons": [ + "publication:blocked" + ], + "state": "awaiting-owner-review" + }, + "source_id": "ie.sfpa.freezer-vessels", + "source_url": "https://www.sfpa.ie/What-We-Do/Seafood-Safety/Registration-Approval-of-Businesses/List-of-Approved-Establishments/Approved-Freezer-Vessels", + "status": { + "acquisition": "artifact_private_only", + "evidence": [ + "docs/country-recon-ie.md", + "docs/countries/ireland/v1-field-crosswalk.json", + "data/manifests/ireland-source-artifacts.json", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Keep vessel rows separate from fixed establishments and review address/rights handling.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + } + ], + "state": "private-candidate-ready", + "summary": "No release approval is implied; inspect source gates below." + }, + "IN": { + "acquisition_counts": { + "blocked": 3, + "not_run": 1 + }, + "basis": [ + "in.cpcb.environmental-compliance: blocked / blocked", + "in.dahd.livestock-statistics: not_run / reconnaissance", + "in.fssai.foscos: blocked / blocked", + "in.mca.company-master: blocked / blocked" + ], + "country_reasons": [ + "acquisition:blocked", + "acquisition:not_run", + "publication:blocked" + ], + "name": "IN", + "owner_review": "awaiting-owner-review", + "publication_state": "blocked", + "source_count": 4, + "sources": [ + { + "access_method": "official portal, published policy, and sector/monitoring systems; no uncontrolled portal extraction", + "attribution": { + "attribution_required": true, + "notice": "Government environmental source; permit, monitoring and enforcement records require source-specific terms, safety and privacy review", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "system/report-specific; unknown nationally", + "country_code": "IN", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "No single national facility export verified; state PCB authority coverage, identifiers, route permissions, coordinate precision and status semantics remain unresolved." + ] + }, + "jurisdiction_scope": "India; Central Pollution Control Board industry monitoring and environmental compliance surfaces", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "in.cpcb.environmental-compliance", + "source_url": "https://cpcb.nic.in/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-in.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Map state/federal routes, identifiers, terms, privacy and status semantics before acquisition.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official reports, spreadsheets, and manuals from DAHD", + "attribution": { + "attribution_required": true, + "notice": "Official aggregate statistics; retain table/report metadata and suppress re-identification of small cells", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "annual BAHS; quinquennial livestock census; revisions and publication lag possible", + "country_code": "IN", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Pin stable report/table download identifiers and machine-readable route; household/holding microdata must remain restricted and is not a facility registry." + ] + }, + "jurisdiction_scope": "India; Department of Animal Husbandry and Dairying livestock census and Basic Animal Husbandry Statistics", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "in.dahd.livestock-statistics", + "source_url": "https://dahd.gov.in/schemes/programmes/animal-husbandry-statistics", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-in.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Pin report/table identifiers and machine-readable downloads; retain aggregate-only livestock context.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official public portal and FSSAI-described FBO search/verification; no automated query performed", + "attribution": { + "attribution_required": true, + "notice": "Government source; portal access does not settle bulk reuse, personal-address exposure, or publication rights", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "operational portal; statistics show publisher update date", + "country_code": "IN", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Search/export contract, stable identifiers, rate limits, privacy boundary, state coverage, and terms require authorized validation; do not scrape login or CAPTCHA surfaces." + ] + }, + "jurisdiction_scope": "India; FSSAI Food Safety Compliance System food-business licensing and registration", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "in.fssai.foscos", + "source_url": "https://foscos.fssai.gov.in/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-in.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Confirm authorized FBO search/export, identifiers, status semantics, terms, privacy, and state coverage.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official MCA public company-master lookup; no bulk extraction or authenticated route used", + "attribution": { + "attribution_required": true, + "notice": "Corporate identity source; registered-office and director-personal fields require minimization and privacy review", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "operational registry; provider-specific", + "country_code": "IN", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Current lookup/API contract, terms, rate limits, historical identity semantics, and safe fields require authorized validation; do not expose director or residential information." + ] + }, + "jurisdiction_scope": "India; Ministry of Corporate Affairs company/LLP master data", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "in.mca.company-master", + "source_url": "https://www.mca.gov.in/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-in.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Confirm public lookup contract and safe corporate fields; exclude director/personal exposure.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + } + ], + "state": "blocked", + "summary": "No release approval is implied; inspect source gates below." + }, + "IT": { + "acquisition_counts": { + "artifact_private_only": 1, + "not_run": 1 + }, + "basis": [ + "it.1069-2009: not_run / reconnaissance", + "it.853-2004: artifact_private_only / awaiting-owner-review" + ], + "country_reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "name": "IT", + "owner_review": "awaiting-owner-review", + "publication_state": "blocked", + "source_count": 2, + "sources": [ + { + "access_method": "not acquired; separate catalog candidate", + "attribution": { + "attribution_required": true, + "notice": "Separate Ministry catalog and Italian Open Data Licence v2.0 indication; no adapter or publication decision.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "daily", + "country_code": "IT", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Assess scope, terms, schema, identity/link semantics, privacy, and whether this source belongs in the project before acquisition." + ] + }, + "jurisdiction_scope": "Italy; Ministry of Health establishments for animal by-products under Regulation (EC) 1069/2009", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "it.1069-2009", + "source_url": "https://www.dati.salute.gov.it/it/dataset/stabilimenti-italiani-i-sottoprodotti-di-origine-animale/", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-it.md", + "pipeline/source_registry.json", + "pipeline/sources/italy/README.md" + ], + "metadata": "verified", + "next_action": "Keep the by-products dataset separate; assess scope, schema, terms, identity links, privacy, and a dedicated adapter before any acquisition or integration.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "stable catalog page -> same-origin dated CSV discovery", + "attribution": { + "attribution_required": true, + "notice": "Ministry of Health catalog identifies Italian Open Data Licence v2.0; terms evidence and privacy review remain required before publication.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "daily", + "country_code": "IT", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Resolve repeated establishment/activity identity, coded values, coordinate provenance, address privacy, coverage, and project publication approval before release review." + ] + }, + "jurisdiction_scope": "Italy; Ministry of Health establishments recognized under Regulation (EC) 853/2004", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": true, + "public_release_allowed": false, + "reasons": [ + "publication:blocked" + ], + "state": "awaiting-owner-review" + }, + "source_id": "it.853-2004", + "source_url": "https://www.dati.salute.gov.it/it/dataset/stabilimenti-italiani-gli-alimenti-di-origine-animale/", + "status": { + "acquisition": "artifact_private_only", + "evidence": [ + "docs/country-recon-it.md", + "pipeline/source_registry.json", + "pipeline/sources/italy/acquire.py", + "pipeline/sources/italy/it_853_adapter.py", + "pipeline/sources/italy/README.md", + "pipeline/common/review_packet.py", + "docs/review-packet-italy.md", + "pipeline/tests/e2e/test_italy_candidate_import.py" + ], + "metadata": "verified", + "next_action": "Keep catalog acquisition, lifecycle, candidate import, and guarded API evidence private; use source-category/activity diagnostics to resolve repeated identity, coordinate/address privacy, coverage, and project approval before release review.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + } + ], + "state": "private-candidate-ready", + "summary": "No release approval is implied; inspect source gates below." + }, + "LB": { + "acquisition_counts": { + "blocked": 5, + "not_run": 1 + }, + "basis": [ + "lb.cas.livestock-statistics: not_run / reconnaissance", + "lb.industry.food-guide: blocked / blocked", + "lb.justice.companies: blocked / blocked", + "lb.moa.approved-food: blocked / blocked", + "lb.moa.farms-livestock: blocked / blocked", + "lb.moe.environment-eia: blocked / blocked" + ], + "country_reasons": [ + "acquisition:blocked", + "acquisition:not_run", + "publication:blocked" + ], + "name": "LB", + "owner_review": "awaiting-owner-review", + "publication_state": "blocked", + "source_count": 6, + "sources": [ + { + "access_method": "official tables/publications; authorized query/download to be pinned", + "attribution": { + "attribution_required": true, + "notice": "Revisions, licensing, and suppression rules require review", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "table-specific; unknown", + "country_code": "LB", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Current table/API identifiers and terms not pinned." + ] + }, + "jurisdiction_scope": "Lebanon; Central Administration of Statistics aggregate livestock indicators", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "lb.cas.livestock-statistics", + "source_url": "https://www.cas.gov.lb/", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-lb.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin aggregate table/API identifiers, cadence, revisions, terms, and suppression rules.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official guide/list; authorized bounded download only", + "attribution": { + "attribution_required": true, + "notice": "Terms, schema, IDs, and safe address boundary require review", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "2022 visible; current unknown", + "country_code": "LB", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Current export and reuse terms not pinned." + ] + }, + "jurisdiction_scope": "Lebanon; Ministry of Industry licensed food factories", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "lb.industry.food-guide", + "source_url": "https://www.industry.gov.lb/IndustrialStatistics/IndustrialGuide", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-lb.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Confirm current list schema, licensing, cadence, and safe address boundary.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official interactive search; do not bypass controls", + "attribution": { + "attribution_required": true, + "notice": "Coverage, fees, terms, and personal-address policy require review", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown", + "country_code": "LB", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Machine route and nationwide coverage not verified." + ] + }, + "jurisdiction_scope": "Lebanon; Ministry of Justice commercial register", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "lb.justice.companies", + "source_url": "https://cr.justice.gov.lb/index.aspx", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-lb.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Confirm authorized access, coverage, identifiers, fees, terms, and personal-address policy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official pages; authorized bounded export/API only", + "attribution": { + "attribution_required": true, + "notice": "Government source; safety, privacy, terms, and conflict-sensitive locations require review", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown", + "country_code": "LB", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "No safe authorized bulk/API contract, IDs, cadence, or terms pinned." + ] + }, + "jurisdiction_scope": "Lebanon; Ministry of Agriculture slaughterhouses and animal-origin food establishments", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "lb.moa.approved-food", + "source_url": "https://www.agriculture.gov.lb/Subjects/Animal-Wealth/laws", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-lb.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin safe authorized route, schema, IDs, cadence, terms, privacy, and conflict-sensitive location policy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official route; aggregate-only until safe access verified", + "attribution": { + "attribution_required": true, + "notice": "Potentially operationally sensitive; privacy and safety review required", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown", + "country_code": "LB", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Fields, export/API, IDs, licensing, and safety controls unknown." + ] + }, + "jurisdiction_scope": "Lebanon; Ministry of Agriculture farms, livestock registration, farmer registry", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "lb.moa.farms-livestock", + "source_url": "https://www.agriculture.gov.lb/Media/News/2025/Summary-Report-%E2%80%93-Farmers-Registry-in-Lebanon", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-lb.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Keep farm evidence aggregate-only until safe access and operational-security review.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official framework/register; authorized API/export only", + "attribution": { + "attribution_required": true, + "notice": "Permit geometry and terms require safety/privacy review", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown", + "country_code": "LB", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "No current public permit API/export or safe geometry route pinned." + ] + }, + "jurisdiction_scope": "Lebanon; Ministry of Environment EIA and environmental review", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "lb.moe.environment-eia", + "source_url": "https://www.moe.gov.lb/MOE%20Site/SEA/SEA%20in%20Lebanon.htm", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-lb.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin current permit route and safe geometry/privacy/terms controls.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + } + ], + "state": "blocked", + "summary": "No release approval is implied; inspect source gates below." + }, + "LT": { + "acquisition_counts": { + "blocked": 3, + "not_run": 4 + }, + "basis": [ + "lt.animal-experiments: blocked / blocked", + "lt.environment.permits: blocked / blocked", + "lt.jar.organizations: not_run / reconnaissance", + "lt.statistics.slaughter: not_run / reconnaissance", + "lt.vmvt.approved-food: not_run / reconnaissance", + "lt.vmvt.farm-aquaculture: blocked / blocked", + "lt.vmvt.inspections: not_run / reconnaissance" + ], + "country_reasons": [ + "acquisition:blocked", + "acquisition:not_run", + "metadata:unknown", + "publication:blocked" + ], + "name": "LT", + "owner_review": "awaiting-owner-review", + "publication_state": "blocked", + "source_count": 7, + "sources": [ + { + "access_method": "official guidance/reports; facility route unverified", + "attribution": { + "attribution_required": true, + "notice": "Sensitive research evidence; aggregate/anonymize and require review.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "annual/report-specific", + "country_code": "LT", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "No public research-facility master verified." + ] + }, + "jurisdiction_scope": "Lithuania; animal experimentation and welfare evidence", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "lt.animal-experiments", + "source_url": "https://vmvt.lrv.lt/lt/atviri-vmvt-duomenys-ir-registrai/atviri-duomenys/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-lt.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Keep aggregate research evidence separate; no facility master verified.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "national open-data/environmental permit routes; not verified", + "attribution": { + "attribution_required": true, + "notice": "Verify publisher, license, sensitive-site handling and geometry.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown", + "country_code": "LT", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "No stable national animal-facility permit export verified." + ] + }, + "jurisdiction_scope": "Lithuania; environmental permits and releases", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "metadata:unknown", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "lt.environment.permits", + "source_url": "https://data.gov.lt/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-lt.md", + "pipeline/source_registry.json" + ], + "metadata": "unknown", + "next_action": "Locate and verify a stable environmental permit route.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official register/API route unverified", + "attribution": { + "attribution_required": true, + "notice": "Identity-only linkage; suppress personal, sole-trader and residential details.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "provider-defined", + "country_code": "LT", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Verify public API/access, terms, quotas and privacy policy." + ] + }, + "jurisdiction_scope": "Lithuania; legal-entity register identity", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "lt.jar.organizations", + "source_url": "https://www.registrucentras.lt/", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-lt.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Verify public API/access, terms and identity-only privacy policy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official statistical tables/API; exact table IDs unverified", + "attribution": { + "attribution_required": true, + "notice": "Official statistics; preserve dimensions, revisions and license metadata.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "table-specific", + "country_code": "LT", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Pin table IDs/API contract; keep aggregates separate from facilities." + ] + }, + "jurisdiction_scope": "Lithuania; official slaughter, livestock and animal-use statistics", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "lt.statistics.slaughter", + "source_url": "https://osp.stat.gov.lt/", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-lt.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin table IDs/API and preserve aggregate revisions.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "open-data API, CSV, JSON and JSONL resources", + "attribution": { + "attribution_required": true, + "notice": "Official VMVT; catalogue states CC BY 4.0; verify current terms and privacy.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "varies by resource; timestamp retrieval", + "country_code": "LT", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Verify current resource URLs, schema, completeness, terms and sensitive-field suppression." + ] + }, + "jurisdiction_scope": "Lithuania; VMVT approved food and veterinary-control establishments", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "lt.vmvt.approved-food", + "source_url": "https://data.gov.lt/dataset/valstybines-veterinarines-kontroles-subjektai", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-lt.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Verify current resource URLs, schema, completeness, terms and privacy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official registers/open-data routes", + "attribution": { + "attribution_required": true, + "notice": "Official authority; animal-holder/property locations require strict privacy review.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "resource-specific", + "country_code": "LT", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Verify public facility scope, current export/API, CRS, terms and coverage." + ] + }, + "jurisdiction_scope": "Lithuania; VMVT animal, herd and aquaculture establishment evidence", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "lt.vmvt.farm-aquaculture", + "source_url": "https://vmvt.lrv.lt/lt/atviri-vmvt-duomenys-ir-registrai/atviri-duomenys/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-lt.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Verify public scope, export/API, CRS, terms and coverage.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "open-data API/CSV/JSON/JSONL", + "attribution": { + "attribution_required": true, + "notice": "Official control evidence; preserve event status and privacy; CC BY 4.0 catalogue claim requires confirmation.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "varies; portal says not uniform", + "country_code": "LT", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Verify current resource/version, event semantics and coverage." + ] + }, + "jurisdiction_scope": "Lithuania; VMVT veterinary-control inspections/enforcement", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "lt.vmvt.inspections", + "source_url": "https://data.gov.lt/dataset/valstybines-veterinarines-kontroles-subjektai", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-lt.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Verify resource version, event semantics and coverage.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + } + ], + "state": "blocked", + "summary": "No release approval is implied; inspect source gates below." + }, + "LV": { + "acquisition_counts": { + "blocked": 2, + "not_run": 4 + }, + "basis": [ + "lv.animal-experiments: blocked / blocked", + "lv.environment.permits: not_run / reconnaissance", + "lv.ldc.slaughter-farms: blocked / blocked", + "lv.pvd.approved-food: not_run / reconnaissance", + "lv.stat.api: not_run / reconnaissance", + "lv.ur.organizations: not_run / reconnaissance" + ], + "country_reasons": [ + "acquisition:blocked", + "acquisition:not_run", + "publication:blocked" + ], + "name": "LV", + "owner_review": "awaiting-owner-review", + "publication_state": "blocked", + "source_count": 6, + "sources": [ + { + "access_method": "official guidance/registers; public facility/statistics route unverified", + "attribution": { + "attribution_required": true, + "notice": "Sensitive research/inspection evidence; aggregate, anonymize and require review.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown", + "country_code": "LV", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "No stable public facility-level route verified." + ] + }, + "jurisdiction_scope": "Latvia; animal experimentation, inspections and enforcement evidence", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "lv.animal-experiments", + "source_url": "https://registri.pvd.gov.lv/en/cr", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-lv.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Keep research/inspection evidence aggregate until a public route is verified.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "open-data CSV/resource and public registers", + "attribution": { + "attribution_required": true, + "notice": "Catalogue states CC0 1.0; preserve source and verify resource scope/geometry.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "dataset-specific", + "country_code": "LV", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Verify current resource URLs, schema, coverage and privacy." + ] + }, + "jurisdiction_scope": "Latvia; VVD environmental permits and public registers", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "lv.environment.permits", + "source_url": "https://data.gov.lv/dati/dataset/izsniegtas-atlaujas-un-licences", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-lv.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Verify resource URLs, schema, coverage, geometry and privacy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "public filtered register and statistics", + "attribution": { + "attribution_required": true, + "notice": "Official register; animal-holder/property privacy review required.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown; timestamp access", + "country_code": "LV", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Verify bulk/API route, coverage, terms and sensitive-field policy." + ] + }, + "jurisdiction_scope": "Latvia; LDC slaughterhouse, herd/location and livestock register evidence", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "lv.ldc.slaughter-farms", + "source_url": "https://registri.ldc.gov.lv/en/slaughterhouses", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-lv.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Verify bulk/API route, coverage, terms and sensitive-field policy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "machine-readable ZIP/CSV and sectioned XLSX lists", + "attribution": { + "attribution_required": true, + "notice": "Official PVD/FVS; verify dataset terms and privacy before release.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "publisher-defined; timestamp retrieval", + "country_code": "LV", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Verify current files, section coverage, IDs, cadence, terms and privacy." + ] + }, + "jurisdiction_scope": "Latvia; PVD/FVS approved and registered food, feed and ABP establishments", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "lv.pvd.approved-food", + "source_url": "https://pakalpojumi.pvd.gov.lv/en/opendata_files/ipvd_object_opendata", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-lv.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Verify current files, section coverage, IDs, cadence, terms and privacy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "PxWeb API v2", + "attribution": { + "attribution_required": true, + "notice": "Official statistics; preserve table IDs, revisions and license/attribution metadata.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "table-specific; 30 requests/10 seconds/IP stated", + "country_code": "LV", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Pin table IDs and metadata; keep aggregate scope separate." + ] + }, + "jurisdiction_scope": "Latvia; official statistics API for slaughter, livestock and animal-use aggregates", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "lv.stat.api", + "source_url": "https://stat.gov.lv/en/api-un-kodu-vardnicas/api-v2", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-lv.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Pin table IDs/metadata and preserve aggregate revisions.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official register/API route unverified", + "attribution": { + "attribution_required": true, + "notice": "Identity-only linkage; suppress personal/sole-trader/residential details.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "provider-defined", + "country_code": "LV", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Verify current API/access, terms and privacy policy." + ] + }, + "jurisdiction_scope": "Latvia; Latvian enterprise registration identifiers", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "lv.ur.organizations", + "source_url": "https://www.ur.gov.lv/en/", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-lv.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Verify current API/access, terms and privacy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + } + ], + "state": "blocked", + "summary": "No release approval is implied; inspect source gates below." + }, + "MD": { + "acquisition_counts": { + "blocked": 5, + "not_run": 1 + }, + "basis": [ + "md.ansa.approved-food: blocked / blocked", + "md.ansa.farms-aquaculture: blocked / blocked", + "md.ansa.inspections-experiments: blocked / blocked", + "md.asp.organizations: blocked / blocked", + "md.environment-permits: blocked / blocked", + "md.stat.statistics: not_run / reconnaissance" + ], + "country_reasons": [ + "acquisition:blocked", + "acquisition:not_run", + "publication:blocked" + ], + "name": "MD", + "owner_review": "awaiting-owner-review", + "publication_state": "blocked", + "source_count": 6, + "sources": [ + { + "access_method": "official ANSA category pages and linked lists/documents", + "attribution": { + "attribution_required": true, + "notice": "Official authority; verify file terms, attribution, privacy and coordinates before acquisition.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown; timestamp retrieval", + "country_code": "MD", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Pin stable ANSA file routes, schemas, cadence, IDs, terms and privacy." + ] + }, + "jurisdiction_scope": "Moldova; ANSA authorized and registered animal-origin food establishments", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "md.ansa.approved-food", + "source_url": "https://www.ansa.gov.md/siguranta-alimentelor.html", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-md.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Pin current ANSA files, schemas, cadence, IDs, terms and privacy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official ANSA registers, checklists and linked documents", + "attribution": { + "attribution_required": true, + "notice": "Verify publisher, license, animal-holder/property privacy and coordinate precision.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "resource-specific", + "country_code": "MD", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Verify complete holding/aquaculture exports, IDs, cadence, terms and privacy." + ] + }, + "jurisdiction_scope": "Moldova; ANSA livestock holdings, fish farms and aquaculture-related evidence", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "md.ansa.farms-aquaculture", + "source_url": "https://www.ansa.gov.md/sanatatea-si-bunastarea-animalelor.html", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-md.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Verify holding/fish-farm exports, IDs, cadence, coordinates, terms and privacy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official control checklists, reports and linked registers", + "attribution": { + "attribution_required": true, + "notice": "Sensitive control/research evidence; aggregate or anonymize and require project review.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "resource/report-specific", + "country_code": "MD", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "No stable public experimental-animal facility master verified; pin current fields and privacy policy." + ] + }, + "jurisdiction_scope": "Moldova; ANSA inspections/enforcement and animal-experimentation evidence", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "md.ansa.inspections-experiments", + "source_url": "https://ansa.gov.md/conducerea/liste-de-verificare.html", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-md.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Locate experimentation facility route; keep sensitive evidence aggregate until verified.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official online extracts and contracted ACCES-Web/statistical services", + "attribution": { + "attribution_required": true, + "notice": "Identity linkage only; contracts, fees and personal/beneficial-owner privacy apply.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "service-defined; online/non-stop services described", + "country_code": "MD", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Pin authorized service/API, fees, fields, cadence, site matching and privacy policy." + ] + }, + "jurisdiction_scope": "Moldova; Public Services Agency State Register of Legal Entities", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "md.asp.organizations", + "source_url": "https://asp.gov.md/en/servicii/persoane-juridice/informatii-afaceri", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-md.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Pin authorized ASP service/API, fees, fields, cadence, matching and privacy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official environmental authority portals, public notices and documents", + "attribution": { + "attribution_required": true, + "notice": "Verify public-read rights, document license, geometry and privacy.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "permit/document-specific", + "country_code": "MD", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Stable public read/API/export and complete facility coverage are unresolved." + ] + }, + "jurisdiction_scope": "Moldova; environmental permits and authorizations", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "md.environment-permits", + "source_url": "https://www.mediu.gov.md/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-md.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Verify public environmental permit route, coverage, documents, geometry and privacy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "National Bureau of Statistics tables/data services; exact route unverified", + "attribution": { + "attribution_required": true, + "notice": "Official statistics; verify table terms, citation requirements and aggregate-only scope.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "table-specific; preserve revisions", + "country_code": "MD", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Pin current slaughter and animal-use table IDs, API/download contracts and cadence." + ] + }, + "jurisdiction_scope": "Moldova; official aggregate slaughter, livestock and animal-use statistics", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "md.stat.statistics", + "source_url": "https://statistica.gov.md/en", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-md.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin official slaughter/animal-use table IDs, APIs, cadence and revisions.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + } + ], + "state": "blocked", + "summary": "No release approval is implied; inspect source gates below." + }, + "MK": { + "acquisition_counts": { + "blocked": 5, + "not_run": 1 + }, + "basis": [ + "mk.crm.organizations: blocked / blocked", + "mk.environment-permits: blocked / blocked", + "mk.fva.approved-food: blocked / blocked", + "mk.fva.farms-aquaculture: blocked / blocked", + "mk.fva.inspections-experiments: blocked / blocked", + "mk.stat.statistics: not_run / reconnaissance" + ], + "country_reasons": [ + "acquisition:blocked", + "acquisition:not_run", + "publication:blocked" + ], + "name": "MK", + "owner_review": "awaiting-owner-review", + "publication_state": "blocked", + "source_count": 6, + "sources": [ + { + "access_method": "authorized online distribution system; paid/prepaid services", + "attribution": { + "attribution_required": true, + "notice": "Central Registry terms restrict commercial reproduction/modification without prior consent; identity linkage only.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "service-defined; current/historical products", + "country_code": "MK", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Confirm commercial permission, fees, API/service contract, quotas, fields and privacy." + ] + }, + "jurisdiction_scope": "North Macedonia; Central Registry legal-entity records", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "mk.crm.organizations", + "source_url": "https://www.crm.com.mk/en/professional-users/lessors/access-to-data-via-the-online-distribution-system", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-mk.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Confirm authorized commercial access, fees, service contract, fields and privacy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official ministry portals, public notices and documents", + "attribution": { + "attribution_required": true, + "notice": "Verify public-read rights, document license, geometry and privacy.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "permit/document-specific", + "country_code": "MK", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Stable public read/API/export route and complete facility coverage are unresolved." + ] + }, + "jurisdiction_scope": "North Macedonia; environmental permits and physical-planning authorizations", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "mk.environment-permits", + "source_url": "https://www.moepp.gov.mk/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-mk.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Verify public environmental permit API/export, coverage, documents, geometry and privacy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official FVA register pages with linked Google Drive documents", + "attribution": { + "attribution_required": true, + "notice": "Official authority; verify file terms, attribution, privacy and coordinates before acquisition.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown; timestamp retrieval", + "country_code": "MK", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Pin stable Drive file IDs, schemas, cadence, IDs, terms and privacy." + ] + }, + "jurisdiction_scope": "North Macedonia; FVA approved and registered food establishments handling animal-origin products", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "mk.fva.approved-food", + "source_url": "https://fva.gov.mk/mk/registri-hrana-zivotinsko-poteklo", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-mk.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Pin stable FVA/Drive files, schemas, cadence, IDs, terms and privacy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official FVA registers and linked documents", + "attribution": { + "attribution_required": true, + "notice": "Verify publisher, license, animal-holder/property privacy and coordinate precision.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "resource-specific", + "country_code": "MK", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Verify current farm/aquaculture files, IDs, cadence, terms and privacy." + ] + }, + "jurisdiction_scope": "North Macedonia; FVA livestock, slaughterhouse, holding and aquaculture-related evidence", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "mk.fva.farms-aquaculture", + "source_url": "https://fva.gov.mk/mk/zdravstvena-zastita-blagosostojba-zivotni-1", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-mk.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Verify farm/aquaculture files, IDs, cadence, coordinates, terms and privacy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official FVA registers and linked documents", + "attribution": { + "attribution_required": true, + "notice": "Sensitive control/research evidence; aggregate or anonymize and require project review.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "resource-specific", + "country_code": "MK", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Pin current register file/API, fields, IDs, privacy and retention policy." + ] + }, + "jurisdiction_scope": "North Macedonia; FVA inspections/enforcement and animal-experimentation institutions", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "mk.fva.inspections-experiments", + "source_url": "https://fva.gov.mk/mk/zdravstvena-zastita-blagosostojba-zivotni-1", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-mk.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Pin experimentation register/API, fields, IDs, privacy and retention policy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "State Statistical Office tables/data services; exact route unverified", + "attribution": { + "attribution_required": true, + "notice": "Official statistics; verify table terms, citation requirements and aggregate-only scope.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "table-specific; preserve revisions", + "country_code": "MK", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Pin current slaughter and animal-use table IDs, API/download contracts and cadence." + ] + }, + "jurisdiction_scope": "North Macedonia; official aggregate slaughter, livestock and animal-use statistics", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "mk.stat.statistics", + "source_url": "https://www.stat.gov.mk/", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-mk.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin official slaughter/animal-use table IDs, APIs, cadence and revisions.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + } + ], + "state": "blocked", + "summary": "No release approval is implied; inspect source gates below." + }, + "MX": { + "acquisition_counts": { + "blocked": 1 + }, + "basis": [ + "mx.locations: blocked / blocked" + ], + "country_reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "name": "MX", + "owner_review": "awaiting-owner-review", + "publication_state": "blocked", + "source_count": 1, + "sources": [ + { + "access_method": "download/API or assisted export", + "attribution": { + "attribution_required": true, + "notice": "unknown; separate official source terms from INEGI-derived research artifacts before reuse", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown", + "country_code": "MX", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Split official source identities from derived DENUE/aquaculture work and establish source-level provenance." + ] + }, + "jurisdiction_scope": "Mexico; mixed official-register and INEGI-derived legacy coverage", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "mx.locations", + "source_url": "unknown", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-mx.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Resolve DENUE token/terms and SENASICA current directory/schema before bounded acquisition.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + } + ], + "state": "blocked", + "summary": "No release approval is implied; inspect source gates below." + }, + "NL": { + "acquisition_counts": { + "artifact_private_only": 7, + "blocked": 1, + "not_run": 1 + }, + "basis": [ + "nl.cbs.livestock: artifact_private_only / awaiting-owner-review", + "nl.cbs.slaughter: artifact_private_only / awaiting-owner-review", + "nl.cokz.dairy-eggs: artifact_private_only / awaiting-owner-review", + "nl.koop.local-permits: artifact_private_only / awaiting-owner-review", + "nl.kvk.hvds: not_run / reconnaissance", + "nl.nvwa.approved-food: artifact_private_only / awaiting-owner-review", + "nl.nvwa.welfare-enforcement: artifact_private_only / awaiting-owner-review", + "nl.pdok.omgevingswet: artifact_private_only / awaiting-owner-review", + "nl.rvo.ir: blocked / blocked" + ], + "country_reasons": [ + "acquisition:blocked", + "acquisition:not_run", + "publication:blocked" + ], + "name": "NL", + "owner_review": "awaiting-owner-review", + "publication_state": "blocked", + "source_count": 9, + "sources": [ + { + "access_method": "public OData API", + "attribution": { + "attribution_required": true, + "notice": "Official CBS; cite CBS and verify dataset-specific reuse terms.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "twice-yearly; latest periods may be provisional", + "country_code": "NL", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Do not interpret as named facilities or coordinates; preserve scope/status." + ] + }, + "jurisdiction_scope": "Netherlands; CBS aggregate livestock statistics", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": true, + "public_release_allowed": false, + "reasons": [ + "publication:blocked" + ], + "state": "awaiting-owner-review" + }, + "source_id": "nl.cbs.livestock", + "source_url": "https://opendata.cbs.nl/ODataApi/OData/84952NED", + "status": { + "acquisition": "artifact_private_only", + "evidence": [ + "docs/country-recon-nl.md", + "data/manifests/nl-source-artifacts.json", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Keep twice-yearly aggregate data separate from named facilities and preserve provisional status.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "public OData API", + "attribution": { + "attribution_required": true, + "notice": "Official CBS; cite CBS and verify dataset-specific reuse terms.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "monthly; new figures about two months after reference month", + "country_code": "NL", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Keep outside facility tables; preserve provisional/unknown/secret flags." + ] + }, + "jurisdiction_scope": "Netherlands; CBS monthly aggregate slaughter statistics", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": true, + "public_release_allowed": false, + "reasons": [ + "publication:blocked" + ], + "state": "awaiting-owner-review" + }, + "source_id": "nl.cbs.slaughter", + "source_url": "https://opendata.cbs.nl/ODataApi/OData/7123slac", + "status": { + "acquisition": "artifact_private_only", + "evidence": [ + "docs/country-recon-nl.md", + "data/manifests/nl-source-artifacts.json", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Keep monthly aggregate claims outside facility tables and retain CBS flags and scope.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official HTML registers", + "attribution": { + "attribution_required": true, + "notice": "Official delegated regulator; terms and machine reuse not verified.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "register-specific update dates; uniform API cadence unknown", + "country_code": "NL", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Confirm all register families, lifecycle, terms and overlap with NVWA." + ] + }, + "jurisdiction_scope": "Netherlands; COKZ approved dairy, farm-dairy, egg and egg-product establishments", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": true, + "public_release_allowed": false, + "reasons": [ + "publication:blocked" + ], + "state": "awaiting-owner-review" + }, + "source_id": "nl.cokz.dairy-eggs", + "source_url": "https://cokz.nl/", + "status": { + "acquisition": "artifact_private_only", + "evidence": [ + "docs/country-recon-nl.md", + "docs/countries/nl/v1-field-crosswalk.json", + "data/manifests/nl-source-artifacts.json", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Confirm COKZ register families, update semantics, terms and overlap with NVWA before modeling.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "KOOP/Overheid.nl SRU and local feeds", + "attribution": { + "attribution_required": true, + "notice": "Official publication evidence; preserve authority, identity, dates and state.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "daily publication surface; local coverage varies", + "country_code": "NL", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Use collection-specific CQL; initial keyword query returned diagnostics; no complete national permit coverage claim." + ] + }, + "jurisdiction_scope": "Netherlands; official publications and local permit/announcement evidence", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": true, + "public_release_allowed": false, + "reasons": [ + "publication:blocked" + ], + "state": "awaiting-owner-review" + }, + "source_id": "nl.koop.local-permits", + "source_url": "https://zoek.officielebekendmakingen.nl/sru/Search", + "status": { + "acquisition": "artifact_private_only", + "evidence": [ + "docs/country-recon-nl.md", + "data/manifests/nl-source-artifacts.json", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Replace keyword query with collection-specific CQL and document incomplete local coverage.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "open dataset API or subscribed APIs; no calls made", + "attribution": { + "attribution_required": true, + "notice": "Open dataset documented CC BY 4.0; API terms and privacy restrictions apply.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "on-demand; provider update cadence per run", + "country_code": "NL", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Confirm terms/privacy; use only for reviewed identity links, never facility proof or UBO." + ] + }, + "jurisdiction_scope": "Netherlands; KVK business and establishment identity/link evidence", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "nl.kvk.hvds", + "source_url": "https://developers.kvk.nl/nl/documentation/open-dataset-basis-bedrijfsgegevens-api", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-nl.md", + "docs/countries/nl/v1-field-crosswalk.json", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Confirm API route and privacy/terms review; use KVK only for reviewed identity links, never UBO or facility proof.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "public control XML plus SOAP POST; bounded private capture", + "attribution": { + "attribution_required": true, + "notice": "Official NVWA; confirm reuse, privacy and project approval before release.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown; control file observed current 2026-09-15", + "country_code": "NL", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Implement SOAP adapter; confirm pagination, coverage, lifecycle semantics, terms, privacy and identity policy." + ] + }, + "jurisdiction_scope": "Netherlands; NVWA approved food establishments, slaughter/cutting lists", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": true, + "public_release_allowed": false, + "reasons": [ + "publication:blocked" + ], + "state": "awaiting-owner-review" + }, + "source_id": "nl.nvwa.approved-food", + "source_url": "https://www.nvwa.nl/site/binaries/content/assets/site-content/webapp-data/lijsten-erkende-bedrijven/stuurbestand-lijsten-erkende-bedrijven", + "status": { + "acquisition": "artifact_private_only", + "evidence": [ + "docs/country-recon-nl.md", + "docs/countries/nl/v1-field-crosswalk.json", + "data/manifests/nl-source-artifacts.json", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Implement SOAP only after confirming coverage, repeated observations, lifecycle, terms, privacy and identity policy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official HTML pages and PDF tables", + "attribution": { + "attribution_required": true, + "notice": "Official NVWA; preserve publication/effective period and confirm reuse/privacy.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "annual summary; detailed tables are period-specific", + "country_code": "NL", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Model inspections and enforcement separately; absence is not closure; review identity links." + ] + }, + "jurisdiction_scope": "Netherlands; NVWA welfare, animal-experiment and red-meat compliance publications", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": true, + "public_release_allowed": false, + "reasons": [ + "publication:blocked" + ], + "state": "awaiting-owner-review" + }, + "source_id": "nl.nvwa.welfare-enforcement", + "source_url": "https://www.nvwa.nl/over-de-nvwa/publicaties/jaarbeeld-2025/dierenwelzijn", + "status": { + "acquisition": "artifact_private_only", + "evidence": [ + "docs/country-recon-nl.md", + "data/manifests/nl-source-artifacts.json", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Keep 2025 aggregate welfare and 2024 detailed compliance evidence dated and separate; review identity links and release terms.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "public PDOK OGC API plus DSO context APIs", + "attribution": { + "attribution_required": true, + "notice": "PDOK metadata states CC0 1.0; DSO terms and legal interpretation remain gates.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "daily; production collections observed updated 2026-09-15", + "country_code": "NL", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Resolve document/legal scope; no national animal-facility master implied." + ] + }, + "jurisdiction_scope": "Netherlands; DSO/PDOK planning geometry/document evidence", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": true, + "public_release_allowed": false, + "reasons": [ + "publication:blocked" + ], + "state": "awaiting-owner-review" + }, + "source_id": "nl.pdok.omgevingswet", + "source_url": "https://api.pdok.nl/omgevingswet/omgevingsdocumenten/ogc/v2?f=html&lang=nl", + "status": { + "acquisition": "artifact_private_only", + "evidence": [ + "docs/country-recon-nl.md", + "data/manifests/nl-source-artifacts.json", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Resolve DSO document identity and legal scope before treating geometry as permit evidence.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "authorized web services and WSDL/XSD", + "attribution": { + "attribution_required": true, + "notice": "Official RVO; holder, location and animal data may be restricted/personal.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "24/7 service; refresh cadence unknown", + "country_code": "NL", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Obtain purpose-specific authorization; do not scrape or expose restricted data." + ] + }, + "jurisdiction_scope": "Netherlands; RVO I&R animal and UBN/location services", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "nl.rvo.ir", + "source_url": "https://www.rvo.nl/form/bestanden-webservices", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-nl.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Do not acquire until purpose-specific authorization and privacy/terms review exists; keep UBN separate.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + } + ], + "state": "blocked", + "summary": "No release approval is implied; inspect source gates below." + }, + "NO": { + "acquisition_counts": { + "blocked": 2, + "not_run": 6 + }, + "basis": [ + "no.brreg.organizations: not_run / reconnaissance", + "no.fiskeridir.aquaculture: not_run / reconnaissance", + "no.landbruksdirektoratet.farm-register: blocked / blocked", + "no.mattilsynet.animal-experiments: not_run / reconnaissance", + "no.mattilsynet.approved-food: not_run / reconnaissance", + "no.mattilsynet.feed-abp: not_run / reconnaissance", + "no.miljodirektoratet.prtr-permits: blocked / blocked", + "no.ssb.meat-and-animal-use: not_run / reconnaissance" + ], + "country_reasons": [ + "acquisition:blocked", + "acquisition:not_run", + "publication:blocked" + ], + "name": "NO", + "owner_review": "awaiting-owner-review", + "publication_state": "blocked", + "source_count": 8, + "sources": [ + { + "access_method": "public REST JSON API and CSV/gzip/XLSX downloads", + "attribution": { + "attribution_required": true, + "notice": "NLOD 2.0 stated by publisher; suppress person roles, birth numbers, sole-trader and mixed residential addresses.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "API/update-feed specific; timestamp each run", + "country_code": "NO", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Verify rate limits, endpoint versions, field semantics and identity-only matching policy." + ] + }, + "jurisdiction_scope": "Norway; Brønnøysundregistrene Central Coordinating Register open organization data", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "no.brreg.organizations", + "source_url": "https://data.brreg.no/enhetsregisteret/api/dokumentasjon/en/index.html", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-no.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Verify endpoint versions, rate limits and identity-only privacy policy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "public REST/API catalogue, CSV/XLSX downloads and ArcGIS feature service", + "attribution": { + "attribution_required": true, + "notice": "Official Fiskeridirektoratet; preserve provider attribution, CRS/geometry semantics and verify current API terms.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "provider-defined; timestamp API and layer metadata", + "country_code": "NO", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Capture OpenAPI/version metadata, pagination, field semantics, geometry precision and release/privacy terms." + ] + }, + "jurisdiction_scope": "Norway; Fiskeridirektoratet Aquaculture Register localities and permits", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "no.fiskeridir.aquaculture", + "source_url": "https://api.fiskeridir.no/catalog/", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-no.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Capture API metadata and validate pagination, identifiers, geometry and terms.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official register guidance and aggregate/downloadable reports", + "attribution": { + "attribution_required": true, + "notice": "Official authority; farm/person/property linkage is sensitive and must not be exposed without purpose and review.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "report/register-specific; exact public export cadence unknown", + "country_code": "NO", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "No public facility-level export verified; obtain authorization and privacy review before any acquisition." + ] + }, + "jurisdiction_scope": "Norway; Landbruksdirektoratet agricultural property/farm register and livestock statistics", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "no.landbruksdirektoratet.farm-register", + "source_url": "https://www.landbruksdirektoratet.no/nb/jordbruk/kart-og-register/landbruksregisteret", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-no.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Do not acquire property/person/farm rows without an authorized purpose-limited route.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "annual reports and ALURES statistical database; reporting system authenticated", + "attribution": { + "attribution_required": true, + "notice": "Sensitive research/animal-use evidence; aggregate/anonymize and keep facility claims out of canonical entities.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "annual", + "country_code": "NO", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "No public research-facility master verified; resolve stable report links and privacy boundary." + ] + }, + "jurisdiction_scope": "Norway; experimental-animal use statistics and reporting", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "no.mattilsynet.animal-experiments", + "source_url": "https://www.mattilsynet.no/dyr/forsoksdyr/bruk-av-dyr-i-forsok", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-no.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Keep annual aggregates separate; no research-facility master is verified.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official section pages with CSV-backed lists", + "attribution": { + "attribution_required": true, + "notice": "Official Norwegian Food Safety Authority; confirm file-specific reuse terms; names/addresses require privacy review.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "most lists stated daily; timestamp each retrieval", + "country_code": "NO", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Verify complete section coverage, direct CSV URLs, headers, terms, privacy and approval-ID semantics." + ] + }, + "jurisdiction_scope": "Norway; Mattilsynet approved animal-origin food establishments", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "no.mattilsynet.approved-food", + "source_url": "https://www.mattilsynet.no/godkjente-produkter-og-virksomheter", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-no.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Verify complete section coverage, CSV contracts, terms, privacy and approval-ID semantics.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official sectioned list pages and CSV/HTML links", + "attribution": { + "attribution_required": true, + "notice": "Official authority; terms and category-specific privacy review required.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "publisher list cadence; verify per section", + "country_code": "NO", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Verify direct files, category boundaries, completeness, terms and privacy before acquisition." + ] + }, + "jurisdiction_scope": "Norway; Mattilsynet approved/registered feed and animal-by-product establishments", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "no.mattilsynet.feed-abp", + "source_url": "https://www.mattilsynet.no/godkjente-produkter-og-virksomheter/forvarefeed-sector-approved-and-registered-feed-companies-tse", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-no.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Verify direct files, category boundaries, completeness, terms and privacy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official environmental register/document references; public bulk route not verified", + "attribution": { + "attribution_required": true, + "notice": "Official environmental authority; terms, sensitive sites and document privacy require review.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "annual/report-specific; unresolved", + "country_code": "NO", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Verify current public search/export/API, stable identifiers, completeness and coordinate/address exposure." + ] + }, + "jurisdiction_scope": "Norway; Norwegian Environment Agency pollutant-release and permit evidence", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "no.miljodirektoratet.prtr-permits", + "source_url": "https://www.miljodirektoratet.no/globalassets/publikasjoner/M138/M138.pdf", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-no.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Verify current public environmental export/API and stable IDs before acquisition.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "open SSB PxWeb/StatBank API plus official annual animal-use reports", + "attribution": { + "attribution_required": true, + "notice": "SSB API states CC BY 4.0; cite table/source and preserve revisions; animal-use reports require separate provenance.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "SSB daily 08:00 update; animal-use annual", + "country_code": "NO", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Pin table IDs and metadata contracts; preserve aggregate scope and do not double-count facility sources." + ] + }, + "jurisdiction_scope": "Norway; Statistics Norway meat production and Mattilsynet animal-use aggregates", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "no.ssb.meat-and-animal-use", + "source_url": "https://www.ssb.no/en/api", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-no.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Pin SSB table IDs and metadata; preserve aggregate scope and revisions.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + } + ], + "state": "blocked", + "summary": "No release approval is implied; inspect source gates below." + }, + "NZ": { + "acquisition_counts": { + "blocked": 1 + }, + "basis": [ + "nz.locations: blocked / blocked" + ], + "country_reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "name": "NZ", + "owner_review": "awaiting-owner-review", + "publication_state": "blocked", + "source_count": 1, + "sources": [ + { + "access_method": "browser-assisted public-register export or download", + "attribution": { + "attribution_required": true, + "notice": "Legacy notes identify MPI; verify current terms, attribution, and whether the register endpoint is stable.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown", + "country_code": "NZ", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Confirm current register coverage, export format, access limits, and publication semantics." + ] + }, + "jurisdiction_scope": "New Zealand; MPI approved premises/register coverage", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "nz.locations", + "source_url": "https://mpi.my.site.com/publicregister/s/RiskMeasureSearch?riskMeasureType=Risk%20Management%20Programme", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-nz.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Confirm MPI permitted acquisition route, list-specific terms, and schema before private staging.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + } + ], + "state": "blocked", + "summary": "No release approval is implied; inspect source gates below." + }, + "PL": { + "acquisition_counts": { + "blocked": 3, + "not_run": 8 + }, + "basis": [ + "pl.arimr.processing-support: not_run / reconnaissance", + "pl.gdos.eia: blocked / blocked", + "pl.geoportal.urban-planning: not_run / reconnaissance", + "pl.gios.ippc: not_run / reconnaissance", + "pl.giw.abp: not_run / reconnaissance", + "pl.giw.approved-food: blocked / blocked", + "pl.giw.registered-food: not_run / reconnaissance", + "pl.giw.rrw: not_run / reconnaissance", + "pl.gus.regon-bir: blocked / blocked", + "pl.gus.slaughter: not_run / reconnaissance", + "pl.krs.open-api: not_run / reconnaissance" + ], + "country_reasons": [ + "acquisition:blocked", + "acquisition:not_run", + "publication:blocked" + ], + "name": "PL", + "owner_review": "awaiting-owner-review", + "publication_state": "blocked", + "source_count": 11, + "sources": [ + { + "access_method": "official call/guidance; beneficiary/project dataset not acquired", + "attribution": { + "attribution_required": true, + "notice": "Polish official funding evidence; beneficiary/privacy terms review required", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "call-specific; 2026 call window observed 1-30 September", + "country_code": "PL", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Do not treat funding as facility authorization, operation or compliance." + ] + }, + "jurisdiction_scope": "Poland; ARiMR agricultural processing investment support", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "pl.arimr.processing-support", + "source_url": "https://www.gov.pl/web/arimr/startuje-wsparcie-dla-przetworcow", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-pl.md", + "docs/countries/pl/source-crosswalk.json", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Acquire only permitted funding/project evidence; do not interpret funding as facility operation or compliance.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official BIP database/search; page states access from outside Poland is blocked", + "attribution": { + "attribution_required": true, + "notice": "Polish official source; document privacy and access restrictions apply", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "authority entry within 30 days; public release cadence unknown", + "country_code": "PL", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Obtain an in-Poland or authorized bounded capture; no facility-master inference." + ] + }, + "jurisdiction_scope": "Poland; GDOŚ environmental-impact-assessment database", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "pl.gdos.eia", + "source_url": "https://www.gov.pl/web/gdos/bazy-danych-o-ocenach-oddzialywania-na-srodowisko", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-pl.md", + "docs/countries/pl/source-crosswalk.json", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Use an in-Poland or authorized context for bounded EIA acquisition; source states outside-Poland access is blocked.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "public WMS/register announcement; service contract not acquired", + "attribution": { + "attribution_required": true, + "notice": "Polish official spatial data; preserve service terms and legal dates", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "regular local-government updates; transition noted for end of September 2026", + "country_code": "PL", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Capture production service metadata and keep zoning context separate from facility identity." + ] + }, + "jurisdiction_scope": "Poland; Geoportal/Urban Register planning data", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "pl.geoportal.urban-planning", + "source_url": "https://www.geoportal.gov.pl/aktualnosci/nowe-uslugi-w-geoportalu-rejestr-urbanistyczny/", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-pl.md", + "docs/countries/pl/source-crosswalk.json", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Capture production Urban Register/WMS metadata and preserve planning act identity and legal/effective dates.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "central official page with 16 regional links; no national export verified", + "attribution": { + "attribution_required": true, + "notice": "Polish official environmental source; regional terms/privacy review required", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "regional/unknown", + "country_code": "PL", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Capture regional schemas and retain permit/installations as evidence overlays." + ] + }, + "jurisdiction_scope": "Poland; GIOŚ and 16 WIOŚ integrated-permit installation registers", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "pl.gios.ippc", + "source_url": "https://www.gov.pl/web/gios/instalacje-wymagajace-uzyskania-pozwolenia-zintegrowanego", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-pl.md", + "docs/countries/pl/source-crosswalk.json", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Capture regional WIOŚ registers one voivodeship at a time and keep permits/installations as dated environmental evidence.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "GIW page plus linked pasze.wetgiw.gov.pl register; exact export unresolved", + "attribution": { + "attribution_required": true, + "notice": "Polish official source under ABP rules; terms/privacy/release review required", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown", + "country_code": "PL", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Verify current list route and preserve 1069/2009 scope separately from food facilities." + ] + }, + "jurisdiction_scope": "Poland; GIW animal by-product establishments and operators", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "pl.giw.abp", + "source_url": "https://www.wetgiw.gov.pl/handel-eksport-import/niespozywcze-produkty-pochodzenia-zwierzecego", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-pl.md", + "docs/countries/pl/source-crosswalk.json", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Verify the current ABP list route/schema and preserve Regulation 1069/2009 scope outside food-facility totals.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official rendered HTML register with filters plus XLS export route; bounded metadata-only observation", + "attribution": { + "attribution_required": true, + "notice": "Polish official source; terms, privacy and project release review required", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown; current views observed 2026-09-16", + "country_code": "PL", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Acquire a bounded source artifact and hash; resolve WNI lifecycle/repeat semantics; do not sum section rows; review privacy/terms/release approval." + ] + }, + "jurisdiction_scope": "Poland; GIW approved animal-origin food establishments under Regulation (EC) 853/2004", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "pl.giw.approved-food", + "source_url": "https://www.wetgiw.gov.pl/handel-eksport-import/listy-zakladow", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-pl.md", + "docs/countries/pl/source-crosswalk.json", + "data/manifests/pl-source-artifacts.json", + "data/raw/poland/20260916T000000Z/metadata.json", + "pipeline/source_registry.json", + "pipeline/poland/test_metadata.py" + ], + "metadata": "verified", + "next_action": "Acquire a bounded GIW XLS/HTML artifact from an allowed context, record source hash/schema, and model WNI activity/species/product observations without summing sections or publishing addresses.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official index with 14 list families; list-specific routes not acquired", + "attribution": { + "attribution_required": true, + "notice": "Polish official source; list-specific terms/privacy review required", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown", + "country_code": "PL", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Capture one list at a time and keep registration separate from 853/2004 approval." + ] + }, + "jurisdiction_scope": "Poland; GIW registered animal-origin food-sector activities", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "pl.giw.registered-food", + "source_url": "https://www.wetgiw.gov.pl/nadzor-weterynaryjny/wykaz-zakladow-rejestrowanych", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-pl.md", + "docs/countries/pl/source-crosswalk.json", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Capture one registered-list family at a time; preserve list-specific identifiers and keep registration separate from 853/2004 approval.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official RRW publication/index; no current file acquired", + "attribution": { + "attribution_required": true, + "notice": "Polish official aggregate/evidence reports; small-cell and case privacy review required", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "annual/report-specific", + "country_code": "PL", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Acquire current reports and model them as dated evidence, not facility master rows." + ] + }, + "jurisdiction_scope": "Poland; GIW veterinary statistical reporting", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "pl.giw.rrw", + "source_url": "https://www.wetgiw.gov.pl/publikacje/rrw-sprawozdawczosc-statystyczna/printpage", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-pl.md", + "docs/countries/pl/source-crosswalk.json", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Acquire current RRW reports and model welfare, controls, enforcement and meat-examination evidence by report/table/year or source-local ID.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "documented SOAP/API service; registration/user key required", + "attribution": { + "attribution_required": true, + "notice": "Official GUS service; respect registration, rate limits and personal-data handling", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "continuously updated register; on-demand queries", + "country_code": "PL", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Obtain authorized access before queries and keep fields restricted." + ] + }, + "jurisdiction_scope": "Poland; GUS REGON BIR1 identity service", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "pl.gus.regon-bir", + "source_url": "https://api.stat.gov.pl/Home/RegonApi?lang=en", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-pl.md", + "docs/countries/pl/source-crosswalk.json", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Obtain authorized BIR1 access before queries, respect documented rate limits, and keep sole-trader/restricted fields private.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official HTML publication with XLSX attachment; page observed", + "attribution": { + "attribution_required": true, + "notice": "GUS official statistics; preserve revisions/flags and source terms", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "monthly collection; 2025 publication dated 2026-03-02", + "country_code": "PL", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Never infer named facilities or join totals to GIW WNI; keep aggregate context separate." + ] + }, + "jurisdiction_scope": "Poland; Statistics Poland slaughter statistics", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "pl.gus.slaughter", + "source_url": "https://stat.gov.pl/obszary-tematyczne/rolnictwo-lesnictwo/produkcja-zwierzeca-zwierzeta-gospodarskie/uboje-zwierzat-gospodarskich-w-2025-r-,16,1.html", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-pl.md", + "docs/countries/pl/source-crosswalk.json", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Keep GUS R-09U aggregates separate from GIW WNI facilities and preserve statistical revisions/flags.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "open API announced by Ministry; no request made", + "attribution": { + "attribution_required": true, + "notice": "Official KRS; RODO filtering and API terms review required", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "on-demand/registry filings", + "country_code": "PL", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Confirm current API contract; exact identity link only, never operating-site inference." + ] + }, + "jurisdiction_scope": "Poland; Ministry of Justice KRS legal-entity identity", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "pl.krs.open-api", + "source_url": "https://prs.ms.gov.pl/krs/openApi", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-pl.md", + "docs/countries/pl/source-crosswalk.json", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Confirm current KRS API contract and RODO filtering; use only for exact organization identity links supplied by an upstream source.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + } + ], + "state": "blocked", + "summary": "No release approval is implied; inspect source gates below." + }, + "PT": { + "acquisition_counts": { + "blocked": 5, + "not_run": 1 + }, + "basis": [ + "pt.apambiente.tua: blocked / blocked", + "pt.dgav.abp: blocked / blocked", + "pt.dgav.approved-food: blocked / blocked", + "pt.dgav.feed: blocked / blocked", + "pt.ifap.snira: blocked / blocked", + "pt.ine.animal-production: not_run / reconnaissance" + ], + "country_reasons": [ + "acquisition:blocked", + "acquisition:not_run", + "publication:blocked" + ], + "name": "PT", + "owner_review": "awaiting-owner-review", + "publication_state": "blocked", + "source_count": 6, + "sources": [ + { + "access_method": "official guidance and electronic-title/document route; no stable national public export verified", + "attribution": { + "attribution_required": true, + "notice": "Portuguese environmental authority source; document reuse, geometry, personal-address, and permit-condition publication terms require review", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "decision/document-specific", + "country_code": "PT", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Locate and verify a current public search/export contract; keep TUA separate from DGAV approval and do not infer facility identity from holder or permit text." + ] + }, + "jurisdiction_scope": "Portugal; Agência Portuguesa do Ambiente Título Único Ambiental and linked environmental licensing decisions", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "pt.apambiente.tua", + "source_url": "https://apambiente.pt/avaliacao-e-gestao-ambiental/titulo-unico-ambiental", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-pt.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Locate a current public TUA search/export contract and safe permit fields; keep environmental decisions as a separate reviewed evidence overlay.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official DGAV public +SIPACE/legacy SIPACE list family; separate ABP section capture required", + "attribution": { + "attribution_required": true, + "notice": "Portuguese government source; ABP-specific reuse, attribution, privacy, and redistribution terms were not verified", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "not stated; timestamp every retrieval", + "country_code": "PT", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Do not union with food or feed lists; exact ABP section routes, codebook, export, cadence, privacy, terms, and coverage remain unresolved." + ] + }, + "jurisdiction_scope": "Portugal; DGAV animal-by-product establishments, installations, and operators approved, registered, or authorized under Regulations (EC) 1069/2009 and 142/2011", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "pt.dgav.abp", + "source_url": "https://maissipace.dgav.pt/Listagens", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-pt.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Capture ABP sections separately from food; resolve approval/registration IDs, category/activity codes, cadence, coverage, terms, privacy, and project approval.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official DGAV public +SIPACE/legacy SIPACE list family; browser-assisted bounded capture only until a stable export is verified", + "attribution": { + "attribution_required": true, + "notice": "Portuguese government source; source-specific reuse, attribution, personal-address, and redistribution terms were not verified", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "not stated; timestamp every retrieval and preserve list/section context", + "country_code": "PT", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "The new +SIPACE route is transitional and no stable bulk/API contract was verified; confirm section coverage, pagination, status/effective-date semantics, codes, privacy, terms, and project approval before acquisition." + ] + }, + "jurisdiction_scope": "Portugal; DGAV approved and registered food establishments, including animal-origin establishments under Regulation (EC) 853/2004", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "pt.dgav.approved-food", + "source_url": "https://maissipace.dgav.pt/Listagens", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-pt.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Obtain an authorized bounded public-list capture; fingerprint section headers, pagination, NCV/NII status semantics, codes, coverage, terms, privacy, and project approval before an adapter.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official DGAV public +SIPACE/legacy SIPACE list family; separate feed section capture required", + "attribution": { + "attribution_required": true, + "notice": "Portuguese government source; feed-list reuse, attribution, privacy, and redistribution terms were not verified", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "not stated; timestamp every retrieval", + "country_code": "PT", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "The feed identifier is not interchangeable with an NCV; confirm current public section, codebook, pagination, export, cadence, private/primary-producer boundary, terms, and coverage." + ] + }, + "jurisdiction_scope": "Portugal; DGAV feed-sector establishments and operators registered or approved under Regulation (EC) 183/2005", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "pt.dgav.feed", + "source_url": "https://maissipace.dgav.pt/Listagens", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-pt.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Confirm the public feed section and NII semantics with an authorized capture; keep feed separate from NCV food/ABP rows and review private/primary-producer exposure.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "restricted IFAP area and credentialed webservice; no public acquisition authorized", + "attribution": { + "attribution_required": true, + "notice": "Restricted animal/holder data; purpose limitation, access control, retention, and privacy review are mandatory", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "operational system; provider-specific", + "country_code": "PT", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Do not scrape or retain rows; obtain explicit authorization only if a narrowly scoped relationship study is approved, and suppress holder, farm, parcel, and precise-location details." + ] + }, + "jurisdiction_scope": "Portugal; IFAP/DGAV SNIRA animal-identification, holding, movement, and herd information", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "pt.ifap.snira", + "source_url": "https://www.ifap.pt/portal/en/registo-area-reservada", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-pt.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Do not acquire without explicit authorization; keep holder, animal, farm, parcel, and precise-location data restricted and assess any future relationship study separately.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official INE metadata/PxWeb statistics route; table-specific API or download to be pinned", + "attribution": { + "attribution_required": true, + "notice": "Official statistics; preserve table metadata, revisions, confidentiality flags, and source terms", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "annual/semestral/monthly by table; metadata checked through 2025/2026", + "country_code": "PT", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Pin reproducible table IDs, API contract, revisions, terms, and confidentiality rules; never infer facility coverage or join confidential slaughter surveys to named establishments." + ] + }, + "jurisdiction_scope": "Portugal; Statistics Portugal aggregate animal-production, meat, and slaughter statistics", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "pt.ine.animal-production", + "source_url": "https://ine.pt/bddXplorer/htdocs/minfo.jsp?lingua=EN&var_cd=0000916&var_cd=0000917&var_cd=0000918", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-pt.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Pin reproducible INE table/API identifiers and revision/confidentiality terms; keep statistics aggregate and separate from named-facility evidence.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + } + ], + "state": "blocked", + "summary": "No release approval is implied; inspect source gates below." + }, + "RO": { + "acquisition_counts": { + "blocked": 4, + "not_run": 2 + }, + "basis": [ + "ro.ansvsa.approved-food: blocked / blocked", + "ro.environment-permits: blocked / blocked", + "ro.farm-aquaculture: blocked / blocked", + "ro.inspections-experiments: blocked / blocked", + "ro.insse.statistics: not_run / reconnaissance", + "ro.onrc.organizations: not_run / reconnaissance" + ], + "country_reasons": [ + "acquisition:blocked", + "acquisition:not_run", + "publication:blocked" + ], + "name": "RO", + "owner_review": "awaiting-owner-review", + "publication_state": "blocked", + "source_count": 6, + "sources": [ + { + "access_method": "official ANSVSA/DSVSA registers and relevant EU approval surfaces; national bulk route unverified", + "attribution": { + "attribution_required": true, + "notice": "Official authority; verify national list license, privacy, attribution and source terms before acquisition.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown; timestamp retrieval", + "country_code": "RO", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Pin current national list/API/export, stable identifiers, cadence, coordinates, terms and privacy." + ] + }, + "jurisdiction_scope": "Romania; ANSVSA approved and registered food establishments handling food of animal origin", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "ro.ansvsa.approved-food", + "source_url": "https://portal.ansvsa.ro/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-ro.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin national list/API/export, identifiers, cadence, coordinates, terms and privacy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "SIM/eFORM official web system and document/publication routes", + "attribution": { + "attribution_required": true, + "notice": "Official environmental authority; verify public-read rights, document license, geometry and privacy.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown; system currently reports technical unavailability", + "country_code": "RO", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Public read/API route and dependable availability are unresolved; ANPM reports SIM technical outage." + ] + }, + "jurisdiction_scope": "Romania; ANPM environmental authorizations and integrated environmental system", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "ro.environment-permits", + "source_url": "https://raportare.anpm.ro/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-ro.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Recheck ANPM availability and verify public read/API, document route, license, geometry and privacy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "national open-data catalogue, agriculture records and official environmental/biodiversity systems", + "attribution": { + "attribution_required": true, + "notice": "Verify publisher, license, animal-holder/property privacy and coordinate precision.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "resource-specific", + "country_code": "RO", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "No complete national farm/aquaculture public export contract verified." + ] + }, + "jurisdiction_scope": "Romania; farm, holding and aquaculture establishment evidence", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "ro.farm-aquaculture", + "source_url": "https://data.gov.ro/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-ro.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Verify national farm/aquaculture scope, export/API, IDs, coordinates, terms and privacy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official portals, reports and statistical publications", + "attribution": { + "attribution_required": true, + "notice": "Sensitive control/research evidence; aggregate or anonymize and require project review.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "report/register-specific", + "country_code": "RO", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "No stable public facility-level inspection or animal-experimentation master verified." + ] + }, + "jurisdiction_scope": "Romania; ANSVSA/DSVSA inspections, enforcement and animal-experimentation evidence", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "ro.inspections-experiments", + "source_url": "https://portal.ansvsa.ro/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-ro.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Locate a stable public facility/event route; keep sensitive evidence aggregate until verified.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official statistical tables/data services; exact table/API route unverified", + "attribution": { + "attribution_required": true, + "notice": "Official statistics; verify table terms, citation requirements and aggregate-only scope.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "table-specific; preserve revisions", + "country_code": "RO", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Pin current slaughter and animal-use table IDs, API/download contracts and cadence." + ] + }, + "jurisdiction_scope": "Romania; INSSE aggregate slaughter, livestock and animal-use statistics", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "ro.insse.statistics", + "source_url": "https://insse.ro/cms/", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-ro.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin official slaughter/animal-use table IDs, APIs, cadence and revisions.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "data.gov.ro CKAN CSV snapshots and API metadata", + "attribution": { + "attribution_required": true, + "notice": "Catalogue snapshots are public and some are CC BY 4.0; verify current file license, field privacy and attribution.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "snapshot-specific; timestamp each release", + "country_code": "RO", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Pin current snapshot, cadence, field dictionary and registered-office publication policy." + ] + }, + "jurisdiction_scope": "Romania; ONRC legal-entity and authorized-activity snapshots", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "ro.onrc.organizations", + "source_url": "https://data.gov.ro/dataset?organiza=&organization=onrc&res_format=csv", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-ro.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Pin current CSV snapshot, cadence, fields, license and registered-office privacy policy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + } + ], + "state": "blocked", + "summary": "No release approval is implied; inspect source gates below." + }, + "RS": { + "acquisition_counts": { + "blocked": 5, + "not_run": 1 + }, + "basis": [ + "rs.apr.organizations: blocked / blocked", + "rs.environment-permits: blocked / blocked", + "rs.farm-aquaculture: blocked / blocked", + "rs.inspections-experiments: blocked / blocked", + "rs.stat.statistics: not_run / reconnaissance", + "rs.veterinary.approved-food: blocked / blocked" + ], + "country_reasons": [ + "acquisition:blocked", + "acquisition:not_run", + "publication:blocked" + ], + "name": "RS", + "owner_review": "awaiting-owner-review", + "publication_state": "blocked", + "source_count": 6, + "sources": [ + { + "access_method": "official browser and authorized web services; automated downloading restricted", + "attribution": { + "attribution_required": true, + "notice": "Identity linkage only; respect APR terms, fees, access restrictions and registered-office privacy.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "provider-defined", + "country_code": "RS", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Pin authorized API/service, fees, quotas, cadence, fields and automation permissions." + ] + }, + "jurisdiction_scope": "Serbia; APR centralized business-entity registers", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "rs.apr.organizations", + "source_url": "https://www.apr.gov.rs/registers/media/data-search.1728.html", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-rs.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Pin authorized APR service, fees, quotas, cadence, fields and automation permissions.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official environmental agency systems, public notices and documents", + "attribution": { + "attribution_required": true, + "notice": "Verify public-read rights, document license, geometry and privacy.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "permit/document-specific", + "country_code": "RS", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Stable public read/API/export route and complete national coverage are unresolved." + ] + }, + "jurisdiction_scope": "Serbia; environmental permits and environmental authorization evidence", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "rs.environment-permits", + "source_url": "https://www.sepa.gov.rs/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-rs.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Verify public environmental permit route, coverage, documents, geometry, license and privacy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "open-data portal, agriculture resources and official fisheries/veterinary registers", + "attribution": { + "attribution_required": true, + "notice": "Verify publisher, license, animal-holder/property privacy and coordinate precision.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "resource-specific", + "country_code": "RS", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "No complete national farm/aquaculture establishment export contract verified." + ] + }, + "jurisdiction_scope": "Serbia; livestock holdings and aquaculture permit/establishment evidence", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "rs.farm-aquaculture", + "source_url": "https://data.gov.rs/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-rs.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Verify farm/aquaculture scope, export/API, IDs, coordinates, terms and privacy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official control plans, reports and statistical publications", + "attribution": { + "attribution_required": true, + "notice": "Sensitive control/research evidence; aggregate or anonymize and require project review.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "report/register-specific", + "country_code": "RS", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "No stable public facility-level inspection or animal-experimentation master verified." + ] + }, + "jurisdiction_scope": "Serbia; veterinary inspections/enforcement and animal-experimentation evidence", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "rs.inspections-experiments", + "source_url": "https://www.minpolj.gov.rs/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-rs.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Locate a stable public facility/event route; keep sensitive evidence aggregate until verified.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official JSON/CSV statistical APIs and data portal", + "attribution": { + "attribution_required": true, + "notice": "Official statistics; verify table terms, citation requirements and aggregate-only scope.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "table-specific; preserve revisions", + "country_code": "RS", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Pin current slaughter and animal-use dataset IDs, API contracts and cadence." + ] + }, + "jurisdiction_scope": "Serbia; Statistical Office aggregate slaughter, livestock and animal-use statistics", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "rs.stat.statistics", + "source_url": "https://www.stat.gov.rs/", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-rs.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Pin official slaughter/animal-use dataset IDs, APIs, cadence and revisions.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official ministry/veterinary registers and relevant EU/control surfaces; national bulk route unverified", + "attribution": { + "attribution_required": true, + "notice": "Official authority; verify list license, attribution, privacy and coordinates before acquisition.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown; timestamp retrieval", + "country_code": "RS", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Pin current national list/API/export, categories, IDs, cadence, terms and privacy." + ] + }, + "jurisdiction_scope": "Serbia; approved and registered food establishments handling food of animal origin", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "rs.veterinary.approved-food", + "source_url": "https://www.minpolj.gov.rs/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-rs.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin veterinary list/API/export, categories, IDs, cadence, terms and privacy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + } + ], + "state": "blocked", + "summary": "No release approval is implied; inspect source gates below." + }, + "SE": { + "acquisition_counts": { + "not_run": 6 + }, + "basis": [ + "se.bolagsverket.company-api: not_run / reconnaissance", + "se.jordbruksverket.animal-experiments: not_run / reconnaissance", + "se.jordbruksverket.feed-abp: not_run / reconnaissance", + "se.jordbruksverket.slaughter-stats: not_run / reconnaissance", + "se.jordbruksverket.slaughterhouses: not_run / reconnaissance", + "se.naturvardsverket.prtr: not_run / reconnaissance" + ], + "country_reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "name": "SE", + "owner_review": "awaiting-owner-review", + "publication_state": "blocked", + "source_count": 6, + "sources": [ + { + "access_method": "official API/download documentation; access not exercised", + "attribution": { + "attribution_required": true, + "notice": "Identity linkage only; API terms, personal-data handling, and sole-trader/mixed-address suppression required.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "provider-defined; auth and quota cadence unknown", + "country_code": "SE", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Confirm account/auth, rate limits, terms, field availability and privacy before automated crosswalk use; never treat corporate identity as facility proof." + ] + }, + "jurisdiction_scope": "Sweden; Bolagsverket corporate identity and organization-number API", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "se.bolagsverket.company-api", + "source_url": "https://bolagsverket.se/apierochoppnadata/hamtaforetagsinformation/apiforatthamtaforetagsinformation.3988.html", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-se.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Confirm API access, quotas, terms, and privacy before identity-only crosswalk use.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official guidance, annual statistics and ALURES summaries", + "attribution": { + "attribution_required": true, + "notice": "Sensitive animal-use/research evidence; anonymization, purpose limitation and privacy review required; terms not fully verified.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "annual statistics; ALURES summaries from 2021", + "country_code": "SE", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "No public national facility export verified; resolve stable statistics route and keep facility-level research claims out of canonical entities." + ] + }, + "jurisdiction_scope": "Sweden; experimental-animal permits, approvals and use statistics", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "se.jordbruksverket.animal-experiments", + "source_url": "https://jordbruksverket.se/djur/ovriga-djur/forsoksdjur-och-djurforsok/verksamhet-med-forsoksdjur", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-se.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Keep anonymized annual/ALURES summaries separate; no facility master is verified.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official landing page with linked XLSX/PDF downloads", + "attribution": { + "attribution_required": true, + "notice": "Official Swedish authority; file-specific reuse terms not verified; operator names/addresses may require privacy review.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "continuous updates stated by publisher", + "country_code": "SE", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Capture and fingerprint each linked file, verify headers/section semantics, direct-download stability, terms, and national coverage." + ] + }, + "jurisdiction_scope": "Sweden; Jordbruksverket feed and animal-by-product facility lists", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "se.jordbruksverket.feed-abp", + "source_url": "https://jordbruksverket.se/djur/foder-och-produkter-fran-djur/listor-over-anlaggningar-for-foder-och-animaliska-biprodukter-och-darav-framstallda-produkter", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-se.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Capture linked XLSX files privately, fingerprint headers/sections, and resolve terms, coverage, and privacy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official XLSX downloads and explanatory HTML", + "attribution": { + "attribution_required": true, + "notice": "Official Swedish authority; source-specific reuse terms not verified; preserve consent-limited naming and aggregate scope.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "weekly reporting; publication/update quarterly", + "country_code": "SE", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Do not infer facility completeness from consent-based named subset; verify file URLs, schema, revisions and terms." + ] + }, + "jurisdiction_scope": "Sweden; Jordbruksverket aggregate slaughter and classification statistics", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "se.jordbruksverket.slaughter-stats", + "source_url": "https://jordbruksverket.se/djur/djurtransportorer-och-slakterier/statistik-om-slaktade-djur-och-klassning", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-se.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Verify current quarterly/annual XLSX links and preserve consent-limited named versus Other scope.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "public live HTML table", + "attribution": { + "attribution_required": true, + "notice": "Official Swedish authority; source-specific reuse terms not verified; privacy review required for names and addresses if later captured.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "not stated; timestamp each retrieval", + "country_code": "SE", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Verify terms, live HTML stability, completeness, effective-date semantics, and privacy before automated acquisition." + ] + }, + "jurisdiction_scope": "Sweden; Jordbruksverket slaughterhouse installation-number table", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "se.jordbruksverket.slaughterhouses", + "source_url": "https://jordbruksverket.se/3608.html", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-se.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Verify live HTML stability, terms, completeness, effective-date semantics, and privacy before automated acquisition.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "public search service and browser-filtered document catalogue", + "attribution": { + "attribution_required": true, + "notice": "Official Swedish environmental authority; bulk reuse terms/API not verified; location and permit-document privacy review required.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "annual PRTR emissions/transfers; document publication varies", + "country_code": "SE", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Verify export/API, stable facility key, completeness, terms, and separation of PRTR from permit/food approval evidence." + ] + }, + "jurisdiction_scope": "Sweden; Swedish Pollutant Release and Transfer Register and environmental document catalogue", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "se.naturvardsverket.prtr", + "source_url": "https://www.naturvardsverket.se/en/services-and-permits/data-databases-and-applications/the-swedish-pollutant-release-and-transfer-register/", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-se.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Verify public export/API, facility key, completeness, terms, and separation from permit evidence.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + } + ], + "state": "acquisition-ready", + "summary": "No release approval is implied; inspect source gates below." + }, + "SI": { + "acquisition_counts": { + "blocked": 2, + "not_run": 3 + }, + "basis": [ + "si.corporate-register: not_run / reconnaissance", + "si.environment.permits: blocked / blocked", + "si.statistics-slaughter: not_run / reconnaissance", + "si.uvhvvr.approved-food-feed: not_run / reconnaissance", + "si.uvhvvr.farms-aquaculture: blocked / blocked" + ], + "country_reasons": [ + "acquisition:blocked", + "acquisition:not_run", + "metadata:unknown", + "publication:blocked" + ], + "name": "SI", + "owner_review": "awaiting-owner-review", + "publication_state": "blocked", + "source_count": 5, + "sources": [ + { + "access_method": "official business-register route; API unverified", + "attribution": { + "attribution_required": true, + "notice": "Identity-only linkage; suppress personal, sole-trader and residential details.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "provider-defined", + "country_code": "SI", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Verify current access, terms, quotas and privacy policy." + ] + }, + "jurisdiction_scope": "Slovenia; corporate identifiers", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "si.corporate-register", + "source_url": "https://www.ajpes.si/", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-si.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Verify current access, terms and identity-only privacy policy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official environmental routes; not verified", + "attribution": { + "attribution_required": true, + "notice": "Verify license, geometry and sensitive-site handling.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown", + "country_code": "SI", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "No stable national animal-facility permit export verified." + ] + }, + "jurisdiction_scope": "Slovenia; environmental permit evidence", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "metadata:unknown", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "si.environment.permits", + "source_url": "https://www.gov.si/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-si.md", + "pipeline/source_registry.json" + ], + "metadata": "unknown", + "next_action": "Locate and verify environmental permit data.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official PxWeb API/table", + "attribution": { + "attribution_required": true, + "notice": "Official statistics; preserve table ID, revisions and license metadata.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "annual/table-specific", + "country_code": "SI", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Pin API metadata and preserve aggregate scope." + ] + }, + "jurisdiction_scope": "Slovenia; official livestock slaughter statistics", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "si.statistics-slaughter", + "source_url": "https://pxweb.stat.si/SiStatData/pxweb/en/Data/-/H202S.px", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-si.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Pin PxWeb table/API metadata and revisions.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official PDF/list registers", + "attribution": { + "attribution_required": true, + "notice": "Official Slovenian authority; verify terms, attribution and privacy.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "publisher-defined; timestamp retrieval", + "country_code": "SI", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Verify current files/API, completeness, terms and coordinates/privacy." + ] + }, + "jurisdiction_scope": "Slovenia; UVHVVR approved food and feed establishments", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "si.uvhvvr.approved-food-feed", + "source_url": "https://www.gov.si/zbirke/storitve/odobritev-zivilskega-obrata/", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-si.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Verify current files/API, completeness, terms and privacy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "open-data catalogue and register systems", + "attribution": { + "attribution_required": true, + "notice": "Animal/property location data require strict privacy review.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "dataset-specific", + "country_code": "SI", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Verify public scope, export/API, terms and sensitive fields." + ] + }, + "jurisdiction_scope": "Slovenia; UVHVVR/OPSI animal holding and aquaculture data", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "si.uvhvvr.farms-aquaculture", + "source_url": "https://podatki.gov.si/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-si.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Verify public scope, export/API, terms and sensitive fields.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + } + ], + "state": "blocked", + "summary": "No release approval is implied; inspect source gates below." + }, + "SK": { + "acquisition_counts": { + "blocked": 3, + "not_run": 2 + }, + "basis": [ + "sk.environment.permits: blocked / blocked", + "sk.statistics-corporate: not_run / reconnaissance", + "sk.svps.approved-food: not_run / reconnaissance", + "sk.svps.farms-aquaculture: blocked / blocked", + "sk.svps.inspections-experiments: blocked / blocked" + ], + "country_reasons": [ + "acquisition:blocked", + "acquisition:not_run", + "metadata:unknown", + "publication:blocked" + ], + "name": "SK", + "owner_review": "awaiting-owner-review", + "publication_state": "blocked", + "source_count": 5, + "sources": [ + { + "access_method": "official environmental routes; not verified", + "attribution": { + "attribution_required": true, + "notice": "Verify license, geometry and sensitive-site handling.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown", + "country_code": "SK", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "No stable national animal-facility permit export verified." + ] + }, + "jurisdiction_scope": "Slovakia; environmental permit evidence", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "metadata:unknown", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "sk.environment.permits", + "source_url": "https://www.minzp.sk/en/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-sk.md", + "pipeline/source_registry.json" + ], + "metadata": "unknown", + "next_action": "Locate and verify environmental permit data.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official statistical/business-register routes; exact APIs unverified", + "attribution": { + "attribution_required": true, + "notice": "Identity-only corporate linkage; aggregate statistics preserve revisions and source terms.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "table/provider-specific", + "country_code": "SK", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Verify current table/API contracts, business-register access, terms and privacy." + ] + }, + "jurisdiction_scope": "Slovakia; official slaughter/livestock statistics and business identifiers", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "sk.statistics-corporate", + "source_url": "https://slovak.statistics.sk/", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-sk.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Verify statistics/business-register APIs, terms and privacy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official filtered lists and linked XSL/XLSX datasets", + "attribution": { + "attribution_required": true, + "notice": "Official Slovak authority; verify terms, attribution and privacy.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "list-specific dated updates", + "country_code": "SK", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Verify direct files, complete categories, stable IDs, terms and address/coordinate fields." + ] + }, + "jurisdiction_scope": "Slovakia; SVPS approved food, slaughter and ABP establishments", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "sk.svps.approved-food", + "source_url": "https://zoznamy.svps.sk/?LANG=EN", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-sk.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Verify direct files, categories, IDs, terms and address/coordinates.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official datasets and register links", + "attribution": { + "attribution_required": true, + "notice": "Animal-holder/site data require privacy review.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "resource-specific", + "country_code": "SK", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Verify public scope, current export/API, terms and sensitive-field policy." + ] + }, + "jurisdiction_scope": "Slovakia; SVPS farm, veterinary and aquaculture registers", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "sk.svps.farms-aquaculture", + "source_url": "https://svps.sk/datasety/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-sk.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Verify public scope, export/API, terms and sensitive fields.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official reports/systems; row-level route unverified", + "attribution": { + "attribution_required": true, + "notice": "Sensitive evidence; aggregate/anonymize and require review.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "annual/report-specific", + "country_code": "SK", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "No stable public facility-level route verified." + ] + }, + "jurisdiction_scope": "Slovakia; SVPS inspections, enforcement and animal-experiment evidence", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "sk.svps.inspections-experiments", + "source_url": "https://svps.sk/english/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-sk.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Keep aggregate/event evidence separate until a public route is verified.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + } + ], + "state": "blocked", + "summary": "No release approval is implied; inspect source gates below." + }, + "TR": { + "acquisition_counts": { + "blocked": 5, + "not_run": 1 + }, + "basis": [ + "tr.cevre.environment-permits: blocked / blocked", + "tr.mersis.organizations: blocked / blocked", + "tr.tarim.approved-food: blocked / blocked", + "tr.tarim.inspections-enforcement: blocked / blocked", + "tr.tarim.livestock-systems: blocked / blocked", + "tr.tuik.animal-statistics: not_run / reconnaissance" + ], + "country_reasons": [ + "acquisition:blocked", + "acquisition:not_run", + "publication:blocked" + ], + "name": "TR", + "owner_review": "awaiting-owner-review", + "publication_state": "blocked", + "source_count": 6, + "sources": [ + { + "access_method": "official route; authorized bounded capture/API only", + "attribution": { + "attribution_required": true, + "notice": "Official Turkish government source; terms, privacy, access controls, and safety review required", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown; verify source-specific", + "country_code": "TR", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Exact machine contract, stable IDs, cadence, licensing, privacy, and automation permissions not pinned." + ] + }, + "jurisdiction_scope": "Turkey; environmental permits and EIA", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "tr.cevre.environment-permits", + "source_url": "https://csb.gov.tr/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-tr.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin authorized route, schema, cadence, IDs, terms, privacy, and automation permissions.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official route; authorized bounded capture/API only", + "attribution": { + "attribution_required": true, + "notice": "Official Turkish government source; terms, privacy, access controls, and safety review required", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown; verify source-specific", + "country_code": "TR", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Exact machine contract, stable IDs, cadence, licensing, privacy, and automation permissions not pinned." + ] + }, + "jurisdiction_scope": "Turkey; MERSIS central company registry", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "tr.mersis.organizations", + "source_url": "https://mersis.ticaret.gov.tr/Portal/KullaniciIslemleri/GirisIslemleri", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-tr.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin authorized route, schema, cadence, IDs, terms, privacy, and automation permissions.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official route; authorized bounded capture/API only", + "attribution": { + "attribution_required": true, + "notice": "Official Turkish government source; terms, privacy, access controls, and safety review required", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown; verify source-specific", + "country_code": "TR", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Exact machine contract, stable IDs, cadence, licensing, privacy, and automation permissions not pinned." + ] + }, + "jurisdiction_scope": "Turkey; Ministry approved food businesses and slaughterhouses", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "tr.tarim.approved-food", + "source_url": "https://www.tarimorman.gov.tr/Konular/Hayvancilik/hayvan-refah%C4%B1-kimliklendirme-ve-i%C5%9Fletme-onay/kesimhaneler", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-tr.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin authorized route, schema, cadence, IDs, terms, privacy, and automation permissions.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official route; authorized bounded capture/API only", + "attribution": { + "attribution_required": true, + "notice": "Official Turkish government source; terms, privacy, access controls, and safety review required", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown; verify source-specific", + "country_code": "TR", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Exact machine contract, stable IDs, cadence, licensing, privacy, and automation permissions not pinned." + ] + }, + "jurisdiction_scope": "Turkey; official controls and public animal-use evidence", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "tr.tarim.inspections-enforcement", + "source_url": "https://www.tarimorman.gov.tr/Konular/Gida-Ve-Yem-Hizmetleri/Gida-Hizmetleri/Resmi-Kontroller", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-tr.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin authorized route, schema, cadence, IDs, terms, privacy, and automation permissions.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official route; authorized bounded capture/API only", + "attribution": { + "attribution_required": true, + "notice": "Official Turkish government source; terms, privacy, access controls, and safety review required", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown; verify source-specific", + "country_code": "TR", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Exact machine contract, stable IDs, cadence, licensing, privacy, and automation permissions not pinned." + ] + }, + "jurisdiction_scope": "Turkey; Ministry animal registration and livestock systems", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "tr.tarim.livestock-systems", + "source_url": "https://www.tarimorman.gov.tr/HAYGEM/Menu/2/Hayvancilik", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-tr.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin authorized route, schema, cadence, IDs, terms, privacy, and automation permissions.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official route; authorized bounded capture/API only", + "attribution": { + "attribution_required": true, + "notice": "Official Turkish government source; terms, privacy, access controls, and safety review required", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown; verify source-specific", + "country_code": "TR", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Exact machine contract, stable IDs, cadence, licensing, privacy, and automation permissions not pinned." + ] + }, + "jurisdiction_scope": "Turkey; aggregate animal-production and slaughter statistics", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "tr.tuik.animal-statistics", + "source_url": "https://veriportali.tuik.gov.tr/en/press/58015", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-tr.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin authorized route, schema, cadence, IDs, terms, privacy, and automation permissions.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + } + ], + "state": "blocked", + "summary": "No release approval is implied; inspect source gates below." + }, + "UA": { + "acquisition_counts": { + "blocked": 5, + "not_run": 1 + }, + "basis": [ + "ua.dpss.approved-food: blocked / blocked", + "ua.edr.organizations: blocked / blocked", + "ua.environment-permits: blocked / blocked", + "ua.farm-aquaculture: blocked / blocked", + "ua.inspections-experiments: blocked / blocked", + "ua.ukrstat.statistics: not_run / reconnaissance" + ], + "country_reasons": [ + "acquisition:blocked", + "acquisition:not_run", + "publication:blocked" + ], + "name": "UA", + "owner_review": "awaiting-owner-review", + "publication_state": "blocked", + "source_count": 6, + "sources": [ + { + "access_method": "official registry/search route; authorized bounded export or API only", + "attribution": { + "attribution_required": true, + "notice": "Official Ukrainian government source; terms, privacy, wartime safety, and attribution require review", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown; verify from authorized metadata", + "country_code": "UA", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Bulk route, schema, stable ID, cadence, terms, privacy, and wartime safety controls are not pinned." + ] + }, + "jurisdiction_scope": "Ukraine; DPSS registered food-market operators and facilities, including animal-origin food", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "ua.dpss.approved-food", + "source_url": "https://dpss.gov.ua/diyalnist/bezpechnist-harchovih-produktiv-ta-veterinarna-medicina/reyestri", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-ua.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin authorized route, schema, cadence, IDs, terms, privacy, and wartime safety controls.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official open-data catalog discovery; authorized API/download only", + "attribution": { + "attribution_required": true, + "notice": "Government open-data source; field-level personal-address, terms, and rate-limit review required", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown", + "country_code": "UA", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Current authorized route, fields, cadence, licensing, privacy policy, and automation permissions not verified." + ] + }, + "jurisdiction_scope": "Ukraine; Unified State Register of legal entities and organizations", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "ua.edr.organizations", + "source_url": "https://data.gov.ua/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-ua.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin authorized route, schema, cadence, IDs, terms, privacy, and wartime safety controls.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official EcoSystem/register route; bounded authorized API/export only", + "attribution": { + "attribution_required": true, + "notice": "Ministry source; license, geometry, privacy, and security review required", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "resource-specific; unknown", + "country_code": "UA", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Exact public permit API/export, coverage, cadence, geometry policy, and wartime security controls not pinned." + ] + }, + "jurisdiction_scope": "Ukraine; Ministry environmental registers and permits", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "ua.environment-permits", + "source_url": "https://mepr.gov.ua/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-ua.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin authorized route, schema, cadence, IDs, terms, privacy, and wartime safety controls.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official registration guidance; no facility acquisition until safety-reviewed export/API exists", + "attribution": { + "attribution_required": true, + "notice": "Official government source; coordinates, privacy, reuse terms, and wartime exposure require review", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown", + "country_code": "UA", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "No safe public bulk route, stable ID, cadence, coordinate policy, or reuse terms verified." + ] + }, + "jurisdiction_scope": "Ukraine; DPSS livestock facilities and operators, including aquaculture scope", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "ua.farm-aquaculture", + "source_url": "https://dpss.gov.ua/diyalnist/bezpechnist-harchovih-produktiv-ta-veterinarna-medicina/dozvoly-ta-reiestratsiia-dlia-biznesu-u-sferakh-veterynarnoi-medytsyny-bezpechnosti-harchovykh-produktiv-ta-kormiv/tvarynnytski-potuzhnosti/derzhavna-reiestratsiia-tvarynnytskykh-potuzhnostei-ta-operatoriv-rynku", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-ua.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin authorized route, schema, cadence, IDs, terms, privacy, and wartime safety controls.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official reports/registers; aggregate-only until a safe public route is authorized", + "attribution": { + "attribution_required": true, + "notice": "Sensitive control/research evidence; privacy, retention, and wartime safety review required", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "resource-specific; unknown", + "country_code": "UA", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "No safe stable public facility/event master verified; stop and escalate on sensitive exposure." + ] + }, + "jurisdiction_scope": "Ukraine; DPSS inspections/enforcement and public animal-experimentation evidence", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "ua.inspections-experiments", + "source_url": "https://dpss.gov.ua/diyalnist/bezpechnist-harchovih-produktiv-ta-veterinarna-medicina/reyestri", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-ua.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin authorized route, schema, cadence, IDs, terms, privacy, and wartime safety controls.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official statistical tables/publications; API or bounded download to be pinned", + "attribution": { + "attribution_required": true, + "notice": "State Statistics Service source; publication terms, revisions, and regional suppression require review", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "table-specific; unknown", + "country_code": "UA", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Current table IDs, machine API, cadence, revision semantics, and suppression rules not pinned." + ] + }, + "jurisdiction_scope": "Ukraine; aggregate livestock, animal-production, and slaughter statistics", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "ua.ukrstat.statistics", + "source_url": "https://ukrstat.gov.ua/", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-ua.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin authorized route, schema, cadence, IDs, terms, privacy, and wartime safety controls.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + } + ], + "state": "blocked", + "summary": "No release approval is implied; inspect source gates below." + }, + "UK": { + "acquisition_counts": { + "artifact_private_only": 1 + }, + "basis": [ + "uk.locations: artifact_private_only / awaiting-owner-review" + ], + "country_reasons": [ + "publication:blocked" + ], + "name": "UK", + "owner_review": "awaiting-owner-review", + "publication_state": "blocked", + "source_count": 1, + "sources": [ + { + "access_method": "monthly CSV download or assisted export", + "attribution": { + "attribution_required": true, + "notice": "Legacy inventory names the Food Standards Agency; confirm the applicable national publication licences and attribution.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "monthly", + "country_code": "UK", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Identify and model each national feed separately before combining; confirm current URLs and terms." + ] + }, + "jurisdiction_scope": "United Kingdom; England, Wales, Northern Ireland, and Scotland legacy coverage", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": true, + "public_release_allowed": false, + "reasons": [ + "publication:blocked" + ], + "state": "awaiting-owner-review" + }, + "source_id": "uk.locations", + "source_url": "unknown", + "status": { + "acquisition": "artifact_private_only", + "evidence": [ + "docs/country-recon-uk.md", + "docs/countries/uk/fss-approved-establishments-source-assessment.md", + "pipeline/sources/uk/fsa_approved/README.md", + "pipeline/sources/uk/fsa_approved/handoff.py", + "pipeline/sources/uk/fsa_approved/refresh.py", + "pipeline/sources/uk/fss_approved/refresh.py", + "pipeline/common/review_packet.py", + "docs/review-packet-united-kingdom.md", + "docs/architecture/disposable-candidate-import.md", + "pipeline/scripts/maintenance/import-candidate.py", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Repeatable source-local refresh dry-run passed on the retained snapshot (5,342 input, 4,300 normalized, 1,042 quarantined, expected schema fingerprint, no drift alarms); use refresh.py dry-run/handoff only in restricted staging. This does not establish production/runtime health or publication eligibility. Complete source-rights, privacy/coordinate, duplicate, coverage, and release review while keeping NI and Scotland separate.", + "publication_eligibility": "blocked", + "runtime_health": "unknown" + } + } + ], + "state": "human-review-ready", + "summary": "No release approval is implied; inspect source gates below." + }, + "US": { + "acquisition_counts": { + "blocked": 1, + "not_run": 12 + }, + "basis": [ + "us.aaalac.accredited: not_run / reconnaissance", + "us.aphis: not_run / reconnaissance", + "us.ca.cdph-lab-animals: not_run / reconnaissance", + "us.dod.acuro: not_run / reconnaissance", + "us.fda.glp-animal-research: not_run / reconnaissance", + "us.fsis: blocked / blocked", + "us.inspections: not_run / reconnaissance", + "us.nasa.nspires: not_run / reconnaissance", + "us.nih.olaw-assurances: not_run / reconnaissance", + "us.nih.reporter: not_run / reconnaissance", + "us.nsf.awards: not_run / reconnaissance", + "us.state-mpi: not_run / reconnaissance", + "us.va.animal-research: not_run / reconnaissance" + ], + "country_reasons": [ + "acquisition:blocked", + "acquisition:not_run", + "publication:blocked" + ], + "name": "US", + "owner_review": "awaiting-owner-review", + "publication_state": "blocked", + "source_count": 13, + "sources": [ + { + "access_method": "official public directory search/result pages; no bulk contract verified", + "attribution": { + "attribution_required": true, + "notice": "Private nonprofit accreditation source; preserve organization/unit distinction, attribution and terms; accreditation is not regulatory registration", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "ongoing directory updates; observation date required", + "country_code": "US", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Confirm directory query/export behavior, unit lifecycle, terms, contact/address minimization and exact parent/unit identity matching before use." + ] + }, + "jurisdiction_scope": "United States; AAALAC International voluntary accreditation directory observations", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "us.aaalac.accredited", + "source_url": "https://www.aaalac.org/accreditation/directory/directory-of-accredited-organizations-search-result/", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/countries/us/research-animal-coverage-recon.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Review directory terms and unit-level identity before any private capture; keep voluntary accreditation separate from regulatory registration and animal-use evidence.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "operator-assisted public-search export with explicit registrations/annual_reports/inspections profile", + "attribution": { + "attribution_required": true, + "notice": "APHIS authority and public-search scope verified; export terms, attribution, and privacy handling remain review gates", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown; record selected report/search date", + "country_code": "US", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Capture and verify the current export schema and terms; no facility/laboratory merge or public release until review." + ] + }, + "jurisdiction_scope": "United States; USDA APHIS Animal Care public search and annual-report data", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "us.aphis", + "source_url": "https://aphis.my.site.com/PublicSearchTool/s/annual-reports", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-us.md", + "docs/countries/us/README.md", + "pipeline/sources/us/aphis/config.json", + "pipeline/sources/us/aphis/adapter.py", + "pipeline/sources/us/aphis/refresh.py", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Use the explicit profile-based assisted export for registrations, annual reports, or inspections; preserve each evidence type separately and complete terms, privacy, schema, and review gates.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official state program page and LAB 139 application/renewal form; no public statewide bulk export verified", + "attribution": { + "attribution_required": true, + "notice": "California state source; minimize responsible-person/address fields and review public-record, privacy and redistribution conditions", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "annual approval renewal; record approval/reporting period", + "country_code": "US", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Federal-regulated laboratories are exempt from this state approval; no public bulk registry was verified; pilot only and never a national denominator." + ] + }, + "jurisdiction_scope": "United States; California Department of Public Health laboratory-animal approval and form evidence", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "us.ca.cdph-lab-animals", + "source_url": "https://www.cdph.ca.gov/Programs/cls/operations/Pages/LaboratoryAnimalUseApprovalProgram.aspx", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/countries/us/research-animal-coverage-recon.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Pilot California approval/count evidence only through an authorized current extract or public-record response; model federal exemptions and do not generalize to a national denominator.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official ACURO policy/reporting/site-visit pages; no national facility or use export verified", + "attribution": { + "attribution_required": true, + "notice": "DoD/DHA official oversight source; public pages may omit sensitive contract, protocol and security details", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "policy and reporting specific; unknown nationally", + "country_code": "US", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "No public national facility/count dataset was verified; do not infer sites from awardee addresses or expose sensitive operational details." + ] + }, + "jurisdiction_scope": "United States; DHA/USAMRDC ACURO animal protocol and site-oversight evidence", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "us.dod.acuro", + "source_url": "https://mrdc.health.mil/index.cfm/resources/research_protections/acuro", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/countries/us/research-animal-coverage-recon.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Keep ACURO protocol/site-oversight material manual and privacy-reviewed; do not infer facilities from DoD awardee addresses or build a national count.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official policy, MOU and facility pages; no national facility export verified", + "attribution": { + "attribution_required": true, + "notice": "FDA official source; distinguish GLP inspection/disqualification authority from USDA/OLAW evidence and review security/privacy before retaining facility details", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "policy/facility-page specific; unknown nationally", + "country_code": "US", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "No public national FDA animal-research register or count route was verified; do not use openFDA adverse-event records as research-use counts." + ] + }, + "jurisdiction_scope": "United States; FDA laboratory-animal/GLP program and public FDA facility evidence", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "us.fda.glp-animal-research", + "source_url": "https://www.fda.gov/about-fda/domestic-mous/mou-225-16-010", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/countries/us/research-animal-coverage-recon.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Keep FDA evidence policy/manual until a public reproducible facility or inspection route is verified; do not treat openFDA adverse events as research-use counts.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "operator-assisted official CSV export; bounded direct fetch only with terms review", + "attribution": { + "attribution_required": true, + "notice": "FSIS authority and directory scope verified; terms/attribution and current export URL must be recorded per run before publication", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "weekly replacement", + "country_code": "US", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Current direct links returned HTTP 403 during reconnaissance; obtain an authorized current export and complete terms, schema, privacy, and project review." + ] + }, + "jurisdiction_scope": "United States; USDA FSIS meat and poultry establishment coverage", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "us.fsis", + "source_url": "https://www.fsis.usda.gov/inspection/establishments/meat-poultry-and-egg-product-inspection-directory", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-us.md", + "docs/countries/us/README.md", + "docs/countries/us/v1-field-crosswalk.json", + "pipeline/sources/us/fsis/config.json", + "pipeline/sources/us/fsis/adapter.py", + "pipeline/sources/us/fsis/refresh.py", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Use the assisted official export contract after the 403 blocker; record URL, retrieval/effective dates, content type, byte size, SHA-256, terms, schema fingerprint, privacy review, and reconciliation before any test-only handoff.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "operator-assisted APHIS public-search inspection export", + "attribution": { + "attribution_required": true, + "notice": "APHIS inspection evidence; export terms, attribution, redaction/privacy handling, and review outcome remain required", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown; record selected search date", + "country_code": "US", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Verify current inspection export schema and use explicit, reviewable identity matching only; absence is not closure." + ] + }, + "jurisdiction_scope": "United States; USDA APHIS inspection-report observations", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "us.inspections", + "source_url": "https://efile.aphis.usda.gov/PublicSearchTool/s/inspection-reports", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-us.md", + "docs/countries/us/README.md", + "pipeline/sources/us/aphis/config.json", + "pipeline/sources/us/aphis/adapter.py", + "pipeline/sources/us/aphis/refresh.py", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Capture the APHIS inspections profile through the documented public-search route; treat rows as observations, not a facility master, and use explicit reviewable identity matching only.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official NSPIRES web search and solicitation/award pages; no bulk animal-use route verified", + "attribution": { + "attribution_required": true, + "notice": "NASA public research-award evidence; review portal access, identifiers, privacy and performance-site semantics", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "solicitation/award specific; record observation date", + "country_code": "US", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Funding evidence does not prove animal use or facility location; the OLAW assurance relationship is policy context, not a count or registry join." + ] + }, + "jurisdiction_scope": "United States; NASA NSPIRES research solicitation and award evidence", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "us.nasa.nspires", + "source_url": "https://www.nasa.gov/hrp/for-prospective-researchers/", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/countries/us/research-animal-coverage-recon.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Keep NASA NSPIRES as funding/project evidence; verify public award identifiers and performance-site semantics before any integration.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official browser lookup table; no public bulk/API contract verified", + "attribution": { + "attribution_required": true, + "notice": "NIH/OLAW public institutional assurance evidence; do not expose restricted assurance documents or infer project-level use", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "current-assurance lookup; observation date required", + "country_code": "US", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Verify durable capture/export semantics, assurance lifecycle, branch/affiliate scope, terms, privacy and exact organization matching; lookup presence is not an animal count or facility census." + ] + }, + "jurisdiction_scope": "United States; NIH OLAW current approved Domestic and Foreign Animal Welfare Assurances lookup", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "us.nih.olaw-assurances", + "source_url": "https://grants.nih.gov/policy-and-compliance/policy-topics/animal-welfare/assurance/assured-institutions", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/countries/us/research-animal-coverage-recon.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Validate an assisted capture contract for the OLAW assured-institutions lookup; preserve Assurance ID/type and branch/affiliate scope, and do not infer protocols or animal counts.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official JSON API and annual/bulk ExPORTER files; rate-limited deterministic acquisition", + "attribution": { + "attribution_required": true, + "notice": "NIH public administrative award data; preserve API/bulk release metadata and review terms, privacy and PI minimization before publication", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "API service with annual consolidated project-file release and later updates", + "country_code": "US", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Funding/project evidence does not prove animal use, physical performance site, operation, or animal counts; validate API version, bulk file refresh, rate limits, text classification, privacy and reviewed organization/site links before integration." + ] + }, + "jurisdiction_scope": "United States; NIH RePORTER and ExPORTER federal research award/project evidence", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "us.nih.reporter", + "source_url": "https://api.reporter.nih.gov/", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/countries/us/research-animal-coverage-recon.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Implement a rate-limited NIH RePORTER API/ExPORTER funding-project adapter with exact award and organization identifiers; keep animal relevance as project evidence, not facility or animal-use counts.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official NSF Award Search API and open-data routes", + "attribution": { + "attribution_required": true, + "notice": "NSF public award evidence; review API terms, identifiers, PI minimization and performance-site semantics", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "API/data-source specific; record response and release metadata", + "country_code": "US", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Funding evidence does not prove animal use or site operation; integrate only with explicit animal-relevance and OLAW-assurance semantics." + ] + }, + "jurisdiction_scope": "United States; NSF research award/project evidence", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "us.nsf.awards", + "source_url": "https://www.nsf.gov/digital/developer", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/countries/us/research-animal-coverage-recon.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Consider NSF Award Search only as a funding/project adjunct after NIH RePORTER; require explicit animal-relevance and performance-site review.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "state-specific HTML/PDF/XLSX/CSV/map routes or authorized operator-assisted capture; FSIS CIS workbooks are a separate overlay", + "attribution": { + "attribution_required": true, + "notice": "State and FSIS authority are documented; public availability does not establish redistribution permission. Review terms, privacy, and category scope per state before acquisition or publication.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "state-specific and often undocumented; record every visible revision/effective date", + "country_code": "US", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "No state roster or CIS workbook was acquired; obtain authorized current exports or operator-assisted captures, resolve terms/schema/privacy, and keep state, federal, CIS, custom-exempt, retail/handler, and inactive/expired populations separate." + ] + }, + "jurisdiction_scope": "United States; state Meat and Poultry Inspection programs and Cooperative Interstate Shipment overlays", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "us.state-mpi", + "source_url": "https://www.fsis.usda.gov/inspection/state-inspection-programs", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/countries/us/state-mpi-source-recon.md", + "docs/countries/us/README.md", + "docs/source-status.md" + ], + "metadata": "verified", + "next_action": "Obtain authorized current state MPI rosters or operator-assisted captures, preserving state-native identifiers, source classes, status/effective dates, terms, address/coordinate provenance, and separate official/CIS/custom-exempt/retail populations before any test-only handoff.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official VA program, policy and facility pages; no national bulk dataset verified", + "attribution": { + "attribution_required": true, + "notice": "VA official source; facility pages and ORO oversight statements are not a national animal-use register", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "page/policy specific; unknown nationally", + "country_code": "US", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Identify a public reproducible national VA facility/count route before acquisition; keep local facility pages, assurance identifiers and oversight events separate." + ] + }, + "jurisdiction_scope": "United States; Veterans Affairs animal-research program and oversight evidence", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "us.va.animal-research", + "source_url": "https://www.research.va.gov/programs/animal_research/default.cfm", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/countries/us/research-animal-coverage-recon.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Retain VA program, facility and ORO pages as separate observations; identify an authorized national facility/count route before acquisition.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + } + ], + "state": "blocked", + "summary": "No release approval is implied; inspect source gates below." + }, + "XK": { + "acquisition_counts": { + "blocked": 5, + "not_run": 1 + }, + "basis": [ + "xk.arbk.organizations: blocked / blocked", + "xk.ask.statistics: not_run / reconnaissance", + "xk.auvk.approved-food: blocked / blocked", + "xk.auvk.farms-aquaculture: blocked / blocked", + "xk.auvk.inspections-experiments: blocked / blocked", + "xk.environment-permits: blocked / blocked" + ], + "country_reasons": [ + "acquisition:blocked", + "acquisition:not_run", + "publication:blocked" + ], + "name": "XK", + "owner_review": "awaiting-owner-review", + "publication_state": "blocked", + "source_count": 6, + "sources": [ + { + "access_method": "official business-register web/admin surface; public API/bulk route unverified", + "attribution": { + "attribution_required": true, + "notice": "Identity linkage only; verify automation permissions, terms, fields and registered-office privacy.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "provider-defined", + "country_code": "XK", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Pin public query/API, authentication, rate limits, cadence, terms and privacy." + ] + }, + "jurisdiction_scope": "Kosovo; ARBK business organizations and corporate identifiers", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "xk.arbk.organizations", + "source_url": "https://arbk.rks-gov.net/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-xk.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin ARBK API/search route, auth, rate limits, cadence, terms and privacy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "Kosovo Agency of Statistics tables/data services; exact route unverified", + "attribution": { + "attribution_required": true, + "notice": "Official statistics; verify table terms, citation requirements and aggregate-only scope.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "table-specific; preserve revisions", + "country_code": "XK", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Pin current slaughter and animal-use table IDs, API/download contracts and cadence." + ] + }, + "jurisdiction_scope": "Kosovo; official aggregate slaughter, livestock and animal-use statistics", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "xk.ask.statistics", + "source_url": "https://ask.rks-gov.net/", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-xk.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin official slaughter/animal-use table IDs, APIs, cadence and revisions.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official AUVK category pages and linked downloads", + "attribution": { + "attribution_required": true, + "notice": "Official authority; verify file terms, attribution, privacy and coordinates before acquisition.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "publisher-defined; timestamp retrieval", + "country_code": "XK", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Pin current AUVK files, schemas, freshness, IDs, terms and privacy." + ] + }, + "jurisdiction_scope": "Kosovo; AUVK approved and registered animal-origin food establishments", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "xk.auvk.approved-food", + "source_url": "https://auvk.rks-gov.net/en/approved-businesses-for-food-of-animal-origin/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-xk.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Pin current AUVK files, schemas, freshness, IDs, terms and privacy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official AUVK registers, linked files and animal-registration guidance", + "attribution": { + "attribution_required": true, + "notice": "Verify publisher, license, animal-holder/property privacy and coordinate precision.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "resource-specific", + "country_code": "XK", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Verify current holding/fishpond files, IDs, cadence, terms and privacy." + ] + }, + "jurisdiction_scope": "Kosovo; AUVK livestock holdings, fishponds and aquaculture-related evidence", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "xk.auvk.farms-aquaculture", + "source_url": "https://auvk.rks-gov.net/shendeti-i-kafsheve/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-xk.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Verify holding/fishpond files, IDs, cadence, coordinates, terms and privacy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official AUVK control summaries, registers and linked documents", + "attribution": { + "attribution_required": true, + "notice": "Sensitive control/research evidence; aggregate or anonymize and require project review.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "resource/report-specific", + "country_code": "XK", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Pin current experimentation register/API, fields, IDs, privacy and retention policy." + ] + }, + "jurisdiction_scope": "Kosovo; AUVK inspections/enforcement and experimental-animal institutions", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "xk.auvk.inspections-experiments", + "source_url": "https://auvk.rks-gov.net/kontrolli-i-brendshem/veterinar/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-xk.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Pin experimentation register/API, fields, IDs, privacy and retention policy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official ministry/environmental authority documents and public notices", + "attribution": { + "attribution_required": true, + "notice": "Verify public-read rights, document license, geometry and privacy.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "permit/document-specific", + "country_code": "XK", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Stable public read/API/export and complete facility coverage are unresolved." + ] + }, + "jurisdiction_scope": "Kosovo; environmental permits and authorizations", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "xk.environment-permits", + "source_url": "https://mmphi.rks-gov.net/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-xk.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Verify public environmental permit route, coverage, documents, geometry and privacy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + } + ], + "state": "blocked", + "summary": "No release approval is implied; inspect source gates below." + } + }, + "country_count": 45, + "country_records": [ + { + "acquisition_counts": { + "blocked": 5, + "not_run": 1 + }, + "basis": [ + "al.aku.approved-food: blocked / blocked", + "al.aku.farms-aquaculture: blocked / blocked", + "al.aku.inspections-experiments: blocked / blocked", + "al.environment-permits: blocked / blocked", + "al.instat.statistics: not_run / reconnaissance", + "al.qkb.organizations: blocked / blocked" + ], + "country_code": "AL", + "country_reasons": [ + "acquisition:blocked", + "acquisition:not_run", + "publication:blocked" + ], + "display_name": "AL", + "owner_review": "awaiting-owner-review", + "publication_state": "blocked", + "readiness_class": "blocked", + "source_count": 6, + "sources": [ + { + "access_method": "official AKU register pages and linked downloads", + "attribution": { + "attribution_required": true, + "notice": "Official authority; verify file terms, attribution, privacy and coordinates before acquisition.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown; timestamp retrieval", + "country_code": "AL", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Pin stable AKU file routes, schemas, cadence, IDs, terms and privacy." + ] + }, + "jurisdiction_scope": "Albania; National Food Authority approved and registered animal-origin food establishments", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "al.aku.approved-food", + "source_url": "https://aku.gov.al/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-al.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin stable AKU files, schemas, cadence, IDs, terms and privacy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official AKU/agriculture/fisheries registers and linked documents", + "attribution": { + "attribution_required": true, + "notice": "Verify publisher, license, animal-holder/property privacy and coordinate precision.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "resource-specific", + "country_code": "AL", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Verify complete farm/aquaculture coverage, exports, IDs, cadence, terms and privacy." + ] + }, + "jurisdiction_scope": "Albania; AKU primary producers, livestock establishments and agriculture/fisheries aquaculture evidence", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "al.aku.farms-aquaculture", + "source_url": "https://aku.gov.al/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-al.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Verify farm/aquaculture coverage, exports, IDs, coordinates, terms and privacy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official AKU registers, control reports and linked documents", + "attribution": { + "attribution_required": true, + "notice": "Sensitive control/research evidence; aggregate or anonymize and require project review.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "resource/report-specific", + "country_code": "AL", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Pin experimentation register/API, fields, IDs, privacy and retention policy." + ] + }, + "jurisdiction_scope": "Albania; AKU inspections/enforcement and experimental-animal institutions", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "al.aku.inspections-experiments", + "source_url": "https://aku.gov.al/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-al.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Pin experimentation register/API, fields, IDs, privacy and retention policy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "environmental authority routes and QKB permit/licence register", + "attribution": { + "attribution_required": true, + "notice": "Verify public-read rights, document license, geometry and privacy.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "permit/document-specific", + "country_code": "AL", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Stable public read/API/export and complete facility coverage are unresolved." + ] + }, + "jurisdiction_scope": "Albania; environmental permits and authorizations", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "al.environment-permits", + "source_url": "https://akm.gov.al/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-al.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Verify public environmental permit route, coverage, documents, geometry and privacy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official statistical tables/data services; exact route unverified", + "attribution": { + "attribution_required": true, + "notice": "Official statistics; verify table terms, citation requirements and aggregate-only scope.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "table-specific; preserve revisions", + "country_code": "AL", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Pin current slaughter and animal-use table IDs, API/download contracts and cadence." + ] + }, + "jurisdiction_scope": "Albania; INSTAT aggregate slaughter, livestock and animal-use statistics", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "al.instat.statistics", + "source_url": "https://www.instat.gov.al/en/", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-al.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin official slaughter/animal-use table IDs, APIs, cadence and revisions.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official public search/services; API/bulk route unverified", + "attribution": { + "attribution_required": true, + "notice": "Identity linkage only; QKB privacy policy limits personal/address disclosure and terms require verification.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "provider-defined", + "country_code": "AL", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Pin API/bulk route, rate limits, cadence, fields, terms and personal-address suppression." + ] + }, + "jurisdiction_scope": "Albania; National Business Center commercial and permit/licence registers", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "al.qkb.organizations", + "source_url": "https://qkb.gov.al/en/home-3/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-al.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Pin QKB API/search route, rate limits, fields, terms and personal-address suppression.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + } + ], + "summary": "No release approval is implied; inspect source gates below." + }, + { + "acquisition_counts": { + "blocked": 5, + "not_run": 1 + }, + "basis": [ + "am.armstat.livestock-statistics: not_run / reconnaissance", + "am.e-register.organizations: blocked / blocked", + "am.environment-permits: blocked / blocked", + "am.snund.approved-food: blocked / blocked", + "am.snund.farms-livestock: blocked / blocked", + "am.snund.inspections-experiments: blocked / blocked" + ], + "country_code": "AM", + "country_reasons": [ + "acquisition:blocked", + "acquisition:not_run", + "publication:blocked" + ], + "display_name": "AM", + "owner_review": "awaiting-owner-review", + "publication_state": "blocked", + "readiness_class": "blocked", + "source_count": 6, + "sources": [ + { + "access_method": "official Armstat PxWeb table/query; API contract to be pinned", + "attribution": { + "attribution_required": true, + "notice": "Statistical Committee source; copyright, revisions, and suppression rules require review", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "table-specific; unknown", + "country_code": "AM", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Pin current table IDs, API/query contract, cadence, revision semantics, and licensing." + ] + }, + "jurisdiction_scope": "Armenia; aggregate livestock and agriculture statistics", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "am.armstat.livestock-statistics", + "source_url": "https://statbank.armstat.am/pxweb/en/ArmStatBank/ArmStatBank__6%20Agriculture%2C%20forestry%20and%20fishing/AF-1-2024.px/", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-am.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin authorized route, schema, cadence, IDs, terms, privacy, and automation permissions.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official search/extract service; do not bypass authentication or payment", + "attribution": { + "attribution_required": true, + "notice": "Government registry; access fees, terms, privacy, and personal-address policy require review", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown", + "country_code": "AM", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Authorized machine route, fields, limits, cadence, and reuse terms not verified; full extracts may require sign-in/payment." + ] + }, + "jurisdiction_scope": "Armenia; electronic register of legal entities", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "am.e-register.organizations", + "source_url": "https://www.e-register.am/en/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-am.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin authorized route, schema, cadence, IDs, terms, privacy, and automation permissions.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official environmental service/register route; authorized API/export only", + "attribution": { + "attribution_required": true, + "notice": "Ministry source; license, geometry, privacy, and terms require review", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "resource-specific; unknown", + "country_code": "AM", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Exact public permit route, API/export, coverage, cadence, license, and privacy not pinned." + ] + }, + "jurisdiction_scope": "Armenia; environmental permits and registers", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "am.environment-permits", + "source_url": "https://env.am/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-am.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin authorized route, schema, cadence, IDs, terms, privacy, and automation permissions.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official FSIB registry/page; authorized bounded export or API only", + "attribution": { + "attribution_required": true, + "notice": "Official Armenian government source; terms, privacy, and location-safety review required", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown; verify from source metadata", + "country_code": "AM", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Exact file/API, schema, stable IDs, cadence, terms, and privacy controls not pinned." + ] + }, + "jurisdiction_scope": "Armenia; Food Safety Inspection Body slaughterhouses and animal-origin food-chain operators", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "am.snund.approved-food", + "source_url": "https://snund.am/en/page/operating-slaughterhouses/106", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-am.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin authorized route, schema, cadence, IDs, terms, privacy, and automation permissions.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official FSIB service/registry route; no facility acquisition until contract review", + "attribution": { + "attribution_required": true, + "notice": "Official government source; scope, personal data, locations, and reuse terms require review", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown", + "country_code": "AM", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Public bulk route, stable IDs, cadence, location policy, and licensing not verified." + ] + }, + "jurisdiction_scope": "Armenia; FSIB food-chain registration and veterinary/livestock-related operators", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "am.snund.farms-livestock", + "source_url": "https://www.snund.am/en", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-am.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin authorized route, schema, cadence, IDs, terms, privacy, and automation permissions.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official reports/plans; aggregate-only until a safe event route is verified", + "attribution": { + "attribution_required": true, + "notice": "Sensitive control/research evidence; privacy, retention, and publication review required", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "resource-specific; unknown", + "country_code": "AM", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "No stable public experimentation/event master verified; stop and escalate on sensitive exposure." + ] + }, + "jurisdiction_scope": "Armenia; FSIB food/veterinary inspections and public animal-experimentation evidence", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "am.snund.inspections-experiments", + "source_url": "https://www.snund.am/en/page/inspection-body/50", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-am.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin authorized route, schema, cadence, IDs, terms, privacy, and automation permissions.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + } + ], + "summary": "No release approval is implied; inspect source gates below." + }, + { + "acquisition_counts": { + "artifact_private_only": 2, + "blocked": 9, + "not_run": 11 + }, + "basis": [ + "au.abr.lookup: not_run / reconnaissance", + "au.act.food-registration: blocked / blocked", + "au.animal-welfare-and-use: not_run / reconnaissance", + "au.asic.company-dataset: blocked / blocked", + "au.daff.export-establishments: blocked / blocked", + "au.npi.facilities: artifact_private_only / awaiting-owner-review", + "au.nsw.animal-use: not_run / reconnaissance", + "au.nsw.epa-poeo: blocked / blocked", + "au.nsw.food-authority: not_run / reconnaissance", + "au.nsw.food-enforcement: not_run / reconnaissance", + "au.nt.epa-licences: not_run / reconnaissance", + "au.nt.meat-licensing: blocked / blocked", + "au.pirsa.meat: not_run / reconnaissance", + "au.primesafe.vic.meat-licences: blocked / blocked", + "au.qld.environmental-authorities: blocked / blocked", + "au.sa.epa.licensed-activities: artifact_private_only / awaiting-owner-review", + "au.safefood.qld.accreditation: blocked / blocked", + "au.tas.biosecurity-meat: not_run / reconnaissance", + "au.tas.epa-listmap: blocked / blocked", + "au.vic.animal-use: not_run / reconnaissance", + "au.vic.epa-permissions: not_run / reconnaissance", + "au.wamia.abattoirs: not_run / reconnaissance" + ], + "country_code": "AU", + "country_reasons": [ + "acquisition:blocked", + "acquisition:not_run", + "metadata:unknown", + "publication:blocked" + ], + "display_name": "AU", + "owner_review": "awaiting-owner-review", + "publication_state": "blocked", + "readiness_class": "blocked", + "source_count": 22, + "sources": [ + { + "access_method": "public lookup/web service; route-specific rate and terms", + "attribution": { + "attribution_required": true, + "notice": "ABR public identity route; use only for exact upstream ABN/ACN crosswalk and never infer operation/ownership", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "hourly update claim for ABN Lookup", + "country_code": "AU", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "No detailed contact/industry fields; individual and sole-trader privacy review required." + ] + }, + "jurisdiction_scope": "Australia; ABN Lookup and ABR public identity data", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "au.abr.lookup", + "source_url": "https://abr.business.gov.au/", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-au.md", + "docs/countries/australia/source-crosswalk.json", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Query only when an upstream source supplies an ABN/ACN; record lookup time and terms, and suppress sole-trader personal details.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "public guidance; dedicated abattoir list not verified", + "attribution": { + "attribution_required": true, + "notice": "ACT food registration guidance only; no facility publication claim", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown", + "country_code": "AU", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Identify ACT Health/local-government facility and environmental routes before acquisition." + ] + }, + "jurisdiction_scope": "Australia; Australian Capital Territory food registration coverage check", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "metadata:unknown", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "au.act.food-registration", + "source_url": "https://www.act.gov.au/business/health-licenses-and-inspections/food-businesses-and-events-registration", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-au.md", + "docs/countries/australia/source-crosswalk.json", + "pipeline/source_registry.json" + ], + "metadata": "unknown", + "next_action": "Identify the ACT authority and any public facility register before claiming ACT coverage; absence is not closure.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "jurisdiction-specific web pages, reports, and case records", + "attribution": { + "attribution_required": true, + "notice": "Government-sourced evidence only; allegations, personal information, sensitive locations, terms, and human review are mandatory gates", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "jurisdiction-specific", + "country_code": "AU", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "No uniform national facility-level feed; do not turn complaints, prosecutions, or aggregate totals into canonical facility facts." + ] + }, + "jurisdiction_scope": "Australia; jurisdiction-specific animal welfare enforcement and animal-use statistics", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "au.animal-welfare-and-use", + "source_url": "https://www.agriculture.gov.au/agriculture-land/animal/welfare/state", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-au.md", + "docs/countries/australia/source-crosswalk.json", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Keep welfare enforcement and animal-use evidence jurisdiction-specific; acquire only deidentified/terms-permitted aggregates or reviewed case records and model them as observations/events, not facility masters.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "public weekly snapshot; large download", + "attribution": { + "attribution_required": true, + "notice": "Catalogue states CC BY 3.0 Australia; preserve snapshot date and delimiter; not beneficial-ownership data", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "weekly Tuesday snapshot", + "country_code": "AU", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Do not bulk-fetch ~371.9 MiB without approved bounded plan; snapshot may lag ASIC Connect." + ] + }, + "jurisdiction_scope": "Australia; ASIC selected company-register snapshot", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "au.asic.company-dataset", + "source_url": "https://data.gov.au/data/en/dataset/asic-companies/resource/5c3914e6-413e-4a2c-b890-bf8efe3eabf2", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-au.md", + "docs/countries/australia/source-crosswalk.json", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Do not bulk-fetch the large ASIC snapshot without an approved bounded plan; use ACN/ABN only as an upstream identity crosswalk, never as proof of site ownership.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "authenticated TradeClear/Export Service or assisted request; public guidance only", + "attribution": { + "attribution_required": true, + "notice": "DAFF authority and export scope verified; authorised access, terms, retention, and privacy review required", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "ongoing/certificate-specific", + "country_code": "AU", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "No verified public meat bulk export; do not scrape authenticated systems or infer public access from guidance pages." + ] + }, + "jurisdiction_scope": "Australia; DAFF export-registered prescribed-goods establishments", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "au.daff.export-establishments", + "source_url": "https://www.agriculture.gov.au/biosecurity-trade/export/from-australia/documentation-registration-licensing/establishment-registration", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-au.md", + "docs/countries/australia/artifact-metadata.json", + "docs/countries/australia/source-crosswalk.json", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Obtain an authorised bounded current DAFF establishment report/export, preserve commodity/list type, record response metadata/hash/bytes, and keep export-only scope and privacy/terms/release gates explicit.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "public catalogue download", + "attribution": { + "attribution_required": true, + "notice": "Catalogue identifies CC BY 4.0; attribute Commonwealth of Australia/DCCEEW and complete privacy/release review", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "catalogue dataset date 2026-04-01; cadence not explicit", + "country_code": "AU", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Observed NPI population is not a complete slaughter registry; no public row release approved." + ] + }, + "jurisdiction_scope": "Australia; National Pollutant Inventory facilities", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": true, + "public_release_allowed": false, + "reasons": [ + "publication:blocked" + ], + "state": "awaiting-owner-review" + }, + "source_id": "au.npi.facilities", + "source_url": "https://data.gov.au/data/dataset/043f58e0-a188-4458-b61c-04e5b540aea4", + "status": { + "acquisition": "artifact_private_only", + "evidence": [ + "docs/country-recon-au.md", + "data/raw/australia/metadata.json", + "docs/countries/australia/artifact-metadata.json", + "docs/countries/australia/source-crosswalk.json", + "pipeline/source_registry.json", + "pipeline/tests/test_australia_source_metadata.py" + ], + "metadata": "verified", + "next_action": "Add a private deterministic NPI facility/report adapter and synthetic contract tests; preserve annual release/correction history and do not present NPI as complete facility coverage.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "public deidentified aggregate CSV/XLSX; facility list not public", + "attribution": { + "attribution_required": true, + "notice": "NSW DPIRD aggregate statistics; CC BY 4.0 stated; small-cell and re-identification review required", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "annual; 2024-2025 published 2026-07-20", + "country_code": "AU", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Do not map aggregate uses or associate them with named establishments." + ] + }, + "jurisdiction_scope": "Australia; NSW animal-use research statistics", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "au.nsw.animal-use", + "source_url": "https://www.dpird.nsw.gov.au/dpi/animals/animal-ethics-infolink/nsw-animal-use-statistics/animal-use-data", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-au.md", + "docs/countries/australia/source-crosswalk.json", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Use deidentified annual aggregates only as context; keep them separate from facility points, slaughter, and individual-animal claims.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "public search and downloadable register", + "attribution": { + "attribution_required": true, + "notice": "NSW EPA public-register route; confirm current export and historical-holder semantics", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "page updated 2026-04-28; transition noted", + "country_code": "AU", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Validate current transition/export scope before adapter work." + ] + }, + "jurisdiction_scope": "Australia; NSW POEO environmental register", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "au.nsw.epa-poeo", + "source_url": "https://apps.epa.nsw.gov.au/prpoeoapp/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-au.md", + "docs/countries/australia/source-crosswalk.json", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Resolve the current transition-aware full-list/search route, capture one bounded edition, and keep licence/application/notice/enforcement record types separate.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "public guidance/forms; complete licensee list not verified", + "attribution": { + "attribution_required": true, + "notice": "NSW Food Authority authority and scope verified; licensing/notification records and privacy restrictions remain distinct", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "risk-based audit; register cadence unknown", + "country_code": "AU", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "No verified public complete licensee export; do not use private notification records." + ] + }, + "jurisdiction_scope": "Australia; NSW Food Authority meat licensing", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "au.nsw.food-authority", + "source_url": "https://www.foodauthority.nsw.gov.au/help/licensing", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-au.md", + "docs/countries/australia/source-crosswalk.json", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Treat current meat-sector category counts as context only; locate a permitted current facility lookup/export before adapter work and do not infer closure or named facilities from aggregates.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "public HTML register", + "attribution": { + "attribution_required": true, + "notice": "Public enforcement register; preserve allegation, penalty, prosecution, and court-outcome semantics and screen personal names", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "weekly stated; penalty register has one-year publication window", + "country_code": "AU", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "No facility-status inference; preserve changing publication windows and event-versus-entity distinction." + ] + }, + "jurisdiction_scope": "Australia; NSW Food Authority enforcement events", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "au.nsw.food-enforcement", + "source_url": "https://www.foodauthority.nsw.gov.au/offences/penalty-notices", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-au.md", + "docs/countries/australia/source-crosswalk.json", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Model penalty/prosecution entries as time-bounded enforcement events; preserve allegation versus court outcome and suppress personal details.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "public HTML category register and document links", + "attribution": { + "attribution_required": true, + "notice": "NT EPA public category register; capture document identifiers and terms", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown", + "country_code": "AU", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Current document identifiers and permitted downloads need verification." + ] + }, + "jurisdiction_scope": "Australia; NT EPA environment-protection licences", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "au.nt.epa-licences", + "source_url": "https://ntepa.nt.gov.au/your-business/public-registers/licences-and-approvals-register/environment-protection-licences", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-au.md", + "docs/countries/australia/source-crosswalk.json", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Capture current NT EPA licence/document identifiers and terms, then link to NT meat licences only through explicit review.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "public guidance plus generic licence search; API/bulk route not verified", + "attribution": { + "attribution_required": true, + "notice": "NT meat licence scope verified; generic register selector and public disclosure terms need capture", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "licence year 1 July-30 June", + "country_code": "AU", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Verify the scheme selector and permitted extract." + ] + }, + "jurisdiction_scope": "Australia; Northern Territory meat industry licensing", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "au.nt.meat-licensing", + "source_url": "https://nt.gov.au/industry/agriculture/meat-industry/domestic-abattoirs-meat-processing", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-au.md", + "docs/countries/australia/source-crosswalk.json", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Validate the NT licence-scheme selector and obtain an authorised bounded extract; treat licence as authorisation evidence only.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "public guidance/application route; current row export not verified", + "attribution": { + "attribution_required": true, + "notice": "PIRSA authority and accreditation scope verified; current register availability and terms need confirmation", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown", + "country_code": "AU", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Obtain an authorised current accreditation export before integration." + ] + }, + "jurisdiction_scope": "Australia; South Australia PIRSA meat accreditation", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "au.pirsa.meat", + "source_url": "https://pir.sa.gov.au/animal-management/food-safety-for-meat-dairy-and-eggs/meat", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-au.md", + "docs/countries/australia/source-crosswalk.json", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Confirm whether PIRSA can provide a current permitted accreditation export; do not derive facility rows from guidance or application forms.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "public search linked by PrimeSafe; annual report PDF", + "attribution": { + "attribution_required": true, + "notice": "PrimeSafe authority and licence categories verified; search route, terms, and confidential complaint handling are separate", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown; annual aggregate report", + "country_code": "AU", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Identify the current search/export route; annual category totals are not facility rows." + ] + }, + "jurisdiction_scope": "Australia; Victoria PrimeSafe meat and seafood licences", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "au.primesafe.vic.meat-licences", + "source_url": "https://www.primesafe.vic.gov.au/licensing/about-your-licence/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-au.md", + "docs/countries/australia/source-crosswalk.json", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Identify the current PrimeSafe search/export route and capture one bounded result with licence, category, status and site fields; keep annual totals separate.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "public search/view/download; portal says not everything is online", + "attribution": { + "attribution_required": true, + "notice": "Queensland public-register route; preserve current/cancelled/surrendered status and information-request gaps", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "refresh observed 2026-09-11; not guaranteed", + "country_code": "AU", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Completeness and terms need an authorised bounded capture." + ] + }, + "jurisdiction_scope": "Australia; Queensland environmental authorities and enforcement", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "au.qld.environmental-authorities", + "source_url": "https://apps.des.qld.gov.au/public-register/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-au.md", + "docs/countries/australia/source-crosswalk.json", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Capture one bounded Queensland EA search/download and validate completeness, activity codes, status history and holder/site identity.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "public GeoJSON download", + "attribution": { + "attribution_required": true, + "notice": "CC BY 3.0 Australia; publisher warns points are approximate and may omit latest information; privacy/release review required", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "resource edition observed 2026-03-18", + "country_code": "AU", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Private artifact exists; validate activity semantics, approximate points, terms, and licence-to-site identity before integration." + ] + }, + "jurisdiction_scope": "Australia; South Australia EPA licensed activities", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": true, + "public_release_allowed": false, + "reasons": [ + "publication:blocked" + ], + "state": "awaiting-owner-review" + }, + "source_id": "au.sa.epa.licensed-activities", + "source_url": "https://data.sa.gov.au/data/dataset/8fdb86ff-d3d1-4f9e-85a5-bed4080d5ee1", + "status": { + "acquisition": "artifact_private_only", + "evidence": [ + "docs/country-recon-au.md", + "docs/countries/australia/artifact-metadata.json", + "docs/countries/australia/source-crosswalk.json", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Validate licence-level aggregation, multi-activity child rows, approximate-point semantics, and privacy/terms before any map or graph integration.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "public browser search; bulk/API route not verified", + "attribution": { + "attribution_required": true, + "notice": "Safe Food Queensland public register; result capture, terms, and address/privacy handling require review", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown", + "country_code": "AU", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Verify an authorised bounded capture, schema, category codebook, and reuse terms." + ] + }, + "jurisdiction_scope": "Australia; Queensland Safe Food accreditation", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "au.safefood.qld.accreditation", + "source_url": "https://hub.safefood.qld.gov.au/registry/s/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-au.md", + "docs/countries/australia/source-crosswalk.json", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Use an assisted register search to capture accreditation-number/business/activity/status fields, confirm bulk/export behavior and terms, and keep annual counts separate from facility rows.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "public HTML/PDF guidance; current row export not verified", + "attribution": { + "attribution_required": true, + "notice": "Biosecurity Tasmania authority and audited-program scope verified; historical lists are explicitly non-current", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "annual/unknown", + "country_code": "AU", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "2019 feasibility appendix is legacy only; confirm current register/export and terms." + ] + }, + "jurisdiction_scope": "Australia; Tasmania Biosecurity Tasmania meat accreditation", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "au.tas.biosecurity-meat", + "source_url": "https://nre.tas.gov.au/biosecurity-tasmania/product-integrity/food-safety/meat-and-poultry", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-au.md", + "docs/countries/australia/source-crosswalk.json", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Locate a current Tasmania accreditation route; keep the 2019 feasibility appendix legacy and separate from current observations.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "public search/LISTmap; stable API not verified", + "attribution": { + "attribution_required": true, + "notice": "Tasmania EPA route; preserve redaction and date limits", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "documents from 2022 onward; ongoing additions", + "country_code": "AU", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "LISTmap service layer and download route unresolved." + ] + }, + "jurisdiction_scope": "Australia; Tasmania EPA regulated premises and monitoring", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "au.tas.epa-listmap", + "source_url": "https://epa.tas.gov.au/about-the-epa/release-of-environmental-monitoring-information/search-for-environmental-monitoring-information", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-au.md", + "docs/countries/australia/source-crosswalk.json", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Resolve the Tasmania LISTmap service layer and permitted download route; preserve redaction/date limits and keep monitoring documents separate.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "public guidance and annual report downloads; complete premises list not verified", + "attribution": { + "attribution_required": true, + "notice": "Agriculture Victoria licence categories and aggregate reports; privacy review required", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "annual returns/report", + "country_code": "AU", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "No complete public premises list verified; keep annual-use reports separate from facilities." + ] + }, + "jurisdiction_scope": "Australia; Victoria scientific-procedure licensing and statistics", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "au.vic.animal-use", + "source_url": "https://agriculture.vic.gov.au/livestock-and-animals/animal-welfare-victoria/animals-used-in-research-and-teaching/licensing-to-use-animals-in-research-or-teaching/about-licensing-to-use-animals-in-research-or-teaching", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-au.md", + "docs/countries/australia/source-crosswalk.json", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Keep Victoria scientific-procedure licence guidance and aggregate reports separate; locate a public premises list before any facility mapping.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "public search plus ArcGIS REST layer", + "attribution": { + "attribution_required": true, + "notice": "EPA Victoria register and ArcGIS metadata observed; validate production service/version and geometry privacy", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "overnight; up to 24-hour delay; public from 2021-07-01", + "country_code": "AU", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Validate service filters, geometry semantics, and pre-2021 coverage." + ] + }, + "jurisdiction_scope": "Australia; EPA Victoria permissions", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "au.vic.epa-permissions", + "source_url": "https://www.epa.vic.gov.au/public-registers?register=permissions", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-au.md", + "docs/countries/australia/source-crosswalk.json", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Capture a bounded ArcGIS REST query and matching public-register record; validate pagination, identifiers, location types, terms, privacy, and permission-to-facility relationship semantics.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "public HTML list", + "attribution": { + "attribution_required": true, + "notice": "WAMIA approval page and 2026 guideline context verified; confirm reuse terms and revision semantics", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "page update observed July 2026", + "country_code": "AU", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Confirm stable approval identifier and capture rules; source names are not global IDs." + ] + }, + "jurisdiction_scope": "Australia; Western Australia WAMIA approved abattoirs", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "au.wamia.abattoirs", + "source_url": "https://wamia.wa.gov.au/abattoir-approvals/", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-au.md", + "docs/countries/australia/source-crosswalk.json", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Capture the current WAMIA HTML list, confirm approval numbers/revision and reuse terms, and keep source names as unresolved candidates until reviewed.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + } + ], + "summary": "No release approval is implied; inspect source gates below." + }, + { + "acquisition_counts": { + "blocked": 5, + "not_run": 1 + }, + "basis": [ + "az.afsa.food-subjects: blocked / blocked", + "az.afsa.inspections-enforcement: blocked / blocked", + "az.afsa.livestock-traceability: blocked / blocked", + "az.eco.environment-permits: blocked / blocked", + "az.stat.livestock-statistics: not_run / reconnaissance", + "az.taxes.organizations: blocked / blocked" + ], + "country_code": "AZ", + "country_reasons": [ + "acquisition:blocked", + "acquisition:not_run", + "publication:blocked" + ], + "display_name": "AZ", + "owner_review": "awaiting-owner-review", + "publication_state": "blocked", + "readiness_class": "blocked", + "source_count": 6, + "sources": [ + { + "access_method": "official AFSA search service; authorized bounded export/API only", + "attribution": { + "attribution_required": true, + "notice": "Official Azerbaijani government source; terms, privacy, and regional safety review required", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown", + "country_code": "AZ", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Bulk/API route, schema, stable IDs, cadence, licensing, privacy, and safe publication boundary not pinned." + ] + }, + "jurisdiction_scope": "Azerbaijan; AFSA registered food subjects and animal-origin food activities", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "az.afsa.food-subjects", + "source_url": "https://afsa.gov.az/az/qida-subyektleri", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-az.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin authorized route, schema, cadence, IDs, terms, privacy, and automation permissions.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official reports/search services; aggregate-only until safe event route is verified", + "attribution": { + "attribution_required": true, + "notice": "Sensitive control/research evidence; privacy, retention, and safety review required", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "resource-specific; unknown", + "country_code": "AZ", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "No stable public event/experimentation master verified; stop and escalate on sensitive exposure." + ] + }, + "jurisdiction_scope": "Azerbaijan; AFSA inspections, violations, veterinary controls, and public experimentation evidence", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "az.afsa.inspections-enforcement", + "source_url": "https://afsa.gov.az/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-az.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin authorized route, schema, cadence, IDs, terms, privacy, and automation permissions.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official notice references AQTIS; login-protected system, do not bypass access controls", + "attribution": { + "attribution_required": true, + "notice": "Sensitive veterinary/farm data; privacy, security, retention, and terms require explicit review", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown", + "country_code": "AZ", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Public export/API and authorization not verified; records may be operationally sensitive." + ] + }, + "jurisdiction_scope": "Azerbaijan; AFSA animal identification/registration and farm-to-table traceability", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "az.afsa.livestock-traceability", + "source_url": "https://afsa.gov.az/az/heyvan-saglamligi-ve-bioloji-tehlukesizlik/xeberler/heyvanlarin-identiklesdirilmesi-baytarliq-nezaretinin-effektivliyini-artirir", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-az.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin authorized route, schema, cadence, IDs, terms, privacy, and automation permissions.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official ministry route; authorized API/export only", + "attribution": { + "attribution_required": true, + "notice": "Ministry source; license, geometry, privacy, and safety review required", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "resource-specific; unknown", + "country_code": "AZ", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Exact public route, API/export, coverage, cadence, license, geometry, and privacy not pinned." + ] + }, + "jurisdiction_scope": "Azerbaijan; environmental permits and registers", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "az.eco.environment-permits", + "source_url": "https://eco.gov.az/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-az.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin authorized route, schema, cadence, IDs, terms, privacy, and automation permissions.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official statistical tables/yearbooks; machine API or bounded download to be pinned", + "attribution": { + "attribution_required": true, + "notice": "State Statistical Committee source; publication terms, revisions, and suppression rules require review", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "annual and table-specific", + "country_code": "AZ", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Current table IDs, machine API, revision semantics, licensing, and suppression rules not pinned." + ] + }, + "jurisdiction_scope": "Azerbaijan; aggregate livestock, slaughter, meat, milk, and fishery statistics", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "az.stat.livestock-statistics", + "source_url": "https://www.stat.gov.az/source/agriculture/?lang=en", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-az.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin authorized route, schema, cadence, IDs, terms, privacy, and automation permissions.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official search/public database; no bypass of access controls", + "attribution": { + "attribution_required": true, + "notice": "Government registry; personal-data, terms, limits, and reuse review required", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown; statistical snapshots exist", + "country_code": "AZ", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Authorized machine route, fields, cadence, licensing, privacy, and automation permissions not verified." + ] + }, + "jurisdiction_scope": "Azerbaijan; State Tax Service commercial legal-entity and taxpayer registration", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "az.taxes.organizations", + "source_url": "https://www.taxes.gov.az/en/page/qeydiyyat", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-az.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin authorized route, schema, cadence, IDs, terms, privacy, and automation permissions.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + } + ], + "summary": "No release approval is implied; inspect source gates below." + }, + { + "acquisition_counts": { + "blocked": 5, + "not_run": 1 + }, + "basis": [ + "ba.bhas.statistics: not_run / reconnaissance", + "ba.bizreg.organizations: blocked / blocked", + "ba.environment-permits: blocked / blocked", + "ba.farm-aquaculture: blocked / blocked", + "ba.inspections-experiments: blocked / blocked", + "ba.veterinary.approved-food: blocked / blocked" + ], + "country_code": "BA", + "country_reasons": [ + "acquisition:blocked", + "acquisition:not_run", + "publication:blocked" + ], + "display_name": "BA", + "owner_review": "awaiting-owner-review", + "publication_state": "blocked", + "readiness_class": "blocked", + "source_count": 6, + "sources": [ + { + "access_method": "Agency for Statistics of BiH and entity statistical tables/data services", + "attribution": { + "attribution_required": true, + "notice": "Official statistics; verify table terms, citation, entity coverage and aggregate-only scope.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "table/authority-specific; preserve revisions", + "country_code": "BA", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Pin state/entity slaughter and animal-use table IDs, APIs/downloads and cadence." + ] + }, + "jurisdiction_scope": "Bosnia and Herzegovina; state/entity official slaughter, livestock and animal-use aggregates", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "ba.bhas.statistics", + "source_url": "https://bhas.gov.ba/", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-ba.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin state/entity slaughter and animal-use table IDs, APIs, cadence and revisions.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official searchable web portal; API/bulk route unverified", + "attribution": { + "attribution_required": true, + "notice": "Identity linkage only; verify terms, automation permissions, fields and registered-office privacy.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "portal-defined; last-update field displayed", + "country_code": "BA", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Pin stable endpoints, query contract, rate limits, cadence and entity-specific field semantics." + ] + }, + "jurisdiction_scope": "Bosnia and Herzegovina; BIZREG registers for Federation BiH, Republika Srpska and Brčko District", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "ba.bizreg.organizations", + "source_url": "https://bizreg.pravosudje.ba/pls/apex/f?p=186%3A%3A2313976059615753%3A%3ANO%3A%3A", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-ba.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Pin BIZREG endpoints, entity fields, rate limits, cadence, terms and automation permissions.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "entity environmental ministries/agencies, public notices and documents", + "attribution": { + "attribution_required": true, + "notice": "Verify public-read rights, document license, geometry, sensitive sites and entity coverage.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "permit/document-specific", + "country_code": "BA", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "No unified public read/API/export route or complete cross-entity coverage verified." + ] + }, + "jurisdiction_scope": "Bosnia and Herzegovina; state/entity environmental permits and integrated controls", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "ba.environment-permits", + "source_url": "https://fmoit.gov.ba/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-ba.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Verify entity environmental permit routes, coverage, documents, geometry, license and privacy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "entity/cantonal agriculture, veterinary, fisheries and official open-data/document resources", + "attribution": { + "attribution_required": true, + "notice": "Verify authority, license, animal-holder/property privacy and coordinate precision by entity.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "authority/resource-specific", + "country_code": "BA", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "All-authority farm/aquaculture export and cross-entity identifiers are unresolved." + ] + }, + "jurisdiction_scope": "Bosnia and Herzegovina; livestock holdings, feed and aquaculture across state/entity/cantonal authorities", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "ba.farm-aquaculture", + "source_url": "https://fuzip.gov.ba/unutrasnja-organizacija/federalni-poljoprivredni-inspektorat/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-ba.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Verify all-authority farm/aquaculture scope, exports, IDs, coordinates, terms and privacy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "state/entity/cantonal control plans, inspection reports and statistical publications", + "attribution": { + "attribution_required": true, + "notice": "Sensitive control/research evidence; aggregate or anonymize and require project review.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "report/authority-specific", + "country_code": "BA", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "No stable public cross-entity facility/event or animal-experimentation master verified." + ] + }, + "jurisdiction_scope": "Bosnia and Herzegovina; veterinary/food inspections, enforcement and animal-experimentation evidence", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "ba.inspections-experiments", + "source_url": "https://fuzip.gov.ba/kontrolne-liste/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-ba.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Locate cross-entity inspection/event routes; keep sensitive evidence aggregate until verified.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "state/entity veterinary registers, inspection portals and official documents", + "attribution": { + "attribution_required": true, + "notice": "Official but fragmented authorities; verify each source license, attribution, privacy and coordinates.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "authority-specific; timestamp retrieval", + "country_code": "BA", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "No unified national establishment export verified; map FBiH, RS, Brčko and cantonal coverage first." + ] + }, + "jurisdiction_scope": "Bosnia and Herzegovina; state/entity veterinary and food authorities, including FBiH and cantonal controls", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "ba.veterinary.approved-food", + "source_url": "https://fuzip.gov.ba/federalni-veterinarski-inspektorat/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-ba.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Map state/entity/cantonal coverage and pin establishment routes, IDs, cadence, terms and privacy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + } + ], + "summary": "No release approval is implied; inspect source gates below." + }, + { + "acquisition_counts": { + "artifact_private_only": 1 + }, + "basis": [ + "be.locations: artifact_private_only / awaiting-owner-review" + ], + "country_code": "BE", + "country_reasons": [ + "publication:blocked" + ], + "display_name": "BE", + "owner_review": "awaiting-owner-review", + "publication_state": "blocked", + "readiness_class": "human-review-ready", + "source_count": 1, + "sources": [ + { + "access_method": "published weekly operator CSV plus companion LAP/PAP activity-code CSV; assisted capture supported", + "attribution": { + "attribution_required": true, + "notice": "CC Attribution 4.0; attribute FASFC and the last update date, do not imply FASFC affiliation/approval, and do not mislead", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "weekly", + "country_code": "BE", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "A current official pair was captured privately and validated through the deterministic adapter. Five exact duplicates and 15 unresolved activity codes remain quarantined; privacy, source-terms interpretation, classification, coverage, and project publication approval remain human gates." + ] + }, + "jurisdiction_scope": "Belgium; FASFC-registered, approved, or authorized operators, including animal-origin food and other food-chain activities", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": true, + "public_release_allowed": false, + "reasons": [ + "publication:blocked" + ], + "state": "awaiting-owner-review" + }, + "source_id": "be.locations", + "source_url": "https://www.static.favv.be/bo-documents/inter_actieve_actoren_EN.csv", + "status": { + "acquisition": "artifact_private_only", + "evidence": [ + "docs/country-recon-be.md", + "data/manifests/de-be-private-candidates-2026-09-17.json", + "pipeline/sources/belgium/adapter.py", + "pipeline/sources/belgium/refresh.py", + "docs/review-packet-belgium.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Review the private current pair: 310,660 input, 310,640 normalized, 20 quarantined; names are not supplied by this snapshot. Keep privacy, attribution, classification, terms, and project approval gates closed.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + } + ], + "summary": "No release approval is implied; inspect source gates below." + }, + { + "acquisition_counts": { + "blocked": 5, + "not_run": 1 + }, + "basis": [ + "bg.bfsa.approved-food: blocked / blocked", + "bg.environment-permits: blocked / blocked", + "bg.farm-aquaculture: blocked / blocked", + "bg.inspections-experiments: blocked / blocked", + "bg.nsi.statistics: not_run / reconnaissance", + "bg.registry-agency.organizations: blocked / blocked" + ], + "country_code": "BG", + "country_reasons": [ + "acquisition:blocked", + "acquisition:not_run", + "publication:blocked" + ], + "display_name": "BG", + "owner_review": "awaiting-owner-review", + "publication_state": "blocked", + "readiness_class": "blocked", + "source_count": 6, + "sources": [ + { + "access_method": "official BFSA public registers and linked EU feed-establishment list", + "attribution": { + "attribution_required": true, + "notice": "Official authority; verify register terms, attribution, privacy and coordinates before acquisition.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "publisher-defined; timestamp retrieval", + "country_code": "BG", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Pin current BFSA list/API/export, slaughter categories, IDs, cadence, terms and privacy." + ] + }, + "jurisdiction_scope": "Bulgaria; BFSA approved and registered food/feed establishments handling animal-origin products", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "bg.bfsa.approved-food", + "source_url": "https://bfsa.egov.bg/wps/portal/bfsa-web/registers", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-bg.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin BFSA list/API/export, categories, IDs, cadence, terms and privacy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "Executive Environment Agency systems and public notices/documents", + "attribution": { + "attribution_required": true, + "notice": "Verify public-read rights, document license, geometry and privacy.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "permit/document-specific", + "country_code": "BG", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Stable public read/API/export route and complete national coverage are unresolved." + ] + }, + "jurisdiction_scope": "Bulgaria; environmental permits and integrated-control authorizations", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "bg.environment-permits", + "source_url": "https://eea.government.bg/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-bg.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Verify public permit API/export, coverage, documents, geometry, license and privacy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official BFSA/agriculture/fisheries registers and open-data resources", + "attribution": { + "attribution_required": true, + "notice": "Verify publisher, license, animal-holder/property privacy and coordinate precision.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "resource-specific", + "country_code": "BG", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "No complete national farm/aquaculture public export contract verified." + ] + }, + "jurisdiction_scope": "Bulgaria; livestock holdings and aquaculture permit/establishment evidence", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "bg.farm-aquaculture", + "source_url": "https://bfsa.egov.bg/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-bg.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Verify national farm/aquaculture scope, export/API, IDs, coordinates, terms and privacy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official control plans, reports and statistical publications", + "attribution": { + "attribution_required": true, + "notice": "Sensitive control/research evidence; aggregate or anonymize and require project review.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "report/register-specific", + "country_code": "BG", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "No stable public facility-level inspection or animal-experimentation master verified." + ] + }, + "jurisdiction_scope": "Bulgaria; BFSA inspections/enforcement and animal-experimentation evidence", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "bg.inspections-experiments", + "source_url": "https://bfsa.egov.bg/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-bg.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Locate a stable public facility/event route; keep sensitive evidence aggregate until verified.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official statistical tables/data services; exact table/API route unverified", + "attribution": { + "attribution_required": true, + "notice": "Official statistics; verify table terms, citation requirements and aggregate-only scope.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "table-specific; preserve revisions", + "country_code": "BG", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Pin current slaughter and animal-use table IDs, API/download contracts and cadence." + ] + }, + "jurisdiction_scope": "Bulgaria; NSI aggregate slaughter, livestock and animal-use statistics", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "bg.nsi.statistics", + "source_url": "https://www.nsi.bg/", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-bg.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin official slaughter/animal-use table IDs, APIs, cadence and revisions.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official public register/service surface; API/bulk route unverified", + "attribution": { + "attribution_required": true, + "notice": "Identity linkage only; verify terms, fields, rate limits and registered-office privacy.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "provider-defined", + "country_code": "BG", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Pin stable query/API route, authentication/CAPTCHA, cadence, terms and privacy policy." + ] + }, + "jurisdiction_scope": "Bulgaria; Registry Agency Commercial Register corporate identifiers", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "bg.registry-agency.organizations", + "source_url": "https://www.registryagency.bg/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-bg.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin corporate query/API route, auth/CAPTCHA, fields, cadence, terms and privacy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + } + ], + "summary": "No release approval is implied; inspect source gates below." + }, + { + "acquisition_counts": { + "artifact_private_only": 2, + "verified": 2 + }, + "basis": [ + "br.sif.export: verified / awaiting-owner-review", + "br.sif.registered: verified / awaiting-owner-review", + "br.sisbi.public: artifact_private_only / awaiting-owner-review", + "br.trase.facilities: artifact_private_only / awaiting-owner-review" + ], + "country_code": "BR", + "country_reasons": [ + "publication:blocked" + ], + "display_name": "BR", + "owner_review": "awaiting-owner-review", + "publication_state": "blocked", + "readiness_class": "human-review-ready", + "source_count": 4, + "sources": [ + { + "access_method": "published semicolon-delimited CSV via MAPA CKAN; bounded GET or assisted capture", + "attribution": { + "attribution_required": true, + "notice": "MAPA catalog displays Creative Commons Attribution; export/product rows remain separate evidence and require terms/privacy review", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "monthly; catalog metadata checked 2026-09-16", + "country_code": "BR", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Do not treat export authorization as facility status or count; terms, effective-date semantics, privacy, reconciliation, and project approval remain pending." + ] + }, + "jurisdiction_scope": "Brazil; MAPA/DIPOA SIF establishments with country/product export authorization observations", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": true, + "public_release_allowed": false, + "reasons": [ + "publication:blocked" + ], + "state": "awaiting-owner-review" + }, + "source_id": "br.sif.export", + "source_url": "https://dados.agricultura.gov.br/dataset/062166e3-b515-4274-8e7d-68aadd64b820/resource/fcb7f87d-0092-4a52-a44b-b3550747b4c2/download/sigsifestabelecimentosnacionais.csv", + "status": { + "acquisition": "verified", + "evidence": [ + "docs/country-recon-br.md", + "docs/countries/br/v1-field-crosswalk.json", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Model country/product authorizations as separate dated observations keyed to SIF; confirm validity/suspension semantics, terms, privacy, and no-double-counting rules before integration.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "published semicolon-delimited CSV via MAPA CKAN; bounded GET or assisted capture", + "attribution": { + "attribution_required": true, + "notice": "MAPA catalog displays Creative Commons Attribution; confirm dataset-specific reuse, attribution, and personal-data handling before publication", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "monthly; catalog last-update metadata checked 2026-09-16", + "country_code": "BR", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Restricted raw/contact/address handling, current code semantics, dataset-specific terms, identity lifecycle, and project publication approval remain pending." + ] + }, + "jurisdiction_scope": "Brazil; MAPA/DIPOA establishments registered under the federal SIF directory", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": true, + "public_release_allowed": false, + "reasons": [ + "publication:blocked" + ], + "state": "awaiting-owner-review" + }, + "source_id": "br.sif.registered", + "source_url": "https://dados.agricultura.gov.br/dataset/062166e3-b515-4274-8e7d-68aadd64b820/resource/97277e92-264a-4dc0-9aea-f87b8ea93798/download/sigsifestabelecimentosregistradosnosif.csv", + "status": { + "acquisition": "verified", + "evidence": [ + "docs/country-recon-br.md", + "docs/countries/br/v1-field-crosswalk.json", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Keep the current private SIF CSV and provenance restricted; validate repeated-row semantics, status/code meanings, source terms, privacy, reconciliation, and release approval before any adapter or publication.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "public JavaScript client with bounded JSON GET routes under sisbi_api and GIS API; no bulk scrape performed", + "attribution": { + "attribution_required": true, + "notice": "MAPA government source; public access does not settle API reuse, retention, privacy, or publication terms", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown; public route and API access verified 2026-09-16", + "country_code": "BR", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Confirm pagination/query contract, code lists, status/effective-date semantics, ID lifecycle, address linkage, update cadence, privacy, terms, and project approval with an authorized operator." + ] + }, + "jurisdiction_scope": "Brazil; public e-SISBI/SISBI-POA service, establishment, product, capacity, and MAPA GIS address routes", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": true, + "public_release_allowed": false, + "reasons": [ + "publication:blocked" + ], + "state": "awaiting-owner-review" + }, + "source_id": "br.sisbi.public", + "source_url": "https://sistemasweb.agricultura.gov.br/sgsi/app/estabelecimentos", + "status": { + "acquisition": "artifact_private_only", + "evidence": [ + "docs/country-recon-br.md", + "docs/countries/br/v1-field-crosswalk.json", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Use bounded GET samples only while an authorized operator confirms pagination, code lists, lifecycle/status, address linkage, cadence, terms, privacy, and project approval.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "published GeoJSON download; private reconnaissance capture only", + "attribution": { + "attribution_required": true, + "notice": "Trase page permits platform charts/maps/representations under CC BY 4.0 and asks commercial data users to contact Trase; raw-data reuse and privacy handling remain review gates", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "dataset page says 2025 data, updated 2026-01-01; future cadence unknown", + "country_code": "BR", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Secondary source must remain separate from MAPA; validate source snapshots, geocoding provenance, terms, privacy, constructed-ID behavior, coverage, and project approval before any use." + ] + }, + "jurisdiction_scope": "Brazil; Trase secondary compilation of SIF, SISBI, SIE, SIM, and CONSORCIO facility/activity rows", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": true, + "public_release_allowed": false, + "reasons": [ + "publication:blocked" + ], + "state": "awaiting-owner-review" + }, + "source_id": "br.trase.facilities", + "source_url": "https://trase.earth/open-data/datasets/brazil-facilities", + "status": { + "acquisition": "artifact_private_only", + "evidence": [ + "docs/country-recon-br.md", + "docs/countries/br/v1-field-crosswalk.json", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Keep Trase as separately labeled secondary evidence; review source lineage, geocoding, constructed IDs, raw-data terms, privacy, coverage, and any exact-ID reconciliation before use.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + } + ], + "summary": "No release approval is implied; inspect source gates below." + }, + { + "acquisition_counts": { + "not_run": 2 + }, + "basis": [ + "ca.cfia.federal-meat: not_run / reconnaissance", + "ca.ontario.meat-plants: not_run / reconnaissance" + ], + "country_code": "CA", + "country_reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "display_name": "CA", + "owner_review": "awaiting-owner-review", + "publication_state": "blocked", + "readiness_class": "acquisition-ready", + "source_count": 2, + "sources": [ + { + "access_method": "bounded registry download or assisted local capture", + "attribution": { + "attribution_required": true, + "notice": "CFIA government source; registry page calls the consolidation a convenience reference and current reuse/privacy review remains required", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown; registry page states list update 2023-12-04", + "country_code": "CA", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Federal registry is not complete provincial coverage; function-code schema, currency, privacy, and project publication approval remain pending." + ] + }, + "jurisdiction_scope": "Canada; CFIA federally registered meat establishments and licensed operators", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "ca.cfia.federal-meat", + "source_url": "https://active.inspection.gc.ca/scripts/meavia/reglist/download.asp?lang=e", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-ca.md", + "docs/countries/canada/meat-plants-pipeline.md", + "pipeline/sources/canada/adapter.py", + "pipeline/sources/canada/acquire.py", + "pipeline/sources/canada/refresh.py", + "pipeline/sources/canada/fixtures/cfia.csv", + "pipeline/common/review_packet.py", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Run the bounded private CFIA registry refresh; validate the live export schema/function codes, keep federal scope separate from provincial scope, and retain publication gates.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "bounded CSV fetch or assisted local capture", + "attribution": { + "attribution_required": true, + "notice": "Government of Ontario dataset; current licence, attribution, privacy, and redistribution review remain explicit gates", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown; dataset metadata checked 2026-09-14", + "country_code": "CA", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Ontario coverage is not national; privacy/coordinate review and project publication approval remain pending." + ] + }, + "jurisdiction_scope": "Canada; Ontario provincial meat plants only", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "ca.ontario.meat-plants", + "source_url": "https://data.ontario.ca/dataset/a763088c-018d-48b7-bf47-3027a8c725b8/resource/ee6d559a-78de-40e6-b2ba-ad3c4a674b96/download/1._all_meat_plants.csv", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-ca.md", + "docs/countries/canada/meat-plants-pipeline.md", + "pipeline/sources/canada/adapter.py", + "pipeline/sources/canada/acquire.py", + "pipeline/sources/canada/refresh.py", + "pipeline/sources/canada/fixtures/ontario.csv", + "pipeline/common/review_packet.py", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Run the bounded private Ontario refresh; keep plant/contact/coordinate fields restricted pending privacy and licence review, and do not generalize Ontario coverage nationally.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + } + ], + "summary": "No release approval is implied; inspect source gates below." + }, + { + "acquisition_counts": { + "blocked": 1 + }, + "basis": [ + "ch.blv.approved-food: blocked / blocked" + ], + "country_code": "CH", + "country_reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "display_name": "CH", + "owner_review": "awaiting-owner-review", + "publication_state": "blocked", + "readiness_class": "blocked", + "source_count": 1, + "sources": [ + { + "access_method": "official multilingual FSVO search/list route; authorized bounded export or browser capture only", + "attribution": { + "attribution_required": true, + "notice": "Swiss government source; public visibility does not settle reuse, attribution, personal-address, coordinate, or publication rights", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "source-specific; current page and list route observed 2026-09-16", + "country_code": "CH", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "No stable bulk/API contract or complete national export verified; federal list may be assembled from cantonal authorities; language normalization, effective/status semantics, privacy, terms, coverage, and project approval remain unresolved." + ] + }, + "jurisdiction_scope": "Switzerland; FSVO list of approved food businesses, including animal-origin establishments and slaughterhouses", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "ch.blv.approved-food", + "source_url": "https://www.blv.admin.ch/de/listen-bewilligter-schweizer-betriebe", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-ch.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Obtain an authorized bounded export or capture; fingerprint multilingual schema/list version, preserve approval/activity observations separately, and complete coverage, privacy, terms, and project-approval review.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + } + ], + "summary": "No release approval is implied; inspect source gates below." + }, + { + "acquisition_counts": { + "blocked": 5, + "not_run": 1 + }, + "basis": [ + "cy.companies.registry: blocked / blocked", + "cy.cystat.livestock-meat: not_run / reconnaissance", + "cy.environment-permits: blocked / blocked", + "cy.vs.approved-food: blocked / blocked", + "cy.vs.farms-livestock: blocked / blocked", + "cy.vs.inspections-enforcement: blocked / blocked" + ], + "country_code": "CY", + "country_reasons": [ + "acquisition:blocked", + "acquisition:not_run", + "publication:blocked" + ], + "display_name": "CY", + "owner_review": "awaiting-owner-review", + "publication_state": "blocked", + "readiness_class": "blocked", + "source_count": 6, + "sources": [ + { + "access_method": "official Republic of Cyprus route; authorized bounded capture/API only", + "attribution": { + "attribution_required": true, + "notice": "Government source; Republic of Cyprus scope, terms, privacy, and safety review required", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown; source-specific", + "country_code": "CY", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Exact machine contract, stable IDs, cadence, licensing, privacy, and territorial completeness not pinned." + ] + }, + "jurisdiction_scope": "Republic of Cyprus; Registrar of Companies and Intellectual Property", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "cy.companies.registry", + "source_url": "https://www.companies.gov.cy/en/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-cy.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin authorized route, schema, cadence, IDs, terms, privacy, and Republic of Cyprus territorial scope.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official Republic of Cyprus route; authorized bounded capture/API only", + "attribution": { + "attribution_required": true, + "notice": "Government source; Republic of Cyprus scope, terms, privacy, and safety review required", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown; source-specific", + "country_code": "CY", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Exact machine contract, stable IDs, cadence, licensing, privacy, and territorial completeness not pinned." + ] + }, + "jurisdiction_scope": "Republic of Cyprus; aggregate livestock and meat statistics", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "cy.cystat.livestock-meat", + "source_url": "https://cystatdb.cystat.gov.cy/pxweb/en/8.CYSTAT-DB/8.CYSTAT-DB__Agriculture%2C%20Livestock%2C%20Fishing__Livestock/0320031E.px/", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-cy.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin authorized route, schema, cadence, IDs, terms, privacy, and Republic of Cyprus territorial scope.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official Republic of Cyprus route; authorized bounded capture/API only", + "attribution": { + "attribution_required": true, + "notice": "Government source; Republic of Cyprus scope, terms, privacy, and safety review required", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown; source-specific", + "country_code": "CY", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Exact machine contract, stable IDs, cadence, licensing, privacy, and territorial completeness not pinned." + ] + }, + "jurisdiction_scope": "Republic of Cyprus; environmental/EIA and waste permits", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "cy.environment-permits", + "source_url": "https://www.moa.gov.cy/moa/environment/environment.nsf", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-cy.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin authorized route, schema, cadence, IDs, terms, privacy, and Republic of Cyprus territorial scope.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official Republic of Cyprus route; authorized bounded capture/API only", + "attribution": { + "attribution_required": true, + "notice": "Government source; Republic of Cyprus scope, terms, privacy, and safety review required", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown; source-specific", + "country_code": "CY", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Exact machine contract, stable IDs, cadence, licensing, privacy, and territorial completeness not pinned." + ] + }, + "jurisdiction_scope": "Republic of Cyprus; Veterinary Services approved animal-origin establishments and slaughterhouses", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "cy.vs.approved-food", + "source_url": "https://www.moa.gov.cy/moa/vs/vs.nsf/All/9F6A5DB7308579ACC225764D001D01AF?OpenDocument=&print=", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-cy.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin authorized route, schema, cadence, IDs, terms, privacy, and Republic of Cyprus territorial scope.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official Republic of Cyprus route; authorized bounded capture/API only", + "attribution": { + "attribution_required": true, + "notice": "Government source; Republic of Cyprus scope, terms, privacy, and safety review required", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown; source-specific", + "country_code": "CY", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Exact machine contract, stable IDs, cadence, licensing, privacy, and territorial completeness not pinned." + ] + }, + "jurisdiction_scope": "Republic of Cyprus; Veterinary Services livestock holdings and animal identification", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "cy.vs.farms-livestock", + "source_url": "https://www.moa.gov.cy/moa/vs/vs.nsf", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-cy.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin authorized route, schema, cadence, IDs, terms, privacy, and Republic of Cyprus territorial scope.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official Republic of Cyprus route; authorized bounded capture/API only", + "attribution": { + "attribution_required": true, + "notice": "Government source; Republic of Cyprus scope, terms, privacy, and safety review required", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown; source-specific", + "country_code": "CY", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Exact machine contract, stable IDs, cadence, licensing, privacy, and territorial completeness not pinned." + ] + }, + "jurisdiction_scope": "Republic of Cyprus; Veterinary Services inspections and animal-use evidence", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "cy.vs.inspections-enforcement", + "source_url": "https://www.moa.gov.cy/moa/vs/vs.nsf", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-cy.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin authorized route, schema, cadence, IDs, terms, privacy, and Republic of Cyprus territorial scope.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + } + ], + "summary": "No release approval is implied; inspect source gates below." + }, + { + "acquisition_counts": { + "blocked": 2, + "not_run": 3 + }, + "basis": [ + "cz.business-register: not_run / reconnaissance", + "cz.environment.permits: blocked / blocked", + "cz.statistics: not_run / reconnaissance", + "cz.svs.approved-food: not_run / reconnaissance", + "cz.svs.farms-aquaculture: blocked / blocked" + ], + "country_code": "CZ", + "country_reasons": [ + "acquisition:blocked", + "acquisition:not_run", + "metadata:unknown", + "publication:blocked" + ], + "display_name": "CZ", + "owner_review": "awaiting-owner-review", + "publication_state": "blocked", + "readiness_class": "blocked", + "source_count": 5, + "sources": [ + { + "access_method": "public register search and CSV export", + "attribution": { + "attribution_required": true, + "notice": "Identity-only linkage; suppress personal, sole-trader and residential details.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "provider-defined", + "country_code": "CZ", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Verify current endpoint, terms, quotas and privacy policy." + ] + }, + "jurisdiction_scope": "Czechia; corporate/statistical register identity", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "cz.business-register", + "source_url": "https://portal.gov.cz/sluzby-vs/ziskani-zverejnenych-informaci-ze-statistickych-registru-S4953", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-cz.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Verify current endpoint, terms and identity-only privacy policy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official environmental registry routes; not verified", + "attribution": { + "attribution_required": true, + "notice": "Verify license, geometry and sensitive-site handling.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown", + "country_code": "CZ", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "No stable national animal-facility permit export verified." + ] + }, + "jurisdiction_scope": "Czechia; environmental permits and releases", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "metadata:unknown", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "cz.environment.permits", + "source_url": "https://www.mzp.cz/en", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-cz.md", + "pipeline/source_registry.json" + ], + "metadata": "unknown", + "next_action": "Locate and verify environmental permit data route.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official statistical tables/API; exact route unverified", + "attribution": { + "attribution_required": true, + "notice": "Official statistics; preserve table IDs, revisions and license metadata.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "table-specific", + "country_code": "CZ", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Pin current table/API contracts and keep aggregate scope separate." + ] + }, + "jurisdiction_scope": "Czechia; official slaughter, livestock and animal-use aggregates", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "cz.statistics", + "source_url": "https://www.czso.cz/csu/czso/statistics", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-cz.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin statistics table/API contracts and revisions.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official filtered lists and linked files", + "attribution": { + "attribution_required": true, + "notice": "Official Czech veterinary authority; verify terms, fields and privacy.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "list-specific; timestamp access", + "country_code": "CZ", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Verify bulk/API/file contracts, completeness, terms, IDs and coordinate availability." + ] + }, + "jurisdiction_scope": "Czechia; SVS approved/registered animal-origin food, ABP, feed and aquaculture establishments", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "cz.svs.approved-food", + "source_url": "https://en.svs.gov.cz/registered-subjects/", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-cz.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Verify bulk/API/file contracts, completeness, terms, IDs and coordinates.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official lists/filter surface", + "attribution": { + "attribution_required": true, + "notice": "Animal-holder and site data require privacy review.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown", + "country_code": "CZ", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Verify national scope, export route, terms and sensitive-field policy." + ] + }, + "jurisdiction_scope": "Czechia; SVS farms, aquaculture and animal-sector registers", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "cz.svs.farms-aquaculture", + "source_url": "https://en.svs.gov.cz/registered-subjects/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-cz.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Verify national scope, export route, terms and sensitive-field policy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + } + ], + "summary": "No release approval is implied; inspect source gates below." + }, + { + "acquisition_counts": { + "artifact_private_only": 1 + }, + "basis": [ + "de.locations: artifact_private_only / awaiting-owner-review" + ], + "country_code": "DE", + "country_reasons": [ + "publication:blocked" + ], + "display_name": "DE", + "owner_review": "awaiting-owner-review", + "publication_state": "blocked", + "readiness_class": "human-review-ready", + "source_count": 1, + "sources": [ + { + "access_method": "BVL portal selected CSV/XLS export or assisted capture", + "attribution": { + "attribution_required": true, + "notice": "unknown; verify BVL reuse and attribution terms", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown", + "country_code": "DE", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "A current public general-list export was captured privately: 15,788 input, 2,691 normalized, and 13,097 quarantined. The export URL is session/request-specific, effective date is unknown, and dataset reuse terms, privacy, coverage, and project approval remain pending human confirmation. Keep raw data private and publication blocked." + ] + }, + "jurisdiction_scope": "Germany", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": true, + "public_release_allowed": false, + "reasons": [ + "publication:blocked" + ], + "state": "awaiting-owner-review" + }, + "source_id": "de.locations", + "source_url": "https://www.bvl.bund.de/bltu", + "status": { + "acquisition": "artifact_private_only", + "evidence": [ + "docs/germany-source-assessment.md", + "data/manifests/de-be-private-candidates-2026-09-17.json", + "pipeline/sources/germany/adapter.py", + "pipeline/sources/germany/refresh.py", + "pipeline/common/review_packet.py", + "docs/review-packet-germany.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Review the private current export: 15,788 input, 2,691 normalized, 13,097 quarantined, with 50-column schema matched and no release created. The session-bound export route, unknown effective date, terms, privacy, coverage, and project approval remain unresolved.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + } + ], + "summary": "No release approval is implied; inspect source gates below." + }, + { + "acquisition_counts": { + "verified": 1 + }, + "basis": [ + "dk.smiley: verified / awaiting-owner-review" + ], + "country_code": "DK", + "country_reasons": [ + "publication:blocked" + ], + "display_name": "DK", + "owner_review": "awaiting-owner-review", + "publication_state": "blocked", + "readiness_class": "human-review-ready", + "source_count": 1, + "sources": [ + { + "access_method": "bulk XML download", + "attribution": { + "attribution_required": true, + "notice": "Official Find Smiley data page records public-data reuse terms: attribute Fødevarestyrelsen, do not use its logo, and keep displayed smileys current; no project publication approval.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "weekly", + "country_code": "DK", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Publisher supplies no dataset effective date; source coverage is limited to data available on Find Smiley and is not a completeness claim. Denmark candidate rows require explicit source-key mapping before disposable DB import." + ] + }, + "jurisdiction_scope": "Denmark; food and animal-related establishments in Find Smiley coverage", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": true, + "public_release_allowed": false, + "reasons": [ + "publication:blocked" + ], + "state": "awaiting-owner-review" + }, + "source_id": "dk.smiley", + "source_url": "https://pub.fvst.dk/publikationer/Smileydata.xml", + "status": { + "acquisition": "verified", + "evidence": [ + "pipeline/sources/denmark/README.md", + "pipeline/contracts/README.md", + "pipeline/common/review_packet.py", + "docs/review-packet-denmark.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Keep the full privately staged artifact and candidate import validation restricted; run a bounded guarded API check with explicit test-only labeling, capture rerun exit output, and resolve coverage/effective-date/category semantics before any release review. This is not a production-health or publication claim.", + "publication_eligibility": "blocked", + "runtime_health": "unknown" + } + } + ], + "summary": "No release approval is implied; inspect source gates below." + }, + { + "acquisition_counts": { + "blocked": 2, + "not_run": 4 + }, + "basis": [ + "ee.ariregister.organizations: not_run / reconnaissance", + "ee.keskkonnaamet.kotkas: blocked / blocked", + "ee.pria.animal-register: not_run / reconnaissance", + "ee.pria.aquaculture: blocked / blocked", + "ee.pta.approved-food: not_run / reconnaissance", + "ee.stat.slaughter: not_run / reconnaissance" + ], + "country_code": "EE", + "country_reasons": [ + "acquisition:blocked", + "acquisition:not_run", + "publication:blocked" + ], + "display_name": "EE", + "owner_review": "awaiting-owner-review", + "publication_state": "blocked", + "readiness_class": "blocked", + "source_count": 6, + "sources": [ + { + "access_method": "official register/API route unverified", + "attribution": { + "attribution_required": true, + "notice": "Identity-only linkage; suppress personal, sole-trader and residential details; verify terms.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "provider-defined", + "country_code": "EE", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Verify current API, auth, quotas and privacy policy." + ] + }, + "jurisdiction_scope": "Estonia; Äriregister corporate identity", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "ee.ariregister.organizations", + "source_url": "https://ariregister.rik.ee/eng", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-ee.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Verify API, auth, quotas, terms and identity-only policy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "KOTKAS/document register and public notices", + "attribution": { + "attribution_required": true, + "notice": "Official environmental authority; verify terms, geometry and sensitive-site handling.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "publication/permit-specific", + "country_code": "EE", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "UI/document extraction and complete national coverage unresolved." + ] + }, + "jurisdiction_scope": "Estonia; Keskkonnaamet environmental permit and public-notice evidence", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "ee.keskkonnaamet.kotkas", + "source_url": "https://keskkonnaamet.ee/keskkonnateadlikkus-avalikustamised/raagi-kaasa/lubade-eelnoude-avalik-valjapanek", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-ee.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Verify document extraction, terms, identifiers, geometry and coverage.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "public query/XML and geospatial web map/service", + "attribution": { + "attribution_required": true, + "notice": "Official PRIA; animal/property location data require strict privacy and purpose review.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "prior-day data described; verify service cadence", + "country_code": "EE", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Verify current service/download, CRS, terms, field suppression and coverage." + ] + }, + "jurisdiction_scope": "Estonia; PRIA public farm-animal and aquaculture establishment data", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "ee.pria.animal-register", + "source_url": "https://www.pria.ee/registrid/avalikud-andmed", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-ee.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Verify current public service/download, CRS, terms and field suppression.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "public register and animal-register services", + "attribution": { + "attribution_required": true, + "notice": "Official authority; separate aquaculture from farm/food entities and review sensitive locations.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "service-specific", + "country_code": "EE", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "No aquaculture-specific bulk contract verified." + ] + }, + "jurisdiction_scope": "Estonia; PRIA aquaculture establishment register", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "ee.pria.aquaculture", + "source_url": "https://www.pria.ee/registrid/kalad-ja-vahid", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-ee.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Verify aquaculture-specific bulk contract and privacy scope.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official registers and linked downloads", + "attribution": { + "attribution_required": true, + "notice": "Official Estonian authority; verify dataset terms and privacy.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "publisher-defined; timestamp retrieval", + "country_code": "EE", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Verify current bulk/API route, completeness, headers, terms and privacy." + ] + }, + "jurisdiction_scope": "Estonia; PTA approved and registered food/animal establishments", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "ee.pta.approved-food", + "source_url": "https://pta.agri.ee/riiklikud-registrid-ja-andmekogud", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-ee.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Verify current bulk/API route, completeness, headers, terms and privacy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official PxWeb/statistical database", + "attribution": { + "attribution_required": true, + "notice": "Statistics Estonia open-data route states CC BY-SA 4.0; preserve table IDs and revisions.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "monthly/table-specific", + "country_code": "EE", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Pin API/table metadata; keep aggregate scope separate from facility evidence." + ] + }, + "jurisdiction_scope": "Estonia; Statistics Estonia slaughter and livestock aggregates", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "ee.stat.slaughter", + "source_url": "https://andmed.stat.ee/en/stat/majandus__pellumajandus__pellumajandussaaduste-tootmine__loomakasvatussaaduste-tootmine/PM190", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-ee.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Pin current PxWeb table metadata and preserve revisions.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + } + ], + "summary": "No release approval is implied; inspect source gates below." + }, + { + "acquisition_counts": { + "blocked": 1 + }, + "basis": [ + "es.locations: blocked / blocked" + ], + "country_code": "ES", + "country_reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "display_name": "ES", + "owner_review": "awaiting-owner-review", + "publication_state": "blocked", + "readiness_class": "blocked", + "source_count": 1, + "sources": [ + { + "access_method": "download or assisted export", + "attribution": { + "attribution_required": true, + "notice": "unknown; confirm competent authority, reuse terms, and attribution", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown", + "country_code": "ES", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Identify the exact competent-authority publication and distinguish source values from legacy transformations." + ] + }, + "jurisdiction_scope": "Spain", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "es.locations", + "source_url": "unknown", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-es.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Resolve a permitted current AESAN/MAPA artifact or query route, then verify export/schema, rights, effective-date semantics, sector coverage, privacy, and legacy/source boundaries before acquisition.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + } + ], + "summary": "No release approval is implied; inspect source gates below." + }, + { + "acquisition_counts": { + "artifact_private_only": 1, + "not_run": 2 + }, + "basis": [ + "eu.eurostat.nl-slaughter: artifact_private_only / awaiting-owner-review", + "eu.traces.approved-establishments: not_run / reconnaissance", + "eu.traces.pl-approved-food: not_run / reconnaissance" + ], + "country_code": "EU", + "country_reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "display_name": "EU", + "owner_review": "awaiting-owner-review", + "publication_state": "blocked", + "readiness_class": "private-candidate-ready", + "source_count": 3, + "sources": [ + { + "access_method": "public dissemination JSON API", + "attribution": { + "attribution_required": true, + "notice": "Official EU mirror; never count in addition to CBS or NVWA.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "dataset-defined; retain last-updated metadata", + "country_code": "EU", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Use only for harmonized aggregate context and preserve flags." + ] + }, + "jurisdiction_scope": "EU statistical mirror; Netherlands aggregate slaughter context", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": true, + "public_release_allowed": false, + "reasons": [ + "publication:blocked" + ], + "state": "awaiting-owner-review" + }, + "source_id": "eu.eurostat.nl-slaughter", + "source_url": "https://ec.europa.eu/eurostat/api/dissemination/statistics/1.0/data/apro_mt_pann?geo=NL&lang=en", + "status": { + "acquisition": "artifact_private_only", + "evidence": [ + "docs/country-recon-nl.md", + "data/manifests/nl-source-artifacts.json", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Use only as harmonized aggregate context; retain dimensions/flags and do not double-count CBS.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "TRACES publication surface and national links", + "attribution": { + "attribution_required": true, + "notice": "Official EU mirror; do not double-count national NVWA observations.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "competent-authority updates; timestamp each access", + "country_code": "EU", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Verify NL export/API and stable IDs before use; keep separate provenance layer." + ] + }, + "jurisdiction_scope": "EU; TRACES/IMSOC approved-establishment mirror", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "eu.traces.approved-establishments", + "source_url": "https://food.ec.europa.eu/food-safety/biological-safety/food-hygiene/approved-eu-food-establishments_en", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-nl.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Verify NL-specific TRACES export and stable IDs only if needed; never merge as additional facilities.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "public TRACES/publication page; Poland export not verified", + "attribution": { + "attribution_required": true, + "notice": "EU official mirror; do not double-count GIW", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "competent-authority updates", + "country_code": "EU", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Verify Poland-specific export and keep mirror provenance separate." + ] + }, + "jurisdiction_scope": "EU mirror; Poland approved-food establishment lineage", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "eu.traces.pl-approved-food", + "source_url": "https://food.ec.europa.eu/food-safety/biological-safety/food-hygiene/approved-eu-food-establishments_en", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-pl.md", + "docs/countries/pl/source-crosswalk.json", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Verify Poland-specific TRACES export only as a lineage/cross-check layer; never double-count GIW.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + } + ], + "summary": "No release approval is implied; inspect source gates below." + }, + { + "acquisition_counts": { + "blocked": 3, + "not_run": 5 + }, + "basis": [ + "fi.animal-experiments: not_run / reconnaissance", + "fi.aquaculture: blocked / blocked", + "fi.luke.agriculture: not_run / reconnaissance", + "fi.prh.ytj.organizations: not_run / reconnaissance", + "fi.ruokavirasto.approved-food: not_run / reconnaissance", + "fi.ruokavirasto.feed-abp: blocked / blocked", + "fi.statfin.pxweb: not_run / reconnaissance", + "fi.syke.environment: blocked / blocked" + ], + "country_code": "FI", + "country_reasons": [ + "acquisition:blocked", + "acquisition:not_run", + "publication:blocked" + ], + "display_name": "FI", + "owner_review": "awaiting-owner-review", + "publication_state": "blocked", + "readiness_class": "blocked", + "source_count": 8, + "sources": [ + { + "access_method": "official guidance and annual reports", + "attribution": { + "attribution_required": true, + "notice": "Sensitive research evidence; aggregate/anonymize and apply strict privacy review.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "annual/report-specific", + "country_code": "FI", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "No public research-facility master verified." + ] + }, + "jurisdiction_scope": "Finland; animal experimentation guidance/statistics", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "fi.animal-experiments", + "source_url": "https://www.ruokavirasto.fi/en/animals/animal-experiments/", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-fi.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Keep annual aggregates separate; no research-facility master verified.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official statistics and environmental/geospatial services", + "attribution": { + "attribution_required": true, + "notice": "Verify provider terms, geometry precision and site privacy before use.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "statistics/service-specific; unknown", + "country_code": "FI", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "No current national public permit/site API verified." + ] + }, + "jurisdiction_scope": "Finland; aquaculture production/sites and environmental evidence", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "fi.aquaculture", + "source_url": "https://www.luke.fi/en/statistics/aquaculture", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-fi.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Verify current public permit/site API, geometry, terms and privacy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official statistics pages/downloads/API where available", + "attribution": { + "attribution_required": true, + "notice": "Official statistics; verify dataset license and preserve revisions/aggregate scope.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "table-specific; timestamp each release", + "country_code": "FI", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Pin table IDs, API/download contracts and license; do not infer facility rows." + ] + }, + "jurisdiction_scope": "Finland; Luke aggregate agriculture, livestock and aquaculture statistics", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "fi.luke.agriculture", + "source_url": "https://www.luke.fi/en/statistics", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-fi.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Pin table/API contracts and preserve aggregate scope.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official open-data/API/download documentation; current route unverified", + "attribution": { + "attribution_required": true, + "notice": "Identity linkage only; suppress personal, sole-trader and residential details; verify terms.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "provider-defined", + "country_code": "FI", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Verify current endpoint, auth, quotas, fields and identity-only policy." + ] + }, + "jurisdiction_scope": "Finland; PRH/YTJ corporate and business identifiers", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "fi.prh.ytj.organizations", + "source_url": "https://www.prh.fi/en/uutislistaus/uutiset/2020/P_23520.html", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-fi.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Verify current endpoint, auth, quotas and identity-only privacy policy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official web pages and linked lists; bulk route unverified", + "attribution": { + "attribution_required": true, + "notice": "Official Finnish Food Authority; verify file-specific reuse terms and privacy before acquisition.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "publisher-defined; timestamp each retrieval", + "country_code": "FI", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Verify current export/API, section coverage, headers, terms and mixed-address privacy." + ] + }, + "jurisdiction_scope": "Finland; Ruokavirasto approved animal-origin food establishments", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "fi.ruokavirasto.approved-food", + "source_url": "https://www.ruokavirasto.fi/en/companies/food-sector/food-establishments/approved-establishments/", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-fi.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Verify current export/API, section coverage, headers, terms and privacy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official guidance/register surface; bulk route unverified", + "attribution": { + "attribution_required": true, + "notice": "Official authority; category-specific terms, coverage and privacy require review.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown; timestamp each access", + "country_code": "FI", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Verify public list/export, category boundaries and source rights before acquisition." + ] + }, + "jurisdiction_scope": "Finland; Ruokavirasto feed and animal-by-product establishments", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "fi.ruokavirasto.feed-abp", + "source_url": "https://www.ruokavirasto.fi/en/companies/feed/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-fi.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Verify public list/export, category boundaries and source rights.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official PxWeb/statistical database API or downloads", + "attribution": { + "attribution_required": true, + "notice": "Official Statistics Finland; verify table license and preserve dimensions/revisions.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "table-specific; timestamp releases/revisions", + "country_code": "FI", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Pin current table IDs and API contracts; keep aggregate scope separate." + ] + }, + "jurisdiction_scope": "Finland; Statistics Finland aggregate slaughter and animal-use context", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "fi.statfin.pxweb", + "source_url": "https://stat.fi/en/services/statistical-data-services/statistical-databases", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-fi.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin current table IDs/API contracts and preserve revisions.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official open-data/service catalogue; route-specific", + "attribution": { + "attribution_required": true, + "notice": "Verify license, attribution, sensitive-site handling and geometry semantics.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "dataset-specific", + "country_code": "FI", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "No stable national animal-facility permit export verified." + ] + }, + "jurisdiction_scope": "Finland; environmental open information and permit evidence", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "fi.syke.environment", + "source_url": "https://www.syke.fi/en-US/Open_information", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-fi.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Verify current environmental export/API, license, geometry and coverage.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + } + ], + "summary": "No release approval is implied; inspect source gates below." + }, + { + "acquisition_counts": { + "not_run": 2 + }, + "basis": [ + "fr.dgal.section-i: not_run / reconnaissance", + "fr.dgal.section-ii: not_run / reconnaissance" + ], + "country_code": "FR", + "country_reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "display_name": "FR", + "owner_review": "awaiting-owner-review", + "publication_state": "blocked", + "readiness_class": "acquisition-ready", + "source_count": 2, + "sources": [ + { + "access_method": "bounded TXT fetch or assisted local capture", + "attribution": { + "attribution_required": true, + "notice": "Ministry page indicates Etalab 2.0 for site content; file-specific terms and attribution confirmation remain pending", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "daily; Ministry page checked 2026-09-15", + "country_code": "FR", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Privacy/coordinate review, file-specific terms, category codebook, and project publication approval remain pending." + ] + }, + "jurisdiction_scope": "France; DGAL Regulation (EC) 853/2004 Section I domestic ungulate establishments", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "fr.dgal.section-i", + "source_url": "https://fichiers-publics.agriculture.gouv.fr/dgal/ListesOfficielles/SSA1_VIAN_ONG_DOM.txt", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-fr.md", + "docs/countries/france/dgal-853-pipeline.md", + "pipeline/sources/france/adapter.py", + "pipeline/sources/france/acquire.py", + "pipeline/sources/france/refresh.py", + "pipeline/sources/france/fixtures/section_i.csv", + "pipeline/common/review_packet.py", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Run the bounded private Section I refresh with an approved terms record or authorized capture; review category semantics, address privacy, schema drift, and release approval.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "bounded TXT fetch or assisted local capture", + "attribution": { + "attribution_required": true, + "notice": "Ministry page indicates Etalab 2.0 for site content; file-specific terms and attribution confirmation remain pending", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "daily; Ministry page checked 2026-09-15", + "country_code": "FR", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Section II is separate from Section I; privacy/coordinate review, terms, codebook, and project publication approval remain pending." + ] + }, + "jurisdiction_scope": "France; DGAL Regulation (EC) 853/2004 Section II poultry and lagomorph establishments", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "fr.dgal.section-ii", + "source_url": "https://fichiers-publics.agriculture.gouv.fr/dgal/ListesOfficielles/SSA1_VIAN_COL_LAGO.txt", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-fr.md", + "docs/countries/france/dgal-853-pipeline.md", + "pipeline/sources/france/adapter.py", + "pipeline/sources/france/acquire.py", + "pipeline/sources/france/refresh.py", + "pipeline/sources/france/fixtures/section_ii.csv", + "pipeline/common/review_packet.py", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Run the bounded private Section II refresh separately from Section I; review category/species semantics, address privacy, schema drift, and release approval.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + } + ], + "summary": "No release approval is implied; inspect source gates below." + }, + { + "acquisition_counts": { + "blocked": 5, + "not_run": 1 + }, + "basis": [ + "ge.geostat.slaughter-statistics: not_run / reconnaissance", + "ge.napr.organizations: blocked / blocked", + "ge.nea.environment-permits: blocked / blocked", + "ge.nfa.approved-food: blocked / blocked", + "ge.nfa.farms-livestock: blocked / blocked", + "ge.nfa.inspections-experiments: blocked / blocked" + ], + "country_code": "GE", + "country_reasons": [ + "acquisition:blocked", + "acquisition:not_run", + "publication:blocked" + ], + "display_name": "GE", + "owner_review": "awaiting-owner-review", + "publication_state": "blocked", + "readiness_class": "blocked", + "source_count": 6, + "sources": [ + { + "access_method": "official Geostat publications and tables; machine API or bounded download to be pinned", + "attribution": { + "attribution_required": true, + "notice": "National Statistics Office source; publication terms, revisions, and suppression rules require review", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "quarterly/annual; publication-specific", + "country_code": "GE", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Current table IDs, machine API, revision semantics, and suppression rules not pinned." + ] + }, + "jurisdiction_scope": "Georgia; aggregate livestock slaughterhouse and animal-production statistics", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "ge.geostat.slaughter-statistics", + "source_url": "https://www.geostat.ge/en/modules/categories/755/section-5-livestock-poultry-and-beehives", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-ge.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin authorized route, schema, cadence, IDs, terms, privacy, and automation permissions.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official NAPR online extract/service; authorized API/bulk access only", + "attribution": { + "attribution_required": true, + "notice": "Official registry service; fees, rate limits, terms, and personal-address policy require review", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown", + "country_code": "GE", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Authorized machine route, fields, cadence, fees, limits, and privacy policy not verified." + ] + }, + "jurisdiction_scope": "Georgia; NAPR entrepreneur and legal-entity identifiers", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "ge.napr.organizations", + "source_url": "https://www.napr.gov.ge/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-ge.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin authorized route, schema, cadence, IDs, terms, privacy, and automation permissions.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official NEA service/register route; authorized API/export only", + "attribution": { + "attribution_required": true, + "notice": "Agency source; license, geometry, privacy, and terms require review", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "resource-specific; unknown", + "country_code": "GE", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Exact permit route, export/API, coverage, cadence, license, geometry, and privacy not pinned." + ] + }, + "jurisdiction_scope": "Georgia; National Environment Agency environmental permits and related registers", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "ge.nea.environment-permits", + "source_url": "https://nea.gov.ge/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-ge.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin authorized route, schema, cadence, IDs, terms, privacy, and automation permissions.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official NFA page with advertised download; authorized bounded capture only", + "attribution": { + "attribution_required": true, + "notice": "Official Georgian government source; terms, privacy, and location-safety review required", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "periodic; exact cadence unknown", + "country_code": "GE", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Exact download URL, format, schema, stable ID, cadence, terms, and privacy policy not pinned." + ] + }, + "jurisdiction_scope": "Georgia; NFA registered slaughterhouses and recognized animal-origin food operators", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "ge.nfa.approved-food", + "source_url": "https://nfa.gov.ge/Ge/Page/List%20of%20Slaughterhouses%20Registered%20in%20Georgia", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-ge.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin authorized route, schema, cadence, IDs, terms, privacy, and automation permissions.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official NFA service/list route; no facility acquisition until contract review", + "attribution": { + "attribution_required": true, + "notice": "Official government source; scope, personal data, coordinates, and reuse terms require review", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown", + "country_code": "GE", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Bulk/API route, stable IDs, cadence, coverage, privacy, and location policy not verified." + ] + }, + "jurisdiction_scope": "Georgia; NFA primary-production, livestock identification/registration, feed, and recognized operators", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "ge.nfa.farms-livestock", + "source_url": "https://nfa.gov.ge/Ge/Page/Primary%20production%20control", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-ge.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin authorized route, schema, cadence, IDs, terms, privacy, and automation permissions.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official dated reports/registers; aggregate-only until safe event route is verified", + "attribution": { + "attribution_required": true, + "notice": "Sensitive control/research evidence; privacy, retention, and publication review required", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "resource-specific; unknown", + "country_code": "GE", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "No stable public event/experimentation master verified; stop and escalate on sensitive exposure." + ] + }, + "jurisdiction_scope": "Georgia; NFA veterinary/food-control findings and public animal-experimentation evidence", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "ge.nfa.inspections-experiments", + "source_url": "https://www.nfa.gov.ge/Ge/Page/Veterinary%20Control", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-ge.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin authorized route, schema, cadence, IDs, terms, privacy, and automation permissions.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + } + ], + "summary": "No release approval is implied; inspect source gates below." + }, + { + "acquisition_counts": { + "blocked": 3, + "not_run": 3 + }, + "basis": [ + "hr.approved-food: not_run / reconnaissance", + "hr.aquaculture-permits: not_run / reconnaissance", + "hr.business-register: blocked / blocked", + "hr.environment-permits: blocked / blocked", + "hr.inspections-experiments: blocked / blocked", + "hr.statistics: not_run / reconnaissance" + ], + "country_code": "HR", + "country_reasons": [ + "acquisition:blocked", + "acquisition:not_run", + "publication:blocked" + ], + "display_name": "HR", + "owner_review": "awaiting-owner-review", + "publication_state": "blocked", + "readiness_class": "blocked", + "source_count": 6, + "sources": [ + { + "access_method": "national CKAN WEB/XLS resources and official register guidance", + "attribution": { + "attribution_required": true, + "notice": "CKAN lists public/open access; verify exact resource license, privacy and attribution before acquisition.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "not defined; timestamp retrieval", + "country_code": "HR", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Pin direct resource route, schema, cadence, identifiers, coordinates, terms and privacy before acquisition." + ] + }, + "jurisdiction_scope": "Croatia; approved and registered food establishments handling food of animal origin", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "hr.approved-food", + "source_url": "https://data.gov.hr/ckan/en/dataset/upisnik-odobrenih-objekata-u-poslovanju-s-hranom-za-zivotinje", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-hr.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Pin direct resource route, schema, cadence, identifiers, terms and privacy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "CKAN XLS resource", + "attribution": { + "attribution_required": true, + "notice": "CKAN marks the dataset open; verify current license, sensitive-site handling and attribution.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "not defined; timestamp retrieval", + "country_code": "HR", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Pin direct XLS URL, schema, update cadence, stable IDs, coordinates and terms." + ] + }, + "jurisdiction_scope": "Croatia; Ministry aquaculture permit register", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "hr.aquaculture-permits", + "source_url": "https://data.gov.hr/ckan/hr/dataset/registar-dozvola-u-akvakulturi", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-hr.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Pin direct XLS URL, schema, cadence, IDs, coordinates and terms.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "public REST API after free registration; XML or JSON", + "attribution": { + "attribution_required": true, + "notice": "Identity linkage only; verify API terms, quotas and suppression of personal/sole-trader/residential details.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "daily on working days", + "country_code": "HR", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Credentials, quota, exact API contract and legal-person-only matching policy require verification." + ] + }, + "jurisdiction_scope": "Croatia; Court Register corporate identity and registered office", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "hr.business-register", + "source_url": "https://sudreg-data.gov.hr/ords/r/srn_rep/vanjski-srn-rep/home", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-hr.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Provision and verify API credentials, quotas, contract and identity-only policy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "national CKAN resources and environmental registers", + "attribution": { + "attribution_required": true, + "notice": "Verify current publisher, resource license, document rights, geometry and privacy.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "dataset-specific", + "country_code": "HR", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Complete national animal-facility coverage and machine-readable route are unresolved." + ] + }, + "jurisdiction_scope": "Croatia; environmental permits and integrated environmental conditions", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "hr.environment-permits", + "source_url": "https://data.gov.hr/ckan/en/dataset/o-evidnik-uporabnih-dozvola-i-rje-enja-o-objedinjenim-uvjetima-za-tite-okoli-a", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-hr.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Verify publisher, resource route, coverage, geometry, licensing and privacy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official register guidance, reports and statistical publications", + "attribution": { + "attribution_required": true, + "notice": "Sensitive control/research evidence; aggregate or anonymize and require project review.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "report/register-specific", + "country_code": "HR", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "No stable public facility-level inspection or animal-experimentation master verified." + ] + }, + "jurisdiction_scope": "Croatia; food/veterinary inspections, enforcement and animal-experimentation evidence", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "hr.inspections-experiments", + "source_url": "https://inspektorat.gov.hr/ustrojstvo-77/7-sektor-sanitarne-inspekcije/evidentiranje-i-vodjenje-registra-subjekta-i-pripadajucih-objekta-u-poslovanju-s-hranom-iz-nadleznosti-iz-nadleznosti-sanitarne-inspekcije/431", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-hr.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Locate a stable public facility/event route; keep sensitive evidence aggregate until verified.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "Croatian Bureau of Statistics tables/data services", + "attribution": { + "attribution_required": true, + "notice": "Official statistics; verify table terms, citation requirements and aggregate-only scope.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "table-specific; preserve revisions", + "country_code": "HR", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Pin slaughter and animal-use table IDs, API/download contracts and release cadence." + ] + }, + "jurisdiction_scope": "Croatia; official slaughter, livestock and animal-use aggregate statistics", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "hr.statistics", + "source_url": "https://dzs.gov.hr/usluge/objavljivanje/program-publiciranja-2026/2439", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-hr.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin official slaughter and animal-use table IDs, APIs and revisions.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + } + ], + "summary": "No release approval is implied; inspect source gates below." + }, + { + "acquisition_counts": { + "artifact_private_only": 4, + "not_run": 12 + }, + "basis": [ + "ie.cro.companies: not_run / reconnaissance", + "ie.cso.livestock-slaughterings: not_run / reconnaissance", + "ie.dafm.animal-welfare-controls: not_run / reconnaissance", + "ie.dafm.approved-establishments: not_run / reconnaissance", + "ie.dafm.former-la-establishments: not_run / reconnaissance", + "ie.dafm.milk-dairy-establishments: not_run / reconnaissance", + "ie.dafm.national-beef-kill: not_run / reconnaissance", + "ie.dafm.seafood-processing-funding: not_run / reconnaissance", + "ie.epa.leap: not_run / reconnaissance", + "ie.fsai.approved-directory: not_run / reconnaissance", + "ie.fsai.enforcement-orders: not_run / reconnaissance", + "ie.hse.low-throughput-meat: artifact_private_only / awaiting-owner-review", + "ie.planning.npad: not_run / reconnaissance", + "ie.sfpa.approved-establishments: artifact_private_only / awaiting-owner-review", + "ie.sfpa.factory-vessels: artifact_private_only / awaiting-owner-review", + "ie.sfpa.freezer-vessels: artifact_private_only / awaiting-owner-review" + ], + "country_code": "IE", + "country_reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "display_name": "IE", + "owner_review": "awaiting-owner-review", + "publication_state": "blocked", + "readiness_class": "private-candidate-ready", + "source_count": 16, + "sources": [ + { + "access_method": "official open-data bulk/API route or CORE search; bounded identity capture", + "attribution": { + "attribution_required": true, + "notice": "CRO open-data company dataset is described as CC BY 4.0; officer/personal details remain restricted and company identity is not proof of a facility or operation", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "daily; catalog describes daily updates", + "country_code": "IE", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Name similarity is not a merge; require reviewed company-number links and keep registered offices distinct from operating premises." + ] + }, + "jurisdiction_scope": "Ireland; Companies Registration Office CORE company/business-name identity register", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "ie.cro.companies", + "source_url": "https://opendata.cro.ie/dataset/companies", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-ie.md", + "docs/countries/ireland/v1-field-crosswalk.json", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Verify the live daily schema and use company number for reviewed identity edges only.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official CSO release/table route; bounded aggregate capture", + "attribution": { + "attribution_required": true, + "notice": "CSO statistics are aggregate context; table-specific terms and attribution must be retained with any downstream use", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "monthly", + "country_code": "IE", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Do not join aggregate statistics to facility rows or interpret totals as a facility census; preserve CSO coverage notes." + ] + }, + "jurisdiction_scope": "Ireland; CSO aggregate livestock slaughterings and meat supply statistics", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "ie.cso.livestock-slaughterings", + "source_url": "https://www.cso.ie/en/statistics/agriculture/livestockslaughterings/", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-ie.md", + "docs/countries/ireland/v1-field-crosswalk.json", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Capture aggregate monthly measures with coverage notes; never treat them as facility rows.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official statement and annual-report routes; bounded contextual capture", + "attribution": { + "attribution_required": true, + "notice": "DAFM statement describes control responsibilities; use only as authority/scope evidence unless a specific current inspection artifact is lawfully obtained and reviewed", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown; no row-level public welfare feed verified", + "country_code": "IE", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Current row-level facility-linked welfare/enforcement observations were not captured; do not convert qualitative controls or aggregate reports into facility claims." + ] + }, + "jurisdiction_scope": "Ireland; DAFM official-veterinary and animal-welfare control context at approved slaughter plants", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "ie.dafm.animal-welfare-controls", + "source_url": "https://www.gov.ie/en/department-of-agriculture-food-and-the-marine/press-releases/statement-on-garda-investigation-into-alleged-offences-of-deception/", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-ie.md", + "docs/countries/ireland/v1-field-crosswalk.json", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Keep the DAFM statement as control-scope context until a current row-level welfare artifact is lawfully captured and reviewed.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official gov.ie publication link; bounded workbook capture or operator-assisted download", + "attribution": { + "attribution_required": true, + "notice": "DAFM government publication; workbook-specific reuse, attribution, personal-data handling, and redistribution terms must be confirmed before release", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown; publication page last updated 2026-09-15", + "country_code": "IE", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Workbook bytes and headers were not captured in this run; do not infer row counts or schema from the publication title." + ] + }, + "jurisdiction_scope": "Ireland; DAFM-approved or registered meat establishments including fish, egg, and dairy under S.I. 22 of 2020", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "ie.dafm.approved-establishments", + "source_url": "https://assets.gov.ie/static/documents/09fe3ad4/AllApprovedPlants_2026_5.xlsx", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-ie.md", + "docs/countries/ireland/v1-field-crosswalk.json", + "data/manifests/ireland-source-artifacts.json", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Obtain an authorized bounded workbook capture, headers, schema fingerprint, row counts, terms, privacy review, and lifecycle semantics.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official gov.ie publication link; bounded workbook capture or operator-assisted download", + "attribution": { + "attribution_required": true, + "notice": "DAFM government publication; workbook-specific reuse, attribution, personal-data handling, and redistribution terms must be confirmed before release", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown; publication page last updated 2026-09-15", + "country_code": "IE", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Keep former-LA coverage separate from the main DAFM workbook until overlap and authority ownership are reviewed." + ] + }, + "jurisdiction_scope": "Ireland; former local-authority meat establishments including fish, egg, and dairy under S.I. 22 of 2020", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "ie.dafm.former-la-establishments", + "source_url": "https://assets.gov.ie/static/documents/09fe3ad4/AllApprovedPlants_2026_Formerly_LA_Plants_1.xlsx", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-ie.md", + "docs/countries/ireland/v1-field-crosswalk.json", + "data/manifests/ireland-source-artifacts.json", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Capture separately from the main DAFM workbook and measure overlap before any deduplication.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official gov.ie publication link; bounded workbook capture or operator-assisted download", + "attribution": { + "attribution_required": true, + "notice": "DAFM government publication; workbook-specific reuse, attribution, personal-data handling, and redistribution terms must be confirmed before release", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown; workbook filename dated 2026-08-11", + "country_code": "IE", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Do not union dairy rows with meat rows without preserving source family and approval-versus-registration semantics." + ] + }, + "jurisdiction_scope": "Ireland; DAFM milk and dairy establishments approved and/or registered under the Hygiene Regulations", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "ie.dafm.milk-dairy-establishments", + "source_url": "https://assets.gov.ie/static/documents/09fe3ad4/1._Milk_Dairy_Establishments_Registered_and_or_Approved_11th_August_2026.xlsx", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-ie.md", + "docs/countries/ireland/v1-field-crosswalk.json", + "data/manifests/ireland-source-artifacts.json", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Capture the dated workbook and preserve approval-versus-registration and dairy activity semantics.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official open-data catalog/resource route; bounded aggregate capture", + "attribution": { + "attribution_required": true, + "notice": "DAFM open-data catalog indicates open-data reuse for the resource; verify current resource terms and preserve its historical cutoff", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "weekly; current catalog resource described as 2020-2024 and last updated 2024-08-01", + "country_code": "IE", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Catalog resource is historical through 2024 in the observed metadata; do not present it as a current facility status feed or inflate facility counts." + ] + }, + "jurisdiction_scope": "Ireland; DAFM national beef kill figures by approved processing plants, aggregate release", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "ie.dafm.national-beef-kill", + "source_url": "https://opendata.agriculture.gov.ie/dataset/national-beef-kill-figures", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-ie.md", + "docs/countries/ireland/v1-field-crosswalk.json", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Retain the observed 2024 historical cutoff and verify the resource before any aggregate use.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official programme/press-release metadata; no facility award rows captured", + "attribution": { + "attribution_required": true, + "notice": "Government/EU programme context; award-level terms, personal data, and beneficiary publication rules require scheme-specific review", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown; programme/competition dependent", + "country_code": "IE", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Keep funding context separate from approvals; do not link by name without a reviewed company identifier and award relationship." + ] + }, + "jurisdiction_scope": "Ireland; DAFM seafood-processing capital-investment funding context under Ireland Seafood Development Programme/EMFAF", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "ie.dafm.seafood-processing-funding", + "source_url": "https://www.gov.ie/en/department-of-agriculture-food-and-the-marine/press-releases/minister-dooley-announces-opening-of-the-seafood-processing-capital-investment-scheme/", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-ie.md", + "docs/countries/ireland/v1-field-crosswalk.json", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Keep programme context separate; capture awards only after scheme-specific privacy and terms review.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official LEAP search and documented public API families; bounded record retrieval after identifier review", + "attribution": { + "attribution_required": true, + "notice": "EPA LEAP terms and conditions govern use of environmental information; do not bulk-copy or expose personal/precise information without legal/privacy review", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown; most records are published 30 calendar days after creation", + "country_code": "IE", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "LEAP is an accountability layer, not a food-establishment census; resolve API identifiers, terms, privacy, and explicit cross-source matching." + ] + }, + "jurisdiction_scope": "Ireland; EPA IE/IPC licensed sites and public LEAP licensing, compliance, and enforcement records", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "ie.epa.leap", + "source_url": "https://www.epa.ie/our-services/compliance--enforcement/whats-happening/leap-online/", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-ie.md", + "docs/countries/ireland/v1-field-crosswalk.json", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Resolve LEAP API identifiers and terms; use it only for reviewed accountability edges and events.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official directory page; bounded browser capture or operator-assisted authority exports", + "attribution": { + "attribution_required": true, + "notice": "FSAI explains the approval obligation and links to DAFM, HSE, and SFPA lists; the directory page is not itself a facility master and reuse/privacy terms require review", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown", + "country_code": "IE", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Do not count this coordinator page as an additional establishment source; verify linked authority exports, terms, privacy, and publication approval." + ] + }, + "jurisdiction_scope": "Ireland; FSAI coordinating directory for competent-authority approved animal-origin food establishments", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "ie.fsai.approved-directory", + "source_url": "https://www.fsai.ie/enforcement-and-legislation/official-controls/mancp/approved-food-premises", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-ie.md", + "docs/countries/ireland/v1-field-crosswalk.json", + "data/manifests/ireland-source-artifacts.json", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Use FSAI only as the coordinator; capture and review the linked DAFM, HSE, and SFPA artifacts separately.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official notice/news route; bounded notice capture after scope review", + "attribution": { + "attribution_required": true, + "notice": "FSAI notice content is enforcement context; person/business names and allegations require legal, privacy, and factual review before any linkage", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown; notice publication cadence is not a source-health guarantee", + "country_code": "IE", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Do not infer absence of an order from absence of a notice; preserve event scope and link only after reviewable identity resolution." + ] + }, + "jurisdiction_scope": "Ireland; FSAI enforcement-order notices and aggregate official-control enforcement context", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "ie.fsai.enforcement-orders", + "source_url": "https://www.fsai.ie/news-and-alerts/latest-news/fourteen-enforcement-orders-served-on-food-bus-%281%29", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-ie.md", + "docs/countries/ireland/v1-field-crosswalk.json", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Treat notices as dated event evidence and require reviewed subject identity before linkage.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "rendered official HSE/FSAI directory; bounded browser capture", + "attribution": { + "attribution_required": true, + "notice": "Public HSE/FSAI directory; address and trading-name fields are restricted pending terms/privacy review and publication approval", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown; page displayed last refreshed 2026-09-16", + "country_code": "IE", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Page-level refresh date is observed but no documented cadence or export contract was found; preserve all repeated child observations and do not publish addresses." + ] + }, + "jurisdiction_scope": "Ireland; low-throughput meat processors under HSE supervision", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": true, + "public_release_allowed": false, + "reasons": [ + "publication:blocked" + ], + "state": "awaiting-owner-review" + }, + "source_id": "ie.hse.low-throughput-meat", + "source_url": "https://oapi.fsai.ie/HSEApprovedEstablishments.aspx", + "status": { + "acquisition": "artifact_private_only", + "evidence": [ + "docs/country-recon-ie.md", + "docs/countries/ireland/v1-field-crosswalk.json", + "data/manifests/ireland-source-artifacts.json", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Use the bounded page observation as metadata only; obtain a lawful repeatable export/capture contract and keep repeated activity/species rows as children.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official map/open-data/service routes; bounded application-level capture", + "attribution": { + "attribution_required": true, + "notice": "MyPlan allows public information distribution/copying with byline credit, subject to data-use conditions; applicant/personal data and approximate GIS locations require review", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "weekly; MyPlan help states data are uploaded weekly", + "country_code": "IE", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Planning applications show development history or intent, not proof of operating approval; keep applications separate from facility entities and do not use approximate geometry for site-specific decisions." + ] + }, + "jurisdiction_scope": "Ireland; National Planning Application Map and local-authority planning application data", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "ie.planning.npad", + "source_url": "https://www.myplan.ie/national-planning-application-map-viewer/", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-ie.md", + "docs/countries/ireland/v1-field-crosswalk.json", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Capture application-level records separately; preserve approximate geometry and applicant privacy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "rendered official SFPA paginated table; bounded browser capture", + "attribution": { + "attribution_required": true, + "notice": "SFPA page is public but footer states copyright/all rights reserved; obtain reuse permission/interpretation and complete privacy review before redistribution", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown; table displayed updated 2026-09-15", + "country_code": "IE", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Pagination/export behavior, rights, and exact facility-versus-approval row semantics require a bounded artifact capture and review." + ] + }, + "jurisdiction_scope": "Ireland; SFPA establishments approved under Regulation (EC) No 853/2004 for fishery products and live bivalve molluscs", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": true, + "public_release_allowed": false, + "reasons": [ + "publication:blocked" + ], + "state": "awaiting-owner-review" + }, + "source_id": "ie.sfpa.approved-establishments", + "source_url": "https://www.sfpa.ie/What-We-Do/Seafood-Safety/Registration-Approval-of-Businesses/List-of-Approved-Establishments/Approved-Establishments", + "status": { + "acquisition": "artifact_private_only", + "evidence": [ + "docs/country-recon-ie.md", + "docs/countries/ireland/v1-field-crosswalk.json", + "data/manifests/ireland-source-artifacts.json", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Capture all paginated rows privately, confirm rights, and validate approval-number variant and facility semantics.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "rendered official SFPA table; bounded browser capture", + "attribution": { + "attribution_required": true, + "notice": "SFPA page is public but footer states copyright/all rights reserved; vessel data requires rights/privacy review", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown; page is live and no cadence is stated", + "country_code": "IE", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Keep factory-vessel rows as a separate entity class; do not infer a fixed premises or blend with freezer-vessel rows." + ] + }, + "jurisdiction_scope": "Ireland; SFPA factory vessels approved under Regulation (EC) No 853/2004", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": true, + "public_release_allowed": false, + "reasons": [ + "publication:blocked" + ], + "state": "awaiting-owner-review" + }, + "source_id": "ie.sfpa.factory-vessels", + "source_url": "https://www.sfpa.ie/What-We-Do/Seafood-Safety/Registration-Approval-of-Businesses/List-of-Approved-Establishments/Approved-Factory-Vessels", + "status": { + "acquisition": "artifact_private_only", + "evidence": [ + "docs/country-recon-ie.md", + "docs/countries/ireland/v1-field-crosswalk.json", + "data/manifests/ireland-source-artifacts.json", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Keep factory-vessel identity separate; capture and validate the one-entry table through a permitted route.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "rendered official SFPA paginated table; bounded browser capture", + "attribution": { + "attribution_required": true, + "notice": "SFPA page is public but footer states copyright/all rights reserved; vessel and address fields are restricted pending rights/privacy review", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown; table displayed updated 2026-09-15", + "country_code": "IE", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Vessel identity is mobile/non-fixed; do not treat the address as a stable facility location or sum vessels with establishments." + ] + }, + "jurisdiction_scope": "Ireland; SFPA freezer vessels approved under Regulation (EC) No 853/2004", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": true, + "public_release_allowed": false, + "reasons": [ + "publication:blocked" + ], + "state": "awaiting-owner-review" + }, + "source_id": "ie.sfpa.freezer-vessels", + "source_url": "https://www.sfpa.ie/What-We-Do/Seafood-Safety/Registration-Approval-of-Businesses/List-of-Approved-Establishments/Approved-Freezer-Vessels", + "status": { + "acquisition": "artifact_private_only", + "evidence": [ + "docs/country-recon-ie.md", + "docs/countries/ireland/v1-field-crosswalk.json", + "data/manifests/ireland-source-artifacts.json", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Keep vessel rows separate from fixed establishments and review address/rights handling.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + } + ], + "summary": "No release approval is implied; inspect source gates below." + }, + { + "acquisition_counts": { + "blocked": 3, + "not_run": 1 + }, + "basis": [ + "in.cpcb.environmental-compliance: blocked / blocked", + "in.dahd.livestock-statistics: not_run / reconnaissance", + "in.fssai.foscos: blocked / blocked", + "in.mca.company-master: blocked / blocked" + ], + "country_code": "IN", + "country_reasons": [ + "acquisition:blocked", + "acquisition:not_run", + "publication:blocked" + ], + "display_name": "IN", + "owner_review": "awaiting-owner-review", + "publication_state": "blocked", + "readiness_class": "blocked", + "source_count": 4, + "sources": [ + { + "access_method": "official portal, published policy, and sector/monitoring systems; no uncontrolled portal extraction", + "attribution": { + "attribution_required": true, + "notice": "Government environmental source; permit, monitoring and enforcement records require source-specific terms, safety and privacy review", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "system/report-specific; unknown nationally", + "country_code": "IN", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "No single national facility export verified; state PCB authority coverage, identifiers, route permissions, coordinate precision and status semantics remain unresolved." + ] + }, + "jurisdiction_scope": "India; Central Pollution Control Board industry monitoring and environmental compliance surfaces", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "in.cpcb.environmental-compliance", + "source_url": "https://cpcb.nic.in/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-in.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Map state/federal routes, identifiers, terms, privacy and status semantics before acquisition.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official reports, spreadsheets, and manuals from DAHD", + "attribution": { + "attribution_required": true, + "notice": "Official aggregate statistics; retain table/report metadata and suppress re-identification of small cells", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "annual BAHS; quinquennial livestock census; revisions and publication lag possible", + "country_code": "IN", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Pin stable report/table download identifiers and machine-readable route; household/holding microdata must remain restricted and is not a facility registry." + ] + }, + "jurisdiction_scope": "India; Department of Animal Husbandry and Dairying livestock census and Basic Animal Husbandry Statistics", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "in.dahd.livestock-statistics", + "source_url": "https://dahd.gov.in/schemes/programmes/animal-husbandry-statistics", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-in.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Pin report/table identifiers and machine-readable downloads; retain aggregate-only livestock context.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official public portal and FSSAI-described FBO search/verification; no automated query performed", + "attribution": { + "attribution_required": true, + "notice": "Government source; portal access does not settle bulk reuse, personal-address exposure, or publication rights", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "operational portal; statistics show publisher update date", + "country_code": "IN", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Search/export contract, stable identifiers, rate limits, privacy boundary, state coverage, and terms require authorized validation; do not scrape login or CAPTCHA surfaces." + ] + }, + "jurisdiction_scope": "India; FSSAI Food Safety Compliance System food-business licensing and registration", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "in.fssai.foscos", + "source_url": "https://foscos.fssai.gov.in/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-in.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Confirm authorized FBO search/export, identifiers, status semantics, terms, privacy, and state coverage.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official MCA public company-master lookup; no bulk extraction or authenticated route used", + "attribution": { + "attribution_required": true, + "notice": "Corporate identity source; registered-office and director-personal fields require minimization and privacy review", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "operational registry; provider-specific", + "country_code": "IN", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Current lookup/API contract, terms, rate limits, historical identity semantics, and safe fields require authorized validation; do not expose director or residential information." + ] + }, + "jurisdiction_scope": "India; Ministry of Corporate Affairs company/LLP master data", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "in.mca.company-master", + "source_url": "https://www.mca.gov.in/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-in.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Confirm public lookup contract and safe corporate fields; exclude director/personal exposure.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + } + ], + "summary": "No release approval is implied; inspect source gates below." + }, + { + "acquisition_counts": { + "artifact_private_only": 1, + "not_run": 1 + }, + "basis": [ + "it.1069-2009: not_run / reconnaissance", + "it.853-2004: artifact_private_only / awaiting-owner-review" + ], + "country_code": "IT", + "country_reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "display_name": "IT", + "owner_review": "awaiting-owner-review", + "publication_state": "blocked", + "readiness_class": "private-candidate-ready", + "source_count": 2, + "sources": [ + { + "access_method": "not acquired; separate catalog candidate", + "attribution": { + "attribution_required": true, + "notice": "Separate Ministry catalog and Italian Open Data Licence v2.0 indication; no adapter or publication decision.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "daily", + "country_code": "IT", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Assess scope, terms, schema, identity/link semantics, privacy, and whether this source belongs in the project before acquisition." + ] + }, + "jurisdiction_scope": "Italy; Ministry of Health establishments for animal by-products under Regulation (EC) 1069/2009", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "it.1069-2009", + "source_url": "https://www.dati.salute.gov.it/it/dataset/stabilimenti-italiani-i-sottoprodotti-di-origine-animale/", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-it.md", + "pipeline/source_registry.json", + "pipeline/sources/italy/README.md" + ], + "metadata": "verified", + "next_action": "Keep the by-products dataset separate; assess scope, schema, terms, identity links, privacy, and a dedicated adapter before any acquisition or integration.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "stable catalog page -> same-origin dated CSV discovery", + "attribution": { + "attribution_required": true, + "notice": "Ministry of Health catalog identifies Italian Open Data Licence v2.0; terms evidence and privacy review remain required before publication.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "daily", + "country_code": "IT", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Resolve repeated establishment/activity identity, coded values, coordinate provenance, address privacy, coverage, and project publication approval before release review." + ] + }, + "jurisdiction_scope": "Italy; Ministry of Health establishments recognized under Regulation (EC) 853/2004", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": true, + "public_release_allowed": false, + "reasons": [ + "publication:blocked" + ], + "state": "awaiting-owner-review" + }, + "source_id": "it.853-2004", + "source_url": "https://www.dati.salute.gov.it/it/dataset/stabilimenti-italiani-gli-alimenti-di-origine-animale/", + "status": { + "acquisition": "artifact_private_only", + "evidence": [ + "docs/country-recon-it.md", + "pipeline/source_registry.json", + "pipeline/sources/italy/acquire.py", + "pipeline/sources/italy/it_853_adapter.py", + "pipeline/sources/italy/README.md", + "pipeline/common/review_packet.py", + "docs/review-packet-italy.md", + "pipeline/tests/e2e/test_italy_candidate_import.py" + ], + "metadata": "verified", + "next_action": "Keep catalog acquisition, lifecycle, candidate import, and guarded API evidence private; use source-category/activity diagnostics to resolve repeated identity, coordinate/address privacy, coverage, and project approval before release review.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + } + ], + "summary": "No release approval is implied; inspect source gates below." + }, + { + "acquisition_counts": { + "blocked": 5, + "not_run": 1 + }, + "basis": [ + "lb.cas.livestock-statistics: not_run / reconnaissance", + "lb.industry.food-guide: blocked / blocked", + "lb.justice.companies: blocked / blocked", + "lb.moa.approved-food: blocked / blocked", + "lb.moa.farms-livestock: blocked / blocked", + "lb.moe.environment-eia: blocked / blocked" + ], + "country_code": "LB", + "country_reasons": [ + "acquisition:blocked", + "acquisition:not_run", + "publication:blocked" + ], + "display_name": "LB", + "owner_review": "awaiting-owner-review", + "publication_state": "blocked", + "readiness_class": "blocked", + "source_count": 6, + "sources": [ + { + "access_method": "official tables/publications; authorized query/download to be pinned", + "attribution": { + "attribution_required": true, + "notice": "Revisions, licensing, and suppression rules require review", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "table-specific; unknown", + "country_code": "LB", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Current table/API identifiers and terms not pinned." + ] + }, + "jurisdiction_scope": "Lebanon; Central Administration of Statistics aggregate livestock indicators", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "lb.cas.livestock-statistics", + "source_url": "https://www.cas.gov.lb/", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-lb.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin aggregate table/API identifiers, cadence, revisions, terms, and suppression rules.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official guide/list; authorized bounded download only", + "attribution": { + "attribution_required": true, + "notice": "Terms, schema, IDs, and safe address boundary require review", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "2022 visible; current unknown", + "country_code": "LB", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Current export and reuse terms not pinned." + ] + }, + "jurisdiction_scope": "Lebanon; Ministry of Industry licensed food factories", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "lb.industry.food-guide", + "source_url": "https://www.industry.gov.lb/IndustrialStatistics/IndustrialGuide", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-lb.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Confirm current list schema, licensing, cadence, and safe address boundary.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official interactive search; do not bypass controls", + "attribution": { + "attribution_required": true, + "notice": "Coverage, fees, terms, and personal-address policy require review", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown", + "country_code": "LB", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Machine route and nationwide coverage not verified." + ] + }, + "jurisdiction_scope": "Lebanon; Ministry of Justice commercial register", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "lb.justice.companies", + "source_url": "https://cr.justice.gov.lb/index.aspx", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-lb.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Confirm authorized access, coverage, identifiers, fees, terms, and personal-address policy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official pages; authorized bounded export/API only", + "attribution": { + "attribution_required": true, + "notice": "Government source; safety, privacy, terms, and conflict-sensitive locations require review", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown", + "country_code": "LB", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "No safe authorized bulk/API contract, IDs, cadence, or terms pinned." + ] + }, + "jurisdiction_scope": "Lebanon; Ministry of Agriculture slaughterhouses and animal-origin food establishments", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "lb.moa.approved-food", + "source_url": "https://www.agriculture.gov.lb/Subjects/Animal-Wealth/laws", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-lb.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin safe authorized route, schema, IDs, cadence, terms, privacy, and conflict-sensitive location policy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official route; aggregate-only until safe access verified", + "attribution": { + "attribution_required": true, + "notice": "Potentially operationally sensitive; privacy and safety review required", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown", + "country_code": "LB", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Fields, export/API, IDs, licensing, and safety controls unknown." + ] + }, + "jurisdiction_scope": "Lebanon; Ministry of Agriculture farms, livestock registration, farmer registry", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "lb.moa.farms-livestock", + "source_url": "https://www.agriculture.gov.lb/Media/News/2025/Summary-Report-%E2%80%93-Farmers-Registry-in-Lebanon", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-lb.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Keep farm evidence aggregate-only until safe access and operational-security review.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official framework/register; authorized API/export only", + "attribution": { + "attribution_required": true, + "notice": "Permit geometry and terms require safety/privacy review", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown", + "country_code": "LB", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "No current public permit API/export or safe geometry route pinned." + ] + }, + "jurisdiction_scope": "Lebanon; Ministry of Environment EIA and environmental review", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "lb.moe.environment-eia", + "source_url": "https://www.moe.gov.lb/MOE%20Site/SEA/SEA%20in%20Lebanon.htm", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-lb.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin current permit route and safe geometry/privacy/terms controls.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + } + ], + "summary": "No release approval is implied; inspect source gates below." + }, + { + "acquisition_counts": { + "blocked": 3, + "not_run": 4 + }, + "basis": [ + "lt.animal-experiments: blocked / blocked", + "lt.environment.permits: blocked / blocked", + "lt.jar.organizations: not_run / reconnaissance", + "lt.statistics.slaughter: not_run / reconnaissance", + "lt.vmvt.approved-food: not_run / reconnaissance", + "lt.vmvt.farm-aquaculture: blocked / blocked", + "lt.vmvt.inspections: not_run / reconnaissance" + ], + "country_code": "LT", + "country_reasons": [ + "acquisition:blocked", + "acquisition:not_run", + "metadata:unknown", + "publication:blocked" + ], + "display_name": "LT", + "owner_review": "awaiting-owner-review", + "publication_state": "blocked", + "readiness_class": "blocked", + "source_count": 7, + "sources": [ + { + "access_method": "official guidance/reports; facility route unverified", + "attribution": { + "attribution_required": true, + "notice": "Sensitive research evidence; aggregate/anonymize and require review.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "annual/report-specific", + "country_code": "LT", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "No public research-facility master verified." + ] + }, + "jurisdiction_scope": "Lithuania; animal experimentation and welfare evidence", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "lt.animal-experiments", + "source_url": "https://vmvt.lrv.lt/lt/atviri-vmvt-duomenys-ir-registrai/atviri-duomenys/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-lt.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Keep aggregate research evidence separate; no facility master verified.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "national open-data/environmental permit routes; not verified", + "attribution": { + "attribution_required": true, + "notice": "Verify publisher, license, sensitive-site handling and geometry.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown", + "country_code": "LT", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "No stable national animal-facility permit export verified." + ] + }, + "jurisdiction_scope": "Lithuania; environmental permits and releases", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "metadata:unknown", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "lt.environment.permits", + "source_url": "https://data.gov.lt/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-lt.md", + "pipeline/source_registry.json" + ], + "metadata": "unknown", + "next_action": "Locate and verify a stable environmental permit route.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official register/API route unverified", + "attribution": { + "attribution_required": true, + "notice": "Identity-only linkage; suppress personal, sole-trader and residential details.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "provider-defined", + "country_code": "LT", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Verify public API/access, terms, quotas and privacy policy." + ] + }, + "jurisdiction_scope": "Lithuania; legal-entity register identity", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "lt.jar.organizations", + "source_url": "https://www.registrucentras.lt/", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-lt.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Verify public API/access, terms and identity-only privacy policy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official statistical tables/API; exact table IDs unverified", + "attribution": { + "attribution_required": true, + "notice": "Official statistics; preserve dimensions, revisions and license metadata.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "table-specific", + "country_code": "LT", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Pin table IDs/API contract; keep aggregates separate from facilities." + ] + }, + "jurisdiction_scope": "Lithuania; official slaughter, livestock and animal-use statistics", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "lt.statistics.slaughter", + "source_url": "https://osp.stat.gov.lt/", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-lt.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin table IDs/API and preserve aggregate revisions.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "open-data API, CSV, JSON and JSONL resources", + "attribution": { + "attribution_required": true, + "notice": "Official VMVT; catalogue states CC BY 4.0; verify current terms and privacy.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "varies by resource; timestamp retrieval", + "country_code": "LT", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Verify current resource URLs, schema, completeness, terms and sensitive-field suppression." + ] + }, + "jurisdiction_scope": "Lithuania; VMVT approved food and veterinary-control establishments", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "lt.vmvt.approved-food", + "source_url": "https://data.gov.lt/dataset/valstybines-veterinarines-kontroles-subjektai", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-lt.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Verify current resource URLs, schema, completeness, terms and privacy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official registers/open-data routes", + "attribution": { + "attribution_required": true, + "notice": "Official authority; animal-holder/property locations require strict privacy review.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "resource-specific", + "country_code": "LT", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Verify public facility scope, current export/API, CRS, terms and coverage." + ] + }, + "jurisdiction_scope": "Lithuania; VMVT animal, herd and aquaculture establishment evidence", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "lt.vmvt.farm-aquaculture", + "source_url": "https://vmvt.lrv.lt/lt/atviri-vmvt-duomenys-ir-registrai/atviri-duomenys/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-lt.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Verify public scope, export/API, CRS, terms and coverage.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "open-data API/CSV/JSON/JSONL", + "attribution": { + "attribution_required": true, + "notice": "Official control evidence; preserve event status and privacy; CC BY 4.0 catalogue claim requires confirmation.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "varies; portal says not uniform", + "country_code": "LT", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Verify current resource/version, event semantics and coverage." + ] + }, + "jurisdiction_scope": "Lithuania; VMVT veterinary-control inspections/enforcement", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "lt.vmvt.inspections", + "source_url": "https://data.gov.lt/dataset/valstybines-veterinarines-kontroles-subjektai", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-lt.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Verify resource version, event semantics and coverage.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + } + ], + "summary": "No release approval is implied; inspect source gates below." + }, + { + "acquisition_counts": { + "blocked": 2, + "not_run": 4 + }, + "basis": [ + "lv.animal-experiments: blocked / blocked", + "lv.environment.permits: not_run / reconnaissance", + "lv.ldc.slaughter-farms: blocked / blocked", + "lv.pvd.approved-food: not_run / reconnaissance", + "lv.stat.api: not_run / reconnaissance", + "lv.ur.organizations: not_run / reconnaissance" + ], + "country_code": "LV", + "country_reasons": [ + "acquisition:blocked", + "acquisition:not_run", + "publication:blocked" + ], + "display_name": "LV", + "owner_review": "awaiting-owner-review", + "publication_state": "blocked", + "readiness_class": "blocked", + "source_count": 6, + "sources": [ + { + "access_method": "official guidance/registers; public facility/statistics route unverified", + "attribution": { + "attribution_required": true, + "notice": "Sensitive research/inspection evidence; aggregate, anonymize and require review.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown", + "country_code": "LV", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "No stable public facility-level route verified." + ] + }, + "jurisdiction_scope": "Latvia; animal experimentation, inspections and enforcement evidence", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "lv.animal-experiments", + "source_url": "https://registri.pvd.gov.lv/en/cr", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-lv.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Keep research/inspection evidence aggregate until a public route is verified.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "open-data CSV/resource and public registers", + "attribution": { + "attribution_required": true, + "notice": "Catalogue states CC0 1.0; preserve source and verify resource scope/geometry.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "dataset-specific", + "country_code": "LV", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Verify current resource URLs, schema, coverage and privacy." + ] + }, + "jurisdiction_scope": "Latvia; VVD environmental permits and public registers", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "lv.environment.permits", + "source_url": "https://data.gov.lv/dati/dataset/izsniegtas-atlaujas-un-licences", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-lv.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Verify resource URLs, schema, coverage, geometry and privacy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "public filtered register and statistics", + "attribution": { + "attribution_required": true, + "notice": "Official register; animal-holder/property privacy review required.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown; timestamp access", + "country_code": "LV", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Verify bulk/API route, coverage, terms and sensitive-field policy." + ] + }, + "jurisdiction_scope": "Latvia; LDC slaughterhouse, herd/location and livestock register evidence", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "lv.ldc.slaughter-farms", + "source_url": "https://registri.ldc.gov.lv/en/slaughterhouses", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-lv.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Verify bulk/API route, coverage, terms and sensitive-field policy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "machine-readable ZIP/CSV and sectioned XLSX lists", + "attribution": { + "attribution_required": true, + "notice": "Official PVD/FVS; verify dataset terms and privacy before release.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "publisher-defined; timestamp retrieval", + "country_code": "LV", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Verify current files, section coverage, IDs, cadence, terms and privacy." + ] + }, + "jurisdiction_scope": "Latvia; PVD/FVS approved and registered food, feed and ABP establishments", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "lv.pvd.approved-food", + "source_url": "https://pakalpojumi.pvd.gov.lv/en/opendata_files/ipvd_object_opendata", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-lv.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Verify current files, section coverage, IDs, cadence, terms and privacy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "PxWeb API v2", + "attribution": { + "attribution_required": true, + "notice": "Official statistics; preserve table IDs, revisions and license/attribution metadata.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "table-specific; 30 requests/10 seconds/IP stated", + "country_code": "LV", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Pin table IDs and metadata; keep aggregate scope separate." + ] + }, + "jurisdiction_scope": "Latvia; official statistics API for slaughter, livestock and animal-use aggregates", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "lv.stat.api", + "source_url": "https://stat.gov.lv/en/api-un-kodu-vardnicas/api-v2", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-lv.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Pin table IDs/metadata and preserve aggregate revisions.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official register/API route unverified", + "attribution": { + "attribution_required": true, + "notice": "Identity-only linkage; suppress personal/sole-trader/residential details.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "provider-defined", + "country_code": "LV", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Verify current API/access, terms and privacy policy." + ] + }, + "jurisdiction_scope": "Latvia; Latvian enterprise registration identifiers", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "lv.ur.organizations", + "source_url": "https://www.ur.gov.lv/en/", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-lv.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Verify current API/access, terms and privacy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + } + ], + "summary": "No release approval is implied; inspect source gates below." + }, + { + "acquisition_counts": { + "blocked": 5, + "not_run": 1 + }, + "basis": [ + "md.ansa.approved-food: blocked / blocked", + "md.ansa.farms-aquaculture: blocked / blocked", + "md.ansa.inspections-experiments: blocked / blocked", + "md.asp.organizations: blocked / blocked", + "md.environment-permits: blocked / blocked", + "md.stat.statistics: not_run / reconnaissance" + ], + "country_code": "MD", + "country_reasons": [ + "acquisition:blocked", + "acquisition:not_run", + "publication:blocked" + ], + "display_name": "MD", + "owner_review": "awaiting-owner-review", + "publication_state": "blocked", + "readiness_class": "blocked", + "source_count": 6, + "sources": [ + { + "access_method": "official ANSA category pages and linked lists/documents", + "attribution": { + "attribution_required": true, + "notice": "Official authority; verify file terms, attribution, privacy and coordinates before acquisition.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown; timestamp retrieval", + "country_code": "MD", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Pin stable ANSA file routes, schemas, cadence, IDs, terms and privacy." + ] + }, + "jurisdiction_scope": "Moldova; ANSA authorized and registered animal-origin food establishments", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "md.ansa.approved-food", + "source_url": "https://www.ansa.gov.md/siguranta-alimentelor.html", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-md.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Pin current ANSA files, schemas, cadence, IDs, terms and privacy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official ANSA registers, checklists and linked documents", + "attribution": { + "attribution_required": true, + "notice": "Verify publisher, license, animal-holder/property privacy and coordinate precision.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "resource-specific", + "country_code": "MD", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Verify complete holding/aquaculture exports, IDs, cadence, terms and privacy." + ] + }, + "jurisdiction_scope": "Moldova; ANSA livestock holdings, fish farms and aquaculture-related evidence", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "md.ansa.farms-aquaculture", + "source_url": "https://www.ansa.gov.md/sanatatea-si-bunastarea-animalelor.html", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-md.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Verify holding/fish-farm exports, IDs, cadence, coordinates, terms and privacy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official control checklists, reports and linked registers", + "attribution": { + "attribution_required": true, + "notice": "Sensitive control/research evidence; aggregate or anonymize and require project review.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "resource/report-specific", + "country_code": "MD", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "No stable public experimental-animal facility master verified; pin current fields and privacy policy." + ] + }, + "jurisdiction_scope": "Moldova; ANSA inspections/enforcement and animal-experimentation evidence", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "md.ansa.inspections-experiments", + "source_url": "https://ansa.gov.md/conducerea/liste-de-verificare.html", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-md.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Locate experimentation facility route; keep sensitive evidence aggregate until verified.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official online extracts and contracted ACCES-Web/statistical services", + "attribution": { + "attribution_required": true, + "notice": "Identity linkage only; contracts, fees and personal/beneficial-owner privacy apply.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "service-defined; online/non-stop services described", + "country_code": "MD", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Pin authorized service/API, fees, fields, cadence, site matching and privacy policy." + ] + }, + "jurisdiction_scope": "Moldova; Public Services Agency State Register of Legal Entities", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "md.asp.organizations", + "source_url": "https://asp.gov.md/en/servicii/persoane-juridice/informatii-afaceri", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-md.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Pin authorized ASP service/API, fees, fields, cadence, matching and privacy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official environmental authority portals, public notices and documents", + "attribution": { + "attribution_required": true, + "notice": "Verify public-read rights, document license, geometry and privacy.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "permit/document-specific", + "country_code": "MD", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Stable public read/API/export and complete facility coverage are unresolved." + ] + }, + "jurisdiction_scope": "Moldova; environmental permits and authorizations", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "md.environment-permits", + "source_url": "https://www.mediu.gov.md/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-md.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Verify public environmental permit route, coverage, documents, geometry and privacy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "National Bureau of Statistics tables/data services; exact route unverified", + "attribution": { + "attribution_required": true, + "notice": "Official statistics; verify table terms, citation requirements and aggregate-only scope.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "table-specific; preserve revisions", + "country_code": "MD", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Pin current slaughter and animal-use table IDs, API/download contracts and cadence." + ] + }, + "jurisdiction_scope": "Moldova; official aggregate slaughter, livestock and animal-use statistics", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "md.stat.statistics", + "source_url": "https://statistica.gov.md/en", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-md.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin official slaughter/animal-use table IDs, APIs, cadence and revisions.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + } + ], + "summary": "No release approval is implied; inspect source gates below." + }, + { + "acquisition_counts": { + "blocked": 5, + "not_run": 1 + }, + "basis": [ + "mk.crm.organizations: blocked / blocked", + "mk.environment-permits: blocked / blocked", + "mk.fva.approved-food: blocked / blocked", + "mk.fva.farms-aquaculture: blocked / blocked", + "mk.fva.inspections-experiments: blocked / blocked", + "mk.stat.statistics: not_run / reconnaissance" + ], + "country_code": "MK", + "country_reasons": [ + "acquisition:blocked", + "acquisition:not_run", + "publication:blocked" + ], + "display_name": "MK", + "owner_review": "awaiting-owner-review", + "publication_state": "blocked", + "readiness_class": "blocked", + "source_count": 6, + "sources": [ + { + "access_method": "authorized online distribution system; paid/prepaid services", + "attribution": { + "attribution_required": true, + "notice": "Central Registry terms restrict commercial reproduction/modification without prior consent; identity linkage only.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "service-defined; current/historical products", + "country_code": "MK", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Confirm commercial permission, fees, API/service contract, quotas, fields and privacy." + ] + }, + "jurisdiction_scope": "North Macedonia; Central Registry legal-entity records", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "mk.crm.organizations", + "source_url": "https://www.crm.com.mk/en/professional-users/lessors/access-to-data-via-the-online-distribution-system", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-mk.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Confirm authorized commercial access, fees, service contract, fields and privacy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official ministry portals, public notices and documents", + "attribution": { + "attribution_required": true, + "notice": "Verify public-read rights, document license, geometry and privacy.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "permit/document-specific", + "country_code": "MK", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Stable public read/API/export route and complete facility coverage are unresolved." + ] + }, + "jurisdiction_scope": "North Macedonia; environmental permits and physical-planning authorizations", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "mk.environment-permits", + "source_url": "https://www.moepp.gov.mk/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-mk.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Verify public environmental permit API/export, coverage, documents, geometry and privacy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official FVA register pages with linked Google Drive documents", + "attribution": { + "attribution_required": true, + "notice": "Official authority; verify file terms, attribution, privacy and coordinates before acquisition.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown; timestamp retrieval", + "country_code": "MK", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Pin stable Drive file IDs, schemas, cadence, IDs, terms and privacy." + ] + }, + "jurisdiction_scope": "North Macedonia; FVA approved and registered food establishments handling animal-origin products", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "mk.fva.approved-food", + "source_url": "https://fva.gov.mk/mk/registri-hrana-zivotinsko-poteklo", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-mk.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Pin stable FVA/Drive files, schemas, cadence, IDs, terms and privacy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official FVA registers and linked documents", + "attribution": { + "attribution_required": true, + "notice": "Verify publisher, license, animal-holder/property privacy and coordinate precision.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "resource-specific", + "country_code": "MK", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Verify current farm/aquaculture files, IDs, cadence, terms and privacy." + ] + }, + "jurisdiction_scope": "North Macedonia; FVA livestock, slaughterhouse, holding and aquaculture-related evidence", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "mk.fva.farms-aquaculture", + "source_url": "https://fva.gov.mk/mk/zdravstvena-zastita-blagosostojba-zivotni-1", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-mk.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Verify farm/aquaculture files, IDs, cadence, coordinates, terms and privacy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official FVA registers and linked documents", + "attribution": { + "attribution_required": true, + "notice": "Sensitive control/research evidence; aggregate or anonymize and require project review.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "resource-specific", + "country_code": "MK", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Pin current register file/API, fields, IDs, privacy and retention policy." + ] + }, + "jurisdiction_scope": "North Macedonia; FVA inspections/enforcement and animal-experimentation institutions", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "mk.fva.inspections-experiments", + "source_url": "https://fva.gov.mk/mk/zdravstvena-zastita-blagosostojba-zivotni-1", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-mk.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Pin experimentation register/API, fields, IDs, privacy and retention policy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "State Statistical Office tables/data services; exact route unverified", + "attribution": { + "attribution_required": true, + "notice": "Official statistics; verify table terms, citation requirements and aggregate-only scope.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "table-specific; preserve revisions", + "country_code": "MK", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Pin current slaughter and animal-use table IDs, API/download contracts and cadence." + ] + }, + "jurisdiction_scope": "North Macedonia; official aggregate slaughter, livestock and animal-use statistics", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "mk.stat.statistics", + "source_url": "https://www.stat.gov.mk/", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-mk.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin official slaughter/animal-use table IDs, APIs, cadence and revisions.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + } + ], + "summary": "No release approval is implied; inspect source gates below." + }, + { + "acquisition_counts": { + "blocked": 1 + }, + "basis": [ + "mx.locations: blocked / blocked" + ], + "country_code": "MX", + "country_reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "display_name": "MX", + "owner_review": "awaiting-owner-review", + "publication_state": "blocked", + "readiness_class": "blocked", + "source_count": 1, + "sources": [ + { + "access_method": "download/API or assisted export", + "attribution": { + "attribution_required": true, + "notice": "unknown; separate official source terms from INEGI-derived research artifacts before reuse", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown", + "country_code": "MX", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Split official source identities from derived DENUE/aquaculture work and establish source-level provenance." + ] + }, + "jurisdiction_scope": "Mexico; mixed official-register and INEGI-derived legacy coverage", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "mx.locations", + "source_url": "unknown", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-mx.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Resolve DENUE token/terms and SENASICA current directory/schema before bounded acquisition.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + } + ], + "summary": "No release approval is implied; inspect source gates below." + }, + { + "acquisition_counts": { + "artifact_private_only": 7, + "blocked": 1, + "not_run": 1 + }, + "basis": [ + "nl.cbs.livestock: artifact_private_only / awaiting-owner-review", + "nl.cbs.slaughter: artifact_private_only / awaiting-owner-review", + "nl.cokz.dairy-eggs: artifact_private_only / awaiting-owner-review", + "nl.koop.local-permits: artifact_private_only / awaiting-owner-review", + "nl.kvk.hvds: not_run / reconnaissance", + "nl.nvwa.approved-food: artifact_private_only / awaiting-owner-review", + "nl.nvwa.welfare-enforcement: artifact_private_only / awaiting-owner-review", + "nl.pdok.omgevingswet: artifact_private_only / awaiting-owner-review", + "nl.rvo.ir: blocked / blocked" + ], + "country_code": "NL", + "country_reasons": [ + "acquisition:blocked", + "acquisition:not_run", + "publication:blocked" + ], + "display_name": "NL", + "owner_review": "awaiting-owner-review", + "publication_state": "blocked", + "readiness_class": "blocked", + "source_count": 9, + "sources": [ + { + "access_method": "public OData API", + "attribution": { + "attribution_required": true, + "notice": "Official CBS; cite CBS and verify dataset-specific reuse terms.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "twice-yearly; latest periods may be provisional", + "country_code": "NL", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Do not interpret as named facilities or coordinates; preserve scope/status." + ] + }, + "jurisdiction_scope": "Netherlands; CBS aggregate livestock statistics", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": true, + "public_release_allowed": false, + "reasons": [ + "publication:blocked" + ], + "state": "awaiting-owner-review" + }, + "source_id": "nl.cbs.livestock", + "source_url": "https://opendata.cbs.nl/ODataApi/OData/84952NED", + "status": { + "acquisition": "artifact_private_only", + "evidence": [ + "docs/country-recon-nl.md", + "data/manifests/nl-source-artifacts.json", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Keep twice-yearly aggregate data separate from named facilities and preserve provisional status.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "public OData API", + "attribution": { + "attribution_required": true, + "notice": "Official CBS; cite CBS and verify dataset-specific reuse terms.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "monthly; new figures about two months after reference month", + "country_code": "NL", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Keep outside facility tables; preserve provisional/unknown/secret flags." + ] + }, + "jurisdiction_scope": "Netherlands; CBS monthly aggregate slaughter statistics", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": true, + "public_release_allowed": false, + "reasons": [ + "publication:blocked" + ], + "state": "awaiting-owner-review" + }, + "source_id": "nl.cbs.slaughter", + "source_url": "https://opendata.cbs.nl/ODataApi/OData/7123slac", + "status": { + "acquisition": "artifact_private_only", + "evidence": [ + "docs/country-recon-nl.md", + "data/manifests/nl-source-artifacts.json", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Keep monthly aggregate claims outside facility tables and retain CBS flags and scope.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official HTML registers", + "attribution": { + "attribution_required": true, + "notice": "Official delegated regulator; terms and machine reuse not verified.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "register-specific update dates; uniform API cadence unknown", + "country_code": "NL", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Confirm all register families, lifecycle, terms and overlap with NVWA." + ] + }, + "jurisdiction_scope": "Netherlands; COKZ approved dairy, farm-dairy, egg and egg-product establishments", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": true, + "public_release_allowed": false, + "reasons": [ + "publication:blocked" + ], + "state": "awaiting-owner-review" + }, + "source_id": "nl.cokz.dairy-eggs", + "source_url": "https://cokz.nl/", + "status": { + "acquisition": "artifact_private_only", + "evidence": [ + "docs/country-recon-nl.md", + "docs/countries/nl/v1-field-crosswalk.json", + "data/manifests/nl-source-artifacts.json", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Confirm COKZ register families, update semantics, terms and overlap with NVWA before modeling.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "KOOP/Overheid.nl SRU and local feeds", + "attribution": { + "attribution_required": true, + "notice": "Official publication evidence; preserve authority, identity, dates and state.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "daily publication surface; local coverage varies", + "country_code": "NL", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Use collection-specific CQL; initial keyword query returned diagnostics; no complete national permit coverage claim." + ] + }, + "jurisdiction_scope": "Netherlands; official publications and local permit/announcement evidence", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": true, + "public_release_allowed": false, + "reasons": [ + "publication:blocked" + ], + "state": "awaiting-owner-review" + }, + "source_id": "nl.koop.local-permits", + "source_url": "https://zoek.officielebekendmakingen.nl/sru/Search", + "status": { + "acquisition": "artifact_private_only", + "evidence": [ + "docs/country-recon-nl.md", + "data/manifests/nl-source-artifacts.json", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Replace keyword query with collection-specific CQL and document incomplete local coverage.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "open dataset API or subscribed APIs; no calls made", + "attribution": { + "attribution_required": true, + "notice": "Open dataset documented CC BY 4.0; API terms and privacy restrictions apply.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "on-demand; provider update cadence per run", + "country_code": "NL", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Confirm terms/privacy; use only for reviewed identity links, never facility proof or UBO." + ] + }, + "jurisdiction_scope": "Netherlands; KVK business and establishment identity/link evidence", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "nl.kvk.hvds", + "source_url": "https://developers.kvk.nl/nl/documentation/open-dataset-basis-bedrijfsgegevens-api", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-nl.md", + "docs/countries/nl/v1-field-crosswalk.json", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Confirm API route and privacy/terms review; use KVK only for reviewed identity links, never UBO or facility proof.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "public control XML plus SOAP POST; bounded private capture", + "attribution": { + "attribution_required": true, + "notice": "Official NVWA; confirm reuse, privacy and project approval before release.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown; control file observed current 2026-09-15", + "country_code": "NL", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Implement SOAP adapter; confirm pagination, coverage, lifecycle semantics, terms, privacy and identity policy." + ] + }, + "jurisdiction_scope": "Netherlands; NVWA approved food establishments, slaughter/cutting lists", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": true, + "public_release_allowed": false, + "reasons": [ + "publication:blocked" + ], + "state": "awaiting-owner-review" + }, + "source_id": "nl.nvwa.approved-food", + "source_url": "https://www.nvwa.nl/site/binaries/content/assets/site-content/webapp-data/lijsten-erkende-bedrijven/stuurbestand-lijsten-erkende-bedrijven", + "status": { + "acquisition": "artifact_private_only", + "evidence": [ + "docs/country-recon-nl.md", + "docs/countries/nl/v1-field-crosswalk.json", + "data/manifests/nl-source-artifacts.json", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Implement SOAP only after confirming coverage, repeated observations, lifecycle, terms, privacy and identity policy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official HTML pages and PDF tables", + "attribution": { + "attribution_required": true, + "notice": "Official NVWA; preserve publication/effective period and confirm reuse/privacy.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "annual summary; detailed tables are period-specific", + "country_code": "NL", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Model inspections and enforcement separately; absence is not closure; review identity links." + ] + }, + "jurisdiction_scope": "Netherlands; NVWA welfare, animal-experiment and red-meat compliance publications", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": true, + "public_release_allowed": false, + "reasons": [ + "publication:blocked" + ], + "state": "awaiting-owner-review" + }, + "source_id": "nl.nvwa.welfare-enforcement", + "source_url": "https://www.nvwa.nl/over-de-nvwa/publicaties/jaarbeeld-2025/dierenwelzijn", + "status": { + "acquisition": "artifact_private_only", + "evidence": [ + "docs/country-recon-nl.md", + "data/manifests/nl-source-artifacts.json", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Keep 2025 aggregate welfare and 2024 detailed compliance evidence dated and separate; review identity links and release terms.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "public PDOK OGC API plus DSO context APIs", + "attribution": { + "attribution_required": true, + "notice": "PDOK metadata states CC0 1.0; DSO terms and legal interpretation remain gates.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "daily; production collections observed updated 2026-09-15", + "country_code": "NL", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Resolve document/legal scope; no national animal-facility master implied." + ] + }, + "jurisdiction_scope": "Netherlands; DSO/PDOK planning geometry/document evidence", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": true, + "public_release_allowed": false, + "reasons": [ + "publication:blocked" + ], + "state": "awaiting-owner-review" + }, + "source_id": "nl.pdok.omgevingswet", + "source_url": "https://api.pdok.nl/omgevingswet/omgevingsdocumenten/ogc/v2?f=html&lang=nl", + "status": { + "acquisition": "artifact_private_only", + "evidence": [ + "docs/country-recon-nl.md", + "data/manifests/nl-source-artifacts.json", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Resolve DSO document identity and legal scope before treating geometry as permit evidence.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "authorized web services and WSDL/XSD", + "attribution": { + "attribution_required": true, + "notice": "Official RVO; holder, location and animal data may be restricted/personal.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "24/7 service; refresh cadence unknown", + "country_code": "NL", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Obtain purpose-specific authorization; do not scrape or expose restricted data." + ] + }, + "jurisdiction_scope": "Netherlands; RVO I&R animal and UBN/location services", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "nl.rvo.ir", + "source_url": "https://www.rvo.nl/form/bestanden-webservices", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-nl.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Do not acquire until purpose-specific authorization and privacy/terms review exists; keep UBN separate.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + } + ], + "summary": "No release approval is implied; inspect source gates below." + }, + { + "acquisition_counts": { + "blocked": 2, + "not_run": 6 + }, + "basis": [ + "no.brreg.organizations: not_run / reconnaissance", + "no.fiskeridir.aquaculture: not_run / reconnaissance", + "no.landbruksdirektoratet.farm-register: blocked / blocked", + "no.mattilsynet.animal-experiments: not_run / reconnaissance", + "no.mattilsynet.approved-food: not_run / reconnaissance", + "no.mattilsynet.feed-abp: not_run / reconnaissance", + "no.miljodirektoratet.prtr-permits: blocked / blocked", + "no.ssb.meat-and-animal-use: not_run / reconnaissance" + ], + "country_code": "NO", + "country_reasons": [ + "acquisition:blocked", + "acquisition:not_run", + "publication:blocked" + ], + "display_name": "NO", + "owner_review": "awaiting-owner-review", + "publication_state": "blocked", + "readiness_class": "blocked", + "source_count": 8, + "sources": [ + { + "access_method": "public REST JSON API and CSV/gzip/XLSX downloads", + "attribution": { + "attribution_required": true, + "notice": "NLOD 2.0 stated by publisher; suppress person roles, birth numbers, sole-trader and mixed residential addresses.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "API/update-feed specific; timestamp each run", + "country_code": "NO", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Verify rate limits, endpoint versions, field semantics and identity-only matching policy." + ] + }, + "jurisdiction_scope": "Norway; Brønnøysundregistrene Central Coordinating Register open organization data", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "no.brreg.organizations", + "source_url": "https://data.brreg.no/enhetsregisteret/api/dokumentasjon/en/index.html", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-no.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Verify endpoint versions, rate limits and identity-only privacy policy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "public REST/API catalogue, CSV/XLSX downloads and ArcGIS feature service", + "attribution": { + "attribution_required": true, + "notice": "Official Fiskeridirektoratet; preserve provider attribution, CRS/geometry semantics and verify current API terms.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "provider-defined; timestamp API and layer metadata", + "country_code": "NO", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Capture OpenAPI/version metadata, pagination, field semantics, geometry precision and release/privacy terms." + ] + }, + "jurisdiction_scope": "Norway; Fiskeridirektoratet Aquaculture Register localities and permits", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "no.fiskeridir.aquaculture", + "source_url": "https://api.fiskeridir.no/catalog/", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-no.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Capture API metadata and validate pagination, identifiers, geometry and terms.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official register guidance and aggregate/downloadable reports", + "attribution": { + "attribution_required": true, + "notice": "Official authority; farm/person/property linkage is sensitive and must not be exposed without purpose and review.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "report/register-specific; exact public export cadence unknown", + "country_code": "NO", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "No public facility-level export verified; obtain authorization and privacy review before any acquisition." + ] + }, + "jurisdiction_scope": "Norway; Landbruksdirektoratet agricultural property/farm register and livestock statistics", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "no.landbruksdirektoratet.farm-register", + "source_url": "https://www.landbruksdirektoratet.no/nb/jordbruk/kart-og-register/landbruksregisteret", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-no.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Do not acquire property/person/farm rows without an authorized purpose-limited route.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "annual reports and ALURES statistical database; reporting system authenticated", + "attribution": { + "attribution_required": true, + "notice": "Sensitive research/animal-use evidence; aggregate/anonymize and keep facility claims out of canonical entities.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "annual", + "country_code": "NO", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "No public research-facility master verified; resolve stable report links and privacy boundary." + ] + }, + "jurisdiction_scope": "Norway; experimental-animal use statistics and reporting", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "no.mattilsynet.animal-experiments", + "source_url": "https://www.mattilsynet.no/dyr/forsoksdyr/bruk-av-dyr-i-forsok", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-no.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Keep annual aggregates separate; no research-facility master is verified.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official section pages with CSV-backed lists", + "attribution": { + "attribution_required": true, + "notice": "Official Norwegian Food Safety Authority; confirm file-specific reuse terms; names/addresses require privacy review.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "most lists stated daily; timestamp each retrieval", + "country_code": "NO", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Verify complete section coverage, direct CSV URLs, headers, terms, privacy and approval-ID semantics." + ] + }, + "jurisdiction_scope": "Norway; Mattilsynet approved animal-origin food establishments", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "no.mattilsynet.approved-food", + "source_url": "https://www.mattilsynet.no/godkjente-produkter-og-virksomheter", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-no.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Verify complete section coverage, CSV contracts, terms, privacy and approval-ID semantics.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official sectioned list pages and CSV/HTML links", + "attribution": { + "attribution_required": true, + "notice": "Official authority; terms and category-specific privacy review required.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "publisher list cadence; verify per section", + "country_code": "NO", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Verify direct files, category boundaries, completeness, terms and privacy before acquisition." + ] + }, + "jurisdiction_scope": "Norway; Mattilsynet approved/registered feed and animal-by-product establishments", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "no.mattilsynet.feed-abp", + "source_url": "https://www.mattilsynet.no/godkjente-produkter-og-virksomheter/forvarefeed-sector-approved-and-registered-feed-companies-tse", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-no.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Verify direct files, category boundaries, completeness, terms and privacy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official environmental register/document references; public bulk route not verified", + "attribution": { + "attribution_required": true, + "notice": "Official environmental authority; terms, sensitive sites and document privacy require review.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "annual/report-specific; unresolved", + "country_code": "NO", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Verify current public search/export/API, stable identifiers, completeness and coordinate/address exposure." + ] + }, + "jurisdiction_scope": "Norway; Norwegian Environment Agency pollutant-release and permit evidence", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "no.miljodirektoratet.prtr-permits", + "source_url": "https://www.miljodirektoratet.no/globalassets/publikasjoner/M138/M138.pdf", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-no.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Verify current public environmental export/API and stable IDs before acquisition.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "open SSB PxWeb/StatBank API plus official annual animal-use reports", + "attribution": { + "attribution_required": true, + "notice": "SSB API states CC BY 4.0; cite table/source and preserve revisions; animal-use reports require separate provenance.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "SSB daily 08:00 update; animal-use annual", + "country_code": "NO", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Pin table IDs and metadata contracts; preserve aggregate scope and do not double-count facility sources." + ] + }, + "jurisdiction_scope": "Norway; Statistics Norway meat production and Mattilsynet animal-use aggregates", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "no.ssb.meat-and-animal-use", + "source_url": "https://www.ssb.no/en/api", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-no.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Pin SSB table IDs and metadata; preserve aggregate scope and revisions.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + } + ], + "summary": "No release approval is implied; inspect source gates below." + }, + { + "acquisition_counts": { + "blocked": 1 + }, + "basis": [ + "nz.locations: blocked / blocked" + ], + "country_code": "NZ", + "country_reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "display_name": "NZ", + "owner_review": "awaiting-owner-review", + "publication_state": "blocked", + "readiness_class": "blocked", + "source_count": 1, + "sources": [ + { + "access_method": "browser-assisted public-register export or download", + "attribution": { + "attribution_required": true, + "notice": "Legacy notes identify MPI; verify current terms, attribution, and whether the register endpoint is stable.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown", + "country_code": "NZ", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Confirm current register coverage, export format, access limits, and publication semantics." + ] + }, + "jurisdiction_scope": "New Zealand; MPI approved premises/register coverage", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "nz.locations", + "source_url": "https://mpi.my.site.com/publicregister/s/RiskMeasureSearch?riskMeasureType=Risk%20Management%20Programme", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-nz.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Confirm MPI permitted acquisition route, list-specific terms, and schema before private staging.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + } + ], + "summary": "No release approval is implied; inspect source gates below." + }, + { + "acquisition_counts": { + "blocked": 3, + "not_run": 8 + }, + "basis": [ + "pl.arimr.processing-support: not_run / reconnaissance", + "pl.gdos.eia: blocked / blocked", + "pl.geoportal.urban-planning: not_run / reconnaissance", + "pl.gios.ippc: not_run / reconnaissance", + "pl.giw.abp: not_run / reconnaissance", + "pl.giw.approved-food: blocked / blocked", + "pl.giw.registered-food: not_run / reconnaissance", + "pl.giw.rrw: not_run / reconnaissance", + "pl.gus.regon-bir: blocked / blocked", + "pl.gus.slaughter: not_run / reconnaissance", + "pl.krs.open-api: not_run / reconnaissance" + ], + "country_code": "PL", + "country_reasons": [ + "acquisition:blocked", + "acquisition:not_run", + "publication:blocked" + ], + "display_name": "PL", + "owner_review": "awaiting-owner-review", + "publication_state": "blocked", + "readiness_class": "blocked", + "source_count": 11, + "sources": [ + { + "access_method": "official call/guidance; beneficiary/project dataset not acquired", + "attribution": { + "attribution_required": true, + "notice": "Polish official funding evidence; beneficiary/privacy terms review required", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "call-specific; 2026 call window observed 1-30 September", + "country_code": "PL", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Do not treat funding as facility authorization, operation or compliance." + ] + }, + "jurisdiction_scope": "Poland; ARiMR agricultural processing investment support", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "pl.arimr.processing-support", + "source_url": "https://www.gov.pl/web/arimr/startuje-wsparcie-dla-przetworcow", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-pl.md", + "docs/countries/pl/source-crosswalk.json", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Acquire only permitted funding/project evidence; do not interpret funding as facility operation or compliance.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official BIP database/search; page states access from outside Poland is blocked", + "attribution": { + "attribution_required": true, + "notice": "Polish official source; document privacy and access restrictions apply", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "authority entry within 30 days; public release cadence unknown", + "country_code": "PL", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Obtain an in-Poland or authorized bounded capture; no facility-master inference." + ] + }, + "jurisdiction_scope": "Poland; GDOŚ environmental-impact-assessment database", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "pl.gdos.eia", + "source_url": "https://www.gov.pl/web/gdos/bazy-danych-o-ocenach-oddzialywania-na-srodowisko", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-pl.md", + "docs/countries/pl/source-crosswalk.json", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Use an in-Poland or authorized context for bounded EIA acquisition; source states outside-Poland access is blocked.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "public WMS/register announcement; service contract not acquired", + "attribution": { + "attribution_required": true, + "notice": "Polish official spatial data; preserve service terms and legal dates", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "regular local-government updates; transition noted for end of September 2026", + "country_code": "PL", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Capture production service metadata and keep zoning context separate from facility identity." + ] + }, + "jurisdiction_scope": "Poland; Geoportal/Urban Register planning data", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "pl.geoportal.urban-planning", + "source_url": "https://www.geoportal.gov.pl/aktualnosci/nowe-uslugi-w-geoportalu-rejestr-urbanistyczny/", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-pl.md", + "docs/countries/pl/source-crosswalk.json", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Capture production Urban Register/WMS metadata and preserve planning act identity and legal/effective dates.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "central official page with 16 regional links; no national export verified", + "attribution": { + "attribution_required": true, + "notice": "Polish official environmental source; regional terms/privacy review required", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "regional/unknown", + "country_code": "PL", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Capture regional schemas and retain permit/installations as evidence overlays." + ] + }, + "jurisdiction_scope": "Poland; GIOŚ and 16 WIOŚ integrated-permit installation registers", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "pl.gios.ippc", + "source_url": "https://www.gov.pl/web/gios/instalacje-wymagajace-uzyskania-pozwolenia-zintegrowanego", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-pl.md", + "docs/countries/pl/source-crosswalk.json", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Capture regional WIOŚ registers one voivodeship at a time and keep permits/installations as dated environmental evidence.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "GIW page plus linked pasze.wetgiw.gov.pl register; exact export unresolved", + "attribution": { + "attribution_required": true, + "notice": "Polish official source under ABP rules; terms/privacy/release review required", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown", + "country_code": "PL", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Verify current list route and preserve 1069/2009 scope separately from food facilities." + ] + }, + "jurisdiction_scope": "Poland; GIW animal by-product establishments and operators", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "pl.giw.abp", + "source_url": "https://www.wetgiw.gov.pl/handel-eksport-import/niespozywcze-produkty-pochodzenia-zwierzecego", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-pl.md", + "docs/countries/pl/source-crosswalk.json", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Verify the current ABP list route/schema and preserve Regulation 1069/2009 scope outside food-facility totals.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official rendered HTML register with filters plus XLS export route; bounded metadata-only observation", + "attribution": { + "attribution_required": true, + "notice": "Polish official source; terms, privacy and project release review required", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown; current views observed 2026-09-16", + "country_code": "PL", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Acquire a bounded source artifact and hash; resolve WNI lifecycle/repeat semantics; do not sum section rows; review privacy/terms/release approval." + ] + }, + "jurisdiction_scope": "Poland; GIW approved animal-origin food establishments under Regulation (EC) 853/2004", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "pl.giw.approved-food", + "source_url": "https://www.wetgiw.gov.pl/handel-eksport-import/listy-zakladow", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-pl.md", + "docs/countries/pl/source-crosswalk.json", + "data/manifests/pl-source-artifacts.json", + "data/raw/poland/20260916T000000Z/metadata.json", + "pipeline/source_registry.json", + "pipeline/poland/test_metadata.py" + ], + "metadata": "verified", + "next_action": "Acquire a bounded GIW XLS/HTML artifact from an allowed context, record source hash/schema, and model WNI activity/species/product observations without summing sections or publishing addresses.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official index with 14 list families; list-specific routes not acquired", + "attribution": { + "attribution_required": true, + "notice": "Polish official source; list-specific terms/privacy review required", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown", + "country_code": "PL", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Capture one list at a time and keep registration separate from 853/2004 approval." + ] + }, + "jurisdiction_scope": "Poland; GIW registered animal-origin food-sector activities", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "pl.giw.registered-food", + "source_url": "https://www.wetgiw.gov.pl/nadzor-weterynaryjny/wykaz-zakladow-rejestrowanych", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-pl.md", + "docs/countries/pl/source-crosswalk.json", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Capture one registered-list family at a time; preserve list-specific identifiers and keep registration separate from 853/2004 approval.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official RRW publication/index; no current file acquired", + "attribution": { + "attribution_required": true, + "notice": "Polish official aggregate/evidence reports; small-cell and case privacy review required", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "annual/report-specific", + "country_code": "PL", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Acquire current reports and model them as dated evidence, not facility master rows." + ] + }, + "jurisdiction_scope": "Poland; GIW veterinary statistical reporting", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "pl.giw.rrw", + "source_url": "https://www.wetgiw.gov.pl/publikacje/rrw-sprawozdawczosc-statystyczna/printpage", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-pl.md", + "docs/countries/pl/source-crosswalk.json", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Acquire current RRW reports and model welfare, controls, enforcement and meat-examination evidence by report/table/year or source-local ID.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "documented SOAP/API service; registration/user key required", + "attribution": { + "attribution_required": true, + "notice": "Official GUS service; respect registration, rate limits and personal-data handling", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "continuously updated register; on-demand queries", + "country_code": "PL", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Obtain authorized access before queries and keep fields restricted." + ] + }, + "jurisdiction_scope": "Poland; GUS REGON BIR1 identity service", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "pl.gus.regon-bir", + "source_url": "https://api.stat.gov.pl/Home/RegonApi?lang=en", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-pl.md", + "docs/countries/pl/source-crosswalk.json", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Obtain authorized BIR1 access before queries, respect documented rate limits, and keep sole-trader/restricted fields private.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official HTML publication with XLSX attachment; page observed", + "attribution": { + "attribution_required": true, + "notice": "GUS official statistics; preserve revisions/flags and source terms", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "monthly collection; 2025 publication dated 2026-03-02", + "country_code": "PL", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Never infer named facilities or join totals to GIW WNI; keep aggregate context separate." + ] + }, + "jurisdiction_scope": "Poland; Statistics Poland slaughter statistics", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "pl.gus.slaughter", + "source_url": "https://stat.gov.pl/obszary-tematyczne/rolnictwo-lesnictwo/produkcja-zwierzeca-zwierzeta-gospodarskie/uboje-zwierzat-gospodarskich-w-2025-r-,16,1.html", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-pl.md", + "docs/countries/pl/source-crosswalk.json", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Keep GUS R-09U aggregates separate from GIW WNI facilities and preserve statistical revisions/flags.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "open API announced by Ministry; no request made", + "attribution": { + "attribution_required": true, + "notice": "Official KRS; RODO filtering and API terms review required", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "on-demand/registry filings", + "country_code": "PL", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Confirm current API contract; exact identity link only, never operating-site inference." + ] + }, + "jurisdiction_scope": "Poland; Ministry of Justice KRS legal-entity identity", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "pl.krs.open-api", + "source_url": "https://prs.ms.gov.pl/krs/openApi", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-pl.md", + "docs/countries/pl/source-crosswalk.json", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Confirm current KRS API contract and RODO filtering; use only for exact organization identity links supplied by an upstream source.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + } + ], + "summary": "No release approval is implied; inspect source gates below." + }, + { + "acquisition_counts": { + "blocked": 5, + "not_run": 1 + }, + "basis": [ + "pt.apambiente.tua: blocked / blocked", + "pt.dgav.abp: blocked / blocked", + "pt.dgav.approved-food: blocked / blocked", + "pt.dgav.feed: blocked / blocked", + "pt.ifap.snira: blocked / blocked", + "pt.ine.animal-production: not_run / reconnaissance" + ], + "country_code": "PT", + "country_reasons": [ + "acquisition:blocked", + "acquisition:not_run", + "publication:blocked" + ], + "display_name": "PT", + "owner_review": "awaiting-owner-review", + "publication_state": "blocked", + "readiness_class": "blocked", + "source_count": 6, + "sources": [ + { + "access_method": "official guidance and electronic-title/document route; no stable national public export verified", + "attribution": { + "attribution_required": true, + "notice": "Portuguese environmental authority source; document reuse, geometry, personal-address, and permit-condition publication terms require review", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "decision/document-specific", + "country_code": "PT", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Locate and verify a current public search/export contract; keep TUA separate from DGAV approval and do not infer facility identity from holder or permit text." + ] + }, + "jurisdiction_scope": "Portugal; Agência Portuguesa do Ambiente Título Único Ambiental and linked environmental licensing decisions", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "pt.apambiente.tua", + "source_url": "https://apambiente.pt/avaliacao-e-gestao-ambiental/titulo-unico-ambiental", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-pt.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Locate a current public TUA search/export contract and safe permit fields; keep environmental decisions as a separate reviewed evidence overlay.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official DGAV public +SIPACE/legacy SIPACE list family; separate ABP section capture required", + "attribution": { + "attribution_required": true, + "notice": "Portuguese government source; ABP-specific reuse, attribution, privacy, and redistribution terms were not verified", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "not stated; timestamp every retrieval", + "country_code": "PT", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Do not union with food or feed lists; exact ABP section routes, codebook, export, cadence, privacy, terms, and coverage remain unresolved." + ] + }, + "jurisdiction_scope": "Portugal; DGAV animal-by-product establishments, installations, and operators approved, registered, or authorized under Regulations (EC) 1069/2009 and 142/2011", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "pt.dgav.abp", + "source_url": "https://maissipace.dgav.pt/Listagens", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-pt.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Capture ABP sections separately from food; resolve approval/registration IDs, category/activity codes, cadence, coverage, terms, privacy, and project approval.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official DGAV public +SIPACE/legacy SIPACE list family; browser-assisted bounded capture only until a stable export is verified", + "attribution": { + "attribution_required": true, + "notice": "Portuguese government source; source-specific reuse, attribution, personal-address, and redistribution terms were not verified", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "not stated; timestamp every retrieval and preserve list/section context", + "country_code": "PT", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "The new +SIPACE route is transitional and no stable bulk/API contract was verified; confirm section coverage, pagination, status/effective-date semantics, codes, privacy, terms, and project approval before acquisition." + ] + }, + "jurisdiction_scope": "Portugal; DGAV approved and registered food establishments, including animal-origin establishments under Regulation (EC) 853/2004", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "pt.dgav.approved-food", + "source_url": "https://maissipace.dgav.pt/Listagens", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-pt.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Obtain an authorized bounded public-list capture; fingerprint section headers, pagination, NCV/NII status semantics, codes, coverage, terms, privacy, and project approval before an adapter.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official DGAV public +SIPACE/legacy SIPACE list family; separate feed section capture required", + "attribution": { + "attribution_required": true, + "notice": "Portuguese government source; feed-list reuse, attribution, privacy, and redistribution terms were not verified", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "not stated; timestamp every retrieval", + "country_code": "PT", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "The feed identifier is not interchangeable with an NCV; confirm current public section, codebook, pagination, export, cadence, private/primary-producer boundary, terms, and coverage." + ] + }, + "jurisdiction_scope": "Portugal; DGAV feed-sector establishments and operators registered or approved under Regulation (EC) 183/2005", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "pt.dgav.feed", + "source_url": "https://maissipace.dgav.pt/Listagens", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-pt.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Confirm the public feed section and NII semantics with an authorized capture; keep feed separate from NCV food/ABP rows and review private/primary-producer exposure.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "restricted IFAP area and credentialed webservice; no public acquisition authorized", + "attribution": { + "attribution_required": true, + "notice": "Restricted animal/holder data; purpose limitation, access control, retention, and privacy review are mandatory", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "operational system; provider-specific", + "country_code": "PT", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Do not scrape or retain rows; obtain explicit authorization only if a narrowly scoped relationship study is approved, and suppress holder, farm, parcel, and precise-location details." + ] + }, + "jurisdiction_scope": "Portugal; IFAP/DGAV SNIRA animal-identification, holding, movement, and herd information", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "pt.ifap.snira", + "source_url": "https://www.ifap.pt/portal/en/registo-area-reservada", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-pt.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Do not acquire without explicit authorization; keep holder, animal, farm, parcel, and precise-location data restricted and assess any future relationship study separately.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official INE metadata/PxWeb statistics route; table-specific API or download to be pinned", + "attribution": { + "attribution_required": true, + "notice": "Official statistics; preserve table metadata, revisions, confidentiality flags, and source terms", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "annual/semestral/monthly by table; metadata checked through 2025/2026", + "country_code": "PT", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Pin reproducible table IDs, API contract, revisions, terms, and confidentiality rules; never infer facility coverage or join confidential slaughter surveys to named establishments." + ] + }, + "jurisdiction_scope": "Portugal; Statistics Portugal aggregate animal-production, meat, and slaughter statistics", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "pt.ine.animal-production", + "source_url": "https://ine.pt/bddXplorer/htdocs/minfo.jsp?lingua=EN&var_cd=0000916&var_cd=0000917&var_cd=0000918", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-pt.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Pin reproducible INE table/API identifiers and revision/confidentiality terms; keep statistics aggregate and separate from named-facility evidence.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + } + ], + "summary": "No release approval is implied; inspect source gates below." + }, + { + "acquisition_counts": { + "blocked": 4, + "not_run": 2 + }, + "basis": [ + "ro.ansvsa.approved-food: blocked / blocked", + "ro.environment-permits: blocked / blocked", + "ro.farm-aquaculture: blocked / blocked", + "ro.inspections-experiments: blocked / blocked", + "ro.insse.statistics: not_run / reconnaissance", + "ro.onrc.organizations: not_run / reconnaissance" + ], + "country_code": "RO", + "country_reasons": [ + "acquisition:blocked", + "acquisition:not_run", + "publication:blocked" + ], + "display_name": "RO", + "owner_review": "awaiting-owner-review", + "publication_state": "blocked", + "readiness_class": "blocked", + "source_count": 6, + "sources": [ + { + "access_method": "official ANSVSA/DSVSA registers and relevant EU approval surfaces; national bulk route unverified", + "attribution": { + "attribution_required": true, + "notice": "Official authority; verify national list license, privacy, attribution and source terms before acquisition.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown; timestamp retrieval", + "country_code": "RO", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Pin current national list/API/export, stable identifiers, cadence, coordinates, terms and privacy." + ] + }, + "jurisdiction_scope": "Romania; ANSVSA approved and registered food establishments handling food of animal origin", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "ro.ansvsa.approved-food", + "source_url": "https://portal.ansvsa.ro/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-ro.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin national list/API/export, identifiers, cadence, coordinates, terms and privacy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "SIM/eFORM official web system and document/publication routes", + "attribution": { + "attribution_required": true, + "notice": "Official environmental authority; verify public-read rights, document license, geometry and privacy.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown; system currently reports technical unavailability", + "country_code": "RO", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Public read/API route and dependable availability are unresolved; ANPM reports SIM technical outage." + ] + }, + "jurisdiction_scope": "Romania; ANPM environmental authorizations and integrated environmental system", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "ro.environment-permits", + "source_url": "https://raportare.anpm.ro/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-ro.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Recheck ANPM availability and verify public read/API, document route, license, geometry and privacy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "national open-data catalogue, agriculture records and official environmental/biodiversity systems", + "attribution": { + "attribution_required": true, + "notice": "Verify publisher, license, animal-holder/property privacy and coordinate precision.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "resource-specific", + "country_code": "RO", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "No complete national farm/aquaculture public export contract verified." + ] + }, + "jurisdiction_scope": "Romania; farm, holding and aquaculture establishment evidence", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "ro.farm-aquaculture", + "source_url": "https://data.gov.ro/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-ro.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Verify national farm/aquaculture scope, export/API, IDs, coordinates, terms and privacy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official portals, reports and statistical publications", + "attribution": { + "attribution_required": true, + "notice": "Sensitive control/research evidence; aggregate or anonymize and require project review.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "report/register-specific", + "country_code": "RO", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "No stable public facility-level inspection or animal-experimentation master verified." + ] + }, + "jurisdiction_scope": "Romania; ANSVSA/DSVSA inspections, enforcement and animal-experimentation evidence", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "ro.inspections-experiments", + "source_url": "https://portal.ansvsa.ro/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-ro.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Locate a stable public facility/event route; keep sensitive evidence aggregate until verified.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official statistical tables/data services; exact table/API route unverified", + "attribution": { + "attribution_required": true, + "notice": "Official statistics; verify table terms, citation requirements and aggregate-only scope.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "table-specific; preserve revisions", + "country_code": "RO", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Pin current slaughter and animal-use table IDs, API/download contracts and cadence." + ] + }, + "jurisdiction_scope": "Romania; INSSE aggregate slaughter, livestock and animal-use statistics", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "ro.insse.statistics", + "source_url": "https://insse.ro/cms/", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-ro.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin official slaughter/animal-use table IDs, APIs, cadence and revisions.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "data.gov.ro CKAN CSV snapshots and API metadata", + "attribution": { + "attribution_required": true, + "notice": "Catalogue snapshots are public and some are CC BY 4.0; verify current file license, field privacy and attribution.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "snapshot-specific; timestamp each release", + "country_code": "RO", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Pin current snapshot, cadence, field dictionary and registered-office publication policy." + ] + }, + "jurisdiction_scope": "Romania; ONRC legal-entity and authorized-activity snapshots", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "ro.onrc.organizations", + "source_url": "https://data.gov.ro/dataset?organiza=&organization=onrc&res_format=csv", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-ro.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Pin current CSV snapshot, cadence, fields, license and registered-office privacy policy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + } + ], + "summary": "No release approval is implied; inspect source gates below." + }, + { + "acquisition_counts": { + "blocked": 5, + "not_run": 1 + }, + "basis": [ + "rs.apr.organizations: blocked / blocked", + "rs.environment-permits: blocked / blocked", + "rs.farm-aquaculture: blocked / blocked", + "rs.inspections-experiments: blocked / blocked", + "rs.stat.statistics: not_run / reconnaissance", + "rs.veterinary.approved-food: blocked / blocked" + ], + "country_code": "RS", + "country_reasons": [ + "acquisition:blocked", + "acquisition:not_run", + "publication:blocked" + ], + "display_name": "RS", + "owner_review": "awaiting-owner-review", + "publication_state": "blocked", + "readiness_class": "blocked", + "source_count": 6, + "sources": [ + { + "access_method": "official browser and authorized web services; automated downloading restricted", + "attribution": { + "attribution_required": true, + "notice": "Identity linkage only; respect APR terms, fees, access restrictions and registered-office privacy.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "provider-defined", + "country_code": "RS", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Pin authorized API/service, fees, quotas, cadence, fields and automation permissions." + ] + }, + "jurisdiction_scope": "Serbia; APR centralized business-entity registers", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "rs.apr.organizations", + "source_url": "https://www.apr.gov.rs/registers/media/data-search.1728.html", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-rs.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Pin authorized APR service, fees, quotas, cadence, fields and automation permissions.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official environmental agency systems, public notices and documents", + "attribution": { + "attribution_required": true, + "notice": "Verify public-read rights, document license, geometry and privacy.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "permit/document-specific", + "country_code": "RS", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Stable public read/API/export route and complete national coverage are unresolved." + ] + }, + "jurisdiction_scope": "Serbia; environmental permits and environmental authorization evidence", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "rs.environment-permits", + "source_url": "https://www.sepa.gov.rs/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-rs.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Verify public environmental permit route, coverage, documents, geometry, license and privacy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "open-data portal, agriculture resources and official fisheries/veterinary registers", + "attribution": { + "attribution_required": true, + "notice": "Verify publisher, license, animal-holder/property privacy and coordinate precision.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "resource-specific", + "country_code": "RS", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "No complete national farm/aquaculture establishment export contract verified." + ] + }, + "jurisdiction_scope": "Serbia; livestock holdings and aquaculture permit/establishment evidence", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "rs.farm-aquaculture", + "source_url": "https://data.gov.rs/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-rs.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Verify farm/aquaculture scope, export/API, IDs, coordinates, terms and privacy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official control plans, reports and statistical publications", + "attribution": { + "attribution_required": true, + "notice": "Sensitive control/research evidence; aggregate or anonymize and require project review.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "report/register-specific", + "country_code": "RS", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "No stable public facility-level inspection or animal-experimentation master verified." + ] + }, + "jurisdiction_scope": "Serbia; veterinary inspections/enforcement and animal-experimentation evidence", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "rs.inspections-experiments", + "source_url": "https://www.minpolj.gov.rs/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-rs.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Locate a stable public facility/event route; keep sensitive evidence aggregate until verified.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official JSON/CSV statistical APIs and data portal", + "attribution": { + "attribution_required": true, + "notice": "Official statistics; verify table terms, citation requirements and aggregate-only scope.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "table-specific; preserve revisions", + "country_code": "RS", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Pin current slaughter and animal-use dataset IDs, API contracts and cadence." + ] + }, + "jurisdiction_scope": "Serbia; Statistical Office aggregate slaughter, livestock and animal-use statistics", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "rs.stat.statistics", + "source_url": "https://www.stat.gov.rs/", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-rs.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Pin official slaughter/animal-use dataset IDs, APIs, cadence and revisions.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official ministry/veterinary registers and relevant EU/control surfaces; national bulk route unverified", + "attribution": { + "attribution_required": true, + "notice": "Official authority; verify list license, attribution, privacy and coordinates before acquisition.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown; timestamp retrieval", + "country_code": "RS", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Pin current national list/API/export, categories, IDs, cadence, terms and privacy." + ] + }, + "jurisdiction_scope": "Serbia; approved and registered food establishments handling food of animal origin", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "rs.veterinary.approved-food", + "source_url": "https://www.minpolj.gov.rs/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-rs.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin veterinary list/API/export, categories, IDs, cadence, terms and privacy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + } + ], + "summary": "No release approval is implied; inspect source gates below." + }, + { + "acquisition_counts": { + "not_run": 6 + }, + "basis": [ + "se.bolagsverket.company-api: not_run / reconnaissance", + "se.jordbruksverket.animal-experiments: not_run / reconnaissance", + "se.jordbruksverket.feed-abp: not_run / reconnaissance", + "se.jordbruksverket.slaughter-stats: not_run / reconnaissance", + "se.jordbruksverket.slaughterhouses: not_run / reconnaissance", + "se.naturvardsverket.prtr: not_run / reconnaissance" + ], + "country_code": "SE", + "country_reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "display_name": "SE", + "owner_review": "awaiting-owner-review", + "publication_state": "blocked", + "readiness_class": "acquisition-ready", + "source_count": 6, + "sources": [ + { + "access_method": "official API/download documentation; access not exercised", + "attribution": { + "attribution_required": true, + "notice": "Identity linkage only; API terms, personal-data handling, and sole-trader/mixed-address suppression required.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "provider-defined; auth and quota cadence unknown", + "country_code": "SE", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Confirm account/auth, rate limits, terms, field availability and privacy before automated crosswalk use; never treat corporate identity as facility proof." + ] + }, + "jurisdiction_scope": "Sweden; Bolagsverket corporate identity and organization-number API", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "se.bolagsverket.company-api", + "source_url": "https://bolagsverket.se/apierochoppnadata/hamtaforetagsinformation/apiforatthamtaforetagsinformation.3988.html", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-se.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Confirm API access, quotas, terms, and privacy before identity-only crosswalk use.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official guidance, annual statistics and ALURES summaries", + "attribution": { + "attribution_required": true, + "notice": "Sensitive animal-use/research evidence; anonymization, purpose limitation and privacy review required; terms not fully verified.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "annual statistics; ALURES summaries from 2021", + "country_code": "SE", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "No public national facility export verified; resolve stable statistics route and keep facility-level research claims out of canonical entities." + ] + }, + "jurisdiction_scope": "Sweden; experimental-animal permits, approvals and use statistics", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "se.jordbruksverket.animal-experiments", + "source_url": "https://jordbruksverket.se/djur/ovriga-djur/forsoksdjur-och-djurforsok/verksamhet-med-forsoksdjur", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-se.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Keep anonymized annual/ALURES summaries separate; no facility master is verified.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official landing page with linked XLSX/PDF downloads", + "attribution": { + "attribution_required": true, + "notice": "Official Swedish authority; file-specific reuse terms not verified; operator names/addresses may require privacy review.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "continuous updates stated by publisher", + "country_code": "SE", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Capture and fingerprint each linked file, verify headers/section semantics, direct-download stability, terms, and national coverage." + ] + }, + "jurisdiction_scope": "Sweden; Jordbruksverket feed and animal-by-product facility lists", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "se.jordbruksverket.feed-abp", + "source_url": "https://jordbruksverket.se/djur/foder-och-produkter-fran-djur/listor-over-anlaggningar-for-foder-och-animaliska-biprodukter-och-darav-framstallda-produkter", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-se.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Capture linked XLSX files privately, fingerprint headers/sections, and resolve terms, coverage, and privacy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official XLSX downloads and explanatory HTML", + "attribution": { + "attribution_required": true, + "notice": "Official Swedish authority; source-specific reuse terms not verified; preserve consent-limited naming and aggregate scope.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "weekly reporting; publication/update quarterly", + "country_code": "SE", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Do not infer facility completeness from consent-based named subset; verify file URLs, schema, revisions and terms." + ] + }, + "jurisdiction_scope": "Sweden; Jordbruksverket aggregate slaughter and classification statistics", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "se.jordbruksverket.slaughter-stats", + "source_url": "https://jordbruksverket.se/djur/djurtransportorer-och-slakterier/statistik-om-slaktade-djur-och-klassning", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-se.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Verify current quarterly/annual XLSX links and preserve consent-limited named versus Other scope.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "public live HTML table", + "attribution": { + "attribution_required": true, + "notice": "Official Swedish authority; source-specific reuse terms not verified; privacy review required for names and addresses if later captured.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "not stated; timestamp each retrieval", + "country_code": "SE", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Verify terms, live HTML stability, completeness, effective-date semantics, and privacy before automated acquisition." + ] + }, + "jurisdiction_scope": "Sweden; Jordbruksverket slaughterhouse installation-number table", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "se.jordbruksverket.slaughterhouses", + "source_url": "https://jordbruksverket.se/3608.html", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-se.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Verify live HTML stability, terms, completeness, effective-date semantics, and privacy before automated acquisition.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "public search service and browser-filtered document catalogue", + "attribution": { + "attribution_required": true, + "notice": "Official Swedish environmental authority; bulk reuse terms/API not verified; location and permit-document privacy review required.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "annual PRTR emissions/transfers; document publication varies", + "country_code": "SE", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Verify export/API, stable facility key, completeness, terms, and separation of PRTR from permit/food approval evidence." + ] + }, + "jurisdiction_scope": "Sweden; Swedish Pollutant Release and Transfer Register and environmental document catalogue", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "se.naturvardsverket.prtr", + "source_url": "https://www.naturvardsverket.se/en/services-and-permits/data-databases-and-applications/the-swedish-pollutant-release-and-transfer-register/", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-se.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Verify public export/API, facility key, completeness, terms, and separation from permit evidence.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + } + ], + "summary": "No release approval is implied; inspect source gates below." + }, + { + "acquisition_counts": { + "blocked": 2, + "not_run": 3 + }, + "basis": [ + "si.corporate-register: not_run / reconnaissance", + "si.environment.permits: blocked / blocked", + "si.statistics-slaughter: not_run / reconnaissance", + "si.uvhvvr.approved-food-feed: not_run / reconnaissance", + "si.uvhvvr.farms-aquaculture: blocked / blocked" + ], + "country_code": "SI", + "country_reasons": [ + "acquisition:blocked", + "acquisition:not_run", + "metadata:unknown", + "publication:blocked" + ], + "display_name": "SI", + "owner_review": "awaiting-owner-review", + "publication_state": "blocked", + "readiness_class": "blocked", + "source_count": 5, + "sources": [ + { + "access_method": "official business-register route; API unverified", + "attribution": { + "attribution_required": true, + "notice": "Identity-only linkage; suppress personal, sole-trader and residential details.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "provider-defined", + "country_code": "SI", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Verify current access, terms, quotas and privacy policy." + ] + }, + "jurisdiction_scope": "Slovenia; corporate identifiers", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "si.corporate-register", + "source_url": "https://www.ajpes.si/", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-si.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Verify current access, terms and identity-only privacy policy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official environmental routes; not verified", + "attribution": { + "attribution_required": true, + "notice": "Verify license, geometry and sensitive-site handling.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown", + "country_code": "SI", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "No stable national animal-facility permit export verified." + ] + }, + "jurisdiction_scope": "Slovenia; environmental permit evidence", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "metadata:unknown", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "si.environment.permits", + "source_url": "https://www.gov.si/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-si.md", + "pipeline/source_registry.json" + ], + "metadata": "unknown", + "next_action": "Locate and verify environmental permit data.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official PxWeb API/table", + "attribution": { + "attribution_required": true, + "notice": "Official statistics; preserve table ID, revisions and license metadata.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "annual/table-specific", + "country_code": "SI", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Pin API metadata and preserve aggregate scope." + ] + }, + "jurisdiction_scope": "Slovenia; official livestock slaughter statistics", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "si.statistics-slaughter", + "source_url": "https://pxweb.stat.si/SiStatData/pxweb/en/Data/-/H202S.px", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-si.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Pin PxWeb table/API metadata and revisions.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official PDF/list registers", + "attribution": { + "attribution_required": true, + "notice": "Official Slovenian authority; verify terms, attribution and privacy.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "publisher-defined; timestamp retrieval", + "country_code": "SI", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Verify current files/API, completeness, terms and coordinates/privacy." + ] + }, + "jurisdiction_scope": "Slovenia; UVHVVR approved food and feed establishments", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "si.uvhvvr.approved-food-feed", + "source_url": "https://www.gov.si/zbirke/storitve/odobritev-zivilskega-obrata/", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-si.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Verify current files/API, completeness, terms and privacy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "open-data catalogue and register systems", + "attribution": { + "attribution_required": true, + "notice": "Animal/property location data require strict privacy review.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "dataset-specific", + "country_code": "SI", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Verify public scope, export/API, terms and sensitive fields." + ] + }, + "jurisdiction_scope": "Slovenia; UVHVVR/OPSI animal holding and aquaculture data", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "si.uvhvvr.farms-aquaculture", + "source_url": "https://podatki.gov.si/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-si.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Verify public scope, export/API, terms and sensitive fields.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + } + ], + "summary": "No release approval is implied; inspect source gates below." + }, + { + "acquisition_counts": { + "blocked": 3, + "not_run": 2 + }, + "basis": [ + "sk.environment.permits: blocked / blocked", + "sk.statistics-corporate: not_run / reconnaissance", + "sk.svps.approved-food: not_run / reconnaissance", + "sk.svps.farms-aquaculture: blocked / blocked", + "sk.svps.inspections-experiments: blocked / blocked" + ], + "country_code": "SK", + "country_reasons": [ + "acquisition:blocked", + "acquisition:not_run", + "metadata:unknown", + "publication:blocked" + ], + "display_name": "SK", + "owner_review": "awaiting-owner-review", + "publication_state": "blocked", + "readiness_class": "blocked", + "source_count": 5, + "sources": [ + { + "access_method": "official environmental routes; not verified", + "attribution": { + "attribution_required": true, + "notice": "Verify license, geometry and sensitive-site handling.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown", + "country_code": "SK", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "No stable national animal-facility permit export verified." + ] + }, + "jurisdiction_scope": "Slovakia; environmental permit evidence", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "metadata:unknown", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "sk.environment.permits", + "source_url": "https://www.minzp.sk/en/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-sk.md", + "pipeline/source_registry.json" + ], + "metadata": "unknown", + "next_action": "Locate and verify environmental permit data.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official statistical/business-register routes; exact APIs unverified", + "attribution": { + "attribution_required": true, + "notice": "Identity-only corporate linkage; aggregate statistics preserve revisions and source terms.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "table/provider-specific", + "country_code": "SK", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Verify current table/API contracts, business-register access, terms and privacy." + ] + }, + "jurisdiction_scope": "Slovakia; official slaughter/livestock statistics and business identifiers", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "sk.statistics-corporate", + "source_url": "https://slovak.statistics.sk/", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-sk.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Verify statistics/business-register APIs, terms and privacy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official filtered lists and linked XSL/XLSX datasets", + "attribution": { + "attribution_required": true, + "notice": "Official Slovak authority; verify terms, attribution and privacy.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "list-specific dated updates", + "country_code": "SK", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Verify direct files, complete categories, stable IDs, terms and address/coordinate fields." + ] + }, + "jurisdiction_scope": "Slovakia; SVPS approved food, slaughter and ABP establishments", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "sk.svps.approved-food", + "source_url": "https://zoznamy.svps.sk/?LANG=EN", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-sk.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Verify direct files, categories, IDs, terms and address/coordinates.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official datasets and register links", + "attribution": { + "attribution_required": true, + "notice": "Animal-holder/site data require privacy review.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "resource-specific", + "country_code": "SK", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Verify public scope, current export/API, terms and sensitive-field policy." + ] + }, + "jurisdiction_scope": "Slovakia; SVPS farm, veterinary and aquaculture registers", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "sk.svps.farms-aquaculture", + "source_url": "https://svps.sk/datasety/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-sk.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Verify public scope, export/API, terms and sensitive fields.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official reports/systems; row-level route unverified", + "attribution": { + "attribution_required": true, + "notice": "Sensitive evidence; aggregate/anonymize and require review.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "annual/report-specific", + "country_code": "SK", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "No stable public facility-level route verified." + ] + }, + "jurisdiction_scope": "Slovakia; SVPS inspections, enforcement and animal-experiment evidence", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "sk.svps.inspections-experiments", + "source_url": "https://svps.sk/english/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-sk.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Keep aggregate/event evidence separate until a public route is verified.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + } + ], + "summary": "No release approval is implied; inspect source gates below." + }, + { + "acquisition_counts": { + "blocked": 5, + "not_run": 1 + }, + "basis": [ + "tr.cevre.environment-permits: blocked / blocked", + "tr.mersis.organizations: blocked / blocked", + "tr.tarim.approved-food: blocked / blocked", + "tr.tarim.inspections-enforcement: blocked / blocked", + "tr.tarim.livestock-systems: blocked / blocked", + "tr.tuik.animal-statistics: not_run / reconnaissance" + ], + "country_code": "TR", + "country_reasons": [ + "acquisition:blocked", + "acquisition:not_run", + "publication:blocked" + ], + "display_name": "TR", + "owner_review": "awaiting-owner-review", + "publication_state": "blocked", + "readiness_class": "blocked", + "source_count": 6, + "sources": [ + { + "access_method": "official route; authorized bounded capture/API only", + "attribution": { + "attribution_required": true, + "notice": "Official Turkish government source; terms, privacy, access controls, and safety review required", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown; verify source-specific", + "country_code": "TR", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Exact machine contract, stable IDs, cadence, licensing, privacy, and automation permissions not pinned." + ] + }, + "jurisdiction_scope": "Turkey; environmental permits and EIA", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "tr.cevre.environment-permits", + "source_url": "https://csb.gov.tr/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-tr.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin authorized route, schema, cadence, IDs, terms, privacy, and automation permissions.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official route; authorized bounded capture/API only", + "attribution": { + "attribution_required": true, + "notice": "Official Turkish government source; terms, privacy, access controls, and safety review required", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown; verify source-specific", + "country_code": "TR", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Exact machine contract, stable IDs, cadence, licensing, privacy, and automation permissions not pinned." + ] + }, + "jurisdiction_scope": "Turkey; MERSIS central company registry", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "tr.mersis.organizations", + "source_url": "https://mersis.ticaret.gov.tr/Portal/KullaniciIslemleri/GirisIslemleri", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-tr.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin authorized route, schema, cadence, IDs, terms, privacy, and automation permissions.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official route; authorized bounded capture/API only", + "attribution": { + "attribution_required": true, + "notice": "Official Turkish government source; terms, privacy, access controls, and safety review required", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown; verify source-specific", + "country_code": "TR", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Exact machine contract, stable IDs, cadence, licensing, privacy, and automation permissions not pinned." + ] + }, + "jurisdiction_scope": "Turkey; Ministry approved food businesses and slaughterhouses", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "tr.tarim.approved-food", + "source_url": "https://www.tarimorman.gov.tr/Konular/Hayvancilik/hayvan-refah%C4%B1-kimliklendirme-ve-i%C5%9Fletme-onay/kesimhaneler", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-tr.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin authorized route, schema, cadence, IDs, terms, privacy, and automation permissions.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official route; authorized bounded capture/API only", + "attribution": { + "attribution_required": true, + "notice": "Official Turkish government source; terms, privacy, access controls, and safety review required", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown; verify source-specific", + "country_code": "TR", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Exact machine contract, stable IDs, cadence, licensing, privacy, and automation permissions not pinned." + ] + }, + "jurisdiction_scope": "Turkey; official controls and public animal-use evidence", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "tr.tarim.inspections-enforcement", + "source_url": "https://www.tarimorman.gov.tr/Konular/Gida-Ve-Yem-Hizmetleri/Gida-Hizmetleri/Resmi-Kontroller", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-tr.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin authorized route, schema, cadence, IDs, terms, privacy, and automation permissions.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official route; authorized bounded capture/API only", + "attribution": { + "attribution_required": true, + "notice": "Official Turkish government source; terms, privacy, access controls, and safety review required", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown; verify source-specific", + "country_code": "TR", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Exact machine contract, stable IDs, cadence, licensing, privacy, and automation permissions not pinned." + ] + }, + "jurisdiction_scope": "Turkey; Ministry animal registration and livestock systems", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "tr.tarim.livestock-systems", + "source_url": "https://www.tarimorman.gov.tr/HAYGEM/Menu/2/Hayvancilik", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-tr.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin authorized route, schema, cadence, IDs, terms, privacy, and automation permissions.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official route; authorized bounded capture/API only", + "attribution": { + "attribution_required": true, + "notice": "Official Turkish government source; terms, privacy, access controls, and safety review required", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown; verify source-specific", + "country_code": "TR", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Exact machine contract, stable IDs, cadence, licensing, privacy, and automation permissions not pinned." + ] + }, + "jurisdiction_scope": "Turkey; aggregate animal-production and slaughter statistics", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "tr.tuik.animal-statistics", + "source_url": "https://veriportali.tuik.gov.tr/en/press/58015", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-tr.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin authorized route, schema, cadence, IDs, terms, privacy, and automation permissions.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + } + ], + "summary": "No release approval is implied; inspect source gates below." + }, + { + "acquisition_counts": { + "blocked": 5, + "not_run": 1 + }, + "basis": [ + "ua.dpss.approved-food: blocked / blocked", + "ua.edr.organizations: blocked / blocked", + "ua.environment-permits: blocked / blocked", + "ua.farm-aquaculture: blocked / blocked", + "ua.inspections-experiments: blocked / blocked", + "ua.ukrstat.statistics: not_run / reconnaissance" + ], + "country_code": "UA", + "country_reasons": [ + "acquisition:blocked", + "acquisition:not_run", + "publication:blocked" + ], + "display_name": "UA", + "owner_review": "awaiting-owner-review", + "publication_state": "blocked", + "readiness_class": "blocked", + "source_count": 6, + "sources": [ + { + "access_method": "official registry/search route; authorized bounded export or API only", + "attribution": { + "attribution_required": true, + "notice": "Official Ukrainian government source; terms, privacy, wartime safety, and attribution require review", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown; verify from authorized metadata", + "country_code": "UA", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Bulk route, schema, stable ID, cadence, terms, privacy, and wartime safety controls are not pinned." + ] + }, + "jurisdiction_scope": "Ukraine; DPSS registered food-market operators and facilities, including animal-origin food", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "ua.dpss.approved-food", + "source_url": "https://dpss.gov.ua/diyalnist/bezpechnist-harchovih-produktiv-ta-veterinarna-medicina/reyestri", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-ua.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin authorized route, schema, cadence, IDs, terms, privacy, and wartime safety controls.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official open-data catalog discovery; authorized API/download only", + "attribution": { + "attribution_required": true, + "notice": "Government open-data source; field-level personal-address, terms, and rate-limit review required", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown", + "country_code": "UA", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Current authorized route, fields, cadence, licensing, privacy policy, and automation permissions not verified." + ] + }, + "jurisdiction_scope": "Ukraine; Unified State Register of legal entities and organizations", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "ua.edr.organizations", + "source_url": "https://data.gov.ua/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-ua.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin authorized route, schema, cadence, IDs, terms, privacy, and wartime safety controls.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official EcoSystem/register route; bounded authorized API/export only", + "attribution": { + "attribution_required": true, + "notice": "Ministry source; license, geometry, privacy, and security review required", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "resource-specific; unknown", + "country_code": "UA", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Exact public permit API/export, coverage, cadence, geometry policy, and wartime security controls not pinned." + ] + }, + "jurisdiction_scope": "Ukraine; Ministry environmental registers and permits", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "ua.environment-permits", + "source_url": "https://mepr.gov.ua/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-ua.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin authorized route, schema, cadence, IDs, terms, privacy, and wartime safety controls.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official registration guidance; no facility acquisition until safety-reviewed export/API exists", + "attribution": { + "attribution_required": true, + "notice": "Official government source; coordinates, privacy, reuse terms, and wartime exposure require review", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown", + "country_code": "UA", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "No safe public bulk route, stable ID, cadence, coordinate policy, or reuse terms verified." + ] + }, + "jurisdiction_scope": "Ukraine; DPSS livestock facilities and operators, including aquaculture scope", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "ua.farm-aquaculture", + "source_url": "https://dpss.gov.ua/diyalnist/bezpechnist-harchovih-produktiv-ta-veterinarna-medicina/dozvoly-ta-reiestratsiia-dlia-biznesu-u-sferakh-veterynarnoi-medytsyny-bezpechnosti-harchovykh-produktiv-ta-kormiv/tvarynnytski-potuzhnosti/derzhavna-reiestratsiia-tvarynnytskykh-potuzhnostei-ta-operatoriv-rynku", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-ua.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin authorized route, schema, cadence, IDs, terms, privacy, and wartime safety controls.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official reports/registers; aggregate-only until a safe public route is authorized", + "attribution": { + "attribution_required": true, + "notice": "Sensitive control/research evidence; privacy, retention, and wartime safety review required", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "resource-specific; unknown", + "country_code": "UA", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "No safe stable public facility/event master verified; stop and escalate on sensitive exposure." + ] + }, + "jurisdiction_scope": "Ukraine; DPSS inspections/enforcement and public animal-experimentation evidence", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "ua.inspections-experiments", + "source_url": "https://dpss.gov.ua/diyalnist/bezpechnist-harchovih-produktiv-ta-veterinarna-medicina/reyestri", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-ua.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin authorized route, schema, cadence, IDs, terms, privacy, and wartime safety controls.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official statistical tables/publications; API or bounded download to be pinned", + "attribution": { + "attribution_required": true, + "notice": "State Statistics Service source; publication terms, revisions, and regional suppression require review", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "table-specific; unknown", + "country_code": "UA", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Current table IDs, machine API, cadence, revision semantics, and suppression rules not pinned." + ] + }, + "jurisdiction_scope": "Ukraine; aggregate livestock, animal-production, and slaughter statistics", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "ua.ukrstat.statistics", + "source_url": "https://ukrstat.gov.ua/", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-ua.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin authorized route, schema, cadence, IDs, terms, privacy, and wartime safety controls.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + } + ], + "summary": "No release approval is implied; inspect source gates below." + }, + { + "acquisition_counts": { + "artifact_private_only": 1 + }, + "basis": [ + "uk.locations: artifact_private_only / awaiting-owner-review" + ], + "country_code": "UK", + "country_reasons": [ + "publication:blocked" + ], + "display_name": "UK", + "owner_review": "awaiting-owner-review", + "publication_state": "blocked", + "readiness_class": "human-review-ready", + "source_count": 1, + "sources": [ + { + "access_method": "monthly CSV download or assisted export", + "attribution": { + "attribution_required": true, + "notice": "Legacy inventory names the Food Standards Agency; confirm the applicable national publication licences and attribution.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "monthly", + "country_code": "UK", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Identify and model each national feed separately before combining; confirm current URLs and terms." + ] + }, + "jurisdiction_scope": "United Kingdom; England, Wales, Northern Ireland, and Scotland legacy coverage", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": true, + "public_release_allowed": false, + "reasons": [ + "publication:blocked" + ], + "state": "awaiting-owner-review" + }, + "source_id": "uk.locations", + "source_url": "unknown", + "status": { + "acquisition": "artifact_private_only", + "evidence": [ + "docs/country-recon-uk.md", + "docs/countries/uk/fss-approved-establishments-source-assessment.md", + "pipeline/sources/uk/fsa_approved/README.md", + "pipeline/sources/uk/fsa_approved/handoff.py", + "pipeline/sources/uk/fsa_approved/refresh.py", + "pipeline/sources/uk/fss_approved/refresh.py", + "pipeline/common/review_packet.py", + "docs/review-packet-united-kingdom.md", + "docs/architecture/disposable-candidate-import.md", + "pipeline/scripts/maintenance/import-candidate.py", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Repeatable source-local refresh dry-run passed on the retained snapshot (5,342 input, 4,300 normalized, 1,042 quarantined, expected schema fingerprint, no drift alarms); use refresh.py dry-run/handoff only in restricted staging. This does not establish production/runtime health or publication eligibility. Complete source-rights, privacy/coordinate, duplicate, coverage, and release review while keeping NI and Scotland separate.", + "publication_eligibility": "blocked", + "runtime_health": "unknown" + } + } + ], + "summary": "No release approval is implied; inspect source gates below." + }, + { + "acquisition_counts": { + "blocked": 1, + "not_run": 12 + }, + "basis": [ + "us.aaalac.accredited: not_run / reconnaissance", + "us.aphis: not_run / reconnaissance", + "us.ca.cdph-lab-animals: not_run / reconnaissance", + "us.dod.acuro: not_run / reconnaissance", + "us.fda.glp-animal-research: not_run / reconnaissance", + "us.fsis: blocked / blocked", + "us.inspections: not_run / reconnaissance", + "us.nasa.nspires: not_run / reconnaissance", + "us.nih.olaw-assurances: not_run / reconnaissance", + "us.nih.reporter: not_run / reconnaissance", + "us.nsf.awards: not_run / reconnaissance", + "us.state-mpi: not_run / reconnaissance", + "us.va.animal-research: not_run / reconnaissance" + ], + "country_code": "US", + "country_reasons": [ + "acquisition:blocked", + "acquisition:not_run", + "publication:blocked" + ], + "display_name": "US", + "owner_review": "awaiting-owner-review", + "publication_state": "blocked", + "readiness_class": "blocked", + "source_count": 13, + "sources": [ + { + "access_method": "official public directory search/result pages; no bulk contract verified", + "attribution": { + "attribution_required": true, + "notice": "Private nonprofit accreditation source; preserve organization/unit distinction, attribution and terms; accreditation is not regulatory registration", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "ongoing directory updates; observation date required", + "country_code": "US", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Confirm directory query/export behavior, unit lifecycle, terms, contact/address minimization and exact parent/unit identity matching before use." + ] + }, + "jurisdiction_scope": "United States; AAALAC International voluntary accreditation directory observations", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "us.aaalac.accredited", + "source_url": "https://www.aaalac.org/accreditation/directory/directory-of-accredited-organizations-search-result/", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/countries/us/research-animal-coverage-recon.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Review directory terms and unit-level identity before any private capture; keep voluntary accreditation separate from regulatory registration and animal-use evidence.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "operator-assisted public-search export with explicit registrations/annual_reports/inspections profile", + "attribution": { + "attribution_required": true, + "notice": "APHIS authority and public-search scope verified; export terms, attribution, and privacy handling remain review gates", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown; record selected report/search date", + "country_code": "US", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Capture and verify the current export schema and terms; no facility/laboratory merge or public release until review." + ] + }, + "jurisdiction_scope": "United States; USDA APHIS Animal Care public search and annual-report data", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "us.aphis", + "source_url": "https://aphis.my.site.com/PublicSearchTool/s/annual-reports", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-us.md", + "docs/countries/us/README.md", + "pipeline/sources/us/aphis/config.json", + "pipeline/sources/us/aphis/adapter.py", + "pipeline/sources/us/aphis/refresh.py", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Use the explicit profile-based assisted export for registrations, annual reports, or inspections; preserve each evidence type separately and complete terms, privacy, schema, and review gates.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official state program page and LAB 139 application/renewal form; no public statewide bulk export verified", + "attribution": { + "attribution_required": true, + "notice": "California state source; minimize responsible-person/address fields and review public-record, privacy and redistribution conditions", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "annual approval renewal; record approval/reporting period", + "country_code": "US", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Federal-regulated laboratories are exempt from this state approval; no public bulk registry was verified; pilot only and never a national denominator." + ] + }, + "jurisdiction_scope": "United States; California Department of Public Health laboratory-animal approval and form evidence", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "us.ca.cdph-lab-animals", + "source_url": "https://www.cdph.ca.gov/Programs/cls/operations/Pages/LaboratoryAnimalUseApprovalProgram.aspx", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/countries/us/research-animal-coverage-recon.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Pilot California approval/count evidence only through an authorized current extract or public-record response; model federal exemptions and do not generalize to a national denominator.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official ACURO policy/reporting/site-visit pages; no national facility or use export verified", + "attribution": { + "attribution_required": true, + "notice": "DoD/DHA official oversight source; public pages may omit sensitive contract, protocol and security details", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "policy and reporting specific; unknown nationally", + "country_code": "US", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "No public national facility/count dataset was verified; do not infer sites from awardee addresses or expose sensitive operational details." + ] + }, + "jurisdiction_scope": "United States; DHA/USAMRDC ACURO animal protocol and site-oversight evidence", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "us.dod.acuro", + "source_url": "https://mrdc.health.mil/index.cfm/resources/research_protections/acuro", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/countries/us/research-animal-coverage-recon.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Keep ACURO protocol/site-oversight material manual and privacy-reviewed; do not infer facilities from DoD awardee addresses or build a national count.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official policy, MOU and facility pages; no national facility export verified", + "attribution": { + "attribution_required": true, + "notice": "FDA official source; distinguish GLP inspection/disqualification authority from USDA/OLAW evidence and review security/privacy before retaining facility details", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "policy/facility-page specific; unknown nationally", + "country_code": "US", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "No public national FDA animal-research register or count route was verified; do not use openFDA adverse-event records as research-use counts." + ] + }, + "jurisdiction_scope": "United States; FDA laboratory-animal/GLP program and public FDA facility evidence", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "us.fda.glp-animal-research", + "source_url": "https://www.fda.gov/about-fda/domestic-mous/mou-225-16-010", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/countries/us/research-animal-coverage-recon.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Keep FDA evidence policy/manual until a public reproducible facility or inspection route is verified; do not treat openFDA adverse events as research-use counts.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "operator-assisted official CSV export; bounded direct fetch only with terms review", + "attribution": { + "attribution_required": true, + "notice": "FSIS authority and directory scope verified; terms/attribution and current export URL must be recorded per run before publication", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "weekly replacement", + "country_code": "US", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Current direct links returned HTTP 403 during reconnaissance; obtain an authorized current export and complete terms, schema, privacy, and project review." + ] + }, + "jurisdiction_scope": "United States; USDA FSIS meat and poultry establishment coverage", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "us.fsis", + "source_url": "https://www.fsis.usda.gov/inspection/establishments/meat-poultry-and-egg-product-inspection-directory", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-us.md", + "docs/countries/us/README.md", + "docs/countries/us/v1-field-crosswalk.json", + "pipeline/sources/us/fsis/config.json", + "pipeline/sources/us/fsis/adapter.py", + "pipeline/sources/us/fsis/refresh.py", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Use the assisted official export contract after the 403 blocker; record URL, retrieval/effective dates, content type, byte size, SHA-256, terms, schema fingerprint, privacy review, and reconciliation before any test-only handoff.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "operator-assisted APHIS public-search inspection export", + "attribution": { + "attribution_required": true, + "notice": "APHIS inspection evidence; export terms, attribution, redaction/privacy handling, and review outcome remain required", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "unknown; record selected search date", + "country_code": "US", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Verify current inspection export schema and use explicit, reviewable identity matching only; absence is not closure." + ] + }, + "jurisdiction_scope": "United States; USDA APHIS inspection-report observations", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "us.inspections", + "source_url": "https://efile.aphis.usda.gov/PublicSearchTool/s/inspection-reports", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-us.md", + "docs/countries/us/README.md", + "pipeline/sources/us/aphis/config.json", + "pipeline/sources/us/aphis/adapter.py", + "pipeline/sources/us/aphis/refresh.py", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Capture the APHIS inspections profile through the documented public-search route; treat rows as observations, not a facility master, and use explicit reviewable identity matching only.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official NSPIRES web search and solicitation/award pages; no bulk animal-use route verified", + "attribution": { + "attribution_required": true, + "notice": "NASA public research-award evidence; review portal access, identifiers, privacy and performance-site semantics", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "solicitation/award specific; record observation date", + "country_code": "US", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Funding evidence does not prove animal use or facility location; the OLAW assurance relationship is policy context, not a count or registry join." + ] + }, + "jurisdiction_scope": "United States; NASA NSPIRES research solicitation and award evidence", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "us.nasa.nspires", + "source_url": "https://www.nasa.gov/hrp/for-prospective-researchers/", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/countries/us/research-animal-coverage-recon.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Keep NASA NSPIRES as funding/project evidence; verify public award identifiers and performance-site semantics before any integration.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official browser lookup table; no public bulk/API contract verified", + "attribution": { + "attribution_required": true, + "notice": "NIH/OLAW public institutional assurance evidence; do not expose restricted assurance documents or infer project-level use", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "current-assurance lookup; observation date required", + "country_code": "US", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Verify durable capture/export semantics, assurance lifecycle, branch/affiliate scope, terms, privacy and exact organization matching; lookup presence is not an animal count or facility census." + ] + }, + "jurisdiction_scope": "United States; NIH OLAW current approved Domestic and Foreign Animal Welfare Assurances lookup", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "us.nih.olaw-assurances", + "source_url": "https://grants.nih.gov/policy-and-compliance/policy-topics/animal-welfare/assurance/assured-institutions", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/countries/us/research-animal-coverage-recon.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Validate an assisted capture contract for the OLAW assured-institutions lookup; preserve Assurance ID/type and branch/affiliate scope, and do not infer protocols or animal counts.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official JSON API and annual/bulk ExPORTER files; rate-limited deterministic acquisition", + "attribution": { + "attribution_required": true, + "notice": "NIH public administrative award data; preserve API/bulk release metadata and review terms, privacy and PI minimization before publication", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "API service with annual consolidated project-file release and later updates", + "country_code": "US", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Funding/project evidence does not prove animal use, physical performance site, operation, or animal counts; validate API version, bulk file refresh, rate limits, text classification, privacy and reviewed organization/site links before integration." + ] + }, + "jurisdiction_scope": "United States; NIH RePORTER and ExPORTER federal research award/project evidence", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "us.nih.reporter", + "source_url": "https://api.reporter.nih.gov/", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/countries/us/research-animal-coverage-recon.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Implement a rate-limited NIH RePORTER API/ExPORTER funding-project adapter with exact award and organization identifiers; keep animal relevance as project evidence, not facility or animal-use counts.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official NSF Award Search API and open-data routes", + "attribution": { + "attribution_required": true, + "notice": "NSF public award evidence; review API terms, identifiers, PI minimization and performance-site semantics", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "API/data-source specific; record response and release metadata", + "country_code": "US", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Funding evidence does not prove animal use or site operation; integrate only with explicit animal-relevance and OLAW-assurance semantics." + ] + }, + "jurisdiction_scope": "United States; NSF research award/project evidence", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "us.nsf.awards", + "source_url": "https://www.nsf.gov/digital/developer", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/countries/us/research-animal-coverage-recon.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Consider NSF Award Search only as a funding/project adjunct after NIH RePORTER; require explicit animal-relevance and performance-site review.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "state-specific HTML/PDF/XLSX/CSV/map routes or authorized operator-assisted capture; FSIS CIS workbooks are a separate overlay", + "attribution": { + "attribution_required": true, + "notice": "State and FSIS authority are documented; public availability does not establish redistribution permission. Review terms, privacy, and category scope per state before acquisition or publication.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "state-specific and often undocumented; record every visible revision/effective date", + "country_code": "US", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "No state roster or CIS workbook was acquired; obtain authorized current exports or operator-assisted captures, resolve terms/schema/privacy, and keep state, federal, CIS, custom-exempt, retail/handler, and inactive/expired populations separate." + ] + }, + "jurisdiction_scope": "United States; state Meat and Poultry Inspection programs and Cooperative Interstate Shipment overlays", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "us.state-mpi", + "source_url": "https://www.fsis.usda.gov/inspection/state-inspection-programs", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/countries/us/state-mpi-source-recon.md", + "docs/countries/us/README.md", + "docs/source-status.md" + ], + "metadata": "verified", + "next_action": "Obtain authorized current state MPI rosters or operator-assisted captures, preserving state-native identifiers, source classes, status/effective dates, terms, address/coordinate provenance, and separate official/CIS/custom-exempt/retail populations before any test-only handoff.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official VA program, policy and facility pages; no national bulk dataset verified", + "attribution": { + "attribution_required": true, + "notice": "VA official source; facility pages and ORO oversight statements are not a national animal-use register", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "page/policy specific; unknown nationally", + "country_code": "US", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Identify a public reproducible national VA facility/count route before acquisition; keep local facility pages, assurance identifiers and oversight events separate." + ] + }, + "jurisdiction_scope": "United States; Veterans Affairs animal-research program and oversight evidence", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "us.va.animal-research", + "source_url": "https://www.research.va.gov/programs/animal_research/default.cfm", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/countries/us/research-animal-coverage-recon.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Retain VA program, facility and ORO pages as separate observations; identify an authorized national facility/count route before acquisition.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + } + ], + "summary": "No release approval is implied; inspect source gates below." + }, + { + "acquisition_counts": { + "blocked": 5, + "not_run": 1 + }, + "basis": [ + "xk.arbk.organizations: blocked / blocked", + "xk.ask.statistics: not_run / reconnaissance", + "xk.auvk.approved-food: blocked / blocked", + "xk.auvk.farms-aquaculture: blocked / blocked", + "xk.auvk.inspections-experiments: blocked / blocked", + "xk.environment-permits: blocked / blocked" + ], + "country_code": "XK", + "country_reasons": [ + "acquisition:blocked", + "acquisition:not_run", + "publication:blocked" + ], + "display_name": "XK", + "owner_review": "awaiting-owner-review", + "publication_state": "blocked", + "readiness_class": "blocked", + "source_count": 6, + "sources": [ + { + "access_method": "official business-register web/admin surface; public API/bulk route unverified", + "attribution": { + "attribution_required": true, + "notice": "Identity linkage only; verify automation permissions, terms, fields and registered-office privacy.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "provider-defined", + "country_code": "XK", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Pin public query/API, authentication, rate limits, cadence, terms and privacy." + ] + }, + "jurisdiction_scope": "Kosovo; ARBK business organizations and corporate identifiers", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "xk.arbk.organizations", + "source_url": "https://arbk.rks-gov.net/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-xk.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin ARBK API/search route, auth, rate limits, cadence, terms and privacy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "Kosovo Agency of Statistics tables/data services; exact route unverified", + "attribution": { + "attribution_required": true, + "notice": "Official statistics; verify table terms, citation requirements and aggregate-only scope.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "table-specific; preserve revisions", + "country_code": "XK", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Pin current slaughter and animal-use table IDs, API/download contracts and cadence." + ] + }, + "jurisdiction_scope": "Kosovo; official aggregate slaughter, livestock and animal-use statistics", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:not_run", + "publication:blocked" + ], + "state": "reconnaissance" + }, + "source_id": "xk.ask.statistics", + "source_url": "https://ask.rks-gov.net/", + "status": { + "acquisition": "not_run", + "evidence": [ + "docs/country-recon-xk.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Pin official slaughter/animal-use table IDs, APIs, cadence and revisions.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official AUVK category pages and linked downloads", + "attribution": { + "attribution_required": true, + "notice": "Official authority; verify file terms, attribution, privacy and coordinates before acquisition.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "publisher-defined; timestamp retrieval", + "country_code": "XK", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Pin current AUVK files, schemas, freshness, IDs, terms and privacy." + ] + }, + "jurisdiction_scope": "Kosovo; AUVK approved and registered animal-origin food establishments", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "xk.auvk.approved-food", + "source_url": "https://auvk.rks-gov.net/en/approved-businesses-for-food-of-animal-origin/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-xk.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Pin current AUVK files, schemas, freshness, IDs, terms and privacy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official AUVK registers, linked files and animal-registration guidance", + "attribution": { + "attribution_required": true, + "notice": "Verify publisher, license, animal-holder/property privacy and coordinate precision.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "resource-specific", + "country_code": "XK", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Verify current holding/fishpond files, IDs, cadence, terms and privacy." + ] + }, + "jurisdiction_scope": "Kosovo; AUVK livestock holdings, fishponds and aquaculture-related evidence", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "xk.auvk.farms-aquaculture", + "source_url": "https://auvk.rks-gov.net/shendeti-i-kafsheve/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-xk.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Verify holding/fishpond files, IDs, cadence, coordinates, terms and privacy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official AUVK control summaries, registers and linked documents", + "attribution": { + "attribution_required": true, + "notice": "Sensitive control/research evidence; aggregate or anonymize and require project review.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "resource/report-specific", + "country_code": "XK", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Pin current experimentation register/API, fields, IDs, privacy and retention policy." + ] + }, + "jurisdiction_scope": "Kosovo; AUVK inspections/enforcement and experimental-animal institutions", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "xk.auvk.inspections-experiments", + "source_url": "https://auvk.rks-gov.net/kontrolli-i-brendshem/veterinar/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-xk.md", + "pipeline/source_registry.json" + ], + "metadata": "verified", + "next_action": "Pin experimentation register/API, fields, IDs, privacy and retention policy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + }, + { + "access_method": "official ministry/environmental authority documents and public notices", + "attribution": { + "attribution_required": true, + "notice": "Verify public-read rights, document license, geometry and privacy.", + "source_origin": "government or documented secondary source; see source registry", + "terms_status": "pending-human-review" + }, + "cadence": "permit/document-specific", + "country_code": "XK", + "coverage": { + "completeness": "not-claimed", + "disappearance_semantics": "not-observed; never inferred as closure", + "limitations": [ + "Stable public read/API/export and complete facility coverage are unresolved." + ] + }, + "jurisdiction_scope": "Kosovo; environmental permits and authorizations", + "owner_review": { + "decision_recorded": false, + "state": "awaiting-owner-review" + }, + "publication": { + "approval_required": true, + "reason": "No owner approval is recorded in the checked-in status baseline.", + "state": "blocked" + }, + "readiness": { + "owner_review": "awaiting-owner-review", + "private_candidate": false, + "public_release_allowed": false, + "reasons": [ + "acquisition:blocked", + "publication:blocked" + ], + "state": "blocked" + }, + "source_id": "xk.environment-permits", + "source_url": "https://mmphi.rks-gov.net/", + "status": { + "acquisition": "blocked", + "evidence": [ + "docs/country-recon-xk.md", + "pipeline/source_registry.json" + ], + "metadata": "partial", + "next_action": "Verify public environmental permit route, coverage, documents, geometry and privacy.", + "publication_eligibility": "blocked", + "runtime_health": "not_run" + } + } + ], + "summary": "No release approval is implied; inspect source gates below." + } + ], + "derived_context": true, + "generated_at": "2026-09-18T22:35:41Z", + "publication_boundary": "This row-free snapshot is operator context only. It cannot approve, promote, or publish a release.", + "readiness_classes": [ + "infrastructure-only", + "acquisition-ready", + "private-candidate-ready", + "human-review-ready", + "publication-eligible", + "blocked" + ], + "readiness_counts": { + "acquisition-ready": 3, + "blocked": 34, + "human-review-ready": 5, + "private-candidate-ready": 3 + }, + "schema_version": "private-review-console-v1", + "source_count": 254, + "source_of_truth": { + "platform_registry": "pipeline/source_registry.json + docs/source-status.json", + "publication_boundary": "private staging only; no release approval or promotion implied" + }, + "states": [ + "infrastructure-only", + "acquisition-ready", + "private-candidate-ready", + "human-review-ready", + "publication-eligible", + "blocked" + ] +} From 88422957082845a39db89afa54f2f9d56701bf25 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Fri, 18 Sep 2026 15:40:38 -0700 Subject: [PATCH 258/311] Align private review console with registry packet --- static/modules/__tests__/privateReview.test.js | 8 ++++++++ static/private-review.html | 4 ++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/static/modules/__tests__/privateReview.test.js b/static/modules/__tests__/privateReview.test.js index 1fb96e0..58b5ecf 100644 --- a/static/modules/__tests__/privateReview.test.js +++ b/static/modules/__tests__/privateReview.test.js @@ -38,6 +38,8 @@ test('readiness asset validates the current registry-driven country set with exa expect(states).toContain(country.state); expect(country).toEqual(expect.objectContaining({ name: expect.any(String), summary: expect.any(String), basis: expect.any(Array) })); } + expect(matrix.countries.BE.sources[0].attribution.terms_status).toBe('pending-human-review'); + expect(matrix.countries.BE.sources[0].status.acquisition).toBe('artifact_private_only'); }); test('safe review page never includes private address, raw payload, geocoder, or requester fields', () => { @@ -49,3 +51,9 @@ test('safe review page never includes private address, raw payload, geocoder, or expect(js).toContain('Withheld by console'); expect(js.toLowerCase()).toContain('observations remain separate'); }); + +test('review packet loader rejects row-shaped payloads and keeps tokens out of storage APIs', async () => { + const { validateReviewPacket } = await import('../../private-review.js'); + expect(() => validateReviewPacket({ schema_version: 'private-review-packet-v2', counts: { input_rows: 1 } })).not.toThrow(); + expect(() => validateReviewPacket({ schema_version: 'private-review-packet-v2', normalized: { records: [] } })).toThrow('rejected safely'); +}); diff --git a/static/private-review.html b/static/private-review.html index f3ad185..cb9f89f 100644 --- a/static/private-review.html +++ b/static/private-review.html @@ -138,7 +138,7 @@

    Candidate preview

    02 / REVIEW WORK QUEUES

    -

    Contradictions, identity, quarantine, statistics

    +

    Evidence queues and safety state

    private graph · separate observations
    @@ -184,7 +184,7 @@

    Entity search and neighborhood

    - + The server contract remains bounded. Direction and depth are sent with every neighborhood read.

    Private graph search is not connected.

    From 898fe019d66647dc4c23e60a42282831156e0230 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Fri, 18 Sep 2026 15:59:26 -0700 Subject: [PATCH 259/311] Build private candidate review console --- docs/api/private-graph-contract.md | 13 +- docs/private-review-console.md | 79 ++++++ pipeline/common/review_console.py | 199 ++++++++++++++ pipeline/common/test_review_console.py | 69 +++++ .../build-review-console-snapshot.py | 29 ++ scripts/dev.py | 4 + src/graph_private.rs | 33 ++- src/lib.rs | 41 ++- .../modules/__tests__/privateReview.test.js | 6 +- static/private-graph.css | 43 ++- static/private-graph.html | 2 +- static/private-graph.js | 2 +- static/private-review.js | 259 +++++++++++++++++- static/private-review/readiness-matrix.json | 2 +- 14 files changed, 742 insertions(+), 39 deletions(-) create mode 100644 docs/private-review-console.md create mode 100644 pipeline/common/review_console.py create mode 100644 pipeline/common/test_review_console.py create mode 100644 pipeline/scripts/diagnostics/build-review-console-snapshot.py diff --git a/docs/api/private-graph-contract.md b/docs/api/private-graph-contract.md index 042c795..532771c 100644 --- a/docs/api/private-graph-contract.md +++ b/docs/api/private-graph-contract.md @@ -6,10 +6,15 @@ authentication returns `404` to avoid endpoint discovery. The token is never accepted on public routes. `GET /entities` supports bounded name lookup (`q`, max `limit=100`) and a -stable UUID cursor. `GET /entities/{id}/neighborhood` supports one-hop -relationship traversal only, with allowlisted type/source/date filters and a -maximum of 100 rows. Queue endpoints expose only structured review metadata: -`contradictions`, `unresolved-identities`, `quarantine`, and `statistics`. +stable UUID cursor. `GET /entities/{id}/neighborhood` supports bounded one- or +two-hop relationship traversal (`direction=in|out|both`, `depth=1|2`) with +allowlisted type/source/date filters and a maximum of 100 rows. The response +echoes the effective direction and depth so an operator can verify the query +that was executed. Queue endpoints expose only structured review metadata: +`contradictions`, `unresolved-identities`, `quarantine`, `claims`, +`rejected-candidates`, `suppression`, and `statistics`. Claim rows include +support/contradiction counts but never claim values or source payloads; +suppression rows expose only opaque case IDs and policy/status metadata. All reads use deterministic tie-breakers, explicit limits, and the private tables. Raw payloads, addresses, geocoder queries, and private notes are not diff --git a/docs/private-review-console.md b/docs/private-review-console.md new file mode 100644 index 0000000..db0c0bd --- /dev/null +++ b/docs/private-review-console.md @@ -0,0 +1,79 @@ +# Private candidate review console + +The local `/private-review.html` page is a read-only operator surface for +candidate evidence. It combines the row-free launch matrix with authenticated +candidate preview, private graph queues, bounded entity traversal, and local +review-packet inspection. It is not a public release view and has no approve, +promote, publish, or export action. + +The page is intentionally framework-free: static HTML, vanilla ES modules, and +custom CSS served by the existing Axum application. Its UX is a read-only +operator workspace with four bounded areas: candidate preview, separate graph +queues, a direction/depth-controlled entity neighborhood, and a derived +country readiness matrix. Tokens stay in page memory; no browser storage or +mutation API is used. + +## Start locally + +1. Build the derived matrix when the source registry or source-status baseline + changes: + + ```text + python scripts/dev.py review-console-snapshot + ``` + +2. Start the development service in loopback mode with a database and the + existing private tokens configured. The candidate preview token is accepted + only by the loopback, test-only endpoint. The graph token is accepted only + by `/api/private/graph/*`. + +3. Open `http://127.0.0.1:8000/private-review.html`. Tokens are held in page + memory only. A missing or invalid token fails closed; the page never treats + an unavailable queue as an empty queue. + +4. Optionally load a row-free `review-packet.json` using the local file picker. + The browser validates the packet schema by allowlist, then displays only + aggregate counts/deltas, quarantine reasons, provenance/schema facts, + geospatial gates, release gates, attribution metadata, and blockers. Unknown + fields, raw values, addresses, coordinates, geocoder fields, contact fields, + and row-shaped payloads are rejected. + +## Matrix semantics + +The matrix is derived by `pipeline/common/review_console.py` from the existing +platform registry; it does not create a second readiness state machine. + +- `blocked`: at least one registered source reports `acquisition=blocked`. +- `acquisition-ready`: all sources have verified/partial metadata and none has + run acquisition yet; this is not permission to acquire. +- `private-candidate-ready`: at least one source is privately acquired while + another registered source is still pending. +- `human-review-ready`: every registered source is privately acquired or + verified and awaits explicit human review; this is not approval. +- `publication-eligible`: only explicit owner approval, release permission, + and an approved/published publication state can produce this label. +- `infrastructure-only`: registry context exists, but acquisition status is + incomplete or not classifiable. + +The current checked-in baseline has 45 countries and 254 sources. It is +conservative: 34 countries are blocked, 3 are acquisition-ready, 3 are +private-candidate-ready, and 5 are human-review-ready. No country is +publication-eligible in this baseline. These are planning labels, not claims +of source completeness, factual truth, privacy clearance, or publication. + +## Evidence boundary + +Preview rows are explicitly labeled `Private development candidate — not +project-approved or published`. Candidate records expose only the fields +already allowed by the authenticated test-only contract, plus coordinate +precision/review state and suppression status. Private graph queues return +opaque IDs and review metadata; claim values, raw source payloads, addresses, +geocoder queries, reviewer identities, and private notes remain excluded. +Claims show supporting/contradicting observation counts, rejected crosswalks +remain visible as rejected candidates, and suppression shows only opaque case +and policy/status metadata. Contradictions and unresolved identity candidates +are never silently merged. + +If a source is missing from the matrix, the snapshot is stale or the platform +registry failed validation; do not infer zero coverage or closure. Regenerate +the snapshot and resolve the contract error before relying on the console. diff --git a/pipeline/common/review_console.py b/pipeline/common/review_console.py new file mode 100644 index 0000000..0999d74 --- /dev/null +++ b/pipeline/common/review_console.py @@ -0,0 +1,199 @@ +"""Row-free data model for the private candidate review console. + +This module derives operator context from the existing platform registry. It +does not add a readiness state machine or make a release decision: the +classification is a conservative display label for acquisition/review +planning, while the source and country contracts remain authoritative. +""" +from __future__ import annotations + +from collections import Counter +from datetime import datetime, timezone +from typing import Any, Iterable, Mapping + + +REVIEW_CONSOLE_SCHEMA_VERSION = "private-review-console-v1" +READINESS_CLASSES = ( + "infrastructure-only", + "acquisition-ready", + "private-candidate-ready", + "human-review-ready", + "publication-eligible", + "blocked", +) +ACQUIRED_STATES = frozenset({"artifact_private_only", "verified"}) + + +def _as_text(value: Any, fallback: str = "unknown") -> str: + return value.strip() if isinstance(value, str) and value.strip() else fallback + + +def classify_country(sources: Iterable[Mapping[str, Any]]) -> str: + """Return a conservative operator label for one country. + + The label deliberately does not mirror ``Readiness.state``. It is a + human-facing matrix classification: blocked acquisition wins; only an + explicit approved/public-release boundary can be publication-eligible. + """ + items = list(sources) + if not items: + return "infrastructure-only" + + statuses = [item.get("status") if isinstance(item.get("status"), Mapping) else {} for item in items] + acquisitions = {_as_text(status.get("acquisition"), "not_run") for status in statuses} + if "blocked" in acquisitions: + return "blocked" + + release_ready = all( + item.get("owner_review", {}).get("state") == "approved" + and item.get("readiness", {}).get("public_release_allowed") is True + and item.get("publication", {}).get("state") in {"approved-for-release", "published"} + for item in items + ) + if release_ready: + return "publication-eligible" + + acquired = acquisitions.intersection(ACQUIRED_STATES) + pending = acquisitions.difference(ACQUIRED_STATES) + if acquired and not pending: + return "human-review-ready" + if acquired: + return "private-candidate-ready" + if acquisitions == {"not_run"} and all( + _as_text(status.get("metadata")) in {"verified", "partial"} for status in statuses + ): + return "acquisition-ready" + return "infrastructure-only" + + +def _source_view(source: Mapping[str, Any]) -> dict[str, Any]: + status = source.get("status") if isinstance(source.get("status"), Mapping) else {} + readiness = source.get("readiness") if isinstance(source.get("readiness"), Mapping) else {} + owner_review = source.get("owner_review") if isinstance(source.get("owner_review"), Mapping) else {} + publication = source.get("publication") if isinstance(source.get("publication"), Mapping) else {} + coverage = source.get("coverage") if isinstance(source.get("coverage"), Mapping) else {} + attribution = source.get("attribution") if isinstance(source.get("attribution"), Mapping) else {} + return { + "source_id": _as_text(source.get("source_id")), + "country_code": _as_text(source.get("country_code")), + "jurisdiction_scope": _as_text(source.get("jurisdiction_scope")), + "source_url": _as_text(source.get("source_url")), + "access_method": _as_text(source.get("access_method")), + "cadence": _as_text(source.get("cadence")), + "attribution": { + "source_origin": _as_text(attribution.get("source_origin")), + "terms_status": _as_text(attribution.get("terms_status")), + "attribution_required": attribution.get("attribution_required") is True, + "notice": _as_text(attribution.get("notice")), + }, + "coverage": { + "completeness": _as_text(coverage.get("completeness")), + "disappearance_semantics": _as_text(coverage.get("disappearance_semantics")), + "limitations": [str(value) for value in coverage.get("limitations", []) if isinstance(value, str)], + }, + "status": { + "metadata": _as_text(status.get("metadata")), + "acquisition": _as_text(status.get("acquisition")), + "runtime_health": _as_text(status.get("runtime_health")), + "publication_eligibility": _as_text(status.get("publication_eligibility")), + "evidence": [str(value) for value in status.get("evidence", []) if isinstance(value, str)], + "next_action": _as_text(status.get("next_action")), + }, + "readiness": { + "state": _as_text(readiness.get("state")), + "owner_review": _as_text(readiness.get("owner_review")), + "private_candidate": readiness.get("private_candidate") is True, + "public_release_allowed": readiness.get("public_release_allowed") is True, + "reasons": [str(value) for value in readiness.get("reasons", []) if isinstance(value, str)], + }, + "owner_review": { + "state": _as_text(owner_review.get("state")), + "decision_recorded": bool(owner_review.get("decision_id")), + }, + "publication": { + "state": _as_text(publication.get("state")), + "approval_required": publication.get("approval_required") is True, + "reason": _as_text(publication.get("reason")), + }, + } + + +def build_review_console_snapshot(registry: Mapping[str, Any], *, generated_at: str | None = None) -> dict[str, Any]: + """Build a deterministic, row-free snapshot for the static operator UI.""" + source_records = [source for source in registry.get("sources", []) if isinstance(source, Mapping)] + countries = [country for country in registry.get("countries", []) if isinstance(country, Mapping)] + by_country: dict[str, list[Mapping[str, Any]]] = {} + for source in source_records: + code = _as_text(source.get("country_code")) + by_country.setdefault(code, []).append(source) + + matrix: list[dict[str, Any]] = [] + for country in countries: + code = _as_text(country.get("country_code")) + sources = sorted(by_country.get(code, []), key=lambda source: _as_text(source.get("source_id"))) + source_views = [_source_view(source) for source in sources] + status_counts = Counter(source["status"]["acquisition"] for source in source_views) + source_summary = [ + f"{view['source_id']}: {view['status']['acquisition']} / {view['readiness']['state']}" + for view in source_views + ] + matrix.append({ + "country_code": code, + "display_name": _as_text(country.get("display_name"), code), + "readiness_class": classify_country(sources), + "source_count": len(source_views), + "acquisition_counts": dict(sorted(status_counts.items())), + "owner_review": _as_text((country.get("owner_review") or {}).get("state")), + "publication_state": _as_text((country.get("publication") or {}).get("state")), + "country_reasons": [ + str(reason) + for reason in ((country.get("readiness") or {}).get("reasons") or []) + if isinstance(reason, str) + ], + "sources": source_views, + "summary": "No release approval is implied; inspect source gates below.", + "basis": source_summary, + }) + + matrix.sort(key=lambda country: country["country_code"]) + counts = Counter(country["readiness_class"] for country in matrix) + return { + "schema_version": REVIEW_CONSOLE_SCHEMA_VERSION, + "generated_at": generated_at or datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z"), + "contract_versions": registry.get("contract_versions", {}), + "source_of_truth": { + "platform_registry": "pipeline/source_registry.json + docs/source-status.json", + "publication_boundary": "private staging only; no release approval or promotion implied", + }, + "derived_context": True, + "country_count": len(matrix), + "source_count": sum(country["source_count"] for country in matrix), + "readiness_counts": dict(sorted(counts.items())), + "readiness_classes": list(READINESS_CLASSES), + "states": list(READINESS_CLASSES), + "classification_notes": { + "blocked": "At least one registered source reports acquisition=blocked; acquisition blockers take precedence.", + "acquisition-ready": "All registered sources have verified or partial metadata and acquisition has not run; this is not an authorization to acquire.", + "private-candidate-ready": "At least one source is privately acquired/verified, while another registered source remains pending; no release is implied.", + "human-review-ready": "All registered sources are privately acquired/verified and await explicit human review; no approval is implied.", + "publication-eligible": "Only explicit owner approval plus a release-allowed contract can produce this label; it is absent from the current baseline unless those fields are recorded.", + "infrastructure-only": "The registry provides infrastructure context, but the acquisition state is incomplete or not yet classifiable.", + }, + "countries": { + country["country_code"]: { + "name": country["display_name"], + "state": country["readiness_class"], + "summary": country["summary"], + "basis": country["basis"], + "source_count": country["source_count"], + "acquisition_counts": country["acquisition_counts"], + "owner_review": country["owner_review"], + "publication_state": country["publication_state"], + "country_reasons": country["country_reasons"], + "sources": country["sources"], + } + for country in matrix + }, + "country_records": matrix, + "publication_boundary": "This row-free snapshot is operator context only. It cannot approve, promote, or publish a release.", + } diff --git a/pipeline/common/test_review_console.py b/pipeline/common/test_review_console.py new file mode 100644 index 0000000..ef3e20b --- /dev/null +++ b/pipeline/common/test_review_console.py @@ -0,0 +1,69 @@ +import unittest + +from pipeline.common.review_console import ( + READINESS_CLASSES, + build_review_console_snapshot, + classify_country, +) +from pipeline.platform_registry import build_platform_registry + + +def source(acquisition="not_run", metadata="verified", *, approved=False, public=False): + return { + "source_id": "aa.synthetic", + "country_code": "AA", + "jurisdiction_scope": "synthetic scope", + "source_url": "https://example.test/source", + "access_method": "synthetic", + "cadence": "unknown", + "status": { + "metadata": metadata, + "acquisition": acquisition, + "runtime_health": "unknown", + "publication_eligibility": "eligible_pending_release_approval" if public else "blocked", + "evidence": [], + "next_action": "review", + }, + "readiness": {"state": "approved-for-release" if approved else "awaiting-owner-review", "owner_review": "approved" if approved else "awaiting-owner-review", "private_candidate": True, "public_release_allowed": public, "reasons": []}, + "owner_review": {"state": "approved" if approved else "awaiting-owner-review", "decision_id": "decision-1" if approved else None}, + "publication": {"state": "approved-for-release" if approved else "blocked", "approval_required": True, "reason": "synthetic"}, + "coverage": {"completeness": "not-claimed", "disappearance_semantics": "not-observed; never inferred as closure", "limitations": []}, + "attribution": {"source_origin": "synthetic", "terms_status": "reviewed", "attribution_required": True, "notice": "synthetic"}, + } + + +class ReviewConsoleTests(unittest.TestCase): + def test_classification_is_conservative_and_ordered(self): + self.assertEqual(classify_country([source("blocked")]), "blocked") + self.assertEqual(classify_country([source("not_run")]), "acquisition-ready") + self.assertEqual(classify_country([source("artifact_private_only"), source("not_run")]), "private-candidate-ready") + self.assertEqual(classify_country([source("verified")]), "human-review-ready") + self.assertEqual(classify_country([source("verified", approved=True, public=True)]), "publication-eligible") + self.assertEqual(classify_country([source("not_run", metadata="unknown")]), "infrastructure-only") + + def test_snapshot_is_row_free_and_covers_all_contract_countries(self): + registry = { + "contract_versions": {"country": "country-contract-v1"}, + "sources": [source("verified")], + "countries": [{"country_code": "AA", "display_name": "AA", "owner_review": {"state": "awaiting-owner-review"}, "publication": {"state": "blocked"}, "readiness": {"reasons": ["review"]}}], + } + snapshot = build_review_console_snapshot(registry, generated_at="2026-09-18T00:00:00Z") + self.assertEqual(snapshot["country_count"], 1) + self.assertEqual(snapshot["source_count"], 1) + self.assertEqual(snapshot["countries"]["AA"]["state"], "human-review-ready") + self.assertEqual(snapshot["generated_at"], "2026-09-18T00:00:00Z") + self.assertTrue(snapshot["derived_context"]) + self.assertEqual(set(snapshot["states"]), set(READINESS_CLASSES)) + serialized = str(snapshot) + for forbidden in ("source_values", "raw_fields", "address", "street_address", "geocoder_query", "reviewer_identity"): + self.assertNotIn(forbidden, serialized) + + def test_checked_in_registry_produces_a_classification_for_every_country(self): + snapshot = build_review_console_snapshot(build_platform_registry(), generated_at="2026-09-18T00:00:00Z") + self.assertEqual(snapshot["country_count"], 45) + self.assertEqual(set(snapshot["countries"]), {record["country_code"] for record in snapshot["country_records"]}) + self.assertTrue(all(record["readiness_class"] in READINESS_CLASSES for record in snapshot["country_records"])) + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/scripts/diagnostics/build-review-console-snapshot.py b/pipeline/scripts/diagnostics/build-review-console-snapshot.py new file mode 100644 index 0000000..8572291 --- /dev/null +++ b/pipeline/scripts/diagnostics/build-review-console-snapshot.py @@ -0,0 +1,29 @@ +"""Build the row-free private review-console readiness snapshot.""" +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[3] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from pipeline.common.review_console import build_review_console_snapshot +from pipeline.platform_registry import build_platform_registry + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("output", type=Path) + args = parser.parse_args() + snapshot = build_review_console_snapshot(build_platform_registry()) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(snapshot, ensure_ascii=False, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(json.dumps({"status": "validated", "country_count": snapshot["country_count"], "source_count": snapshot["source_count"], "output": str(args.output)}, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/dev.py b/scripts/dev.py index 0e08263..6c893c8 100644 --- a/scripts/dev.py +++ b/scripts/dev.py @@ -81,6 +81,8 @@ def main() -> int: sub.add_parser("demo", help="run the safe synthetic/private tooling demo") sub.add_parser("preflight", help="alias for doctor: verify local prerequisites before a run") sub.add_parser("platform-registry", help="validate the joined country/source registry") + rc = sub.add_parser("review-console-snapshot", help="build the row-free private review-console readiness snapshot") + rc.add_argument("output", default=str(ROOT / "static" / "private-review" / "readiness-matrix.json"), nargs="?") pf = sub.add_parser("private-frontend", help="rehearse a candidate against the private frontend preview boundary") pf.add_argument("manifest"); pf.add_argument("output"); pf.add_argument("--root", default=str(ROOT)); pf.add_argument("--base-url"); pf.add_argument("--token") rp = sub.add_parser("review-packet", help="generate a private row-free review packet") @@ -107,6 +109,8 @@ def main() -> int: code = run([sys.executable, "-m", "unittest", "pipeline.common.test_graph_candidates", "pipeline.common.test_review_packet"], capture=args.json) elif args.command == "platform-registry": code = run([sys.executable, "-c", "import json; from pipeline.platform_registry import build_platform_registry; r=build_platform_registry(); print(json.dumps({'countries':r['country_count'],'sources':r['source_count'],'status':'validated'}))"], capture=args.json) + elif args.command == "review-console-snapshot": + code = run([sys.executable, str(ROOT / "pipeline/scripts/diagnostics/build-review-console-snapshot.py"), args.output], capture=args.json) elif args.command == "private-frontend": cmd = [sys.executable, str(ROOT / "pipeline/scripts/maintenance/rehearse_candidate_private_frontend.py"), "--manifest", args.manifest, "--root", args.root, "--output", args.output] if args.base_url: cmd.extend(["--base-url", args.base_url]) diff --git a/src/graph_private.rs b/src/graph_private.rs index 58bcdea..6578a59 100644 --- a/src/graph_private.rs +++ b/src/graph_private.rs @@ -21,11 +21,13 @@ pub struct GraphQuery { pub min_confidence: Option, pub from: Option, pub to: Option, + pub direction: Option, + pub depth: Option, pub limit: Option, pub cursor: Option, } -fn authorized(headers: &HeaderMap) -> bool { +pub(crate) fn authorized(headers: &HeaderMap) -> bool { let Some(expected) = std::env::var("UEC_PRIVATE_GRAPH_TOKEN") .ok() .filter(|v| !v.is_empty()) @@ -99,6 +101,11 @@ pub async fn neighborhood( let Ok(limit) = limit(p.limit) else { return bad("limit must be between 1 and 100"); }; + let direction = p.direction.as_deref().unwrap_or("both"); + if !["in", "out", "both"].contains(&direction) { + return bad("direction must be in, out, or both"); + } + let depth = p.depth.unwrap_or(1).clamp(1, 2); if p.relationship_type.as_deref().is_some_and(|v| { ![ "operator", @@ -127,9 +134,9 @@ pub async fn neighborhood( "private graph database unavailable", ); }; - let rows = match client.query("SELECT relationship_observation_id, from_organization_id, target_facility_id, target_organization_id, relationship_type, assertion_status, observed_at, confidence, review_state, storage_state, privacy_status, publication_status, source_id, source_record_id FROM uec.organization_relationship_observations WHERE ($1 = from_organization_id OR $1 = target_facility_id OR $1 = target_organization_id) AND ($2::text IS NULL OR relationship_type=$2) AND ($3::text IS NULL OR source_id=$3) AND ($4::timestamptz IS NULL OR observed_at >= $4) AND ($5::timestamptz IS NULL OR observed_at < $5) ORDER BY observed_at DESC, relationship_observation_id DESC LIMIT $6", &[&entity_id, &p.relationship_type, &p.source_id, &p.from.map(|d| d.and_hms_opt(0,0,0).unwrap().and_utc()), &p.to.map(|d| d.and_hms_opt(0,0,0).unwrap().and_utc()), &limit]).await { Ok(v) => v, Err(_) => return v2_error(StatusCode::SERVICE_UNAVAILABLE, "private_graph_query_failed", "private graph query failed") }; + let rows = match client.query("WITH RECURSIVE observations AS (SELECT relationship_observation_id, from_organization_id, target_facility_id, target_organization_id, relationship_type, assertion_status, observed_at, confidence, review_state, storage_state, privacy_status, publication_status, source_id, source_record_id FROM uec.organization_relationship_observations WHERE ($4::text IS NULL OR relationship_type=$4) AND ($5::text IS NULL OR source_id=$5) AND ($6::timestamptz IS NULL OR observed_at >= $6) AND ($7::timestamptz IS NULL OR observed_at < $7)), edges AS (SELECT o.*, 'organization'::text AS from_type, o.from_organization_id AS from_id, CASE WHEN o.target_facility_id IS NOT NULL THEN 'facility'::text ELSE 'organization'::text END AS to_type, COALESCE(o.target_facility_id, o.target_organization_id) AS to_id FROM observations o), seed AS (SELECT 'organization'::text AS node_type, $1::uuid AS node_id, 0 AS hop WHERE EXISTS (SELECT 1 FROM uec.organizations WHERE organization_id=$1) UNION ALL SELECT 'facility'::text, $1::uuid, 0 WHERE EXISTS (SELECT 1 FROM uec.facilities WHERE facility_id=$1)), walk(node_type, node_id, hop) AS (SELECT node_type, node_id, hop FROM seed UNION SELECT CASE WHEN $2 IN ('out','both') THEN e.to_type ELSE e.from_type END, CASE WHEN $2 IN ('out','both') THEN e.to_id ELSE e.from_id END, w.hop + 1 FROM walk w JOIN edges e ON (($2 IN ('out','both') AND e.from_type=w.node_type AND e.from_id=w.node_id) OR ($2 IN ('in','both') AND e.to_type=w.node_type AND e.to_id=w.node_id)) WHERE w.hop < $3), matched AS (SELECT DISTINCT e.relationship_observation_id FROM edges e JOIN walk w ON w.hop < $3 AND (($2 IN ('out','both') AND e.from_type=w.node_type AND e.from_id=w.node_id) OR ($2 IN ('in','both') AND e.to_type=w.node_type AND e.to_id=w.node_id))) SELECT o.relationship_observation_id, o.from_organization_id, o.target_facility_id, o.target_organization_id, o.relationship_type, o.assertion_status, o.observed_at, o.confidence, o.review_state, o.storage_state, o.privacy_status, o.publication_status, o.source_id, o.source_record_id FROM observations o JOIN matched m USING (relationship_observation_id) ORDER BY o.observed_at DESC, o.relationship_observation_id DESC LIMIT $8", &[&entity_id, &direction, &depth, &p.relationship_type, &p.source_id, &p.from.map(|d| d.and_hms_opt(0,0,0).unwrap().and_utc()), &p.to.map(|d| d.and_hms_opt(0,0,0).unwrap().and_utc()), &limit]).await { Ok(v) => v, Err(_) => return v2_error(StatusCode::SERVICE_UNAVAILABLE, "private_graph_query_failed", "private graph query failed") }; let data: Vec<_> = rows.into_iter().map(|r| json!({"relationship_observation_id":r.get::<_,Uuid>(0),"from_organization_id":r.get::<_,Option>(1),"target_facility_id":r.get::<_,Option>(2),"target_organization_id":r.get::<_,Option>(3),"relationship_type":r.get::<_,Option>(4),"assertion_status":r.get::<_,String>(5),"observed_at":r.get::<_,chrono::DateTime>(6),"confidence":r.get::<_,Option>(7),"review_state":r.get::<_,String>(8),"storage_state":r.get::<_,String>(9),"privacy_status":r.get::<_,String>(10),"publication_status":r.get::<_,String>(11),"source_id":r.get::<_,String>(12),"source_record_id":r.get::<_,Uuid>(13)})).collect(); - Json(json!({"api_version":"private-graph-v1","data":data,"meta":{"private":true,"entity_id":entity_id,"limit":limit}})).into_response() + Json(json!({"api_version":"private-graph-v1","data":data,"meta":{"private":true,"entity_id":entity_id,"limit":limit,"direction":direction,"depth":depth,"bounded":true}})).into_response() } pub async fn queue( @@ -148,6 +155,9 @@ pub async fn queue( "contradictions", "unresolved-identities", "quarantine", + "claims", + "rejected-candidates", + "suppression", "statistics", ] .contains(&kind.as_str()) @@ -174,16 +184,25 @@ pub async fn queue( }; let sql = match kind.as_str() { "contradictions" => { - "SELECT claim_id, facility_id, organization_id, claim_domain, claim_kind, observed_at, confidence FROM uec.claim_current c WHERE c.review_state IN ('disputed','review_required') ORDER BY observed_at DESC, claim_id DESC LIMIT $1" + "SELECT claim_id::text, facility_id::text, organization_id::text, claim_domain, claim_kind, observed_at::text, confidence::text, review_state, privacy_status, publication_status FROM uec.claim_current c WHERE c.review_state IN ('disputed','review_required') ORDER BY observed_at DESC, claim_id DESC LIMIT $1" } "unresolved-identities" => { - "SELECT crosswalk_id, left_identifier_id, right_identifier_id, assertion_status, confidence, observed_at FROM uec.source_entity_crosswalks WHERE assertion_status IN ('candidate','review_required','disputed') ORDER BY observed_at DESC, crosswalk_id DESC LIMIT $1" + "SELECT crosswalk_id::text, left_identifier_id::text, right_identifier_id::text, assertion_status, confidence::text, observed_at::text, source_id, source_record_id::text, review_state, privacy_status, publication_status FROM uec.source_entity_crosswalks WHERE assertion_status IN ('candidate','review_required','disputed') ORDER BY observed_at DESC, crosswalk_id DESC LIMIT $1" } "quarantine" => { - "SELECT source_record_id, source_id, source_state, received_at FROM uec.source_records WHERE source_state IN ('quarantined','rejected') ORDER BY received_at DESC, source_record_id DESC LIMIT $1" + "SELECT source_record_id::text, source_id, source_state, received_at::text FROM uec.source_records WHERE source_state IN ('quarantined','rejected') ORDER BY received_at DESC, source_record_id DESC LIMIT $1" + } + "claims" => { + "SELECT c.claim_id::text, c.source_id, c.claim_domain, c.claim_kind, c.value_state, c.unknown_reason, c.observed_at::text, c.confidence::text, c.review_state, c.storage_state, c.privacy_status, c.publication_status, COUNT(s.claim_support_id)::text AS support_count, COUNT(s.claim_support_id) FILTER (WHERE s.support_role = 'contradicting')::text AS contradicting_support_count FROM uec.claim_current c LEFT JOIN uec.claim_support s ON s.claim_id = c.claim_id GROUP BY c.claim_id, c.source_id, c.claim_domain, c.claim_kind, c.value_state, c.unknown_reason, c.observed_at, c.confidence, c.review_state, c.storage_state, c.privacy_status, c.publication_status ORDER BY c.observed_at DESC, c.claim_id DESC LIMIT $1" + } + "rejected-candidates" => { + "SELECT crosswalk_id::text, source_id, source_record_id::text, assertion_status, confidence::text, observed_at::text, review_state, privacy_status, publication_status FROM uec.source_entity_crosswalks WHERE assertion_status = 'rejected' OR review_state = 'rejected' ORDER BY observed_at DESC, crosswalk_id DESC LIMIT $1" + } + "suppression" => { + "SELECT cases.case_id::text, cases.status, current_case.event_type, current_case.reason_category, current_case.policy_version, current_case.occurred_at::text FROM uec.suppression_cases cases JOIN uec.suppression_case_current current_case ON current_case.case_id = cases.case_id ORDER BY current_case.occurred_at DESC, cases.case_id DESC LIMIT $1" } _ => { - "SELECT 'claims' AS metric, count(*)::bigint AS value FROM uec.claims UNION ALL SELECT 'relationships', count(*) FROM uec.organization_relationship_observations UNION ALL SELECT 'quarantined_records', count(*) FROM uec.source_records WHERE source_state IN ('quarantined','rejected')" + "SELECT metric, value::text FROM (SELECT 'claims' AS metric, count(*)::bigint AS value FROM uec.claims UNION ALL SELECT 'relationships', count(*) FROM uec.organization_relationship_observations UNION ALL SELECT 'quarantined_records', count(*) FROM uec.source_records WHERE source_state IN ('quarantined','rejected') UNION ALL SELECT 'suppression_cases', count(*) FROM uec.suppression_cases) metrics ORDER BY metric LIMIT $1" } }; let rows = match client.query(sql, &[&limit]).await { diff --git a/src/lib.rs b/src/lib.rs index f730885..7f3b62a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -55,8 +55,16 @@ pub struct PrivateGraphSearchParams { /// Private evidence-only graph search. This is intentionally not a public projection. pub async fn get_private_graph_search_handler( State(state): State, + headers: HeaderMap, Query(params): Query, ) -> impl IntoResponse { + if !graph_private::authorized(&headers) { + return v2_error( + StatusCode::NOT_FOUND, + "private_graph_unavailable", + "private graph unavailable", + ); + } let Some(pool) = state.database else { return v2_error( StatusCode::SERVICE_UNAVAILABLE, @@ -91,8 +99,16 @@ pub struct PrivateGraphTraverseParams { pub async fn get_private_graph_traverse_handler( State(state): State, + headers: HeaderMap, Query(params): Query, ) -> impl IntoResponse { + if !graph_private::authorized(&headers) { + return v2_error( + StatusCode::NOT_FOUND, + "private_graph_unavailable", + "private graph unavailable", + ); + } let Some(pool) = state.database else { return v2_error( StatusCode::SERVICE_UNAVAILABLE, @@ -121,7 +137,7 @@ pub async fn get_private_graph_traverse_handler( ); } }; - let rows = match client.query("SELECT relationship_observation_id, source_id, source_record_id, from_organization_id, target_facility_id, target_organization_id, relationship_type, assertion_status, unknown_reason, valid_from, valid_to, observed_at, confidence, review_state, storage_state, privacy_status, publication_status, note FROM uec.organization_relationship_observations WHERE (($1='organization' AND ((($3 IN ('out','both')) AND from_organization_id=$2) OR (($3 IN ('in','both')) AND target_organization_id=$2))) OR ($1='facility' AND target_facility_id=$2) ORDER BY observed_at DESC LIMIT 200", &[¶ms.entity_type, ¶ms.entity_id, &direction]).await { Ok(r) => r, Err(_) => return v2_error(StatusCode::SERVICE_UNAVAILABLE, "private_graph_query_failed", "private graph traversal failed") }; + let rows = match client.query("WITH RECURSIVE observations AS (SELECT relationship_observation_id, source_id, source_record_id, from_organization_id, target_facility_id, target_organization_id, relationship_type, assertion_status, unknown_reason, valid_from, valid_to, observed_at, confidence, review_state, storage_state, privacy_status, publication_status, note FROM uec.organization_relationship_observations), edges AS (SELECT o.*, 'organization'::text AS from_type, o.from_organization_id AS from_id, CASE WHEN o.target_facility_id IS NOT NULL THEN 'facility'::text ELSE 'organization'::text END AS to_type, COALESCE(o.target_facility_id, o.target_organization_id) AS to_id FROM observations o), walk(node_type, node_id, hop) AS (SELECT $1::text, $2::uuid, 0 UNION SELECT CASE WHEN $3 IN ('out','both') THEN e.to_type ELSE e.from_type END, CASE WHEN $3 IN ('out','both') THEN e.to_id ELSE e.from_id END, w.hop + 1 FROM walk w JOIN edges e ON (($3 IN ('out','both') AND e.from_type=w.node_type AND e.from_id=w.node_id) OR ($3 IN ('in','both') AND e.to_type=w.node_type AND e.to_id=w.node_id)) WHERE w.hop < $4), matched AS (SELECT DISTINCT e.relationship_observation_id FROM edges e JOIN walk w ON w.hop < $4 AND (($3 IN ('out','both') AND e.from_type=w.node_type AND e.from_id=w.node_id) OR ($3 IN ('in','both') AND e.to_type=w.node_type AND e.to_id=w.node_id))) SELECT o.relationship_observation_id, o.source_id, o.source_record_id, o.from_organization_id, o.target_facility_id, o.target_organization_id, o.relationship_type, o.assertion_status, o.unknown_reason, o.valid_from, o.valid_to, o.observed_at, o.confidence, o.review_state, o.storage_state, o.privacy_status, o.publication_status, o.note FROM observations o JOIN matched m USING (relationship_observation_id) ORDER BY o.observed_at DESC LIMIT 200", &[¶ms.entity_type, ¶ms.entity_id, &direction, &depth]).await { Ok(r) => r, Err(_) => return v2_error(StatusCode::SERVICE_UNAVAILABLE, "private_graph_query_failed", "private graph traversal failed") }; let data: Vec = rows.into_iter().map(|r| json!({"relationship_observation_id":r.get::<_,uuid::Uuid>(0),"source_id":r.get::<_,String>(1),"source_record_id":r.get::<_,uuid::Uuid>(2),"from_organization_id":r.get::<_,Option>(3),"target_facility_id":r.get::<_,Option>(4),"target_organization_id":r.get::<_,Option>(5),"relationship_type":r.get::<_,Option>(6),"assertion_status":r.get::<_,String>(7),"unknown_reason":r.get::<_,Option>(8),"valid_from":r.get::<_,Option>(9),"valid_to":r.get::<_,Option>(10),"observed_at":r.get::<_,chrono::DateTime>(11),"confidence":r.get::<_,Option>(12),"review_state":r.get::<_,String>(13),"storage_state":r.get::<_,String>(14),"privacy_status":r.get::<_,String>(15),"publication_status":r.get::<_,String>(16),"note":r.get::<_,Option>(17)})).collect(); Json(json!({"api_version":"private-graph-v1","data":data,"meta":{"scope":"private_evidence_only","bounded":true,"depth":depth,"direction":direction,"contradictions_preserved":true,"public_projection":false}})).into_response() } @@ -902,6 +918,7 @@ pub async fn get_dev_candidate_preview_handler( let rows = match client.query(r#" SELECT r.release_id, r.status, o.source_record_id, o.facility_id, f.canonical_name, f.country_code, f.city, o.classification_category, + o.coordinate_precision, o.coordinate_review_status, ST_Y(g.result::geometry), ST_X(g.result::geometry), source.origin_type, source.source_id, source.name, source.official_url, artifact.retrieved_at, review.factual_review_status, @@ -947,16 +964,20 @@ pub async fn get_dev_candidate_preview_handler( "city": row.get::<_, Option>(6), "category": row.get::<_, String>(7), "display_precision": "exact", - "latitude": row.get::<_, Option>(8), - "longitude": row.get::<_, Option>(9), - "source_type": row.get::<_, String>(10), - "provenance_source_id": row.get::<_, String>(11), - "provenance_source_name": row.get::<_, String>(12), - "provenance_source_url": row.get::<_, String>(13), - "provenance_retrieved_at": row.get::<_, chrono::DateTime>(14), - "factual_review_status": row.get::<_, String>(15), - "privacy_screening_status": row.get::<_, String>(16), + "latitude": row.get::<_, Option>(10), + "longitude": row.get::<_, Option>(11), + "coordinate_precision": row.get::<_, Option>(8), + "coordinate_review_status": row.get::<_, Option>(9), + "source_type": row.get::<_, String>(12), + "provenance_source_id": row.get::<_, String>(13), + "provenance_source_name": row.get::<_, String>(14), + "provenance_source_url": row.get::<_, String>(15), + "provenance_retrieved_at": row.get::<_, chrono::DateTime>(16), + "factual_review_status": row.get::<_, String>(17), + "privacy_screening_status": row.get::<_, String>(18), "project_approval": false, + "maintainer_approval": row.get::<_, String>(19), + "suppression_state": "not_currently_restricted", "release_id": row.get::<_, String>(0), "release_status": row.get::<_, String>(1), "preview_label": "Private development candidate — not project-approved or published" diff --git a/static/modules/__tests__/privateReview.test.js b/static/modules/__tests__/privateReview.test.js index 58b5ecf..382d5e4 100644 --- a/static/modules/__tests__/privateReview.test.js +++ b/static/modules/__tests__/privateReview.test.js @@ -53,7 +53,11 @@ test('safe review page never includes private address, raw payload, geocoder, or }); test('review packet loader rejects row-shaped payloads and keeps tokens out of storage APIs', async () => { - const { validateReviewPacket } = await import('../../private-review.js'); + const { validateReviewPacket, validateReadinessPayload } = await import('../../private-review.js'); expect(() => validateReviewPacket({ schema_version: 'private-review-packet-v2', counts: { input_rows: 1 } })).not.toThrow(); expect(() => validateReviewPacket({ schema_version: 'private-review-packet-v2', normalized: { records: [] } })).toThrow('rejected safely'); + expect(() => validateReviewPacket({ schema_version: 'private-review-packet-v2', counts: { input_rows: 1 }, provenance: { raw_payload_alias: 'withheld' } })).toThrow('rejected safely'); + expect(() => validateReviewPacket({ schema_version: 'private-review-packet-v2', quarantine: { reasons: { raw_payload_alias: 'withheld' } } })).toThrow('rejected safely'); + expect(() => validateReadinessPayload(JSON.parse(root('static/private-review/readiness-matrix.json')))).not.toThrow(); + expect(() => validateReadinessPayload({ schema_version: 'private-review-console-v1', derived_context: true, countries: {} })).toThrow('rejected safely'); }); diff --git a/static/private-graph.css b/static/private-graph.css index eb5880e..1bf7f49 100644 --- a/static/private-graph.css +++ b/static/private-graph.css @@ -1 +1,42 @@ -:root{font:16px/1.45 system-ui,sans-serif;color:#17251e;background:#eef2ed}*{box-sizing:border-box}body{margin:0}main{max-width:1200px;margin:auto;padding:2rem}header,.search,.workspace{background:#fff;border:1px solid #cbd7ce;border-radius:12px;padding:1.25rem;margin-bottom:1rem}.eyebrow{font-size:.75rem;letter-spacing:.12em;color:#567263}h1,h2{margin:.2rem 0 .7rem}.notice{border-left:4px solid #ba6b25;padding:.7rem 1rem;background:#fff6e9}.search div{display:flex;gap:.5rem}.search input{flex:1;padding:.7rem;border:1px solid #9daf9f;border-radius:6px}.search button{padding:.7rem 1rem;background:#1e5940;color:#fff;border:0;border-radius:6px}.workspace{display:grid;grid-template-columns:290px 1fr;gap:1rem;min-height:480px}aside{border-right:1px solid #d5ded8;padding-right:1rem}ul{list-style:none;padding:0;margin:0}li{padding:.65rem;border-bottom:1px solid #e4ebe5;cursor:pointer}li:hover,li.active{background:#e8f1eb}.toolbar{display:flex;align-items:center;gap:1rem;flex-wrap:wrap}.toolbar h2{margin-right:auto}.toolbar select{margin-left:.3rem;padding:.35rem}.empty{padding:2rem;color:#607267}.edge{border:1px solid #d5ded8;border-radius:8px;padding:1rem;margin:.7rem 0}.edge header{border:0;padding:0;margin:0;background:none}.chips{display:flex;gap:.4rem;flex-wrap:wrap}.chip{font-size:.78rem;padding:.2rem .45rem;border-radius:99px;background:#e8f1eb}.warn{background:#fff0de;color:#874912}@media(max-width:700px){main{padding:.7rem}.workspace{grid-template-columns:1fr}aside{border-right:0;border-bottom:1px solid #d5ded8;padding:0 0 1rem}} +:root { + font: 16px/1.45 system-ui, sans-serif; + color: #17251e; + background: #eef2ed; +} + +* { box-sizing: border-box; } +body { margin: 0; } +main { max-width: 1200px; margin: auto; padding: 2rem; } +header, .search, .workspace { + background: #fff; + border: 1px solid #cbd7ce; + border-radius: 12px; + padding: 1.25rem; + margin-bottom: 1rem; +} +.eyebrow { font-size: .75rem; letter-spacing: .12em; color: #567263; } +h1, h2 { margin: .2rem 0 .7rem; } +.notice { border-left: 4px solid #ba6b25; padding: .7rem 1rem; background: #fff6e9; } +.search div { display: flex; gap: .5rem; } +.search input { flex: 1; padding: .7rem; border: 1px solid #9daf9f; border-radius: 6px; } +.search > input { display: block; width: 100%; margin: .35rem 0 .8rem; } +.search button { padding: .7rem 1rem; background: #1e5940; color: #fff; border: 0; border-radius: 6px; } +.workspace { display: grid; grid-template-columns: 290px 1fr; gap: 1rem; min-height: 480px; } +aside { border-right: 1px solid #d5ded8; padding-right: 1rem; } +ul { list-style: none; padding: 0; margin: 0; } +li { padding: .65rem; border-bottom: 1px solid #e4ebe5; cursor: pointer; } +li:hover, li.active { background: #e8f1eb; } +.toolbar { display: flex; align-items: center; gap: 1rem; flex-wrap: wrap; } +.toolbar h2 { margin-right: auto; } +.toolbar select { margin-left: .3rem; padding: .35rem; } +.empty { padding: 2rem; color: #607267; } +.edge { border: 1px solid #d5ded8; border-radius: 8px; padding: 1rem; margin: .7rem 0; } +.edge header { border: 0; padding: 0; margin: 0; background: none; } +.chips { display: flex; gap: .4rem; flex-wrap: wrap; } +.chip { font-size: .78rem; padding: .2rem .45rem; border-radius: 99px; background: #e8f1eb; } +.warn { background: #fff0de; color: #874912; } +@media (max-width: 700px) { + main { padding: .7rem; } + .workspace { grid-template-columns: 1fr; } + aside { border-right: 0; border-bottom: 1px solid #d5ded8; padding: 0 0 1rem; } +} diff --git a/static/private-graph.html b/static/private-graph.html index b0aee7f..8fe0da0 100644 --- a/static/private-graph.html +++ b/static/private-graph.html @@ -1 +1 @@ -Private accountability graph

    RESTRICTED RESEARCH INTERFACE

    Accountability graph explorer

    Private evidence only. This view is not a public release, does not establish ownership or operational status, and must not be used to target people or locations.

    Search requires a configured private graph database.

    Select an entity

    Relationships, history, contradictions, and quarantine notes appear here.
    +Private accountability graph

    RESTRICTED RESEARCH INTERFACE

    Accountability graph explorer

    Private evidence only. This view is not a public release, does not establish ownership or operational status, and must not be used to target people or locations.

    Enter the configured private graph token to begin.

    Select an entity

    Relationships, history, contradictions, and quarantine notes appear here.
    diff --git a/static/private-graph.js b/static/private-graph.js index 9762432..3bdfa21 100644 --- a/static/private-graph.js +++ b/static/private-graph.js @@ -1,6 +1,6 @@ const $=s=>document.querySelector(s); const esc=v=>String(v??'unknown').replace(/[&<>"']/g,c=>({'&':'&','<':'<','>':'>','"':'"',"'":'''}[c])); let entities=[]; let selected=null; -async function api(path,params){const u=new URL(path,location.href);Object.entries(params).forEach(([k,v])=>u.searchParams.set(k,v));const r=await fetch(u,{headers:{Accept:'application/json'}});const b=await r.json();if(!r.ok)throw Error(b.error?.message||`HTTP ${r.status}`);return b} +async function api(path,params){const u=new URL(path,location.href);Object.entries(params).forEach(([k,v])=>u.searchParams.set(k,v));const token=$('#token').value.trim();const r=await fetch(u,{headers:{Accept:'application/json','X-UEC-Private-Graph-Token':token}});const b=await r.json();if(!r.ok)throw Error(b.error?.message||`HTTP ${r.status}`);return b} function renderEntities(){const list=$('#entities');list.innerHTML=entities.map((e,i)=>`
  • ${esc(e.display_name)}
    ${esc(e.entity_type)} · ${esc(e.source_id)} · ${esc(e.source_identifier)}
    • `).join('')||'
  • No entities found.
    • ';list.querySelectorAll('li[data-index]').forEach(x=>x.onclick=()=>select(Number(x.dataset.index)))} async function select(i){selected=i;renderEntities();const e=entities[i];$('#selected').textContent=e.display_name;$('#graph').innerHTML='

    Loading bounded evidence…

    ';try{const b=await api('/api/private/graph/traverse',{entity_type:e.entity_type,entity_id:e.entity_id,direction:$('#direction').value,depth:$('#depth').value});renderEdges(b.data)}catch(err){$('#graph').innerHTML=`

    ${esc(err.message)}

    `}} function renderEdges(edges){if(!edges.length){$('#graph').innerHTML='

    No retained relationship observations for this entity.

    ';return}$('#graph').innerHTML=edges.map(e=>`
    ${esc(e.relationship_type||'unknown relationship')} · ${esc(e.assertion_status)}

    ${esc(e.from_organization_id||'unknown')} → ${esc(e.target_facility_id||e.target_organization_id||'unknown')}

    observed ${esc(e.observed_at)}confidence ${esc(e.confidence)}review ${esc(e.review_state)}privacy ${esc(e.privacy_status)}publication ${esc(e.publication_status)}
    ${e.unknown_reason?`

    Unknown reason: ${esc(e.unknown_reason)}

    `:''}${e.note?`

    ${esc(e.note)}

    `:''}Source ID: ${esc(e.source_id)} · Source record: ${esc(e.source_record_id)}
    `).join('')} diff --git a/static/private-review.js b/static/private-review.js index 89f573a..9eee668 100644 --- a/static/private-review.js +++ b/static/private-review.js @@ -222,24 +222,253 @@ function renderCandidateDetail() { `; } -const FORBIDDEN_PACKET_KEYS = new Set([ - 'source_values', 'raw_fields', 'address', 'street', 'street_address', 'latitude', 'longitude', - 'coordinates', 'geocoder_query', 'geocoder_response', 'phone', 'email', 'requester', - 'requester_contact', 'reviewer_identity', 'records', 'rows', +const PACKET_TOP_KEYS = new Set([ + 'schema_version', 'source_id', 'run_dir_digest', 'run_id', 'classification', 'review_required', 'reasons', + 'provenance', 'schema', 'counts', 'review_metrics', 'facility_observation', 'classification', 'geospatial', + 'quarantine', 'run', 'release_diff', 'graph_candidates', 'gates', 'platform', 'publication_boundary', + 'blockers', 'prior_eligible_release', 'release_promotion_allowed', 'public_exposure', 'operator_actions', ]); +const PROVENANCE_KEYS = new Set(['source_url', 'retrieved_at_utc', 'publication_date', 'effective_date', 'sha256', 'checksum_sha256', 'byte_size', 'code_version', 'config_version', 'redirects']); +const SCHEMA_KEYS = new Set(['adapter_version', 'schema_version', 'schema_fingerprint', 'schema_status']); +const COUNT_KEYS = new Set(['input_rows', 'normalized_rows', 'quarantined_rows', 'reconciles', 'qa_matches_manifest']); +const METRIC_FACILITY_KEYS = new Set(['schema_version', 'source_row_unit', 'input_observations', 'accepted_observations', 'quarantined_observations', 'distinct_provisional_facility_keys', 'accepted_distinct_provisional_facility_keys', 'distinct_observation_keys', 'repeated_provisional_facility_groups', 'repeated_observation_keys', 'max_observations_per_provisional_facility', 'identity_semantics', 'disappearance_semantics']); +const METRIC_CLASSIFICATION_KEYS = new Set(['schema_version', 'rows', 'review_state_counts', 'field_presence', 'status_state_counts', 'interpretation']); +const METRIC_GEOSPATIAL_KEYS = new Set(['schema_version', 'coordinate_state_counts', 'precision_counts', 'coordinate_gate_counts', 'interpretation']); +const QUARANTINE_KEYS = new Set(['rows', 'reasons']); +const RUN_KEYS = new Set(['status', 'publication_state', 'release_state', 'input_rows', 'normalized_rows', 'quarantined_rows', 'drift_alarms']); +const RELEASE_DIFF_KEYS = new Set(['schema_version', 'status', 'delta_version', 'publication_state', 'release_promoted', 'public_surfaces', 'geocoding', 'prior_eligible_release', 'disappearance_semantics', 'error_type', 'error', 'counts', 'previous', 'current', 'added', 'changed', 'not_observed', 'suppressed']); +const RELEASE_SUMMARY_KEYS = new Set(['checksum_sha256', 'schema_fingerprint', 'config_fingerprint', 'mapping_version', 'adapter_version', 'schema_version', 'input_rows', 'normalized_rows', 'quarantined_rows']); +const GRAPH_CANDIDATE_KEYS = new Set(['status', 'storage_state', 'review_state', 'publication_status']); +const GATE_KEYS = new Set(['release_state', 'publication_state', 'release_promoted', 'public_surfaces', 'geocoding']); +const PLATFORM_KEYS = new Set(['registered', 'country_code', 'coverage', 'attribution', 'readiness', 'owner_review', 'publication']); +const COVERAGE_KEYS = new Set(['completeness', 'disappearance_semantics', 'limitations']); +const ATTRIBUTION_KEYS = new Set(['source_origin', 'terms_status', 'attribution_required', 'notice']); +const READINESS_KEYS = new Set(['state', 'owner_review', 'private_candidate', 'public_release_allowed', 'reasons']); +const OWNER_REVIEW_KEYS = new Set(['state', 'decision_recorded', 'decision_id']); +const PUBLICATION_KEYS = new Set(['state', 'approval_required', 'reason']); +const CONTRACT_VERSION_KEYS = new Set(['country', 'readiness']); + +function packetError(path) { + throw new Error(`Review packet rejected safely at ${path}.`); +} + +function assertObjectKeys(value, allowed, path) { + if (!isRecord(value)) packetError(path); + if (Object.keys(value).some((key) => !allowed.has(key))) packetError(path); +} + +function assertScalar(value, path) { + if (!(value === null || typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean')) packetError(path); +} + +function assertStringList(value, path) { + if (!Array.isArray(value) || value.some((item) => typeof item !== 'string')) packetError(path); +} + +function assertAggregateMap(value, path, mode = 'scalar') { + if (!isRecord(value)) packetError(path); + for (const [key, child] of Object.entries(value)) { + if (!/^[A-Za-z][A-Za-z0-9_.-]{0,79}$/.test(key)) packetError(`${path}.${key}`); + if (mode === 'string-list') assertStringList(child, `${path}.${key}`); + else if (mode === 'number') { + if (!Number.isInteger(child) || child < 0) packetError(`${path}.${key}`); + } else assertScalar(child, `${path}.${key}`); + } +} + +function validateMetricObject(value, path, allowed) { + if (value === undefined || value === null) return; + assertObjectKeys(value, allowed, path); + for (const [key, child] of Object.entries(value)) { + const childPath = `${path}.${key}`; + if (['review_state_counts', 'field_presence', 'status_state_counts', 'coordinate_state_counts', 'precision_counts', 'coordinate_gate_counts'].includes(key)) assertAggregateMap(child, childPath, 'number'); + else if (key === 'schema_version' || key === 'source_row_unit' || key === 'identity_semantics' || key === 'disappearance_semantics' || key === 'interpretation') assertScalar(child, childPath); + else if (typeof child !== 'number' || !Number.isFinite(child) || child < 0) packetError(childPath); + } +} + +function validateReviewPacketBranch(value, path = 'packet') { + if (!isRecord(value)) packetError(path); + assertObjectKeys(value, PACKET_TOP_KEYS, path); + for (const [key, child] of Object.entries(value)) { + const childPath = `${path}.${key}`; + if (['schema_version', 'source_id', 'run_dir_digest', 'run_id', 'classification', 'publication_boundary'].includes(key)) assertScalar(child, childPath); + else if (key === 'review_required' || key === 'release_promotion_allowed' || key === 'public_exposure') { + if (typeof child !== 'boolean') packetError(childPath); + } else if (key === 'reasons' || key === 'operator_actions') assertStringList(child, childPath); + else if (key === 'provenance') { + assertObjectKeys(child, PROVENANCE_KEYS, childPath); + Object.entries(child).forEach(([nestedKey, nestedValue]) => nestedKey === 'redirects' ? assertStringList(nestedValue, `${childPath}.${nestedKey}`) : assertScalar(nestedValue, `${childPath}.${nestedKey}`)); + } else if (key === 'schema') { + assertObjectKeys(child, SCHEMA_KEYS, childPath); + Object.entries(child).forEach(([nestedKey, nestedValue]) => assertScalar(nestedValue, `${childPath}.${nestedKey}`)); + } else if (key === 'counts') { + assertObjectKeys(child, COUNT_KEYS, childPath); + Object.entries(child).forEach(([nestedKey, nestedValue]) => { + if (nestedKey === 'reconciles' || nestedKey === 'qa_matches_manifest') { + if (typeof nestedValue !== 'boolean') packetError(`${childPath}.${nestedKey}`); + } else if (!Number.isInteger(nestedValue) || nestedValue < 0) packetError(`${childPath}.${nestedKey}`); + }); + } else if (key === 'review_metrics') { + assertObjectKeys(child, new Set(['schema_version', 'facility_observation', 'classification', 'geospatial']), childPath); + assertScalar(child.schema_version, `${childPath}.schema_version`); + validateMetricObject(child.facility_observation, `${childPath}.facility_observation`, METRIC_FACILITY_KEYS); + validateMetricObject(child.classification, `${childPath}.classification`, METRIC_CLASSIFICATION_KEYS); + validateMetricObject(child.geospatial, `${childPath}.geospatial`, METRIC_GEOSPATIAL_KEYS); + } else if (key === 'facility_observation') validateMetricObject(child, childPath, METRIC_FACILITY_KEYS); + else if (key === 'classification') validateMetricObject(child, childPath, METRIC_CLASSIFICATION_KEYS); + else if (key === 'geospatial') validateMetricObject(child, childPath, METRIC_GEOSPATIAL_KEYS); + else if (key === 'quarantine') { + assertObjectKeys(child, QUARANTINE_KEYS, childPath); + if (child.rows !== undefined && (!Number.isInteger(child.rows) || child.rows < 0)) packetError(`${childPath}.rows`); + if (child.reasons !== undefined) assertAggregateMap(child.reasons, `${childPath}.reasons`, 'number'); + } else if (key === 'run') { + assertObjectKeys(child, RUN_KEYS, childPath); + for (const [nestedKey, nestedValue] of Object.entries(child)) { + if (nestedKey === 'drift_alarms') assertStringList(nestedValue, `${childPath}.${nestedKey}`); + else assertScalar(nestedValue, `${childPath}.${nestedKey}`); + } + } else if (key === 'release_diff') { + assertObjectKeys(child, RELEASE_DIFF_KEYS, childPath); + for (const [nestedKey, nestedValue] of Object.entries(child)) { + if (nestedKey === 'counts') { + assertObjectKeys(nestedValue, new Set(['added', 'changed', 'not_observed', 'suppressed']), `${childPath}.counts`); + Object.values(nestedValue).forEach((count) => { if (count !== null && (!Number.isInteger(count) || count < 0)) packetError(`${childPath}.counts`); }); + } else if (nestedKey === 'public_surfaces') { + assertObjectKeys(nestedValue, new Set(['api', 'map', 'export', 'cache', 'history']), `${childPath}.public_surfaces`); + Object.values(nestedValue).forEach((flag) => { if (typeof flag !== 'boolean') packetError(`${childPath}.public_surfaces`); }); + } else if (nestedKey === 'previous' || nestedKey === 'current') { + assertObjectKeys(nestedValue, RELEASE_SUMMARY_KEYS, `${childPath}.${nestedKey}`); + Object.values(nestedValue).forEach((summaryValue) => assertScalar(summaryValue, `${childPath}.${nestedKey}`)); + } else if (nestedKey === 'release_promoted') { + if (typeof nestedValue !== 'boolean') packetError(`${childPath}.${nestedKey}`); + } else assertScalar(nestedValue, `${childPath}.${nestedKey}`); + } + } else if (key === 'graph_candidates') { + assertObjectKeys(child, GRAPH_CANDIDATE_KEYS, childPath); + Object.values(child).forEach((nestedValue) => assertScalar(nestedValue, childPath)); + } else if (key === 'gates') { + assertObjectKeys(child, GATE_KEYS, childPath); + Object.entries(child).forEach(([nestedKey, nestedValue]) => { + if (nestedKey === 'public_surfaces') { + assertObjectKeys(nestedValue, new Set(['api', 'map', 'export', 'cache', 'history']), `${childPath}.public_surfaces`); + Object.values(nestedValue).forEach((flag) => { if (typeof flag !== 'boolean') packetError(`${childPath}.public_surfaces`); }); + } else if (nestedKey === 'release_promoted') { + if (typeof nestedValue !== 'boolean') packetError(`${childPath}.${nestedKey}`); + } else assertScalar(nestedValue, `${childPath}.${nestedKey}`); + }); + } else if (key === 'platform') validatePlatformPacket(child, childPath); + else if (key === 'blockers') assertAggregateMap(child, childPath, 'string-list'); + else if (key === 'prior_eligible_release') assertScalar(child, childPath); + } + return value; +} + +function validatePlatformPacket(value, path) { + assertObjectKeys(value, PLATFORM_KEYS, path); + for (const [key, child] of Object.entries(value)) { + const childPath = `${path}.${key}`; + if (key === 'registered') { + if (typeof child !== 'boolean') packetError(childPath); + } else if (key === 'country_code') assertScalar(child, childPath); + else if (key === 'coverage') { + assertObjectKeys(child, COVERAGE_KEYS, childPath); + if (child.limitations !== undefined) assertStringList(child.limitations, `${childPath}.limitations`); + if (child.completeness !== undefined) assertScalar(child.completeness, `${childPath}.completeness`); + if (child.disappearance_semantics !== undefined) assertScalar(child.disappearance_semantics, `${childPath}.disappearance_semantics`); + } else if (key === 'attribution') { + assertObjectKeys(child, ATTRIBUTION_KEYS, childPath); + Object.values(child).forEach((nestedValue) => assertScalar(nestedValue, childPath)); + } else if (key === 'readiness') { + assertObjectKeys(child, READINESS_KEYS, childPath); + if (child.reasons !== undefined) assertStringList(child.reasons, `${childPath}.reasons`); + Object.entries(child).forEach(([nestedKey, nestedValue]) => { + if (nestedKey !== 'reasons' && !['state', 'owner_review'].includes(nestedKey) && typeof nestedValue !== 'boolean') assertScalar(nestedValue, `${childPath}.${nestedKey}`); + }); + } else if (key === 'owner_review') { + if (isRecord(child)) { + assertObjectKeys(child, OWNER_REVIEW_KEYS, childPath); + Object.values(child).forEach((nestedValue) => assertScalar(nestedValue, childPath)); + } else assertScalar(child, childPath); + } else if (key === 'publication') { + assertObjectKeys(child, PUBLICATION_KEYS, childPath); + Object.values(child).forEach((nestedValue) => assertScalar(nestedValue, childPath)); + } + } +} export function validateReviewPacket(value, path = 'packet') { - if (Array.isArray(value)) { - value.forEach((child, index) => validateReviewPacket(child, `${path}[${index}]`)); - return value; + validateReviewPacketBranch(value, path); + if (!/^private-review-packet-v[12]$/.test(String(value.schema_version || ''))) packetError(`${path}.schema_version`); + return value; +} + +export function validateReadinessPayload(value, path = 'readiness') { + if (!isRecord(value)) packetError(path); + const allowed = new Set(['schema_version', 'generated_at', 'contract_versions', 'source_of_truth', 'derived_context', 'country_count', 'source_count', 'readiness_counts', 'readiness_classes', 'states', 'classification_notes', 'countries', 'country_records', 'publication_boundary']); + assertObjectKeys(value, allowed, path); + if (value.schema_version !== 'private-review-console-v1' || value.derived_context !== true) packetError(path); + if (value.generated_at !== undefined) assertScalar(value.generated_at, `${path}.generated_at`); + if (value.contract_versions !== undefined) { + assertObjectKeys(value.contract_versions, CONTRACT_VERSION_KEYS, `${path}.contract_versions`); + Object.values(value.contract_versions).forEach((item) => assertScalar(item, `${path}.contract_versions`)); + } + if (value.source_of_truth !== undefined) { + assertObjectKeys(value.source_of_truth, new Set(['platform_registry', 'publication_boundary']), `${path}.source_of_truth`); + Object.values(value.source_of_truth).forEach((item) => assertScalar(item, `${path}.source_of_truth`)); } - if (!isRecord(value)) return value; - const leaked = Object.keys(value).filter((key) => FORBIDDEN_PACKET_KEYS.has(key.toLowerCase())); - if (leaked.length) throw new Error(`Review packet rejected safely at ${path}.`); - Object.entries(value).forEach(([key, child]) => validateReviewPacket(child, `${path}.${key}`)); + if (value.readiness_counts !== undefined) assertAggregateMap(value.readiness_counts, `${path}.readiness_counts`, 'number'); + if (value.classification_notes !== undefined) { + assertObjectKeys(value.classification_notes, new Set(READINESS_STATES), `${path}.classification_notes`); + Object.values(value.classification_notes).forEach((item) => { if (typeof item !== 'string') packetError(`${path}.classification_notes`); }); + } + if (value.publication_boundary !== undefined) assertScalar(value.publication_boundary, `${path}.publication_boundary`); + if (!Number.isInteger(value.country_count) || value.country_count < 1 || !Number.isInteger(value.source_count) || value.source_count < 1) packetError(path); + if (!Array.isArray(value.readiness_classes) || value.readiness_classes.length !== READINESS_STATES.length || value.readiness_classes.some((item, index) => item !== READINESS_STATES[index])) packetError(`${path}.readiness_classes`); + if (!Array.isArray(value.states) || value.states.join('|') !== READINESS_STATES.join('|')) packetError(`${path}.states`); + if (!isRecord(value.countries) || Object.keys(value.countries).length !== value.country_count) packetError(`${path}.countries`); + for (const [code, country] of Object.entries(value.countries)) { + if (!/^[A-Z]{2}$/.test(code) || !isRecord(country)) packetError(`${path}.countries.${code}`); + assertObjectKeys(country, new Set(['name', 'country_code', 'display_name', 'state', 'readiness_class', 'summary', 'basis', 'source_count', 'acquisition_counts', 'country_reasons', 'owner_review', 'publication_state', 'sources']), `${path}.countries.${code}`); + if (typeof country.name !== 'string' || !READINESS_STATES.includes(country.state) || typeof country.summary !== 'string' || !Array.isArray(country.basis) || !Number.isInteger(country.source_count) || country.source_count < 1 || !Array.isArray(country.sources) || country.sources.length !== country.source_count) packetError(`${path}.countries.${code}`); + if (country.acquisition_counts !== undefined) assertAggregateMap(country.acquisition_counts, `${path}.countries.${code}.acquisition_counts`, 'number'); + if (country.country_reasons !== undefined) assertStringList(country.country_reasons, `${path}.countries.${code}.country_reasons`); + if (country.owner_review !== undefined) assertScalar(country.owner_review, `${path}.countries.${code}.owner_review`); + if (country.publication_state !== undefined) assertScalar(country.publication_state, `${path}.countries.${code}.publication_state`); + country.sources.forEach((source, index) => validateReadinessSource(source, `${path}.countries.${code}.sources[${index}]`, code)); + } + if (!Array.isArray(value.country_records) || value.country_records.length !== value.country_count) packetError(`${path}.country_records`); + value.country_records.forEach((record, index) => { + const recordPath = `${path}.country_records[${index}]`; + if (!isRecord(record)) packetError(recordPath); + assertObjectKeys(record, new Set(['country_code', 'display_name', 'readiness_class', 'source_count', 'sources', 'owner_review', 'publication_state', 'basis', 'summary', 'acquisition_counts', 'country_reasons']), recordPath); + if (typeof record.country_code !== 'string' || !READINESS_STATES.includes(record.readiness_class) || !Number.isInteger(record.source_count) || !Array.isArray(record.sources) || record.sources.length !== record.source_count) packetError(recordPath); + record.sources.forEach((source, sourceIndex) => validateReadinessSource(source, `${recordPath}.sources[${sourceIndex}]`, record.country_code)); + }); return value; } +function validateReadinessSource(value, path, countryCode) { + const allowed = new Set(['source_id', 'country_code', 'jurisdiction_scope', 'source_url', 'access_method', 'cadence', 'status', 'readiness', 'owner_review', 'publication', 'coverage', 'attribution']); + assertObjectKeys(value, allowed, path); + if (value.country_code !== countryCode || typeof value.source_id !== 'string' || typeof value.status !== 'object' || typeof value.attribution !== 'object' || typeof value.coverage !== 'object') packetError(path); + assertObjectKeys(value.status, new Set(['metadata', 'acquisition', 'runtime_health', 'publication_eligibility', 'evidence', 'next_action']), `${path}.status`); + if (value.status.evidence !== undefined) assertStringList(value.status.evidence, `${path}.status.evidence`); + Object.entries(value.status).forEach(([key, child]) => { if (key !== 'evidence') assertScalar(child, `${path}.status.${key}`); }); + assertObjectKeys(value.readiness, READINESS_KEYS, `${path}.readiness`); + if (value.readiness.reasons !== undefined) assertStringList(value.readiness.reasons, `${path}.readiness.reasons`); + Object.entries(value.readiness).forEach(([key, child]) => { if (key !== 'reasons') assertScalar(child, `${path}.readiness.${key}`); }); + assertObjectKeys(value.owner_review, new Set(['state', 'decision_id', 'decision_recorded']), `${path}.owner_review`); + Object.values(value.owner_review).forEach((child) => assertScalar(child, `${path}.owner_review`)); + assertObjectKeys(value.publication, new Set(['state', 'approval_required', 'reason']), `${path}.publication`); + Object.values(value.publication).forEach((child) => assertScalar(child, `${path}.publication`)); + assertObjectKeys(value.coverage, COVERAGE_KEYS, `${path}.coverage`); + if (value.coverage.limitations !== undefined) assertStringList(value.coverage.limitations, `${path}.coverage.limitations`); + Object.entries(value.coverage).forEach(([key, child]) => { if (key !== 'limitations') assertScalar(child, `${path}.coverage.${key}`); }); + assertObjectKeys(value.attribution, ATTRIBUTION_KEYS, `${path}.attribution`); + Object.values(value.attribution).forEach((child) => assertScalar(child, `${path}.attribution`)); +} + function packetEntries(value) { if (!isRecord(value)) return '
  • Unavailable
    • '; const entries = Object.entries(value); @@ -266,7 +495,11 @@ function renderReviewPacket() { const diff = packet.release_diff || {}; const diffCounts = diff.counts || {}; const geospatial = packet.geospatial || {}; - panel.innerHTML = `

    LOCAL ROW-FREE PACKET

    ${escapeHtml(displayValue(packet.source_id, 'Source unavailable'))}

    ${escapeHtml(displayValue(packet.publication_boundary, 'Packet is evidence only; no approval is implied.'))}

    inspection only

    Counts / deltas

      ${packetEntries({ input: counts.input_rows, normalized: counts.normalized_rows, quarantined: counts.quarantined_rows, reconciles: counts.reconciles, added: diffCounts.added, changed: diffCounts.changed, not_observed: diffCounts.not_observed })}

    Quarantine reasons

      ${packetEntries(packet.quarantine?.reasons)}

    Coordinate / privacy gates

      ${packetEntries(geospatial.coordinate_gate_counts || geospatial.precision_counts)}

    Publication blockers

      ${packetEntries(packet.blockers)}
    `; + const provenance = packet.provenance || {}; + const schema = packet.schema || {}; + const gates = packet.gates || {}; + const platform = packet.platform || {}; + panel.innerHTML = `

    LOCAL ROW-FREE PACKET

    ${escapeHtml(displayValue(packet.source_id, 'Source unavailable'))}

    ${escapeHtml(displayValue(packet.publication_boundary, 'Packet is evidence only; no approval is implied.'))}

    inspection only

    Counts / deltas

      ${packetEntries({ input: counts.input_rows, normalized: counts.normalized_rows, quarantined: counts.quarantined_rows, reconciles: counts.reconciles, added: diffCounts.added, changed: diffCounts.changed, not_observed: diffCounts.not_observed })}

    Quarantine reasons

      ${packetEntries(packet.quarantine?.reasons)}

    Coordinate / privacy gates

      ${packetEntries(geospatial.coordinate_gate_counts || geospatial.precision_counts)}

    Provenance / schema

      ${packetEntries({ retrieved: provenance.retrieved_at_utc, effective: provenance.effective_date, sha256: provenance.sha256 || provenance.checksum_sha256, bytes: provenance.byte_size, adapter: schema.adapter_version, schema: schema.schema_version, fingerprint: schema.schema_fingerprint, schema_status: schema.schema_status })}

    Release gates

      ${packetEntries({ release: gates.release_state, publication: gates.publication_state, promoted: gates.release_promoted, geocoding: gates.geocoding, public_surfaces: gates.public_surfaces })}

    Platform / attribution

      ${packetEntries({ owner_review: platform.owner_review?.state || platform.owner_review, terms: platform.attribution?.terms_status, source_origin: platform.attribution?.source_origin, coverage: platform.coverage?.completeness })}

    Publication blockers

      ${packetEntries(packet.blockers)}
    `; } async function loadReviewPacket(event) { @@ -418,7 +651,7 @@ async function loadReadiness() { const response = await fetch(READINESS_PATH, { cache: 'no-store', headers: { Accept: 'application/json' } }); if (!response.ok) throw new Error('unavailable'); const payload = await response.json(); - if (!isRecord(payload) || payload.derived_context !== true || !isRecord(payload.countries)) throw new Error('invalid'); + validateReadinessPayload(payload); state.readiness = payload; state.readinessError = false; setAuthStatus('readiness-status', 'Readiness context loaded · derived only', 'ready'); diff --git a/static/private-review/readiness-matrix.json b/static/private-review/readiness-matrix.json index ab36411..7a55f3c 100644 --- a/static/private-review/readiness-matrix.json +++ b/static/private-review/readiness-matrix.json @@ -28578,7 +28578,7 @@ } ], "derived_context": true, - "generated_at": "2026-09-18T22:35:41Z", + "generated_at": "2026-09-18T22:53:58Z", "publication_boundary": "This row-free snapshot is operator context only. It cannot approve, promote, or publish a release.", "readiness_classes": [ "infrastructure-only", From 6367a176b36efada21f967ec1153f701124bb7be Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Fri, 18 Sep 2026 15:50:37 -0700 Subject: [PATCH 260/311] Add country geocoding reconnaissance profiles --- docs/architecture/country-source-platform.md | 22 +- docs/geocoding-reconnaissance.md | 130 ++++++++ .../contracts/geocoding-profile.schema.json | 36 ++ pipeline/contracts/geocoding_profile.py | 307 ++++++++++++++++++ pipeline/geocoding/profiles.json | 207 ++++++++++++ pipeline/geocoding/recon.py | 101 ++++++ pipeline/geocoding/test_recon.py | 99 ++++++ 7 files changed, 901 insertions(+), 1 deletion(-) create mode 100644 docs/geocoding-reconnaissance.md create mode 100644 pipeline/contracts/geocoding-profile.schema.json create mode 100644 pipeline/contracts/geocoding_profile.py create mode 100644 pipeline/geocoding/profiles.json create mode 100644 pipeline/geocoding/recon.py create mode 100644 pipeline/geocoding/test_recon.py diff --git a/docs/architecture/country-source-platform.md b/docs/architecture/country-source-platform.md index cb69c2e..f301ce8 100644 --- a/docs/architecture/country-source-platform.md +++ b/docs/architecture/country-source-platform.md @@ -11,12 +11,32 @@ existing sources of truth: and next-action status for each source. `pipeline/platform_registry.py` joins those files into country contracts and -source records. It currently materializes 234 sources across the registered +source records. It currently materializes the current 254-source registry across the registered country/cross-border prefixes and is designed for the 100+ country / hundreds of source target without copying every source row into a second hand-maintained registry. `pipeline/platform_registry.json` records the grouping rule and scale target. +## Geocoding reconnaissance profile + +`pipeline/geocoding/profiles.json` is a separate, provider-neutral, row-free +reconnaissance registry. It references source IDs from the authoritative source +registry and is validated by +`pipeline/contracts/geocoding_profile.py`. Profiles record source-coordinate +availability and semantics, official address authority candidates, government +and open-data options, regional/global fallbacks, language/address structure, +privacy-minimized query construction, exact/coarse/restricted/unmapped fallback +states, provider-specific confidence mapping, rate/cost unknowns, terms/logging/ +residency questions, outage/replacement behavior, and human-review gates. + +`pipeline/geocoding/recon.py` ranks all registered sources deterministically +from checked-in readiness/status signals and emits a row-free backlog plus +aggregate profile summaries. It does not fetch a provider, send a source +record, change readiness, create a geocode result, approve a release, or grant +publication eligibility. The public-service Nominatim endpoint is not a bulk +default; reusable global fallback candidates are self-hosted Nominatim and +Pelias, with third-party services retained as unreviewed discovery candidates. + ## Contracts `pipeline/contracts/country_contract.py` validates each country contract. Every diff --git a/docs/geocoding-reconnaissance.md b/docs/geocoding-reconnaissance.md new file mode 100644 index 0000000..748d165 --- /dev/null +++ b/docs/geocoding-reconnaissance.md @@ -0,0 +1,130 @@ +# Country/source geocoding reconnaissance + +Status: public-documentation reconnaissance and contract work only. Checked +2026-09-18 under [`docs/ETHICS.md`](ETHICS.md). No private facility record, +address, coordinate, geocoder query, or raw provider response is committed; +no provider was called with a project record. This report does not approve a +source, a geocoder, a coordinate, a release, or publication. + +## Result + +`pipeline/geocoding/profiles.json` is the machine-readable profile registry. +`pipeline/contracts/geocoding-profile.schema.json` documents its external +shape, while `pipeline/contracts/geocoding_profile.py` validates the stronger +privacy, fallback, provenance, terms-review, and publication boundaries. +`pipeline/geocoding/recon.py` joins the profiles to the checked-in source and +status registries and produces a deterministic row-free JSON report: + +```powershell +python -m pipeline.geocoding.recon --output data/reports/geocoding-recon.json +``` + +The report currently ranks 254 registered sources across 45 country/scope +prefixes. Its score uses only checked-in status signals: acquisition (0–50), +metadata (0–15), adapter implementation (0–14), runtime health (0–5), a +blocked-acquisition penalty, and a small assessed-profile tie-break bonus. +Ties sort by `source_id`. It ranks source readiness, not factual quality or +publication eligibility. + +## Deep tranche and order + +The first tranche is deliberately bounded. It covers the highest-readiness +source lanes that have private artifacts or implemented source adapters, plus +the UK lane whose FSA/FSS child feeds have materially stronger private staging +than the aggregate legacy registry row indicates. + +1. Denmark — `dk.smiley`: source `Geo_Lat`/`Geo_Lng` may be present, but point + semantics and privacy remain review questions. Prefer the official DAWA + address service; its current documentation warns that DAWA is closing, so + replacement planning is part of this lane. +2. Belgium — `be.locations`: FASFC operator and activity-code artifacts are + privately validated, but the live operator schema does not establish a + stable coordinate contract. Prefer BeSt-Address and its regional registers; + preserve Dutch/French/German variants and regional position metadata. +3. Italy — `it.853-2004`: the catalog supplies optional coordinate fields and + a location-status value, with some coordinates attributed to OSM. Prefer + ANNCSU for civic-number/access validation; never treat an OSM-derived + catalog point as automatically publishable. +4. Brazil — `br.sif.registered`, `br.sif.export`: SIF registered and export + evidence are separate source families; the assessed SIF CSVs supplied no + source coordinates. Prefer CEP/municipality validation, then a reviewed + exact provider. e-SISBI GIS and Trase coordinates remain separate future or + secondary evidence, not silent repairs. +5. Germany — `de.locations`: the normalized adapter deliberately withholds + coordinates pending review. Prefer BKG/AdV structured geocoding and use its + score, result type, and hit class together; a high score alone is not + acceptance. +6. United Kingdom — `uk.locations`: FSA England/Wales, FSS Scotland and + Northern Ireland are separate feed boundaries. Prefer licensed OS Places / + AddressBase where a project-specific contract permits it; use UPRN and ONS + postcode data only for reviewed linking or coarse display. + +The implementation order is a recommendation for private contract and test +work. Every profile remains `publication: blocked` and `no_private_records_queried: +true`. + +## Reusable global backups + +The default global candidate is `global.nominatim-self-hosted`, with +`global.pelias-self-hosted` as the second architecture candidate. Both require +project-controlled deployment, source-index review, update/capacity +benchmarks, attribution, suppression-aware caching, and a documented data +residency decision. They are not enabled by this change. + +The public `nominatim.openstreetmap.org` service is intentionally not the +default. Its policy sets an absolute maximum of one request per second, +discourages recurring bulk work, requires a valid identifying User-Agent or +Referer and attribution, requires caching for permitted bulk work, and says +not to submit personal or confidential data. It also prohibits systematic +queries and reselling geocoding results. See the [Nominatim usage +policy](https://operations.osmfoundation.org/policies/nominatim/). + +The OSM wiki's [alternatives / third-party providers +catalog](https://wiki.openstreetmap.org/wiki/Nominatim#Alternatives_.2F_Third-party_providers) +is recorded as a discovery source only. OpenCage, Stadia Maps, LocationIQ and +Geoapify are listed in `global_fallback_policy.provider_discovery_catalog` as +unreviewed candidates. Their current privacy, retention, residency, licensing, +rate, bulk and cost terms must be reviewed separately before any one becomes +an approved provider. A hosted global provider is never a reason to transmit a +private candidate address by default. + +## Common fallback and acceptance policy + +Every profile carries the same explicit state chain: + +`exact → coarse → restricted → unmapped` + +An exact result requires a unique address-level match, country/locality +agreement, provider-specific confidence evidence, a privacy decision, and a +human acceptance path. A coarse result is a separately labeled locality or +postcode reference; it is never an invented facility point. Restricted values +stay out of provider queries and public projections. Unmapped means only that +no eligible result was observed; it does not mean closure or absence. + +Provider responses are append-only evidence. Source coordinates are preserved +as source evidence and are never overwritten by a geocoder result. Provider +disagreement, moved points, stale/retired addresses and outages create new +events or review states; they do not silently update a current point. Failed +acquisitions must leave the previous validated release available, subject to +current suppression. + +## Backlog + +The full deterministic backlog is generated from the 254-source registry and +retains explicit unknowns and next actions from `docs/source-status.json`. +Sources with private or verified acquisition but no deep profile are the next +tranche; metadata-only and not-run sources remain reconnaissance or unstarted +backlog. The report is source-level and aggregate-only: it contains no +facility rows, address strings, coordinates, provider responses, or release +approval decisions. + +The current profile validator and tests cover: + +- profile schema and registry/source-ID integration; +- deterministic score and tie ordering; +- explicit unknown/not-observed coordinate states; +- exact/coarse/restricted/unmapped fallback ordering; +- privacy-minimized query construction and no-private-record network boundary; +- provider terms/rate/logging/residency review fields; +- source-coordinate preservation and disagreement/moved-point handling; and +- separation of geocoding evidence from approval and publication. diff --git a/pipeline/contracts/geocoding-profile.schema.json b/pipeline/contracts/geocoding-profile.schema.json new file mode 100644 index 0000000..f48fb28 --- /dev/null +++ b/pipeline/contracts/geocoding-profile.schema.json @@ -0,0 +1,36 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://untileverycage.example/contracts/country-geocoding-profile-v1.json", + "title": "Until Every Cage country geocoding profile", + "description": "Provider-neutral, row-free reconnaissance contract. Validation also enforces publication and privacy gates in Python.", + "type": "object", + "required": ["schema_version", "profiles"], + "properties": { + "schema_version": {"const": "country-geocoding-profile-registry-v1"}, + "profiles": { + "type": "array", + "items": {"$ref": "#/$defs/profile"} + } + }, + "$defs": { + "profile": { + "type": "object", + "required": [ + "profile_version", "profile_id", "country_code", "display_name", "source_ids", + "source_coordinate_evidence", "address_model", "national_address_authority", + "government_open_data_options", "providers", "query_minimization", "fallback_chain", + "confidence_policy", "review_gates", "resilience", "operational_estimate", + "unresolved_questions", "publication_boundary" + ], + "properties": { + "profile_version": {"const": "country-geocoding-profile-v1"}, + "country_code": {"type": "string", "pattern": "^[A-Z]{2,3}$"}, + "source_ids": {"type": "array", "minItems": 1, "items": {"type": "string"}}, + "source_coordinate_evidence": {"type": "array", "minItems": 1}, + "providers": {"type": "array", "minItems": 1}, + "fallback_chain": {"type": "array", "minItems": 4, "maxItems": 4}, + "publication_boundary": {"const": "blocked; profile is reconnaissance evidence only; no release eligibility is granted"} + } + } + } +} diff --git a/pipeline/contracts/geocoding_profile.py b/pipeline/contracts/geocoding_profile.py new file mode 100644 index 0000000..5f83096 --- /dev/null +++ b/pipeline/contracts/geocoding_profile.py @@ -0,0 +1,307 @@ +"""Provider-neutral country geocoding reconnaissance contracts. + +Profiles describe evidence and gates; they are not provider credentials, a +geocoding queue, a cache, or publication approval. The validator is kept +dependency-free so CI can fail closed before a profile is used by a pipeline. +""" +from __future__ import annotations + +from collections.abc import Iterable, Mapping +from typing import Any + + +GEOCODING_PROFILE_VERSION = "country-geocoding-profile-v1" +GEOCODING_REGISTRY_VERSION = "country-geocoding-profile-registry-v1" +RECON_REPORT_VERSION = "country-geocoding-recon-report-v1" + +_AVAILABILITY = frozenset({"present", "mixed", "not_observed", "unknown"}) +_PROVIDER_ROLES = frozenset({"national_authority", "government_lookup", "regional_fallback", "global_fallback"}) +_PROVIDER_STATUS = frozenset({"reviewed", "conditional", "candidate", "blocked", "unknown"}) +_FALLBACK_STATES = ("exact", "coarse", "restricted", "unmapped") +_ACQUISITION_POINTS = {"verified": 50, "artifact_private_only": 45, "not_run": 25, "blocked": 5} +_METADATA_POINTS = {"verified": 15, "partial": 8, "unknown": 0} +_ADAPTER_POINTS = {"implemented": 14, "implemented_partial": 12, "reference_only": 4, "not_started": 0} +_HEALTH_POINTS = {"healthy": 5, "unknown": 2, "not_run": 0} + + +class GeocodingProfileError(ValueError): + """Raised when a geocoding profile or report is incomplete or unsafe.""" + + +def _mapping(value: Any, path: str) -> Mapping[str, Any]: + if not isinstance(value, Mapping): + raise GeocodingProfileError(f"{path} must be an object") + return value + + +def _text(value: Mapping[str, Any], key: str, path: str) -> str: + result = value.get(key) + if not isinstance(result, str) or not result.strip(): + raise GeocodingProfileError(f"{path}.{key} must be a non-empty string") + return result + + +def _list(value: Mapping[str, Any], key: str, path: str, *, nonempty: bool = True) -> list[Any]: + result = value.get(key) + if not isinstance(result, list) or (nonempty and not result): + raise GeocodingProfileError(f"{path}.{key} must be a{' non-empty' if nonempty else 'n'} list") + return result + + +def _bool(value: Mapping[str, Any], key: str, path: str, *, required: bool = True) -> None: + if key not in value and not required: + return + if not isinstance(value.get(key), bool): + raise GeocodingProfileError(f"{path}.{key} must be boolean") + + +def _reject_private_payloads(value: Any, path: str = "profile") -> None: + """Reject shapes that could accidentally turn recon into a record store.""" + forbidden_keys = { + "facility_rows", "record_rows", "source_values", "raw_response", "response_payload", + "query_value", "raw_address", "latitude", "longitude", "lat", "lon", + } + if isinstance(value, Mapping): + for key, child in value.items(): + if str(key).lower() in forbidden_keys: + raise GeocodingProfileError(f"{path}.{key} is not allowed in a row-free profile") + _reject_private_payloads(child, f"{path}.{key}") + elif isinstance(value, list): + for index, child in enumerate(value): + _reject_private_payloads(child, f"{path}[{index}]") + + +def _validate_provider(provider: Mapping[str, Any], path: str, provider_ids: set[str]) -> None: + provider_id = _text(provider, "provider_id", path) + if provider_id in provider_ids: + raise GeocodingProfileError(f"duplicate provider_id: {provider_id}") + provider_ids.add(provider_id) + if provider.get("role") not in _PROVIDER_ROLES: + raise GeocodingProfileError(f"{path}.role is unknown") + if provider.get("status") not in _PROVIDER_STATUS: + raise GeocodingProfileError(f"{path}.status is unknown") + for key in ("name", "authority_or_operator", "coverage", "endpoint", "notes"): + _text(provider, key, path) + access = _mapping(provider.get("access"), f"{path}.access") + for key in ("mode", "bulk_mode", "export_or_download", "cost_model"): + _text(access, key, f"{path}.access") + terms = _mapping(provider.get("terms"), f"{path}.terms") + for key in ("license_status", "attribution", "caching", "redistribution", "retention", "logging", "residency"): + _text(terms, key, f"{path}.terms") + throughput = _mapping(provider.get("throughput"), f"{path}.throughput") + for key in ("documented_limit", "recommended_rate", "duration_or_cost_estimate"): + _text(throughput, key, f"{path}.throughput") + confidence = _mapping(provider.get("confidence_mapping"), f"{path}.confidence_mapping") + for key in ("accepted", "human_review", "rejected"): + _list(confidence, key, f"{path}.confidence_mapping") + docs = _list(provider, "evidence", path) + if any(not isinstance(item, str) or not item.startswith(("https://", "http://", "docs/", "pipeline/")) for item in docs): + raise GeocodingProfileError(f"{path}.evidence must contain URLs or repository paths") + + +def validate_profile(profile: Mapping[str, Any], *, known_source_ids: Iterable[str] | None = None) -> None: + """Validate one country profile and its publication boundary.""" + if not isinstance(profile, Mapping): + raise GeocodingProfileError("profile must be an object") + path = f"profile[{profile.get('profile_id', '?')}]" + if profile.get("profile_version") != GEOCODING_PROFILE_VERSION: + raise GeocodingProfileError(f"{path}.profile_version must be {GEOCODING_PROFILE_VERSION}") + for key in ("profile_id", "country_code", "display_name", "assessed_at", "assessment_scope", "publication_boundary"): + _text(profile, key, path) + source_ids = _list(profile, "source_ids", path) + if any(not isinstance(item, str) or not item.strip() for item in source_ids) or len(set(source_ids)) != len(source_ids): + raise GeocodingProfileError(f"{path}.source_ids must be unique non-empty strings") + if known_source_ids is not None: + unknown = sorted(set(source_ids) - set(known_source_ids)) + if unknown: + raise GeocodingProfileError(f"{path}.source_ids are not in the source registry: {unknown}") + + if profile.get("country_code") != str(profile["country_code"]).upper(): + raise GeocodingProfileError(f"{path}.country_code must be uppercase") + if profile.get("publication_boundary") != "blocked; profile is reconnaissance evidence only; no release eligibility is granted": + raise GeocodingProfileError(f"{path}.publication_boundary must keep publication blocked") + _bool(profile, "no_private_records_queried", path) + if profile.get("no_private_records_queried") is not True: + raise GeocodingProfileError(f"{path}.no_private_records_queried must be true") + + source_coordinates = _list(profile, "source_coordinate_evidence", path) + seen_source_ids: set[str] = set() + for index, item in enumerate(source_coordinates): + item = _mapping(item, f"{path}.source_coordinate_evidence[{index}]") + source_id = _text(item, "source_id", f"{path}.source_coordinate_evidence[{index}]") + if source_id not in source_ids or source_id in seen_source_ids: + raise GeocodingProfileError(f"{path}.source_coordinate_evidence has invalid or duplicate source_id: {source_id}") + seen_source_ids.add(source_id) + if item.get("availability") not in _AVAILABILITY: + raise GeocodingProfileError(f"{path}.source_coordinate_evidence[{index}].availability is unknown") + for key in ("field_names", "semantics", "precision", "expected_coverage", "review_state"): + if key == "field_names": + fields = _list(item, key, f"{path}.source_coordinate_evidence[{index}]") + if any(not isinstance(field, str) for field in fields): + raise GeocodingProfileError(f"{path}.source_coordinate_evidence[{index}].field_names must be strings") + else: + _text(item, key, f"{path}.source_coordinate_evidence[{index}]") + _bool(item, "never_overwrite_source", f"{path}.source_coordinate_evidence[{index}]") + if item.get("never_overwrite_source") is not True: + raise GeocodingProfileError(f"{path}.source_coordinate_evidence[{index}] must preserve source coordinates") + if seen_source_ids != set(source_ids): + raise GeocodingProfileError(f"{path}.source_coordinate_evidence must cover every source_id") + + address_model = _mapping(profile.get("address_model"), f"{path}.address_model") + for key in ("structure", "language_scripts", "transliteration", "administrative_hierarchy", "locality_notes"): + _text(address_model, key, f"{path}.address_model") + + authority = _mapping(profile.get("national_address_authority"), f"{path}.national_address_authority") + authority_provider = _text(authority, "provider_id", f"{path}.national_address_authority") + for key in ("authority", "role", "availability", "coordinate_semantics", "terms_status", "notes"): + _text(authority, key, f"{path}.national_address_authority") + + providers = _list(profile, "providers", path) + provider_ids: set[str] = set() + for index, item in enumerate(providers): + _validate_provider(_mapping(item, f"{path}.providers[{index}]"), f"{path}.providers[{index}]", provider_ids) + if authority_provider not in provider_ids: + raise GeocodingProfileError(f"{path}.national_address_authority.provider_id is not in providers") + if not any(item.get("provider_id") == authority_provider and item.get("role") == "national_authority" for item in providers): + raise GeocodingProfileError(f"{path}.national_address_authority provider must have national_authority role") + global_provider_ids = {str(item.get("provider_id")) for item in providers if str(item.get("provider_id", "")).startswith("global.")} + if not {"global.nominatim-self-hosted", "global.pelias-self-hosted"}.issubset(global_provider_ids): + raise GeocodingProfileError(f"{path} must carry both reusable global fallback candidates") + + lookup_options = _list(profile, "government_open_data_options", path) + for index, item in enumerate(lookup_options): + item = _mapping(item, f"{path}.government_open_data_options[{index}]") + _text(item, "provider_id", f"{path}.government_open_data_options[{index}]") + _text(item, "scope", f"{path}.government_open_data_options[{index}]") + _text(item, "access_and_export", f"{path}.government_open_data_options[{index}]") + _text(item, "limitations", f"{path}.government_open_data_options[{index}]") + + query = _mapping(profile.get("query_minimization"), f"{path}.query_minimization") + for key in ("allowed_components", "construction_steps", "never_send", "logging_boundary"): + _list(query, key, f"{path}.query_minimization") + _bool(query, "private_record_network_authorized", f"{path}.query_minimization") + if query.get("private_record_network_authorized") is not False: + raise GeocodingProfileError(f"{path}.query_minimization.private_record_network_authorized must be false") + + fallback = _list(profile, "fallback_chain", path) + if [item.get("state") for item in fallback if isinstance(item, Mapping)] != list(_FALLBACK_STATES): + raise GeocodingProfileError(f"{path}.fallback_chain must be ordered exact/coarse/restricted/unmapped") + for index, item in enumerate(fallback): + item = _mapping(item, f"{path}.fallback_chain[{index}]") + if item.get("state") not in _FALLBACK_STATES or item.get("order") != index + 1: + raise GeocodingProfileError(f"{path}.fallback_chain[{index}] has invalid state/order") + ids = _list(item, "provider_ids", f"{path}.fallback_chain[{index}", nonempty=False) + if any(provider_id not in provider_ids for provider_id in ids): + raise GeocodingProfileError(f"{path}.fallback_chain[{index}] references an unknown provider") + for key in ("entry_conditions", "exit_conditions"): + _text(item, key, f"{path}.fallback_chain[{index}]") + + confidence = _mapping(profile.get("confidence_policy"), f"{path}.confidence_policy") + for key in ("exact_acceptance", "coarse_acceptance", "human_review_threshold", "disagreement_rule", "moved_point_rule"): + _text(confidence, key, f"{path}.confidence_policy") + overrides = _list(confidence, "provider_overrides", f"{path}.confidence_policy") + for index, item in enumerate(overrides): + item = _mapping(item, f"{path}.confidence_policy.provider_overrides[{index}]") + _text(item, "provider_id", f"{path}.confidence_policy.provider_overrides[{index}]") + for key in ("accept", "review", "reject"): + _text(item, key, f"{path}.confidence_policy.provider_overrides[{index}]") + + gates = _mapping(profile.get("review_gates"), f"{path}.review_gates") + for key in ("terms_review", "privacy_review", "source_coordinate_review", "human_acceptance", "publication_separation"): + _text(gates, key, f"{path}.review_gates") + if gates.get("publication_separation") != "geocoding evidence never grants project approval or publication": + raise GeocodingProfileError(f"{path}.review_gates.publication_separation is unsafe") + + resilience = _mapping(profile.get("resilience"), f"{path}.resilience") + for key in ("disagreement", "moved_point", "outage", "replacement"): + _text(resilience, key, f"{path}.resilience") + estimate = _mapping(profile.get("operational_estimate"), f"{path}.operational_estimate") + for key in ("unit", "duration_formula", "cost_formula", "parallelism_policy"): + _text(estimate, key, f"{path}.operational_estimate") + _list(profile, "unresolved_questions", path) + _list(profile, "evidence", path) + _reject_private_payloads(profile) + + +def validate_profiles(payload: Mapping[str, Any], *, known_source_ids: Iterable[str] | None = None) -> None: + """Validate a profile registry and reject duplicate source ownership.""" + if payload.get("schema_version") != GEOCODING_REGISTRY_VERSION: + raise GeocodingProfileError(f"schema_version must be {GEOCODING_REGISTRY_VERSION}") + global_policy = _mapping(payload.get("global_fallback_policy"), "registry.global_fallback_policy") + for key in ("default_candidate_provider", "backup_candidate_provider", "activation_state", "rationale"): + _text(global_policy, key, "registry.global_fallback_policy") + if global_policy.get("default_candidate_provider") != "global.nominatim-self-hosted" or global_policy.get("backup_candidate_provider") != "global.pelias-self-hosted": + raise GeocodingProfileError("registry global fallback candidates must remain provider-neutral and self-hosted") + profiles = _list(payload, "profiles", "registry") + seen_profile_ids: set[str] = set() + seen_source_ids: set[str] = set() + for profile in profiles: + profile = _mapping(profile, "registry.profile") + validate_profile(profile, known_source_ids=known_source_ids) + profile_id = str(profile["profile_id"]) + if profile_id in seen_profile_ids: + raise GeocodingProfileError(f"duplicate profile_id: {profile_id}") + seen_profile_ids.add(profile_id) + overlap = seen_source_ids.intersection(profile["source_ids"]) + if overlap: + raise GeocodingProfileError(f"source IDs assigned to multiple profiles: {sorted(overlap)}") + seen_source_ids.update(profile["source_ids"]) + + +def rank_sources(source_registry: Mapping[str, Any], status_registry: Mapping[str, Any], profiles: Iterable[Mapping[str, Any]] = ()) -> list[dict[str, Any]]: + """Rank every registered source using only deterministic status signals.""" + statuses = {item.get("source_id"): item for item in status_registry.get("sources", []) if isinstance(item, Mapping)} + profiled = {source_id for profile in profiles for source_id in profile.get("source_ids", [])} + ranked: list[dict[str, Any]] = [] + for source in source_registry.get("sources", []): + source_id = str(source["source_id"]) + status = statuses.get(source_id, {}) + acquisition = str(status.get("acquisition", "not_run")) + metadata = str(status.get("metadata", "unknown")) + adapter = str(source.get("adapter_status", "not_started")) + health = str(status.get("runtime_health", "not_run")) + score = _ACQUISITION_POINTS.get(acquisition, 0) + _METADATA_POINTS.get(metadata, 0) + _ADAPTER_POINTS.get(adapter, 0) + _HEALTH_POINTS.get(health, 0) + if acquisition == "blocked": + score -= 10 + if source_id in profiled: + score += 3 + if acquisition == "verified" and adapter in {"implemented", "implemented_partial"}: + band = "deep-tranche" + elif acquisition == "artifact_private_only" and adapter in {"implemented", "implemented_partial"}: + band = "deep-tranche" + elif acquisition in {"verified", "artifact_private_only"}: + band = "next-tranche" + elif metadata in {"verified", "partial"}: + band = "reconnaissance-backlog" + else: + band = "unstarted-backlog" + ranked.append({ + "source_id": source_id, + "country_code": source_id.split(".", 1)[0].upper(), + "score": score, + "band": band, + "profile_state": "deeply-assessed" if source_id in profiled else "profile-pending", + "signals": {"metadata": metadata, "acquisition": acquisition, "adapter_status": adapter, "runtime_health": health}, + "next_action": str(status.get("next_action") or "Create a source-specific geocoding profile before acquisition or geocoding."), + }) + return sorted(ranked, key=lambda item: (-item["score"], item["source_id"])) + + +def build_recon_report(source_registry: Mapping[str, Any], status_registry: Mapping[str, Any], profile_registry: Mapping[str, Any]) -> dict[str, Any]: + """Build the row-free, deterministic ranking report.""" + validate_profiles(profile_registry, known_source_ids={item["source_id"] for item in source_registry.get("sources", [])}) + ranked = rank_sources(source_registry, status_registry, profile_registry["profiles"]) + deep = [item for item in ranked if item["profile_state"] == "deeply-assessed"] + return { + "report_version": RECON_REPORT_VERSION, + "rows_included": False, + "facility_or_address_payloads_included": False, + "source_count": len(ranked), + "profile_count": len(profile_registry["profiles"]), + "deep_assessment_source_count": len(deep), + "publication_boundary": "reconnaissance only; no profile, provider, source, or geocode result grants publication eligibility", + "ranking_method": "status-only deterministic score: acquisition 0-50, metadata 0-15, adapter 0-14, runtime 0-5, blocked-acquisition penalty 10, assessed-profile tie-break bonus 3; ties sort by source_id", + "ranked_backlog": ranked, + "deep_tranche": [item["source_id"] for item in deep], + "fallback_policy": "country/regional authority first; government/open-data coarse lookup second; reviewed regional candidate third; self-hosted or global fallback only after terms/privacy/rate review; otherwise restricted or unmapped", + } diff --git a/pipeline/geocoding/profiles.json b/pipeline/geocoding/profiles.json new file mode 100644 index 0000000..6e4747b --- /dev/null +++ b/pipeline/geocoding/profiles.json @@ -0,0 +1,207 @@ +{ + "schema_version": "country-geocoding-profile-registry-v1", + "contract": "pipeline/contracts/geocoding-profile.schema.json", + "policy_basis": "docs/ETHICS.md", + "row_free": true, + "global_fallback_policy": { + "default_candidate_provider": "global.nominatim-self-hosted", + "backup_candidate_provider": "global.pelias-self-hosted", + "activation_state": "not_enabled; deployment, licensing, privacy, update, capacity, and human terms review required", + "rationale": "A self-hosted global service can provide one reusable fallback without sending private candidate addresses to a public endpoint. The public Nominatim service is not a bulk or recurring default; its policy requires a deliberate informed choice, caching, attribution, a valid User-Agent, and no personal or confidential data.", + "public_service_state": "blocked for unattended private-record or recurring bulk use", + "provider_discovery_catalog": [ + {"provider_id": "global.opencage", "name": "OpenCage Geocoder", "status": "discovery_only; terms, privacy, residency, pricing, rate and bulk review not performed", "source": "https://wiki.openstreetmap.org/wiki/Nominatim#Alternatives_.2F_Third-party_providers"}, + {"provider_id": "global.stadia", "name": "Stadia Maps Geocoding API", "status": "discovery_only; terms, privacy, residency, pricing, rate and bulk review not performed", "source": "https://wiki.openstreetmap.org/wiki/Nominatim#Alternatives_.2F_Third-party_providers"}, + {"provider_id": "global.locationiq", "name": "LocationIQ", "status": "discovery_only; terms, privacy, residency, pricing, rate and bulk review not performed", "source": "https://wiki.openstreetmap.org/wiki/Nominatim#Alternatives_.2F_Third-party_providers"}, + {"provider_id": "global.geoapify", "name": "Geoapify Geocoding API", "status": "discovery_only; terms, privacy, residency, pricing, rate and bulk review not performed", "source": "https://wiki.openstreetmap.org/wiki/Nominatim#Alternatives_.2F_Third-party_providers"}, + {"provider_id": "global.nominatim-public", "name": "Public Nominatim service", "status": "not a project default; public policy limits and privacy restrictions apply", "source": "https://operations.osmfoundation.org/policies/nominatim/"} + ] + }, + "profiles": [ + { + "profile_version": "country-geocoding-profile-v1", + "profile_id": "dk-facility-geocoding", + "country_code": "DK", + "display_name": "Denmark", + "source_ids": ["dk.smiley"], + "assessed_at": "2026-09-18", + "assessment_scope": "Deep reconnaissance for the privately validated Find Smiley lane and its address/coordinate derivation boundary.", + "no_private_records_queried": true, + "publication_boundary": "blocked; profile is reconnaissance evidence only; no release eligibility is granted", + "source_coordinate_evidence": [ + {"source_id": "dk.smiley", "availability": "mixed", "field_names": ["Geo_Lat", "Geo_Lng"], "semantics": "The source snapshot exposes source-labeled point fields when supplied; the public evidence does not establish whether each point is an entrance, building, parcel, or provider-derived centroid.", "precision": "source precision unknown; preserve source values and provenance separately", "expected_coverage": "source-field presence must be measured per snapshot; no national or source completeness claim", "review_state": "source evidence only; privacy and coordinate review required", "never_overwrite_source": true} + ], + "address_model": {"structure": "Danish street name plus house number, optional floor/door, four-digit postal code and locality; source uses an address string plus separate postal/locality fields.", "language_scripts": "Danish Latin script with diacritics and abbreviations.", "transliteration": "Do not transliterate before the provider query; retain source spelling and use Unicode normalization only as a query candidate.", "administrative_hierarchy": "Municipality, postal locality, region; municipality is a disambiguation constraint, not a facility identity.", "locality_notes": "DAWA distinguishes address and access-address concepts; a matched access point is not automatically the operating facility point."}, + "national_address_authority": {"provider_id": "dk.dawa", "authority": "Danish Address Register (DAR) exposed by Dataforsyningen/DAWA", "role": "Official address lookup and address-data API; current docs warn that DAWA is closing, so replacement planning is required.", "availability": "public documented API with JSON and download capabilities; current operational contract must be rechecked", "coordinate_semantics": "Address/access-address geometry from the national address dataset; exact point semantics and source update lineage must be retained.", "terms_status": "provider terms and sunset/replacement status require maintainer confirmation before recurring geocoding", "notes": "Use structured street, house number and postal code parameters with mini response structure. Keep the existing adapter as development-only until a replacement or continued service contract is confirmed."}, + "government_open_data_options": [ + {"provider_id": "dk.dawa", "scope": "National address, access-address, street, locality and postal data.", "access_and_export": "HTTP JSON API and address-data download documented; use bounded, cached requests or a reviewed local copy.", "limitations": "DAWA closure warning, unspecified provider retention/rate terms, and no source-row publication permission."}, + {"provider_id": "dk.dawa", "scope": "Municipality/locality reference for coarse display only.", "access_and_export": "Use official place/locality endpoints where a city-level result is permitted.", "limitations": "Never invent a city point for an unresolved facility; only use a separately sourced reviewed city reference."} + ], + "providers": [ + {"provider_id": "dk.dawa", "role": "national_authority", "name": "DAWA address API", "authority_or_operator": "Dataforsyningen / Danish Address Register", "status": "conditional", "coverage": "Denmark national addresses and access addresses", "endpoint": "https://api.dataforsyningen.dk/adresser", "access": {"mode": "public HTTPS JSON", "bulk_mode": "download or bounded individual lookup; exact bulk contract to confirm", "export_or_download": "documented", "cost_model": "no price claim; confirm current terms"}, "terms": {"license_status": "not fully verified for this project", "attribution": "source attribution required unless terms confirm otherwise", "caching": "cache policy not verified; use bounded local cache only after review", "redistribution": "not assessed", "retention": "not documented", "logging": "not verified", "residency": "not verified"}, "throughput": {"documented_limit": "not found in public reconnaissance", "recommended_rate": "one request at a time with conservative delay until current provider guidance is confirmed", "duration_or_cost_estimate": "unique query count divided by the approved rate plus retries; cost unknown"}, "confidence_mapping": {"accepted": ["one current exact address/access-address match", "country/postal/house constraints agree", "no privacy restriction and no source-coordinate disagreement"], "human_review": ["multiple matches", "fuzzy-only match", "access-address versus facility-site ambiguity", "moved or retired address"], "rejected": ["zero match", "country mismatch", "provider result conflicts with protected or restricted status"]}, "evidence": ["https://dawadocs.dataforsyningen.dk/dok/api/adresse", "https://dawadocs.dataforsyningen.dk/dok/api/generelt", "https://dawadocs.dataforsyningen.dk/dok/guide/introduktion"], "notes": "Existing DAWA adapter is a development lane; it must not silently replace source coordinates."}, + {"provider_id": "global.nominatim-self-hosted", "role": "global_fallback", "name": "Self-hosted Nominatim over an approved OpenStreetMap extract", "authority_or_operator": "Project-controlled deployment using OpenStreetMap data", "status": "candidate", "coverage": "Global where the selected OSM extract has address coverage; not an authoritative Danish register", "endpoint": "project-controlled endpoint to be assigned after deployment review", "access": {"mode": "self-hosted HTTP API", "bulk_mode": "offline/import-backed batch permitted subject to capacity and ODbL review", "export_or_download": "OSM extract and derived database; source-specific licensing review required", "cost_model": "infrastructure, storage, update and operations cost; no amount claimed"}, "terms": {"license_status": "OpenStreetMap ODbL and Nominatim software terms require review", "attribution": "OpenStreetMap attribution required", "caching": "project-controlled cache permitted subject to ODbL and privacy controls", "redistribution": "derived-data and database-sharing obligations require legal review", "retention": "project policy and source update/removal process required", "logging": "project-controlled; never log raw queries or protected payloads by default", "residency": "choose and document deployment region before processing"}, "throughput": {"documented_limit": "self-hosted capacity limit must be benchmarked", "recommended_rate": "bounded single-worker start with backpressure", "duration_or_cost_estimate": "unique query count divided by measured sustainable rate; infrastructure estimate required"}, "confidence_mapping": {"accepted": ["exact house-level result with matching locality and country", "provider result passes independent address-token comparison"], "human_review": ["street/locality-only result", "multiple candidates", "low or absent address completeness", "OSM freshness or source disagreement"], "rejected": ["place-only result for an exact claim", "country mismatch", "privacy-screened query"]}, "evidence": ["https://operations.osmfoundation.org/policies/nominatim/"], "notes": "Use only after a deliberate project-owned deployment. Do not use the public Nominatim endpoint for recurring bulk work."}, + {"provider_id": "global.pelias-self-hosted", "role": "global_fallback", "name": "Self-hosted Pelias with reviewed address sources", "authority_or_operator": "Project-controlled Pelias deployment and explicitly licensed source indexes", "status": "candidate", "coverage": "Global multi-source search where installed datasets cover the country", "endpoint": "project-controlled endpoint to be assigned after deployment review", "access": {"mode": "self-hosted HTTP API", "bulk_mode": "offline or bounded batch subject to source-index terms", "export_or_download": "source-specific; no redistribution assumption", "cost_model": "infrastructure, index build and update cost; no amount claimed"}, "terms": {"license_status": "each source index requires independent review", "attribution": "per-source attribution required", "caching": "project-controlled with suppression-aware invalidation", "redistribution": "not assumed; source-specific", "retention": "source and project retention schedule required", "logging": "project-controlled; query minimization required", "residency": "choose and document deployment region"}, "throughput": {"documented_limit": "self-hosted capacity limit must be benchmarked", "recommended_rate": "bounded queue with measured concurrency", "duration_or_cost_estimate": "unique query count divided by measured rate plus index operations; cost unknown"}, "confidence_mapping": {"accepted": ["exact address candidate with consistent country/locality and provider source agreement"], "human_review": ["multiple source candidates", "coarse-only result", "source disagreement or stale index"], "rejected": ["no address-level support", "country mismatch", "restricted query" ]}, "evidence": ["https://opendata.bosa.be/index.fr.html"], "notes": "Pelias is a backup architecture candidate, not enabled provider approval."} + ], + "query_minimization": {"allowed_components": ["source postal code", "source locality", "source street token and house number after privacy screening", "country code"], "construction_steps": ["screen and suppress private/residential-risk values before any network request", "prefer structured provider parameters over free text", "send the minimum fields needed for the selected match tier", "hash the normalized query for internal deduplication; do not expose the hash as a public identifier"], "never_send": ["names, contacts, registration identifiers, free-text remarks, full source payloads, restricted addresses, suppressed records"], "logging_boundary": ["provider query is evidence metadata, not a public field", "store provider, timestamp, precision, decision and query hash only in restricted evidence"], "private_record_network_authorized": false}, + "fallback_chain": [ + {"order": 1, "state": "exact", "provider_ids": ["dk.dawa", "global.nominatim-self-hosted", "global.pelias-self-hosted"], "entry_conditions": "privacy and terms gates pass; source address is eligible for an exact lookup", "exit_conditions": "one exact, current, non-conflicting candidate passes provider-specific and human thresholds"}, + {"order": 2, "state": "coarse", "provider_ids": ["dk.dawa", "global.nominatim-self-hosted", "global.pelias-self-hosted"], "entry_conditions": "exact lookup fails but locality evidence is sufficient and a coarse reference is separately approved", "exit_conditions": "reviewed locality reference exists without inventing a facility point"}, + {"order": 3, "state": "restricted", "provider_ids": [], "entry_conditions": "privacy, source terms, disagreement or moved-point concern remains unresolved", "exit_conditions": "authorized reviewer resolves the restriction or the record remains restricted"}, + {"order": 4, "state": "unmapped", "provider_ids": [], "entry_conditions": "no eligible exact/coarse result", "exit_conditions": "new source evidence or reviewed provider result; absence is not closure"} + ], + "confidence_policy": {"exact_acceptance": "Accept only a unique exact address/access-address result with matching country and locality, no unresolved provider disagreement, and separate privacy approval.", "coarse_acceptance": "Accept only a reviewed city/locality reference; never fabricate a centroid from an exact-looking name.", "human_review_threshold": "Any multiple result, fuzzy-only match, source-coordinate disagreement, moved/retired address, or residential/mixed-use signal.", "disagreement_rule": "retain every provider/source result as separate evidence; do not average, overwrite, or choose by convenience", "moved_point_rule": "create a new dated geocoding event and review whether the source record moved, changed address, or was incorrectly matched", "provider_overrides": [{"provider_id": "dk.dawa", "accept": "single current exact result", "review": "multiple or fuzzy result", "reject": "empty or country-mismatched result"}, {"provider_id": "global.nominatim-self-hosted", "accept": "address-level result with matching tokens", "review": "street/locality-only or multiple candidates", "reject": "place-only or stale/conflicting result"}]}, + "review_gates": {"terms_review": "Confirm DAWA sunset/replacement, caching, redistribution and attribution before network use; separately review OSM/Pelias source terms.", "privacy_review": "No geocoding until address/coordinate privacy screening is complete; withheld or mixed residential/business values remain restricted.", "source_coordinate_review": "Source Geo_Lat/Geo_Lng remains evidence and is never overwritten by a geocoder result.", "human_acceptance": "Authorized reviewer must approve exact/coarse acceptance thresholds and disagreement decisions.", "publication_separation": "geocoding evidence never grants project approval or publication"}, + "resilience": {"disagreement": "append provider attempts and retain source coordinates; quarantine conflicts for review", "moved_point": "version the result by query/provider/time and mark prior point superseded without deleting evidence", "outage": "pause that provider, preserve the last eligible release, and route to the next approved provider or unmapped state", "replacement": "replace DAWA only through a provider-profile change, synthetic contract tests, terms review and a bounded shadow run"}, + "operational_estimate": {"unit": "unique privacy-eligible normalized address query, never raw row count", "duration_formula": "unique queries / approved sustainable provider rate + retry and review budget", "cost_formula": "provider tariff or self-hosted infrastructure estimate; unknown values remain unknown", "parallelism_policy": "single bounded worker until provider-specific concurrency and privacy review authorize more"}, + "unresolved_questions": ["What replaces DAWA and when is the current endpoint retired?", "What provider logging, caching and redistribution terms apply to DAWA and any replacement?", "Are Find Smiley source points entrance, building, or other geometry, and what is their effective date?", "What precision is safe for the project profile after residential/mixed-use screening?"], + "evidence": ["pipeline/sources/denmark/stages/normalize-denmark-smiley.py", "pipeline/sources/denmark/stages/geocode-denmark-dawa.py", "docs/current-geospatial-readiness.md", "https://dawadocs.dataforsyningen.dk/dok/adresser", "https://dawadocs.dataforsyningen.dk/dok/guide/introduktion", "https://operations.osmfoundation.org/policies/nominatim/"] + }, + { + "profile_version": "country-geocoding-profile-v1", + "profile_id": "it-853-geocoding", + "country_code": "IT", + "display_name": "Italy", + "source_ids": ["it.853-2004"], + "assessed_at": "2026-09-18", + "assessment_scope": "Deep reconnaissance for the privately acquired Ministry of Health 853/2004 catalog; the 1069/2009 lane remains separate.", + "no_private_records_queried": true, + "publication_boundary": "blocked; profile is reconnaissance evidence only; no release eligibility is granted", + "source_coordinate_evidence": [{"source_id": "it.853-2004", "availability": "mixed", "field_names": ["latitudine", "longitudine", "stato_localizzazione"], "semantics": "The catalog dictionary supplies optional location fields and a geolocation status; repository evidence notes that some coordinates came from OpenStreetMap contributors. Treat them as source observations, not automatically authoritative facility points.", "precision": "source precision and OSM-derived provenance vary; retain supplied status and values privately", "expected_coverage": "mixed across catalog records; calculate only from a private sanitized aggregate audit", "review_state": "source coordinate evidence pending privacy and precision review", "never_overwrite_source": true}], + "address_model": {"structure": "Italian street type/name, civic number and optional suffix/interior, locality/municipality, province/region and postal code; the source also includes ISTAT municipality code.", "language_scripts": "Italian Latin script with diacritics, abbreviations and regional toponyms.", "transliteration": "Preserve Italian spelling; normalize punctuation only for a derived query candidate and retain source text privately.", "administrative_hierarchy": "Comune, province/metropolitan city, region; ISTAT municipality code is a strong disambiguator.", "locality_notes": "Recognized establishments may have multiple activity observations; address and geocoding must attach to the source observation and recognition identity, not a silently merged facility."}, + "national_address_authority": {"provider_id": "it.anncsu", "authority": "ANNCSU, jointly maintained by Agenzia delle Entrate and ISTAT with municipal updates", "role": "National street and civic-number reference with public point lookup and open-data access.", "availability": "public point queries and open-data API/bulk services documented; technical access and current license details require validation", "coordinate_semantics": "Coordinates represent physical access points when supplied by a municipality; they are not necessarily a facility centroid and are not historicized by ANNCSU.", "terms_status": "open-data and service-specific terms must be reviewed before caching or redistribution", "notes": "Prefer ANNCSU to a general geocoder for address existence and access-point validation; do not treat coordinate presence as publication permission."}, + "government_open_data_options": [{"provider_id": "it.anncsu", "scope": "National streets and civic numbers, including municipal coordinate contributions where available.", "access_and_export": "Public address verification, point API, and regional/national bulk open-data routes documented.", "limitations": "Coordinate availability varies by municipality, coordinates can change without historicization, and exact endpoint/authentication terms require pinning."}, {"provider_id": "it.istat-places", "scope": "ISTAT municipality and administrative reference for disambiguation/coarse display.", "access_and_export": "Open administrative geography and reference downloads.", "limitations": "Not an exact address geocoder and cannot supply a facility point."}], + "providers": [ + {"provider_id": "it.anncsu", "role": "national_authority", "name": "ANNCSU address archive and services", "authority_or_operator": "Agenzia delle Entrate and ISTAT with municipalities", "status": "conditional", "coverage": "Italian streets and civic-number access points; municipal completion and coordinate coverage vary", "endpoint": "https://www.anncsu.gov.it/it/", "access": {"mode": "public web lookup plus documented API/open-data services", "bulk_mode": "monthly regional/national open-data download; point services updated daily according to public documentation", "export_or_download": "documented", "cost_model": "public access described as free; confirm service terms and operational prerequisites"}, "terms": {"license_status": "open-data basis documented; exact current dataset license and any API terms require review", "attribution": "Agenzia delle Entrate/ISTAT/municipal attribution required as specified", "caching": "review update and coordinate non-historicization behavior before cache retention", "redistribution": "review current open-data conditions and third-party municipal rights", "retention": "project retention and removal policy must be applied", "logging": "provider behavior not independently verified; minimize queries and retain only restricted evidence metadata", "residency": "provider hosting and service processing region not verified"}, "throughput": {"documented_limit": "not found in public reconnaissance", "recommended_rate": "bounded point lookups or local open-data build after access review", "duration_or_cost_estimate": "local index build and unique-query rate must be benchmarked; provider price not claimed"}, "confidence_mapping": {"accepted": ["current civic number exists and municipality/administrative code agrees", "coordinate is explicitly supplied for that access and passes privacy review"], "human_review": ["no coordinate supplied", "multiple civic-number variants", "municipality or province disagreement", "source catalog coordinate differs from ANNCSU access point"], "rejected": ["retired/non-current access for a current exact claim", "country mismatch", "private or restricted address"]}, "evidence": ["https://www.anncsu.gov.it/it/", "https://www.anncsu.gov.it/it/consultazione-dellarchivio/georeferenziazione-numeri-civici/", "https://www.anncsu.gov.it/it/progetto/adesione-anncsu/index.html"], "notes": "ANNCSU is the preferred national address reference; it is not a project approval or facility registry."}, + {"provider_id": "it.istat-places", "role": "government_lookup", "name": "ISTAT administrative and locality reference", "authority_or_operator": "ISTAT", "status": "reviewed", "coverage": "Italian municipalities, provinces, regions and statistical localities", "endpoint": "https://www.istat.it/", "access": {"mode": "public downloads and reference data", "bulk_mode": "local reference build", "export_or_download": "documented for administrative reference data", "cost_model": "public-data access; exact terms require source review"}, "terms": {"license_status": "source-specific terms require review", "attribution": "ISTAT attribution required", "caching": "local reference cache appropriate after terms review", "redistribution": "not assumed", "retention": "retain versioned metadata and source dates", "logging": "local only; no private query values", "residency": "not verified"}, "throughput": {"documented_limit": "not applicable to local reference data", "recommended_rate": "download once per approved version", "duration_or_cost_estimate": "local download and build cost; no provider API cost claimed"}, "confidence_mapping": {"accepted": ["municipality/province code agrees"], "human_review": ["historic boundary or bilingual locality conflict"], "rejected": ["administrative mismatch"]}, "evidence": ["https://www.istat.it/"], "notes": "Use for disambiguation and coarse fallback only."}, + {"provider_id": "global.nominatim-self-hosted", "role": "global_fallback", "name": "Self-hosted Nominatim over an approved OpenStreetMap extract", "authority_or_operator": "Project-controlled deployment using OpenStreetMap data", "status": "candidate", "coverage": "Global where the extract has address coverage; not an Italian authority", "endpoint": "project-controlled endpoint to be assigned after deployment review", "access": {"mode": "self-hosted HTTP API", "bulk_mode": "offline/import-backed batch subject to capacity and ODbL review", "export_or_download": "OSM extract and derived database; source-specific licensing review required", "cost_model": "infrastructure, update and operations cost; no amount claimed"}, "terms": {"license_status": "OpenStreetMap ODbL and software terms require review", "attribution": "OpenStreetMap attribution required", "caching": "suppression-aware project cache", "redistribution": "source-specific ODbL obligations require legal review", "retention": "project schedule and update/removal process", "logging": "project-controlled; raw queries excluded by default", "residency": "deployment region must be documented"}, "throughput": {"documented_limit": "self-hosted capacity to benchmark", "recommended_rate": "bounded single-worker start", "duration_or_cost_estimate": "unique queries / measured rate; infrastructure estimate required"}, "confidence_mapping": {"accepted": ["exact address result with matching municipality and country"], "human_review": ["multiple, fuzzy-only, or OSM-stale result"], "rejected": ["place-only or country-mismatched result"]}, "evidence": ["https://operations.osmfoundation.org/policies/nominatim/"], "notes": "Do not use the public endpoint for recurring bulk work."}, + {"provider_id": "global.pelias-self-hosted", "role": "global_fallback", "name": "Self-hosted Pelias with reviewed sources", "authority_or_operator": "Project-controlled deployment", "status": "candidate", "coverage": "Global multi-source fallback where installed indexes cover Italy", "endpoint": "project-controlled endpoint to be assigned after deployment review", "access": {"mode": "self-hosted HTTP API", "bulk_mode": "offline/bounded batch", "export_or_download": "source-specific", "cost_model": "infrastructure and index update cost; no amount claimed"}, "terms": {"license_status": "independent source-index review required", "attribution": "per-source attribution", "caching": "suppression-aware cache", "redistribution": "not assumed", "retention": "source-specific and project policy", "logging": "minimized local logs", "residency": "deployment region to be documented"}, "throughput": {"documented_limit": "self-hosted capacity to benchmark", "recommended_rate": "bounded queue", "duration_or_cost_estimate": "unique queries / measured rate plus index maintenance; cost unknown"}, "confidence_mapping": {"accepted": ["exact address with source agreement"], "human_review": ["multiple source candidates or locality-only match"], "rejected": ["no address-level result or country mismatch"]}, "evidence": ["https://www.anncsu.gov.it/it/consultazione-dellarchivio/open-data/"], "notes": "Backup architecture candidate only."} + ], + "query_minimization": {"allowed_components": ["civic number", "street name", "municipality", "postal code", "ISTAT municipality code", "country code"], "construction_steps": ["screen source values for residential/private risk", "prefer ANNCSU structured lookup and municipality code", "send only fields needed for the current match tier", "deduplicate by restricted query hash and preserve attempt history"], "never_send": ["names, VAT/tax identifiers, contacts, remarks, raw catalog rows, restricted addresses, source payloads"], "logging_boundary": ["retain provider/query hash/time/precision/review state in restricted evidence only", "do not expose provider query text in public projections"], "private_record_network_authorized": false}, + "fallback_chain": [{"order": 1, "state": "exact", "provider_ids": ["it.anncsu", "global.nominatim-self-hosted", "global.pelias-self-hosted"], "entry_conditions": "privacy and terms gates pass and an exact eligible address is available", "exit_conditions": "unique current access/address match passes thresholds"}, {"order": 2, "state": "coarse", "provider_ids": ["it.istat-places", "it.anncsu", "global.nominatim-self-hosted", "global.pelias-self-hosted"], "entry_conditions": "exact match fails but municipality evidence is reviewed", "exit_conditions": "separate coarse locality reference is approved"}, {"order": 3, "state": "restricted", "provider_ids": [], "entry_conditions": "privacy, terms, disagreement or moved-point concern unresolved", "exit_conditions": "authorized review resolves or restriction remains"}, {"order": 4, "state": "unmapped", "provider_ids": [], "entry_conditions": "no eligible exact/coarse result", "exit_conditions": "new source evidence or reviewed provider result; no closure inference"}], + "confidence_policy": {"exact_acceptance": "Unique current civic-number/access match with municipality and country agreement, source/provider provenance retained, and privacy approval.", "coarse_acceptance": "Municipality or locality only, separately labeled and never used as an exact facility point.", "human_review_threshold": "Any missing ANNCSU coordinate, multiple civic numbers, source-vs-provider disagreement, moved/retired access, or residential/mixed-use signal.", "disagreement_rule": "retain both source catalog coordinate and provider result as separate versioned evidence; do not overwrite or average", "moved_point_rule": "append a new dated event and review whether the address or source identity changed", "provider_overrides": [{"provider_id": "it.anncsu", "accept": "current civic number and municipality agree", "review": "coordinate missing or changed", "reject": "retired or mismatched address"}, {"provider_id": "global.nominatim-self-hosted", "accept": "exact address with municipality agreement", "review": "fuzzy/multiple result", "reject": "place-only or mismatch"}]}, + "review_gates": {"terms_review": "Confirm ANNCSU open-data/API terms, coordinate provenance, caching and redistribution; separately review OSM/Pelias.", "privacy_review": "Coordinates and addresses remain restricted until field-level privacy screening; source publication does not override ETHICS.md.", "source_coordinate_review": "Catalog coordinates remain source evidence and cannot be silently replaced by ANNCSU or global output.", "human_acceptance": "Authorized reviewer must assess OSM-derived coordinates and repeated activity/address identity.", "publication_separation": "geocoding evidence never grants project approval or publication"}, + "resilience": {"disagreement": "append catalog, ANNCSU and fallback attempts; quarantine conflict", "moved_point": "version by provider/time and preserve prior evidence as superseded", "outage": "keep last eligible release and move to the next approved provider or unmapped", "replacement": "provider replacement requires profile version, contract tests, bounded shadow run and terms review"}, + "operational_estimate": {"unit": "unique privacy-eligible normalized address query", "duration_formula": "unique queries / approved sustainable rate + retry and human-review budget", "cost_formula": "provider tariff or self-hosted infra estimate; unknown remains unknown", "parallelism_policy": "bounded single worker until concurrency is reviewed"}, + "unresolved_questions": ["What are the current public ANNCSU API authentication, rate and caching terms?", "Which current open-data license and attribution text applies to coordinates and bulk files?", "How should ANNCSU non-historicized coordinate changes be represented against source snapshots?", "What exact precision and source labels are acceptable for OSM-derived catalog points?"], + "evidence": ["pipeline/sources/italy/README.md", "docs/country-recon-it.md", "https://www.anncsu.gov.it/it/consultazione-dellarchivio/georeferenziazione-numeri-civici/", "https://www.anncsu.gov.it/it/progetto/adesione-anncsu/index.html", "https://operations.osmfoundation.org/policies/nominatim/"] + }, + { + "profile_version": "country-geocoding-profile-v1", + "profile_id": "be-fasfc-geocoding", + "country_code": "BE", + "display_name": "Belgium", + "source_ids": ["be.locations"], + "assessed_at": "2026-09-18", + "assessment_scope": "Deep reconnaissance for the privately validated FASFC operator/codebook pair; operators, activities and regional address registers remain distinct evidence.", + "no_private_records_queried": true, + "publication_boundary": "blocked; profile is reconnaissance evidence only; no release eligibility is granted", + "source_coordinate_evidence": [{"source_id": "be.locations", "availability": "unknown", "field_names": ["latitude", "longitude", "postal code", "municipality"], "semantics": "The checked-in adapter accepts possible coordinate fields, but the captured FASFC operator schema evidence does not establish a stable coordinate field or point semantics. Postal/municipality values are source evidence only.", "precision": "unknown; no coordinate is emitted by the normalized adapter", "expected_coverage": "unknown until a current schema fingerprint confirms coordinate presence", "review_state": "coordinate availability and privacy review pending", "never_overwrite_source": true}], + "address_model": {"structure": "Belgian address with multilingual street name, house number/suffix, four-digit postal code and municipality; regional registers supply address, street, municipality and postal objects.", "language_scripts": "Dutch, French and German Latin scripts, with region-specific naming and bilingual municipalities.", "transliteration": "Do not translate or collapse language variants; query the source language variant and preserve the original regional value.", "administrative_hierarchy": "Region, province, municipality/NIS code, postal information, street and address; Walloon municipality parts are a separate object.", "locality_notes": "BeSt coordinates can be Lambert regional data with derived WGS84; source documentation warns that regional register quality differs, so coordinate transformation and status must be retained."}, + "national_address_authority": {"provider_id": "be.best", "authority": "BeSt-Address service coordinated by Belgian FPS BOSA from the three regional address registers", "role": "National service boundary over Brussels, Flanders and Wallonia regional address sources.", "availability": "public open-data files, hosted web API and self-hosted best-in-a-box documented", "coordinate_semantics": "Regional address positions with Lambert and derived WGS84 forms; position method/specification and region must be retained.", "terms_status": "address data CC BY 4.0 is documented; API availability is free without guarantee and project field/privacy review remains required", "notes": "Use BeSt exact address lookup before a global fallback. Do not infer that a FASFC operator is a slaughterhouse or that a registered address is a public operating site."}, + "government_open_data_options": [{"provider_id": "be.best", "scope": "National service exposing regional municipalities, streets, addresses and postal information.", "access_and_export": "Hosted API, weekly XML full download and regional CSV/GeoPackage/GeoJSON derivatives documented.", "limitations": "Regional source quality/coverage differences, coordinate conversion semantics, API rate terms and personal-data boundaries require review."}, {"provider_id": "be.regional-registers", "scope": "Flanders AR, Walloon ICAR and Brussels Paradigm address registers.", "access_and_export": "Regional authoritative data and anomaly reporting routes; use only when BeSt coverage/quality requires a region-specific check.", "limitations": "Different schemas, languages, update paths and terms; do not silently union regional IDs."}], + "providers": [ + {"provider_id": "be.best", "role": "national_authority", "name": "BeSt-Address Web API and open data", "authority_or_operator": "FPS BOSA with Belgian regional address registers", "status": "reviewed", "coverage": "Belgian official addresses and regional administrative hierarchy", "endpoint": "https://best.pr.fedservices.be/api/opendata/best/v1/belgianAddress/v2/", "access": {"mode": "hosted JSON API plus weekly XML/CSV downloads", "bulk_mode": "weekly full download or paginated API; bounded access preferred", "export_or_download": "documented", "cost_model": "open-data license; hosted business endpoint described as free without guarantee"}, "terms": {"license_status": "CC BY 4.0 for address data documented", "attribution": "Belgian Regions/BOSA attribution required", "caching": "local cache must preserve version, region and update metadata", "redistribution": "CC BY allows reuse subject to third-party/data-rights review", "retention": "project retention and suppression policy applies", "logging": "provider logging not verified; query minimization required", "residency": "hosted service residency not verified; self-host where required"}, "throughput": {"documented_limit": "no numeric hosted API rate limit found", "recommended_rate": "paginate conservatively and respect provider responses; benchmark local download", "duration_or_cost_estimate": "unique queries / approved rate or local download time; no price claim"}, "confidence_mapping": {"accepted": ["one current address match with matching postal code, municipality/NIS and region", "position method is known and no privacy restriction"], "human_review": ["multiple language variants", "regional coordinate conversion", "proposed/retired address", "source/operator location mismatch"], "rejected": ["no exact address", "country mismatch", "private or restricted address"]}, "evidence": ["https://opendata.bosa.be/index.fr.html", "https://opendata.bosa.be/download/best/bestinabox/best-webapi-user-guide.pdf", "https://urbisdownload.datastore.brussels/UrbIS/TechSpec/Geoloc_TechSpec_EN20260224.pdf"], "notes": "The service is an address authority, not a facility review or publication decision."}, + {"provider_id": "be.regional-registers", "role": "government_lookup", "name": "Belgian regional address registers", "authority_or_operator": "Brussels Paradigm, Flemish AR and Walloon ICAR", "status": "conditional", "coverage": "Region-specific official address and coordinate records", "endpoint": "https://opendata.bosa.be/index.fr.html", "access": {"mode": "regional open data and services", "bulk_mode": "regional weekly downloads where offered", "export_or_download": "documented at BOSA index", "cost_model": "source-specific open-data terms"}, "terms": {"license_status": "regional/source-specific terms require review", "attribution": "regional attribution required", "caching": "version and region required", "redistribution": "not assumed beyond confirmed license", "retention": "project schedule and suppression rules", "logging": "not verified", "residency": "regional service hosting not verified"}, "throughput": {"documented_limit": "not found", "recommended_rate": "download or bounded lookup", "duration_or_cost_estimate": "local build/query estimate; cost unknown"}, "confidence_mapping": {"accepted": ["region-native current address with municipality agreement"], "human_review": ["cross-region language or coordinate transformation", "regional source disagreement"], "rejected": ["retired/mismatched address"]}, "evidence": ["https://opendata.bosa.be/index.fr.html"], "notes": "Use as a source-qualified fallback, never as a silent merge."}, + {"provider_id": "global.nominatim-self-hosted", "role": "global_fallback", "name": "Self-hosted Nominatim over an approved OpenStreetMap extract", "authority_or_operator": "Project-controlled deployment using OpenStreetMap data", "status": "candidate", "coverage": "Global OSM address coverage; not a Belgian authoritative register", "endpoint": "project-controlled endpoint to be assigned after deployment review", "access": {"mode": "self-hosted HTTP API", "bulk_mode": "offline/import-backed batch", "export_or_download": "source extract and derived database subject to ODbL review", "cost_model": "infrastructure and update cost; no amount claimed"}, "terms": {"license_status": "ODbL/software terms require review", "attribution": "OpenStreetMap attribution", "caching": "suppression-aware", "redistribution": "source-specific ODbL review", "retention": "project schedule and update/removal process", "logging": "no raw queries by default", "residency": "deployment region documented before use"}, "throughput": {"documented_limit": "self-host capacity to benchmark", "recommended_rate": "bounded single worker", "duration_or_cost_estimate": "unique queries / measured rate; infrastructure estimate required"}, "confidence_mapping": {"accepted": ["exact multilingual address with postal/municipality agreement"], "human_review": ["multiple variants or coarse result"], "rejected": ["place-only, mismatch or privacy-screened query"]}, "evidence": ["https://operations.osmfoundation.org/policies/nominatim/"], "notes": "The public endpoint is not a recurring bulk default."}, + {"provider_id": "global.pelias-self-hosted", "role": "global_fallback", "name": "Self-hosted Pelias with reviewed address sources", "authority_or_operator": "Project-controlled deployment", "status": "candidate", "coverage": "Global multi-source fallback including reviewed Belgian indexes", "endpoint": "project-controlled endpoint to be assigned after deployment review", "access": {"mode": "self-hosted HTTP API", "bulk_mode": "offline or bounded batch", "export_or_download": "source-specific", "cost_model": "infrastructure/index/update cost; no amount claimed"}, "terms": {"license_status": "each source index independently reviewed", "attribution": "per-source", "caching": "suppression-aware", "redistribution": "not assumed", "retention": "source/project schedule", "logging": "minimized local logging", "residency": "deployment region documented"}, "throughput": {"documented_limit": "self-host capacity to benchmark", "recommended_rate": "bounded queue", "duration_or_cost_estimate": "unique queries / measured rate plus index maintenance; unknown"}, "confidence_mapping": {"accepted": ["exact result from a reviewed source index"], "human_review": ["source disagreement, multiple results or coarse result"], "rejected": ["unreviewed result used as exact facility point"]}, "evidence": ["https://opendata.bosa.be/index.fr.html"], "notes": "Backup architecture candidate only."} + ], + "query_minimization": {"allowed_components": ["street and house number after privacy screening", "postal code", "municipality/NIS code", "country and source region"], "construction_steps": ["screen operator/address fields before network use", "keep Dutch/French/German variants source-qualified", "query BeSt structured fields before free search", "deduplicate by restricted query hash"], "never_send": ["enterprise number, operator name, contacts, remarks, raw CSV, restricted address, source row", "unreviewed coordinates"], "logging_boundary": ["restricted evidence stores provider, query hash, timestamp, precision, and review state", "public outputs contain no provider query or raw response"], "private_record_network_authorized": false}, + "fallback_chain": [{"order": 1, "state": "exact", "provider_ids": ["be.best", "be.regional-registers", "global.nominatim-self-hosted", "global.pelias-self-hosted"], "entry_conditions": "terms/privacy gates pass and source address is eligible", "exit_conditions": "unique current address with region/postal/municipality agreement"}, {"order": 2, "state": "coarse", "provider_ids": ["be.best", "be.regional-registers", "global.nominatim-self-hosted", "global.pelias-self-hosted"], "entry_conditions": "exact match unavailable but municipality/postal evidence is reviewed", "exit_conditions": "reviewed coarse reference only"}, {"order": 3, "state": "restricted", "provider_ids": [], "entry_conditions": "privacy, terms, regional disagreement or moved-point issue unresolved", "exit_conditions": "authorized review resolves or remains restricted"}, {"order": 4, "state": "unmapped", "provider_ids": [], "entry_conditions": "no eligible exact/coarse result", "exit_conditions": "new evidence; absence is not closure"}], + "confidence_policy": {"exact_acceptance": "Unique current BeSt/region address with postal, municipality and language-region agreement; precision and privacy state recorded.", "coarse_acceptance": "Municipality/postal locality only, explicitly coarse and not converted to a point.", "human_review_threshold": "Language variant conflict, multiple addresses, proposed/retired status, coordinate conversion issue, or private/mixed-use signal.", "disagreement_rule": "retain BeSt, regional and fallback outputs as separate evidence; never average or overwrite source values", "moved_point_rule": "append a dated provider event and review identity/lifecycle rather than interpreting change as closure", "provider_overrides": [{"provider_id": "be.best", "accept": "current exact address and regional position metadata", "review": "regional/language/position disagreement", "reject": "no exact or country mismatch"}, {"provider_id": "global.nominatim-self-hosted", "accept": "exact address with postal/municipality agreement", "review": "multiple or locality-only", "reject": "place-only/mismatch"}]}, + "review_gates": {"terms_review": "Confirm CC BY attribution, hosted API terms, caching, regional data rights and no-guarantee status.", "privacy_review": "FASFC operator/postal/address/coordinate fields remain private until field-level screening; registration is not operating-site permission.", "source_coordinate_review": "Any source or provider point is retained as separate evidence and does not replace a source field.", "human_acceptance": "Review ambiguous multilingual, mixed-use and source/operator classification cases.", "publication_separation": "geocoding evidence never grants project approval or publication"}, + "resilience": {"disagreement": "quarantine provider/source disagreement and retain source-qualified results", "moved_point": "version by region/provider/time; review currentness and address lifecycle", "outage": "preserve last eligible release and use next approved lane or unmapped", "replacement": "provider replacement requires schema/terms/privacy tests and bounded shadow comparison"}, + "operational_estimate": {"unit": "unique privacy-eligible normalized address query", "duration_formula": "unique queries / approved rate + retry/review budget; weekly full downloads are separate", "cost_formula": "provider price or self-hosted estimate; no zero-cost assumption", "parallelism_policy": "single bounded worker until hosted API and regional terms authorize more"}, + "unresolved_questions": ["What hosted BeSt API rate limits and logging apply to this use?", "Which regional address fields and coordinate methods are safe to cache and redistribute?", "Can FASFC postal/municipality values be used for address lookup without exposing operator identity?", "What human threshold distinguishes registered office/mixed-use location from operating facility evidence?"], + "evidence": ["docs/country-recon-be.md", "pipeline/sources/belgium/adapter.py", "https://opendata.bosa.be/index.fr.html", "https://opendata.bosa.be/download/best/bestinabox/best-webapi-user-guide.pdf", "https://operations.osmfoundation.org/policies/nominatim/"] + }, + { + "profile_version": "country-geocoding-profile-v1", + "profile_id": "br-mapa-geocoding", + "country_code": "BR", + "display_name": "Brazil", + "source_ids": ["br.sif.registered", "br.sif.export"], + "assessed_at": "2026-09-18", + "assessment_scope": "Deep reconnaissance for MAPA SIF registered and export-authorization evidence; e-SISBI GIS is a separate future source lane and Trase remains secondary evidence.", + "no_private_records_queried": true, + "publication_boundary": "blocked; profile is reconnaissance evidence only; no release eligibility is granted", + "source_coordinate_evidence": [{"source_id": "br.sif.registered", "availability": "not_observed", "field_names": ["LOGRADOURO", "BAIRRO", "CEP", "MUNICIPIO", "UF"], "semantics": "The private reconnaissance crosswalk reports no source coordinates and no geocoding was performed; address components are source values and may identify private or mixed-use locations.", "precision": "no source point supplied", "expected_coverage": "none observed in the assessed file shape; do not infer from address presence", "review_state": "no coordinate evidence; privacy/terms gates remain", "never_overwrite_source": true}, {"source_id": "br.sif.export", "availability": "not_observed", "field_names": ["UF", "MUNICIPIO", "SIF"], "semantics": "Export-authorized observations carry establishment and locality fields but no source coordinate field in the assessed crosswalk; this feed is not a facility-location master.", "precision": "no source point supplied", "expected_coverage": "not applicable to export authorization observations", "review_state": "not a geocoding source; keep separate from registered-facility evidence", "never_overwrite_source": true}], + "address_model": {"structure": "Brazilian address uses logradouro/type, number and optional complement, bairro, municipality, UF and CEP; the source registry uses Portuguese field names and IBGE municipality context where available.", "language_scripts": "Brazilian Portuguese Latin script with diacritics, abbreviations and local naming variation.", "transliteration": "Preserve source Portuguese and diacritics; normalize only derived query candidates.", "administrative_hierarchy": "Municipality, state/UF, neighborhood/bairro, CEP; IBGE municipality code is a disambiguator when present.", "locality_notes": "CEP can identify postal locality/address components but is not a precise facility coordinate; informal/peripheral addresses may be incomplete or non-standard."}, + "national_address_authority": {"provider_id": "br.cep-cadastro", "authority": "Cadastro Base de Endereço using Correios-managed CEP data through Conecta gov.br", "role": "National postal/address component lookup; not a national coordinate geocoder.", "availability": "government API catalog and authenticated/restricted access route documented", "coordinate_semantics": "CEP/address components are returned; no exact coordinate output is documented for this service, so it supports validation and coarse disambiguation only.", "terms_status": "access, IP allowlisting, service agreement, privacy and reuse terms require confirmation", "notes": "For MAPA SIF, prefer source-provided municipality/CEP validation first. The e-SISBI GIS route is not silently folded into this profile; it needs a separate source/provider contract."}, + "government_open_data_options": [{"provider_id": "br.cep-cadastro", "scope": "National CEP-linked minimum address data.", "access_and_export": "Conecta API lookup by CEP; access is described as restricted to registered IPs/eligible integrators.", "limitations": "No documented national exact-coordinate output, rate limit or open redistribution terms."}, {"provider_id": "br.ibge-cnefe", "scope": "IBGE statistical address context and aggregate/coarse geography.", "access_and_export": "Public statistical products with confidentiality safeguards.", "limitations": "Not a named-facility geocoder; do not use microdata or residential detail for exact facility mapping."}, {"provider_id": "br.pbh-geocoder", "scope": "Belo Horizonte regional address/geocoder service.", "access_and_export": "Public regional API with official local address base.", "limitations": "Only Belo Horizonte; no national coverage or general fallback claim."}], + "providers": [ + {"provider_id": "br.cep-cadastro", "role": "national_authority", "name": "Cadastro Base de Endereço / CEP API", "authority_or_operator": "MGI/SGD Conecta gov.br with Correios-managed CEP data", "status": "conditional", "coverage": "Brazilian postal/address components keyed by CEP", "endpoint": "https://apigateway.conectagov.estaleiro.serpro.gov.br/api-cep/v1/consulta/cep/", "access": {"mode": "government API with access onboarding/IP registration", "bulk_mode": "not documented; do not assume bulk", "export_or_download": "not documented", "cost_model": "government service; access conditions and any charges require confirmation"}, "terms": {"license_status": "not verified for project reuse", "attribution": "Correios/MGI/Conecta attribution to confirm", "caching": "provider cache and postal update terms unknown", "redistribution": "not assumed", "retention": "not documented", "logging": "provider behavior not verified; do not send private records", "residency": "Serpro/Conecta processing region and subcontractors require review"}, "throughput": {"documented_limit": "not found", "recommended_rate": "only after access approval; bounded individual CEP validation", "duration_or_cost_estimate": "unique CEP queries / approved rate; no cost claim"}, "confidence_mapping": {"accepted": ["CEP and municipality/UF components agree with source"], "human_review": ["CEP type is not a local address, missing number, or conflicting municipality"], "rejected": ["no CEP match, country mismatch, private or restricted address"]}, "evidence": ["https://www.gov.br/conecta/catalogo/apis/cep-codigo-de-enderecamento-postal"], "notes": "Validation/coarse lookup only; it does not produce a facility point."}, + {"provider_id": "br.ibge-cnefe", "role": "government_lookup", "name": "IBGE CNEFE and geographic references", "authority_or_operator": "Instituto Brasileiro de Geografia e Estatística", "status": "conditional", "coverage": "statistical address context and public aggregate geography", "endpoint": "https://www.ibge.gov.br/estatisticas/sociais/populacao/38734-cadastro-nacional-de-enderecos-para-fins-estatisticos.html", "access": {"mode": "public statistical downloads and reference products", "bulk_mode": "aggregate/coarse downloads", "export_or_download": "documented", "cost_model": "public statistical data; field-specific rights and confidentiality review"}, "terms": {"license_status": "source-specific terms and statistical confidentiality apply", "attribution": "IBGE attribution required", "caching": "versioned aggregate cache only", "redistribution": "do not redistribute protected microdata", "retention": "project policy with suppression", "logging": "local only", "residency": "not verified"}, "throughput": {"documented_limit": "not an address geocoder", "recommended_rate": "download official releases", "duration_or_cost_estimate": "local download/build; cost unknown"}, "confidence_mapping": {"accepted": ["coarse municipality/state context only"], "human_review": ["any attempted exact use"], "rejected": ["microdata/residential inference or exact facility claim"]}, "evidence": ["https://www.ibge.gov.br/estatisticas/sociais/populacao/38734-cadastro-nacional-de-enderecos-para-fins-estatisticos.html"], "notes": "Coarse context only, never an exact facility geocoder."}, + {"provider_id": "br.pbh-geocoder", "role": "regional_fallback", "name": "Belo Horizonte official geocoder", "authority_or_operator": "Prefeitura de Belo Horizonte", "status": "candidate", "coverage": "Belo Horizonte municipal address base only", "endpoint": "https://geocoder.pbh.gov.br/geocoder/v2/address", "access": {"mode": "public local REST API", "bulk_mode": "not documented; individual bounded lookup only", "export_or_download": "not documented", "cost_model": "public municipal service; terms and rate limits unknown"}, "terms": {"license_status": "not verified", "attribution": "municipal attribution to confirm", "caching": "not documented", "redistribution": "not assumed", "retention": "not documented", "logging": "not verified", "residency": "Brazil municipal hosting presumed but not verified"}, "throughput": {"documented_limit": "not found", "recommended_rate": "single bounded worker after review", "duration_or_cost_estimate": "unique local queries / measured rate; cost unknown"}, "confidence_mapping": {"accepted": ["official local address match with municipality agreement"], "human_review": ["non-Belo Horizonte use, multiple or incomplete result"], "rejected": ["outside coverage or country mismatch"]}, "evidence": ["https://geocoder.pbh.gov.br/"], "notes": "A regional option, not a national fallback."}, + {"provider_id": "global.nominatim-self-hosted", "role": "global_fallback", "name": "Self-hosted Nominatim over an approved OpenStreetMap extract", "authority_or_operator": "Project-controlled deployment using OpenStreetMap data", "status": "candidate", "coverage": "Global OSM coverage where the extract is sufficiently mapped; Brazil coverage is uneven and not authoritative", "endpoint": "project-controlled endpoint to be assigned after deployment review", "access": {"mode": "self-hosted HTTP API", "bulk_mode": "offline/import-backed batch", "export_or_download": "extract/database subject to ODbL and source review", "cost_model": "infrastructure/update/index cost; no amount claimed"}, "terms": {"license_status": "ODbL/software terms require review", "attribution": "OpenStreetMap attribution", "caching": "suppression-aware project cache", "redistribution": "source-specific ODbL review", "retention": "project schedule and update/removal process", "logging": "minimized local logging", "residency": "deployment region documented"}, "throughput": {"documented_limit": "self-host capacity to benchmark", "recommended_rate": "bounded single worker", "duration_or_cost_estimate": "unique queries / measured rate; infrastructure estimate required"}, "confidence_mapping": {"accepted": ["exact address with municipality/UF agreement and independent review"], "human_review": ["rural/informal address, multiple or coarse result"], "rejected": ["place-only, mismatch or restricted query"]}, "evidence": ["https://operations.osmfoundation.org/policies/nominatim/"], "notes": "Never treat OSM success as source authority or publication permission."}, + {"provider_id": "global.pelias-self-hosted", "role": "global_fallback", "name": "Self-hosted Pelias with reviewed address sources", "authority_or_operator": "Project-controlled deployment", "status": "candidate", "coverage": "Global multi-source search, dependent on reviewed source indexes", "endpoint": "project-controlled endpoint to be assigned after deployment review", "access": {"mode": "self-hosted HTTP API", "bulk_mode": "offline/bounded batch", "export_or_download": "source-specific", "cost_model": "infrastructure/index/update cost; no amount claimed"}, "terms": {"license_status": "each source index review required", "attribution": "per-source", "caching": "suppression-aware", "redistribution": "not assumed", "retention": "source/project schedule", "logging": "local minimization", "residency": "deployment region documented"}, "throughput": {"documented_limit": "self-host capacity to benchmark", "recommended_rate": "bounded queue", "duration_or_cost_estimate": "unique queries / measured rate + index maintenance; unknown"}, "confidence_mapping": {"accepted": ["exact result from reviewed source index"], "human_review": ["multiple source candidates, informal/rural address or stale index"], "rejected": ["no address-level result or mismatch"]}, "evidence": ["https://www.gov.br/conecta/catalogo/apis/cep-codigo-de-enderecamento-postal"], "notes": "Backup architecture candidate only."} + ], + "query_minimization": {"allowed_components": ["logradouro and number after privacy screening", "CEP", "municipality", "UF", "IBGE municipality code", "country"], "construction_steps": ["screen CNPJ, contacts, names and full address risk before any query", "validate CEP/municipality with the official postal service if authorized", "use regional geocoder only within declared coverage", "deduplicate by restricted query hash"], "never_send": ["CNPJ, legal/trade names, contacts, occurrence text, raw SIF rows, restricted address, export authorization details"], "logging_boundary": ["provider, timestamp, query hash, precision, result class and review state stay restricted", "no raw provider response in row-free reports"], "private_record_network_authorized": false}, + "fallback_chain": [{"order": 1, "state": "exact", "provider_ids": ["br.cep-cadastro", "br.pbh-geocoder", "global.nominatim-self-hosted", "global.pelias-self-hosted"], "entry_conditions": "terms/privacy gate passes and source address is eligible; regional provider only if in coverage", "exit_conditions": "unique exact address result with municipality/UF agreement and human acceptance"}, {"order": 2, "state": "coarse", "provider_ids": ["br.cep-cadastro", "br.ibge-cnefe", "global.nominatim-self-hosted", "global.pelias-self-hosted"], "entry_conditions": "exact result unavailable but municipality/UF evidence is reviewed", "exit_conditions": "coarse locality reference separately approved"}, {"order": 3, "state": "restricted", "provider_ids": [], "entry_conditions": "privacy, terms, informal-address or provider disagreement unresolved", "exit_conditions": "authorized review resolves or remains restricted"}, {"order": 4, "state": "unmapped", "provider_ids": [], "entry_conditions": "no eligible exact/coarse result", "exit_conditions": "new evidence; absence is not closure"}], + "confidence_policy": {"exact_acceptance": "No national provider in this profile supplies a project-ready exact point. Accept only after a reviewed provider result, source/municipality agreement, privacy screen and human decision.", "coarse_acceptance": "Municipality/UF/CEP context only; never use a postal centroid as a facility point.", "human_review_threshold": "Every global fallback result, informal/rural address, multiple result, source discrepancy, and any source-family join requires human review.", "disagreement_rule": "keep SIF registered and export observations separate; never use export authorization to repair or locate the registered source", "moved_point_rule": "append provider/date event and review source lifecycle; no closure inference", "provider_overrides": [{"provider_id": "br.cep-cadastro", "accept": "CEP and municipality/UF agree", "review": "CEP type/number conflict", "reject": "no match or restricted address"}, {"provider_id": "global.nominatim-self-hosted", "accept": "exact address with independent municipality review", "review": "rural/informal/multiple/coarse", "reject": "place-only/mismatch"}]}, + "review_gates": {"terms_review": "Confirm MAPA/Correios/Conecta and regional provider terms, access conditions, logging and redistribution before acquisition or geocoding.", "privacy_review": "SIF address, CNPJ, contacts and occurrences are restricted until field-level review; source publication is not permission to expose.", "source_coordinate_review": "No SIF source point observed; e-SISBI GIS and Trase points remain separate source/provider evidence.", "human_acceptance": "Authorized reviewer must decide whether an exact point is safe and source-supported; do not infer from global geocoder alone.", "publication_separation": "geocoding evidence never grants project approval or publication"}, + "resilience": {"disagreement": "preserve source-family separation and quarantine provider conflicts", "moved_point": "version exact provider event and source snapshot; never overwrite", "outage": "retain last eligible release and use next approved provider or unmapped", "replacement": "replace a provider only after access, terms, privacy and synthetic contract validation"}, + "operational_estimate": {"unit": "unique privacy-eligible normalized address query, not SIF activity/export row count", "duration_formula": "unique queries / approved rate + review/retry budget", "cost_formula": "access tariff or self-hosted infrastructure estimate; unknown remains unknown", "parallelism_policy": "single bounded worker until provider and privacy review allow more"}, + "unresolved_questions": ["Can Conecta/Correios address validation be used by this project and under what authentication, rate and reuse terms?", "What official national or state coordinate service can be used without treating a postal address as a facility point?", "Can e-SISBI GIS coordinates be retained and published under MAPA terms and privacy review?", "How should informal/rural addresses and residential overlap be suppressed?"], + "evidence": ["docs/country-recon-br.md", "docs/countries/br/v1-field-crosswalk.json", "https://www.gov.br/conecta/catalogo/apis/cep-codigo-de-enderecamento-postal", "https://www.ibge.gov.br/estatisticas/sociais/populacao/38734-cadastro-nacional-de-enderecos-para-fins-estatisticos.html", "https://operations.osmfoundation.org/policies/nominatim/"] + }, + { + "profile_version": "country-geocoding-profile-v1", + "profile_id": "de-bltu-geocoding", + "country_code": "DE", + "display_name": "Germany", + "source_ids": ["de.locations"], + "assessed_at": "2026-09-18", + "assessment_scope": "Deep reconnaissance for the privately captured BVL BLtU general-list export and official BKG/AdV geocoding options.", + "no_private_records_queried": true, + "publication_boundary": "blocked; profile is reconnaissance evidence only; no release eligibility is granted", + "source_coordinate_evidence": [{"source_id": "de.locations", "availability": "not_observed", "field_names": ["street", "postal code", "city", "state"], "semantics": "The current adapter deliberately emits no source coordinate and only retains a source-present address state pending privacy review; source export coordinate semantics are not established.", "precision": "not supplied in normalized output; raw-source precision unknown", "expected_coverage": "unknown until an allowed source export schema is independently assessed", "review_state": "address/coordinate terms and privacy review pending", "never_overwrite_source": true}], + "address_model": {"structure": "German street name, house number/suffix, postal code, locality and optional district/administrative fields; German official geocoder supports structured attributes.", "language_scripts": "German Latin script with umlauts, ß, abbreviations and compound names.", "transliteration": "Preserve German spelling and diacritics; only create controlled query variants and retain the source form.", "administrative_hierarchy": "Gemeinde, Kreis, Regierungsbezirk, Bundesland; AGS/region codes can constrain national service lookups.", "locality_notes": "BKG documents address, street and place types, scores and hit classes; a high score can still return a higher-level place if the requested house attributes are not matched."}, + "national_address_authority": {"provider_id": "de.bkg-gdz", "authority": "Bundesamt für Kartographie und Geodäsie (BKG) / AdV geocoding service", "role": "Federal geocoding service backed by official German house-coordinate/address data.", "availability": "public OpenSearch, structured geocode and WFS interfaces documented; usage/cost terms require confirmation", "coordinate_semantics": "Official house coordinates and address objects; quality flags distinguish building/parcel/other coordinate sources where supplied.", "terms_status": "source attribution and any usage-dependent charges must be confirmed; no rate limit was found in the reviewed document", "notes": "BKG is the preferred German exact lookup. Use its score and type/hit fields together; do not accept score alone."}, + "government_open_data_options": [{"provider_id": "de.bkg-gdz", "scope": "National address/geoname geocoding and structured filters.", "access_and_export": "OpenSearch/geocode/WFS HTTP interfaces with regional and administrative filters.", "limitations": "Rate limits, tariffs and exact source-use eligibility are not established; technical logs may include IP/user/service/cost metadata."}, {"provider_id": "de.hauskoordinaten", "scope": "Official German house coordinates (HK-DE) based on state cadastral sources.", "access_and_export": "CSV/ZIP metadata and data-service access documented.", "limitations": "Access is conditioned by V GeoBund/V GeoLänder and source terms; not a casual public bulk default."}], + "providers": [ + {"provider_id": "de.bkg-gdz", "role": "national_authority", "name": "BKG geocoding service", "authority_or_operator": "Bundesamt für Kartographie und Geodäsie", "status": "reviewed", "coverage": "Germany addresses, streets, places and geonames", "endpoint": "https://sg.geodatenzentrum.de/gdz_geokodierung/geocode", "access": {"mode": "documented OpenSearch/structured geocode/WFS service", "bulk_mode": "individual requests; asynchronous server-side bulk is explicitly not implemented", "export_or_download": "WFS/feature access; source-specific", "cost_model": "usage-dependent terms may apply; do not assume free"}, "terms": {"license_status": "BKG source/terms and V GeoBund/V GeoLänder eligibility require review", "attribution": "BKG/AdV source notice required", "caching": "server does not retain semantic query parameters according to documentation; project caching still requires review", "redistribution": "not assumed", "retention": "project retention and source corrections/removals", "logging": "technical cost/access logs may include time, IP, user ID, service and object counts; no semantic query storage is documented", "residency": "BKG infrastructure; operational hosting/residency should be confirmed"}, "throughput": {"documented_limit": "maximum WFS features 1000; no numeric request rate found", "recommended_rate": "single-worker bounded requests with retry/backoff until service terms are confirmed", "duration_or_cost_estimate": "unique queries / approved rate plus review; usage-dependent cost unknown"}, "confidence_mapping": {"accepted": ["score > 0.95 with requested house type and hit class T/unique", "all supplied address attributes have secure agreement"], "human_review": ["0.90 < score <= 0.95", "multiple equal-score results", "type is not Haus", "score high but house attributes absent"], "rejected": ["score <= 0.90", "country/administrative mismatch", "restricted/private address"]}, "evidence": ["https://sgx.geodatenzentrum.de/public/gdz/dokumentation/deu/geokodierungsdienst.pdf", "https://advmis.geodatenzentrum.de/trefferanzeige?docuuid=31A2D5D2-4742-4008-AFDD-F9203E269985"], "notes": "Provider-specific score and hit mapping is captured from the official document; project acceptance remains human-gated."}, + {"provider_id": "de.hauskoordinaten", "role": "government_lookup", "name": "Amtliche Hauskoordinaten Deutschland (HK-DE)", "authority_or_operator": "BKG/AdV and state cadastral authorities", "status": "conditional", "coverage": "German addressed buildings/house coordinates", "endpoint": "https://advmis.geodatenzentrum.de/", "access": {"mode": "official metadata and eligible data-service access", "bulk_mode": "CSV ZIP / licensed data access", "export_or_download": "documented metadata", "cost_model": "eligibility and source-specific terms; no price claim"}, "terms": {"license_status": "V GeoBund/V GeoLänder terms and source notice require review", "attribution": "BKG/AdV/state source attribution", "caching": "versioned local cache only after access review", "redistribution": "not assumed", "retention": "project schedule and correction/removal", "logging": "not verified", "residency": "German service infrastructure expected; confirm"}, "throughput": {"documented_limit": "bulk service terms not fully assessed", "recommended_rate": "prefer approved local dataset over repeated requests where licensed", "duration_or_cost_estimate": "download/import time plus license/infrastructure cost; unknown"}, "confidence_mapping": {"accepted": ["house coordinate matches current official address identity"], "human_review": ["quality flag B/C/P or source-date disagreement"], "rejected": ["unlicensed/ineligible access or privacy restriction"]}, "evidence": ["https://advmis.geodatenzentrum.de/trefferanzeige?docuuid=31A2D5D2-4742-4008-AFDD-F9203E269985"], "notes": "A data source option, not automatically available to this project."}, + {"provider_id": "global.nominatim-self-hosted", "role": "global_fallback", "name": "Self-hosted Nominatim over an approved OpenStreetMap extract", "authority_or_operator": "Project-controlled deployment", "status": "candidate", "coverage": "Global OSM coverage; not German official address authority", "endpoint": "project-controlled endpoint to be assigned after deployment review", "access": {"mode": "self-hosted HTTP API", "bulk_mode": "offline/import-backed batch", "export_or_download": "ODbL/source review", "cost_model": "infrastructure/update cost; no amount claimed"}, "terms": {"license_status": "ODbL/software terms require review", "attribution": "OpenStreetMap attribution", "caching": "suppression-aware", "redistribution": "source-specific ODbL review", "retention": "project schedule", "logging": "minimized local", "residency": "deployment region documented"}, "throughput": {"documented_limit": "self-host capacity to benchmark", "recommended_rate": "bounded worker", "duration_or_cost_estimate": "unique queries / measured rate; infrastructure estimate required"}, "confidence_mapping": {"accepted": ["exact address with locality and country agreement"], "human_review": ["multiple/fuzzy/stale result"], "rejected": ["place-only/mismatch/restricted"]}, "evidence": ["https://operations.osmfoundation.org/policies/nominatim/"], "notes": "Never substitute OSM confidence for BKG source confidence."}, + {"provider_id": "global.pelias-self-hosted", "role": "global_fallback", "name": "Self-hosted Pelias with reviewed address sources", "authority_or_operator": "Project-controlled deployment", "status": "candidate", "coverage": "Global multi-source fallback", "endpoint": "project-controlled endpoint to be assigned after deployment review", "access": {"mode": "self-hosted HTTP API", "bulk_mode": "offline/bounded batch", "export_or_download": "source-specific", "cost_model": "infrastructure/index/update cost; unknown"}, "terms": {"license_status": "source-index review required", "attribution": "per-source", "caching": "suppression-aware", "redistribution": "not assumed", "retention": "source/project schedule", "logging": "minimized local", "residency": "deployment region documented"}, "throughput": {"documented_limit": "self-host capacity to benchmark", "recommended_rate": "bounded queue", "duration_or_cost_estimate": "unique queries / measured rate; cost unknown"}, "confidence_mapping": {"accepted": ["exact result from reviewed index"], "human_review": ["multiple/coarse/source disagreement"], "rejected": ["unreviewed or mismatched result"]}, "evidence": ["https://sgx.geodatenzentrum.de/public/gdz/dokumentation/deu/geokodierungsdienst.pdf"], "notes": "Backup architecture candidate only."} + ], + "query_minimization": {"allowed_components": ["street", "house number", "postal code", "locality", "state/administrative code", "country"], "construction_steps": ["screen raw address and facility/person ambiguity", "prefer structured BKG fields and administrative filters", "avoid sending name/contact/remarks", "deduplicate by restricted query hash"], "never_send": ["facility name, operator, contacts, free text, raw BLtU values, restricted address, source payload"], "logging_boundary": ["provider/query hash/time/precision/score/hit type/review state restricted", "no semantic query in public reports"], "private_record_network_authorized": false}, + "fallback_chain": [{"order": 1, "state": "exact", "provider_ids": ["de.bkg-gdz", "de.hauskoordinaten", "global.nominatim-self-hosted", "global.pelias-self-hosted"], "entry_conditions": "terms/privacy access and exact address eligible", "exit_conditions": "BKG/official unique house result or reviewed global exact result"}, {"order": 2, "state": "coarse", "provider_ids": ["de.bkg-gdz", "global.nominatim-self-hosted", "global.pelias-self-hosted"], "entry_conditions": "exact result unavailable but locality/admin evidence is reviewed", "exit_conditions": "separate coarse locality reference"}, {"order": 3, "state": "restricted", "provider_ids": [], "entry_conditions": "privacy, terms, score/type disagreement or moved-point concern", "exit_conditions": "authorized review resolves or remains restricted"}, {"order": 4, "state": "unmapped", "provider_ids": [], "entry_conditions": "no eligible exact/coarse result", "exit_conditions": "new evidence; no closure inference"}], + "confidence_policy": {"exact_acceptance": "Use BKG score >0.95 only when type/hit class and requested house attributes agree; a high score alone is insufficient.", "coarse_acceptance": "City/place result only with explicit coarse precision and no exact map point.", "human_review_threshold": "0.90 < score <= 0.95, multiple equal scores, non-house type, source-coordinate disagreement, quality flag B/C/P, or privacy signal.", "disagreement_rule": "retain source and provider evidence separately; do not average or overwrite", "moved_point_rule": "append dated new provider event and review source lifecycle", "provider_overrides": [{"provider_id": "de.bkg-gdz", "accept": "score > 0.95 + exact house/hit agreement", "review": "0.90-0.95 or ambiguity/type mismatch", "reject": "<=0.90 or mismatch"}, {"provider_id": "global.nominatim-self-hosted", "accept": "exact address token and locality agreement", "review": "fuzzy/multiple/coarse", "reject": "place-only/mismatch"}]}, + "review_gates": {"terms_review": "Confirm BKG/AdV source attribution, usage-dependent terms, HK-DE eligibility, caching and technical log implications.", "privacy_review": "BLtU address values remain restricted until field-level screening; no geocoding of unresolved or private locations.", "source_coordinate_review": "The adapter's source-coordinate null is not permission to infer or publish a new point.", "human_acceptance": "Human reviewer must assess score/type/hit and source lifecycle semantics.", "publication_separation": "geocoding evidence never grants project approval or publication"}, + "resilience": {"disagreement": "append BKG/HK-DE/global attempts and quarantine conflict", "moved_point": "version point and address identity by provider/time", "outage": "preserve last eligible release and move to next approved provider/unmapped", "replacement": "profile version, source terms review, synthetic score mapping tests and bounded comparison required"}, + "operational_estimate": {"unit": "unique privacy-eligible normalized address query", "duration_formula": "unique queries / approved service rate + retry/review budget", "cost_formula": "BKG/HK-DE usage or self-host infrastructure estimate; unknown remains unknown", "parallelism_policy": "single bounded worker until BKG terms/capacity are confirmed"}, + "unresolved_questions": ["What current BKG rate limits and usage pricing apply to this project?", "Can the project qualify for HK-DE data access and what redistribution/caching terms apply?", "Which BLtU fields are safe to query after privacy screening?", "What effective-date and source-coordinate semantics are supplied by the BVL export?"], + "evidence": ["docs/germany-source-assessment.md", "pipeline/sources/germany/adapter.py", "https://sgx.geodatenzentrum.de/public/gdz/dokumentation/deu/geokodierungsdienst.pdf", "https://advmis.geodatenzentrum.de/trefferanzeige?docuuid=31A2D5D2-4742-4008-AFDD-F9203E269985", "https://operations.osmfoundation.org/policies/nominatim/"] + }, + { + "profile_version": "country-geocoding-profile-v1", + "profile_id": "uk-approved-establishments-geocoding", + "country_code": "GB", + "display_name": "United Kingdom", + "source_ids": ["uk.locations"], + "assessed_at": "2026-09-18", + "assessment_scope": "Deep reconnaissance for the registry's UK aggregate lane, with FSA England/Wales and FSS Scotland private staging treated as separate national feeds; Northern Ireland remains separate/unresolved.", + "no_private_records_queried": true, + "publication_boundary": "blocked; profile is reconnaissance evidence only; no release eligibility is granted", + "source_coordinate_evidence": [{"source_id": "uk.locations", "availability": "unknown", "field_names": ["address lines", "postcode", "X/Y if present in a source feed", "UPRN if available"], "semantics": "The FSA/FSS adapters deliberately suppress coordinates and the aggregate legacy source does not establish a single current coordinate contract; any source X/Y or UPRN must remain feed-specific evidence.", "precision": "unknown by national feed; do not infer from postcode or UPRN without reviewed source metadata", "expected_coverage": "feed-specific and not nationally uniform", "review_state": "source schema, privacy and coordinate review pending", "never_overwrite_source": true}], + "address_model": {"structure": "UK address lines, locality, county/nation and postcode; UPRN/USRN are preferred stable identifiers where legally and operationally available.", "language_scripts": "English, Welsh and Scottish Gaelic Latin scripts; Northern Ireland/Islands have separate feed and authority boundaries.", "transliteration": "Do not translate Welsh/Gaelic names; preserve source variants and use UPRN/postcode for disambiguation where reviewed.", "administrative_hierarchy": "Nation, local authority, locality/post town, postcode, property/UPRN; GeoPlace/OS and national food authorities are distinct roles.", "locality_notes": "FSA England/Wales, FSS Scotland and Northern Ireland must not be silently merged; postcode centroids are coarse only and OS AddressBase/Places is licensed."}, + "national_address_authority": {"provider_id": "uk.os-places", "authority": "Ordnance Survey AddressBase/OS Places with GeoPlace/local-authority address identifiers", "role": "Authoritative licensed GB/Islands address verification and geometry service; not a free general-purpose dataset for this project.", "availability": "commercial/public-sector licensed API and downloads; open UPRN/grid-reference products provide only a narrower identity/coarse fallback", "coordinate_semantics": "AddressBase property-level geometry linked to UPRN; coverage and product geography differ between GB and Islands.", "terms_status": "licensed product, pricing/contract and public redistribution rules require project-specific review", "notes": "Use OS Places only after terms/privacy/cost approval. FSA/FSS source coordinates, if any, remain separate evidence."}, + "government_open_data_options": [{"provider_id": "uk.os-places", "scope": "UK/Islands address, geometry and UPRN lookup; product coverage and update cadence vary.", "access_and_export": "API, downloads and public-sector licences documented; AddressBase Premium updated every six weeks.", "limitations": "Licensed terms, pricing, PAF/third-party rights, Northern Ireland and island boundaries require review."}, {"provider_id": "uk.uprn-open", "scope": "Open UPRN/grid-reference identifiers for property linking and coarse validation.", "access_and_export": "OS/GOV.UK open data routes.", "limitations": "Not a complete address string or an automatic publication permission; exact coordinate use needs source and privacy review."}, {"provider_id": "uk.ons-postcode", "scope": "ONS postcode centroids for coarse locality fallback.", "access_and_export": "Public ONS directory downloads/API.", "limitations": "Centroid is not the facility point; licensing includes OS/ONS rights and must be checked."}], + "providers": [ + {"provider_id": "uk.os-places", "role": "national_authority", "name": "Ordnance Survey Places API / AddressBase", "authority_or_operator": "Ordnance Survey, with GeoPlace/local-authority source relationships", "status": "conditional", "coverage": "Great Britain and Islands product-dependent address/UPRN/geometry data", "endpoint": "https://www.ordnancesurvey.co.uk/products/os-places-api", "access": {"mode": "licensed API and download products", "bulk_mode": "contract/product-specific; not assumed", "export_or_download": "documented for licensed products", "cost_model": "commercial/public-sector licence and usage pricing; obtain current quote/contract"}, "terms": {"license_status": "licensed AddressBase/Places product; public redistribution not assumed", "attribution": "OS/GeoPlace attribution and contract terms", "caching": "contract-specific; suppression-aware cache required", "redistribution": "contract-specific and likely restricted", "retention": "contract and project policy; remove/restrict on source/privacy decision", "logging": "provider/API logging and account telemetry require review", "residency": "provider processing/residency requires contract review"}, "throughput": {"documented_limit": "product/API-specific; no universal limit established", "recommended_rate": "contract-approved bounded rate", "duration_or_cost_estimate": "unique queries / licensed rate + contract/minimum charges; price unknown"}, "confidence_mapping": {"accepted": ["UPRN/address/postcode match with current property geometry and feed/nation agreement"], "human_review": ["multiple UPRNs, lifecycle/prebuild/retired property, national feed mismatch, residential/private signal"], "rejected": ["postcode centroid used as exact point, unlicensed result, country mismatch"]}, "evidence": ["https://www.ordnancesurvey.co.uk/products/os-places-api", "https://www.ordnancesurvey.co.uk/products/addressbase-premium", "https://www.gov.uk/government/publications/open-standards-for-government/identifying-property-and-street-information"], "notes": "Preferred UK exact provider only after procurement and terms/privacy review."}, + {"provider_id": "uk.uprn-open", "role": "government_lookup", "name": "Open UPRN/grid-reference and GOV.UK address standards", "authority_or_operator": "Ordnance Survey / GeoPlace / UK government", "status": "conditional", "coverage": "GB property/street identifiers and grid references; full address data/product scope varies", "endpoint": "https://www.gov.uk/government/publications/open-standards-for-government/identifying-property-and-street-information", "access": {"mode": "open data and government standards", "bulk_mode": "download/local reference", "export_or_download": "documented", "cost_model": "open product-specific terms"}, "terms": {"license_status": "OGL/open-product terms require field-specific review", "attribution": "OS/GeoPlace/GOV.UK attribution", "caching": "versioned local cache", "redistribution": "field/product-specific", "retention": "project schedule", "logging": "local only", "residency": "local deployment chosen by project"}, "throughput": {"documented_limit": "not a free-text geocoder", "recommended_rate": "local lookup", "duration_or_cost_estimate": "download/build time; no API cost claim"}, "confidence_mapping": {"accepted": ["identifier is verified against an authoritative product"], "human_review": ["identifier has multiple/current lifecycle states"], "rejected": ["identifier alone treated as an exact public point"]}, "evidence": ["https://www.gov.uk/government/publications/open-standards-for-government/identifying-property-and-street-information"], "notes": "Use for linking/disambiguation, not silent exact geocoding."}, + {"provider_id": "uk.ons-postcode", "role": "government_lookup", "name": "ONS Postcode Directory", "authority_or_operator": "Office for National Statistics", "status": "conditional", "coverage": "UK live postcode centroids and administrative relationships", "endpoint": "https://www.data.gov.uk/dataset/b1c6d498-278a-4b0b-b53c-4d58d0d0646e/online-ons-postcode-directory-live2", "access": {"mode": "public downloads and ArcGIS service", "bulk_mode": "local directory download", "export_or_download": "documented", "cost_model": "licence/source-rights review required"}, "terms": {"license_status": "data.gov.uk entry reports OS/ONS intellectual-property rights; licence needs confirmation", "attribution": "ONS/OS attribution", "caching": "versioned coarse reference", "redistribution": "not assumed", "retention": "project schedule", "logging": "local only", "residency": "project-controlled"}, "throughput": {"documented_limit": "not a geocoder", "recommended_rate": "local lookup", "duration_or_cost_estimate": "directory build time; no API cost claim"}, "confidence_mapping": {"accepted": ["coarse postcode/locality display only"], "human_review": ["any attempted exact use"], "rejected": ["centroid used as facility point"]}, "evidence": ["https://www.data.gov.uk/dataset/b1c6d498-278a-4b0b-b53c-4d58d0d0646e/online-ons-postcode-directory-live2"], "notes": "Coarse fallback only."}, + {"provider_id": "global.nominatim-self-hosted", "role": "global_fallback", "name": "Self-hosted Nominatim over an approved OpenStreetMap extract", "authority_or_operator": "Project-controlled deployment", "status": "candidate", "coverage": "Global OSM coverage; not a UK official address authority", "endpoint": "project-controlled endpoint to be assigned after deployment review", "access": {"mode": "self-hosted HTTP API", "bulk_mode": "offline/import-backed batch", "export_or_download": "ODbL/source review", "cost_model": "infrastructure/update cost; unknown"}, "terms": {"license_status": "ODbL/software terms require review", "attribution": "OpenStreetMap attribution", "caching": "suppression-aware", "redistribution": "source-specific ODbL review", "retention": "project schedule", "logging": "minimized local", "residency": "deployment region documented"}, "throughput": {"documented_limit": "self-host capacity to benchmark", "recommended_rate": "bounded worker", "duration_or_cost_estimate": "unique queries / measured rate; infrastructure estimate required"}, "confidence_mapping": {"accepted": ["exact address with nation/postcode agreement"], "human_review": ["multiple/fuzzy/lifecycle result"], "rejected": ["place-only/mismatch/restricted"]}, "evidence": ["https://operations.osmfoundation.org/policies/nominatim/"], "notes": "Do not use public Nominatim for recurring bulk work."}, + {"provider_id": "global.pelias-self-hosted", "role": "global_fallback", "name": "Self-hosted Pelias with reviewed address sources", "authority_or_operator": "Project-controlled deployment", "status": "candidate", "coverage": "Global multi-source fallback", "endpoint": "project-controlled endpoint to be assigned after deployment review", "access": {"mode": "self-hosted HTTP API", "bulk_mode": "offline/bounded batch", "export_or_download": "source-specific", "cost_model": "infrastructure/index/update cost; unknown"}, "terms": {"license_status": "source-index review required", "attribution": "per-source", "caching": "suppression-aware", "redistribution": "not assumed", "retention": "source/project schedule", "logging": "minimized local", "residency": "deployment region documented"}, "throughput": {"documented_limit": "self-host capacity to benchmark", "recommended_rate": "bounded queue", "duration_or_cost_estimate": "unique queries / measured rate + index maintenance; unknown"}, "confidence_mapping": {"accepted": ["exact result from reviewed index"], "human_review": ["multiple/coarse/source disagreement"], "rejected": ["unreviewed/mismatched result"]}, "evidence": ["https://www.ordnancesurvey.co.uk/products/os-places-api"], "notes": "Backup architecture candidate only."} + ], + "query_minimization": {"allowed_components": ["address lines after privacy screen", "postcode", "local authority/nation", "UPRN only if already source-supplied and eligible", "country"], "construction_steps": ["keep FSA/FSS/NI feeds separate", "suppress AddressWithheld/remarks/residential-risk values before network use", "prefer UPRN/postcode/structured lookup over free text", "deduplicate by restricted query hash"], "never_send": ["trading name, operator, remarks, source payload, withheld address, unreviewed coordinates, worker/resident details"], "logging_boundary": ["provider/query hash/time/precision/UPRN match/review state restricted", "public report contains no query or raw response"], "private_record_network_authorized": false}, + "fallback_chain": [{"order": 1, "state": "exact", "provider_ids": ["uk.os-places", "uk.uprn-open", "global.nominatim-self-hosted", "global.pelias-self-hosted"], "entry_conditions": "feed-specific terms/privacy gate passes and exact address is eligible", "exit_conditions": "current exact address/UPRN/geometry passes provider and human thresholds"}, {"order": 2, "state": "coarse", "provider_ids": ["uk.ons-postcode", "uk.uprn-open", "global.nominatim-self-hosted", "global.pelias-self-hosted"], "entry_conditions": "exact result unavailable but postcode/local authority evidence is reviewed", "exit_conditions": "coarse locality reference separately approved"}, {"order": 3, "state": "restricted", "provider_ids": [], "entry_conditions": "privacy, licensing, national-feed conflict, lifecycle or moved-point issue unresolved", "exit_conditions": "authorized review resolves or remains restricted"}, {"order": 4, "state": "unmapped", "provider_ids": [], "entry_conditions": "no eligible exact/coarse result", "exit_conditions": "new evidence; absence is not closure"}], + "confidence_policy": {"exact_acceptance": "Require licensed/approved OS exact address or reviewed UPRN/geometry agreement; a global fallback never overrides AddressWithheld or feed privacy gates.", "coarse_acceptance": "Postcode/local-authority reference only; never use postcode centroid as exact point.", "human_review_threshold": "Every global fallback result, multiple UPRNs, lifecycle/prebuild/retired result, national feed mismatch or privacy signal.", "disagreement_rule": "retain source feed and geocoder outputs separately; do not merge FSA/FSS/NI or overwrite source fields", "moved_point_rule": "append provider/date event and review UPRN/property lifecycle and source snapshot", "provider_overrides": [{"provider_id": "uk.os-places", "accept": "current licensed address/UPRN/geometry agreement", "review": "lifecycle/nation/multiple match", "reject": "unlicensed or privacy-restricted"}, {"provider_id": "global.nominatim-self-hosted", "accept": "exact address + nation/postcode agreement", "review": "fuzzy/multiple/lifecycle", "reject": "place-only/mismatch"}]}, + "review_gates": {"terms_review": "Confirm OS/GeoPlace licence, costs, public redistribution, PAF/third-party terms, ONS/OS rights and Nominatim/Pelias source terms.", "privacy_review": "AddressWithheld, remarks and residential/mixed-use signals remain restricted; public source availability is not privacy clearance.", "source_coordinate_review": "Any FSA/FSS source point or OS point remains separate evidence with provider/time/precision/review state.", "human_acceptance": "Review national-feed boundaries and exact geometry before any release decision.", "publication_separation": "geocoding evidence never grants project approval or publication"}, + "resilience": {"disagreement": "preserve FSA/FSS/NI and provider evidence separately; quarantine conflicts", "moved_point": "version UPRN/property lifecycle and source observation", "outage": "keep last eligible release and use next approved provider/coarse/unmapped", "replacement": "contract/profile version, terms review, synthetic national-feed tests and bounded comparison"}, + "operational_estimate": {"unit": "unique privacy-eligible normalized address query, not feed row count", "duration_formula": "unique queries / licensed or measured rate + retry/review budget", "cost_formula": "OS contract or self-host infra estimate; no free assumption", "parallelism_policy": "single bounded worker until contract and provider capacity are confirmed"}, + "unresolved_questions": ["What OS Places plan/licence and per-query or subscription cost applies?", "Which OS/GeoPlace/ONS fields can be cached or redistributed in a public release?", "What current address/coordinate fields are supplied by each FSA, FSS and NI feed?", "How should UPRN lifecycle and source suppression propagate through historical releases?"], + "evidence": ["docs/country-recon-uk.md", "docs/countries/uk/fss-approved-establishments-source-assessment.md", "pipeline/sources/uk/fsa_approved/adapter.py", "pipeline/sources/uk/fss_approved/adapter.py", "https://www.ordnancesurvey.co.uk/products/os-places-api", "https://www.gov.uk/government/publications/open-standards-for-government/identifying-property-and-street-information", "https://operations.osmfoundation.org/policies/nominatim/"] + } + ] +} diff --git a/pipeline/geocoding/recon.py b/pipeline/geocoding/recon.py new file mode 100644 index 0000000..9f63882 --- /dev/null +++ b/pipeline/geocoding/recon.py @@ -0,0 +1,101 @@ +"""Load, validate and rank country geocoding reconnaissance offline.""" +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path +from typing import Any + +from pipeline.contracts.geocoding_profile import ( + build_recon_report, + validate_profiles, +) +from pipeline.source_registry import load_registry + + +ROOT = Path(__file__).resolve().parents[2] +PROFILE_PATH = Path(__file__).with_name("profiles.json") +STATUS_PATH = ROOT / "docs" / "source-status.json" + + +def _load_json(path: Path) -> dict[str, Any]: + value = json.loads(path.read_text(encoding="utf-8-sig")) + if not isinstance(value, dict): + raise ValueError(f"expected JSON object: {path}") + return value + + +def load_profile_registry(path: Path = PROFILE_PATH, *, known_source_ids: set[str] | None = None) -> dict[str, Any]: + payload = _load_json(path) + validate_profiles(payload, known_source_ids=known_source_ids) + return payload + + +def build_geocoding_recon(*, source_path: Path | None = None, status_path: Path | None = None, profile_path: Path | None = None) -> dict[str, Any]: + source_registry = load_registry(source_path or (ROOT / "pipeline" / "source_registry.json")) + status_registry = _load_json(status_path or STATUS_PATH) + profile_registry = load_profile_registry(profile_path or PROFILE_PATH, known_source_ids={item["source_id"] for item in source_registry["sources"]}) + report = build_recon_report(source_registry, status_registry, profile_registry) + rank_by_source = {item["source_id"]: item for item in report["ranked_backlog"]} + summaries: list[dict[str, Any]] = [] + for profile in profile_registry["profiles"]: + source_ids = list(profile["source_ids"]) + source_ranks = [rank_by_source[source_id] for source_id in source_ids] + coordinate = [ + { + "source_id": item["source_id"], + "availability": item["availability"], + "expected_coverage": item["expected_coverage"], + "review_state": item["review_state"], + } + for item in profile["source_coordinate_evidence"] + ] + provider_ids = [item["provider_id"] for item in profile["providers"]] + country_provider_ids = [provider_id for provider_id in provider_ids if not provider_id.startswith("global.")] + burden = "high" if len(profile["unresolved_questions"]) >= 3 else "medium" + summaries.append({ + "profile_id": profile["profile_id"], + "country_code": profile["country_code"], + "source_ids": source_ids, + "best_source_score": max(item["score"] for item in source_ranks), + "current_coordinate_evidence": coordinate, + "expected_geocoding_coverage": profile["national_address_authority"]["availability"], + "review_burden": burden, + "provider_dependencies": country_provider_ids + ["global.nominatim-self-hosted", "global.pelias-self-hosted"], + "recommended_implementation_order": 0, + "publication_state": "blocked", + }) + summaries.sort(key=lambda item: (-item["best_source_score"], item["profile_id"])) + for index, summary in enumerate(summaries, start=1): + summary["recommended_implementation_order"] = index + report["profile_summaries"] = summaries + report["platform_integration"] = { + "source_registry": "pipeline/source_registry.json", + "status_registry": "docs/source-status.json", + "profile_registry": "pipeline/geocoding/profiles.json", + "publication_effect": "none; profile validation and ranking do not alter source status, readiness, release approval or publication eligibility", + } + return report + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", type=Path, help="write the row-free report JSON to this path") + args = parser.parse_args() + report = build_geocoding_recon() + rendered = json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True) + "\n" + if args.output: + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(rendered, encoding="utf-8") + else: + try: + sys.stdout.reconfigure(encoding="utf-8") + except AttributeError: + pass + print(rendered, end="") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/pipeline/geocoding/test_recon.py b/pipeline/geocoding/test_recon.py new file mode 100644 index 0000000..67b7c45 --- /dev/null +++ b/pipeline/geocoding/test_recon.py @@ -0,0 +1,99 @@ +import json +import tempfile +import unittest +from pathlib import Path + +from pipeline.contracts.geocoding_profile import ( + GeocodingProfileError, + build_recon_report, + rank_sources, + validate_profile, + validate_profiles, +) +from pipeline.geocoding.recon import build_geocoding_recon, load_profile_registry + + +ROOT = Path(__file__).parents[2] + + +class GeocodingProfileTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.payload = json.loads((ROOT / "pipeline/geocoding/profiles.json").read_text(encoding="utf-8")) + cls.source_registry = json.loads((ROOT / "pipeline/source_registry.json").read_text(encoding="utf-8-sig")) + cls.status_registry = json.loads((ROOT / "docs/source-status.json").read_text(encoding="utf-8-sig")) + cls.known_ids = {item["source_id"] for item in cls.source_registry["sources"]} + + def test_checked_in_profiles_validate_and_reference_existing_registry_ids(self): + validate_profiles(self.payload, known_source_ids=self.known_ids) + self.assertEqual(len(self.payload["profiles"]), 6) + self.assertEqual(self.payload["global_fallback_policy"]["default_candidate_provider"], "global.nominatim-self-hosted") + discovery_ids = { + item["provider_id"] for item in self.payload["global_fallback_policy"]["provider_discovery_catalog"] + } + self.assertTrue( + {"global.opencage", "global.stadia", "global.locationiq", "global.geoapify", "global.nominatim-public"} + .issubset(discovery_ids) + ) + + def test_profile_rejects_publication_or_private_query_boundary(self): + profile = json.loads(json.dumps(self.payload["profiles"][0])) + profile["publication_boundary"] = "approved-for-release" + with self.assertRaises(GeocodingProfileError): + validate_profile(profile, known_source_ids=self.known_ids) + profile = json.loads(json.dumps(self.payload["profiles"][0])) + profile["query_minimization"]["private_record_network_authorized"] = True + with self.assertRaises(GeocodingProfileError): + validate_profile(profile, known_source_ids=self.known_ids) + + def test_profile_rejects_payload_shaped_fields(self): + profile = json.loads(json.dumps(self.payload["profiles"][0])) + profile["providers"][0]["raw_response"] = {"unexpected": True} + with self.assertRaises(GeocodingProfileError): + validate_profile(profile, known_source_ids=self.known_ids) + + def test_fallback_order_is_explicit_and_global_candidates_are_present(self): + for profile in self.payload["profiles"]: + self.assertEqual([item["state"] for item in profile["fallback_chain"]], ["exact", "coarse", "restricted", "unmapped"]) + provider_ids = {item["provider_id"] for item in profile["providers"]} + self.assertTrue({"global.nominatim-self-hosted", "global.pelias-self-hosted"}.issubset(provider_ids)) + + def test_ranking_is_deterministic_and_tie_breaks_by_source_id(self): + first = rank_sources(self.source_registry, self.status_registry, self.payload["profiles"]) + second = rank_sources(self.source_registry, self.status_registry, list(reversed(self.payload["profiles"]))) + self.assertEqual(first, second) + self.assertEqual(first[0]["source_id"], "dk.smiley") + scores = [item["score"] for item in first] + self.assertEqual(scores, sorted(scores, reverse=True)) + + def test_report_is_row_free_and_keeps_publication_separate(self): + report = build_recon_report(self.source_registry, self.status_registry, self.payload) + self.assertFalse(report["rows_included"]) + self.assertFalse(report["facility_or_address_payloads_included"]) + self.assertIn("dk.smiley", report["deep_tranche"]) + self.assertNotIn("approved-for-release", json.dumps(report)) + + def test_integration_build_has_254_source_backlog_and_profile_summaries(self): + report = build_geocoding_recon() + self.assertEqual(report["source_count"], 254) + self.assertEqual(report["profile_count"], 6) + self.assertEqual(len(report["profile_summaries"]), 6) + self.assertEqual(report["platform_integration"]["publication_effect"].split(";")[0], "none") + + def test_unknowns_are_explicit_not_empty_guesses(self): + profile = next(item for item in self.payload["profiles"] if item["country_code"] == "BR") + self.assertIn(profile["source_coordinate_evidence"][0]["availability"], {"unknown", "not_observed"}) + self.assertTrue(any(token in json.dumps(profile).lower() for token in ("unknown", "not observed", "not verified"))) + + def test_loader_rejects_missing_profile_source(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "profiles.json" + broken = json.loads(json.dumps(self.payload)) + broken["profiles"][0]["source_ids"] = ["made.up.source"] + path.write_text(json.dumps(broken), encoding="utf-8") + with self.assertRaises(GeocodingProfileError): + load_profile_registry(path, known_source_ids=self.known_ids) + + +if __name__ == "__main__": + unittest.main() From 19e1e59f87eb2049e3848c15e47e8a0b05615ddd Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Fri, 18 Sep 2026 16:36:40 -0700 Subject: [PATCH 261/311] Harden unattended geocoding worker --- docs/geocoder-sprint-lane-ledger.md | 11 +- pipeline/scripts/stages/geocode-worker.py | 121 +++++++++++++++--- .../tests/e2e/test_suppression_lifecycle.py | 69 ++++++++++ 3 files changed, 175 insertions(+), 26 deletions(-) diff --git a/docs/geocoder-sprint-lane-ledger.md b/docs/geocoder-sprint-lane-ledger.md index 9446abb..8f1c474 100644 --- a/docs/geocoder-sprint-lane-ledger.md +++ b/docs/geocoder-sprint-lane-ledger.md @@ -4,20 +4,19 @@ This ledger prevents completed work from being mistaken for active work or being omitted from consolidation. It records implementation state only; no entry grants publication approval or authorizes live provider calls. -## Ready for consolidation or recovery +## Consolidated and focused-tested | Lane | Evidence | State | Required consolidation action | | --- | --- | --- | --- | -| Private operator review console | `b74c1a36` | committed, not in `eli/front-end-overhaul` | review and cherry-pick; rerun frontend, Python, Rust, and browser checks | -| Country geocoding reconnaissance | `7dc7fd0a` | committed, not in `eli/front-end-overhaul` | review and cherry-pick provider profiles/schema; resolve overlap with hosted-provider implementation | -| Geoapify adapter | integration worktree changes | implemented locally, focused tests pass, uncommitted | combine with adversarial adapter changes and commit | -| Geocoder adversarial tests | agent worktree at `2f796b1` | tests and narrow DAWA fixes complete; commit blocked by worktree Git metadata permissions | recover patch, review, commit, and run database E2E | +| Private operator review console | `316d549b`, `88422957`, `898fe019` | consolidated; frontend and focused Python tests pass | include in full consolidation gate | +| Country geocoding reconnaissance | `6367a176` | consolidated; profile/schema tests pass | include in full consolidation gate | +| Geoapify adapter and adversarial provider hardening | `d1ba77d2` | consolidated; 16 adapter/import tests pass | run database E2E and full consolidation gate | +| Durable worker | pending checkpoint commit | transactional claims, stale recovery, budgets, bounded attempts and suppression recheck implemented; seven Docker E2E tests pass | include in full consolidation gate | ## In flight | Lane | Scope | Exit requirement | | --- | --- | --- | -| Durable worker | concurrency-safe leasing, stale recovery, retry and daily budget, suppression race protection | committed changes plus focused and database-backed tests | | Geocoder operator tooling | aggregate status/ETA, safe logs, secret configuration, local/container background operation | committed changes plus privacy tests and operator documentation | ## Consolidation gate diff --git a/pipeline/scripts/stages/geocode-worker.py b/pipeline/scripts/stages/geocode-worker.py index ea3f7ff..f041cdc 100644 --- a/pipeline/scripts/stages/geocode-worker.py +++ b/pipeline/scripts/stages/geocode-worker.py @@ -17,29 +17,92 @@ from pipeline.geocoding.registry import get_adapter -def run(database_url: str, provider_id: str, limit: int | None, delay: float, retries: int) -> int: +def _daily_started(connection, provider_id: str) -> int: + return connection.execute(""" + SELECT count(*) + FROM uec.geocode_job_events event + JOIN uec.geocode_jobs job ON job.job_id = event.job_id + WHERE job.provider_id = %s + AND event.event_type = 'started' + AND event.occurred_at >= date_trunc('day', now() AT TIME ZONE 'UTC') AT TIME ZONE 'UTC' + """, (provider_id,)).fetchone()[0] + + +def _claim_job(connection, provider_id: str, worker_id: str, max_attempts: int, lease_timeout: int): + with connection.transaction(): + job = connection.execute(""" + SELECT job.job_id, job.source_record_id, job.query, + COALESCE(current.attempt_number, 0) + FROM uec.geocode_jobs AS job + LEFT JOIN uec.geocode_job_current AS current ON current.job_id = job.job_id + WHERE job.provider_id = %s + AND COALESCE(current.attempt_number, 0) < %s + AND ( + current.event_type IS NULL + OR current.event_type = 'queued' + OR (current.event_type = 'failed' AND current.retryable) + OR (current.event_type = 'started' + AND current.occurred_at < now() - (%s * interval '1 second')) + ) + AND NOT EXISTS ( + SELECT 1 FROM uec.public_access_restricted restricted + WHERE restricted.source_record_id = job.source_record_id + ) + ORDER BY job.created_at, job.job_id + FOR UPDATE OF job SKIP LOCKED + LIMIT 1 + """, (provider_id, max_attempts, lease_timeout)).fetchone() + if not job: + return None + job_id, source_record_id, query, prior_attempt = job + attempt = prior_attempt + 1 + connection.execute(""" + INSERT INTO uec.geocode_job_events + (job_id, event_type, attempt_number, worker_id, details, occurred_at) + VALUES (%s, 'started', %s, %s, %s, %s) + """, ( + job_id, attempt, worker_id, + json.dumps({"lease_timeout_seconds": lease_timeout}), + datetime.now(timezone.utc), + )) + return job_id, source_record_id, query, attempt + + +def _is_restricted(connection, source_record_id) -> bool: + return connection.execute(""" + SELECT EXISTS ( + SELECT 1 FROM uec.public_access_restricted + WHERE source_record_id = %s + ) + """, (source_record_id,)).fetchone()[0] + + +def run( + database_url: str, + provider_id: str, + limit: int | None, + delay: float, + retries: int, + *, + daily_budget: int = 2800, + max_attempts: int = 5, + lease_timeout: int = 900, + worker_id: str | None = None, +) -> int: + if daily_budget < 1 or max_attempts < 1 or lease_timeout < 1 or retries < 1: + raise ValueError("budgets, attempts, lease timeout, and retries must be positive") adapter = get_adapter(provider_id) + worker_id = worker_id or f"geocoder-{uuid.uuid4().hex[:12]}" processed = 0 with psycopg.connect(database_url) as connection: while limit is None or processed < limit: - job = connection.execute(""" - SELECT job.job_id, job.source_record_id, job.provider_id, job.query, COALESCE(current.attempt_number, 0) - FROM uec.geocode_jobs AS job - LEFT JOIN uec.geocode_job_current AS current ON current.job_id = job.job_id - WHERE job.provider_id = %s - AND (current.event_type IS NULL OR current.event_type = 'queued' OR (current.event_type = 'failed' AND current.retryable)) - AND NOT EXISTS ( - SELECT 1 FROM uec.public_access_restricted restricted - WHERE restricted.source_record_id = job.source_record_id - ) - ORDER BY job.created_at, job.job_id LIMIT 1 - """, (provider_id,)).fetchone() + if _daily_started(connection, provider_id) >= daily_budget: + print(f"provider={provider_id} status=daily_budget_reached processed={processed}", flush=True) + break + job = _claim_job(connection, provider_id, worker_id, max_attempts, lease_timeout) if not job: break - job_id, source_record_id, _, query, prior_attempt = job - attempt = prior_attempt + 1 - with connection.transaction(): - connection.execute("INSERT INTO uec.geocode_job_events (job_id, event_type, attempt_number, occurred_at) VALUES (%s, 'started', %s, %s)", (job_id, attempt, datetime.now(timezone.utc))) + job_id, source_record_id, query, attempt = job outcome = None for retry in range(retries): outcome = adapter.geocode(query) @@ -50,14 +113,23 @@ def run(database_url: str, provider_id: str, limit: int | None, delay: float, re queried_at = datetime.now(timezone.utc) result_id = uuid.uuid5(uuid.NAMESPACE_URL, f"urn:uec:geocode:{job_id}:{attempt}") with connection.transaction(): + if _is_restricted(connection, source_record_id): + connection.execute(""" + INSERT INTO uec.geocode_job_events + (job_id, event_type, attempt_number, retryable, worker_id, details, occurred_at) + VALUES (%s, 'cancelled', %s, false, %s, %s, %s) + """, (job_id, attempt, worker_id, json.dumps({"reason": "restricted_during_attempt"}), queried_at)) + processed += 1 + print(f"provider={provider_id} status=cancelled_restricted processed={processed}", flush=True) + continue connection.execute(""" INSERT INTO uec.geocode_results (geocode_result_id, source_record_id, provider_id, query, provider_address_id, result, precision, match_method, status, attempt_number, retryable, response, queried_at) VALUES (%s, %s, %s, %s, %s, ST_SetSRID(ST_MakePoint(%s, %s), 4326)::geography, %s, %s, %s, %s, %s, %s, %s) ON CONFLICT (geocode_result_id) DO NOTHING """, (result_id, source_record_id, provider_id, query, outcome.provider_address_id, outcome.longitude, outcome.latitude, outcome.precision, outcome.match_method, outcome.status, attempt, outcome.retryable, json.dumps(outcome.response, ensure_ascii=False), queried_at)) - connection.execute("INSERT INTO uec.geocode_job_events (job_id, event_type, attempt_number, retryable, details, occurred_at) VALUES (%s, %s, %s, %s, %s, %s)", (job_id, outcome.status, attempt, outcome.retryable, json.dumps({"acceptance": outcome.acceptance}, ensure_ascii=False), queried_at)) + connection.execute("INSERT INTO uec.geocode_job_events (job_id, event_type, attempt_number, retryable, worker_id, details, occurred_at) VALUES (%s, %s, %s, %s, %s, %s, %s)", (job_id, outcome.status, attempt, outcome.retryable, worker_id, json.dumps({"acceptance": outcome.acceptance}, ensure_ascii=False), queried_at)) processed += 1 - print(f"job={job_id} attempt={attempt} status={outcome.status}", flush=True) + print(f"provider={provider_id} status={outcome.status} processed={processed}", flush=True) if limit is None or processed < limit: time.sleep(delay) return processed @@ -70,5 +142,14 @@ def run(database_url: str, provider_id: str, limit: int | None, delay: float, re parser.add_argument("--limit", type=int) parser.add_argument("--delay", type=float, default=1.0) parser.add_argument("--retries", type=int, default=3) + parser.add_argument("--daily-budget", type=int, default=2800) + parser.add_argument("--max-attempts", type=int, default=5) + parser.add_argument("--lease-timeout", type=int, default=900, help="Seconds before an abandoned started event can be reclaimed") + parser.add_argument("--worker-id", default=os.environ.get("UEC_GEOCODE_WORKER_ID")) args = parser.parse_args() - print(f"Processed {run(args.database_url, args.provider, args.limit, args.delay, args.retries)} geocoding jobs") + count = run( + args.database_url, args.provider, args.limit, args.delay, args.retries, + daily_budget=args.daily_budget, max_attempts=args.max_attempts, + lease_timeout=args.lease_timeout, worker_id=args.worker_id, + ) + print(f"provider={args.provider} status=complete processed={count}") diff --git a/pipeline/tests/e2e/test_suppression_lifecycle.py b/pipeline/tests/e2e/test_suppression_lifecycle.py index 0d39e72..fc96ab9 100644 --- a/pipeline/tests/e2e/test_suppression_lifecycle.py +++ b/pipeline/tests/e2e/test_suppression_lifecycle.py @@ -15,6 +15,7 @@ from unittest.mock import patch import psycopg +from pipeline.geocoding.base import GeocodeOutcome try: from .fixture import E2EEnvironment @@ -127,6 +128,74 @@ def seed_synthetic_fixture(cls): ) cls.env.build_public_read_model("e2e-suppression-old") + def _queue_geocode(self, provider, query, *, started_at=None): + job_id = uuid.uuid4() + with psycopg.connect(self.env.database_url) as db: + with db.transaction(): + db.execute( + "INSERT INTO uec.geocode_jobs (job_id,source_record_id,provider_id,query) VALUES (%s,%s,%s,%s)", + (job_id, self.source_record_id, provider, query), + ) + event_type = "started" if started_at else "queued" + db.execute( + "INSERT INTO uec.geocode_job_events (job_id,event_type,attempt_number,occurred_at) VALUES (%s,%s,1,%s)", + (job_id, event_type, started_at or datetime.now(timezone.utc)), + ) + return job_id + + def test_a_worker_claims_job_and_records_append_only_result(self): + job_id = self._queue_geocode("synthetic-lease", "private lease query") + + class Adapter: + def geocode(self, _query): + return GeocodeOutcome("review_required", "review_provider_candidate", 55.6, 12.5, "fixture", "building", "fixture", False, {"fixture": True}) + + with patch.object(WORKER, "get_adapter", return_value=Adapter()): + count = WORKER.run(self.env.database_url, "synthetic-lease", 1, 0, 1, worker_id="e2e-worker") + self.assertEqual(count, 1) + with psycopg.connect(self.env.database_url) as db: + events = db.execute( + "SELECT event_type,worker_id FROM uec.geocode_job_events WHERE job_id=%s ORDER BY occurred_at,event_id", + (job_id,), + ).fetchall() + self.assertEqual([row[0] for row in events], ["queued", "started", "review_required"]) + self.assertEqual(events[-1][1], "e2e-worker") + self.assertEqual(db.execute("SELECT count(*) FROM uec.geocode_results WHERE source_record_id=%s AND provider_id='synthetic-lease'", (self.source_record_id,)).fetchone()[0], 1) + + def test_b_worker_recovers_an_expired_lease(self): + job_id = self._queue_geocode( + "synthetic-stale", "private stale query", + started_at=datetime.now(timezone.utc) - timedelta(hours=1), + ) + + class Adapter: + def geocode(self, _query): + return GeocodeOutcome("unresolved", "unresolved", None, None, None, None, "fixture", False, []) + + with patch.object(WORKER, "get_adapter", return_value=Adapter()): + count = WORKER.run(self.env.database_url, "synthetic-stale", 1, 0, 1, lease_timeout=60, worker_id="recovery-worker") + self.assertEqual(count, 1) + with psycopg.connect(self.env.database_url) as db: + attempts = db.execute("SELECT attempt_number,event_type FROM uec.geocode_job_events WHERE job_id=%s ORDER BY occurred_at,event_id", (job_id,)).fetchall() + self.assertEqual(attempts[-2:], [(2, "started"), (2, "unresolved")]) + + def test_c_daily_budget_stops_before_provider_call(self): + self._queue_geocode("synthetic-budget", "already counted", started_at=datetime.now(timezone.utc)) + self._queue_geocode("synthetic-budget", "must remain queued") + + class Adapter: + calls = 0 + + def geocode(self, _query): + self.calls += 1 + raise AssertionError("daily budget must stop before provider call") + + adapter = Adapter() + with patch.object(WORKER, "get_adapter", return_value=adapter): + count = WORKER.run(self.env.database_url, "synthetic-budget", 1, 0, 1, daily_budget=1) + self.assertEqual(count, 0) + self.assertEqual(adapter.calls, 0) + def get_json(self, path): with urllib.request.urlopen(f"http://localhost:{self.env.api_port}{path}", timeout=10) as response: return response.status, json.loads(response.read()) From a45ebe55f7291378db1f1b43d9cc7240372664cc Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Fri, 18 Sep 2026 16:49:02 -0700 Subject: [PATCH 262/311] Add geocoding operator tooling --- docker-compose.pipeline.yml | 15 +++ docs/geocoder-sprint-lane-ledger.md | 11 +- docs/geocoding-operator.md | 29 +++++ .../diagnostics/geocode-operator-status.py | 113 ++++++++++++++++++ .../tests/test_geocode_operator_status.py | 29 +++++ 5 files changed, 192 insertions(+), 5 deletions(-) create mode 100644 docs/geocoding-operator.md create mode 100644 pipeline/scripts/diagnostics/geocode-operator-status.py create mode 100644 pipeline/tests/test_geocode_operator_status.py diff --git a/docker-compose.pipeline.yml b/docker-compose.pipeline.yml index 472bd7a..0cad874 100644 --- a/docker-compose.pipeline.yml +++ b/docker-compose.pipeline.yml @@ -16,5 +16,20 @@ services: volumes: - uec-postgres-data:/var/lib/postgresql/data + geocode-worker: + profiles: [geocoding] + build: . + command: + - python + - pipeline/scripts/stages/geocode-worker.py + - --provider + - geoapify + environment: + UEC_DATABASE_URL: postgresql://uec:uec-local-development-only@postgres:5432/uec + GEOAPIFY_API_KEY: ${GEOAPIFY_API_KEY:-} + depends_on: + postgres: + condition: service_healthy + volumes: uec-postgres-data: diff --git a/docs/geocoder-sprint-lane-ledger.md b/docs/geocoder-sprint-lane-ledger.md index 8f1c474..b6b0353 100644 --- a/docs/geocoder-sprint-lane-ledger.md +++ b/docs/geocoder-sprint-lane-ledger.md @@ -11,13 +11,12 @@ entry grants publication approval or authorizes live provider calls. | Private operator review console | `316d549b`, `88422957`, `898fe019` | consolidated; frontend and focused Python tests pass | include in full consolidation gate | | Country geocoding reconnaissance | `6367a176` | consolidated; profile/schema tests pass | include in full consolidation gate | | Geoapify adapter and adversarial provider hardening | `d1ba77d2` | consolidated; 16 adapter/import tests pass | run database E2E and full consolidation gate | -| Durable worker | pending checkpoint commit | transactional claims, stale recovery, budgets, bounded attempts and suppression recheck implemented; seven Docker E2E tests pass | include in full consolidation gate | +| Durable worker | `19e1e59f` | transactional claims, stale recovery, budgets, bounded attempts and suppression recheck implemented; seven Docker E2E tests pass | full consolidation gate passed | +| Geocoder operator tooling | pending checkpoint commit | aggregate status/ETA, safe logs, environment-only secret configuration, optional container operation, privacy tests, and operator documentation implemented | full consolidation gate passed | ## In flight -| Lane | Scope | Exit requirement | -| --- | --- | --- | -| Geocoder operator tooling | aggregate status/ETA, safe logs, secret configuration, local/container background operation | committed changes plus privacy tests and operator documentation | +No lanes remain in flight for this sprint. ## Consolidation gate @@ -31,5 +30,7 @@ entry grants publication approval or authorizes live provider calls. 5. A successful geocode cannot create privacy approval, publication approval, release membership, or graph certainty. 6. Focused geocoder tests, database E2E, standard gate, frontend tests, and the - Docker E2E suite pass on the consolidated branch. + Docker E2E suite pass on the consolidated branch. Verified 2026-09-18: + 263 standard Python tests (15 skipped), 61 SQL contract tests, 83 Rust tests, + 25 frontend tests, and the seven-test suppression/geocoder lifecycle suite. 7. The branch is clean before a checkpoint push and human CI review. diff --git a/docs/geocoding-operator.md b/docs/geocoding-operator.md new file mode 100644 index 0000000..c05ae50 --- /dev/null +++ b/docs/geocoding-operator.md @@ -0,0 +1,29 @@ +# Unattended geocoding operations + +This lane performs private enrichment only. It cannot approve or publish a +record. Geocoding jobs, events, and provider evidence remain append-only, and +the worker checks current restrictions before and after each provider request. + +After reviewing the provider's terms and rate limits, supply the API key only +through the environment and start the optional worker profile: + +```powershell +$env:GEOAPIFY_API_KEY = "..." +docker compose -f docker-compose.pipeline.yml --profile geocoding up -d geocode-worker +``` + +Never place the key in source control, Compose files, command arguments, +reports, or logs. Worker output is limited to provider, aggregate status, and +processed counts. + +Inspect the queue without exposing facility records: + +```powershell +python pipeline/scripts/diagnostics/geocode-operator-status.py --pretty +``` + +The report includes aggregate state, provider and country counts; recent daily +usage; retry-state wording; and an estimated completion time based on the last +seven days. It excludes addresses, queries, provider payloads, job and record +identifiers, and credentials. A null ETA means recent throughput is +insufficient to estimate completion; it is not a completion promise. diff --git a/pipeline/scripts/diagnostics/geocode-operator-status.py b/pipeline/scripts/diagnostics/geocode-operator-status.py new file mode 100644 index 0000000..9890d56 --- /dev/null +++ b/pipeline/scripts/diagnostics/geocode-operator-status.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 +"""Print privacy-safe aggregate geocoding queue, usage, retry, and ETA status.""" + +import argparse +import json +import os +from datetime import datetime, timezone + +import psycopg + + +def status(database_url: str, provider: str | None = None) -> dict: + parameters = (provider,) if provider else () + job_filter = "WHERE current.provider_id = %s" if provider else "" + result_filter = "WHERE result.provider_id = %s" if provider else "" + + with psycopg.connect(database_url) as database: + states = database.execute( + f""" + SELECT COALESCE(current.event_type, 'unknown'), current.provider_id, count(*) + FROM uec.geocode_job_current AS current + {job_filter} + GROUP BY 1, 2 + ORDER BY 1, 2 + """, + parameters, + ).fetchall() + countries = database.execute( + f""" + SELECT record.country_code, count(*) + FROM uec.geocode_job_current AS current + JOIN uec.source_records AS record USING (source_record_id) + {job_filter} + GROUP BY 1 + ORDER BY 1 + """, + parameters, + ).fetchall() + daily_usage = database.execute( + f""" + SELECT date_trunc('day', result.queried_at)::date, + result.provider_id, + count(*) + FROM uec.geocode_results AS result + {result_filter} + GROUP BY 1, 2 + ORDER BY 1 DESC, 2 + """, + parameters, + ).fetchall() + pending = database.execute( + f""" + SELECT count(*) + FROM uec.geocode_job_current AS current + WHERE current.event_type IN ('queued', 'started') + {('AND current.provider_id = %s' if provider else '')} + """, + parameters, + ).fetchone()[0] + completed = database.execute( + f""" + SELECT count(*) + FROM uec.geocode_results AS result + WHERE result.queried_at >= now() - interval '7 days' + {('AND result.provider_id = %s' if provider else '')} + """, + parameters, + ).fetchone()[0] + + daily_rate = completed / 7 + return { + "schema_version": "geocode-operator-status-v1", + "generated_at": datetime.now(timezone.utc).isoformat(), + "privacy_boundary": "aggregate only; no addresses, queries, payloads, identifiers, or keys", + "states": [ + {"state": state, "provider": provider_id, "count": count} + for state, provider_id, count in states + ], + "countries": [ + {"country_code": country_code, "count": count} + for country_code, count in countries + ], + "daily_usage": [ + {"day": str(day), "provider": provider_id, "count": count} + for day, provider_id, count in daily_usage + ], + "retry_state": "retryable failures are included in state counts; details are intentionally omitted", + "pending": pending, + "completed_last_7_days": completed, + "eta_days": round(pending / daily_rate, 1) if daily_rate else None, + "eta_basis": "7-day completed geocode result average; null means insufficient usage", + } + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--database-url", + default=os.environ.get( + "UEC_DATABASE_URL", + "postgresql://uec:uec-local-development-only@localhost:5433/uec", + ), + ) + parser.add_argument("--provider") + parser.add_argument("--pretty", action="store_true") + arguments = parser.parse_args() + print( + json.dumps( + status(arguments.database_url, arguments.provider), + indent=2 if arguments.pretty else None, + default=str, + ) + ) diff --git a/pipeline/tests/test_geocode_operator_status.py b/pipeline/tests/test_geocode_operator_status.py new file mode 100644 index 0000000..88b32f6 --- /dev/null +++ b/pipeline/tests/test_geocode_operator_status.py @@ -0,0 +1,29 @@ +import unittest +from pathlib import Path + + +ROOT = Path(__file__).parents[1] + + +class GeocodeOperatorStatusContractTests(unittest.TestCase): + def test_operator_status_is_aggregate_only_and_eta_is_documented(self): + source = (ROOT / "scripts/diagnostics/geocode-operator-status.py").read_text() + self.assertIn("privacy_boundary", source) + self.assertIn("eta_days", source) + self.assertIn("addresses, queries, payloads", source) + + def test_worker_does_not_log_identifiers_or_queries(self): + source = (ROOT / "scripts/stages/geocode-worker.py").read_text() + self.assertNotIn("job={job_id}", source) + self.assertNotIn("source_record_id=", source) + self.assertNotIn("query=", source) + self.assertIn("status=", source) + + def test_compose_requires_environment_secret(self): + source = (ROOT.parent / "docker-compose.pipeline.yml").read_text() + self.assertIn("GEOAPIFY_API_KEY: ${GEOAPIFY_API_KEY:-}", source) + self.assertNotIn("--api-key", source) + + +if __name__ == "__main__": + unittest.main() From 8032e564db183fc42b34abdd6e7ea0237300323d Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Fri, 18 Sep 2026 19:46:22 -0700 Subject: [PATCH 263/311] Fix private graph traversal and integration gates --- .dockerignore | 13 ++ .github/workflows/tests.yml | 15 ++- Dockerfile.context-test | 8 ++ docker-compose.pipeline.yml | 5 +- pipeline/tests/db_schema_preflight.py | 50 +++++++ pipeline/tests/e2e/fixture.py | 74 +++++++++- pipeline/tests/e2e/run-suite.ps1 | 1 + pipeline/tests/e2e/test_private_graph.py | 127 ++++++++++++++++++ pipeline/tests/run-standard.ps1 | 4 +- pipeline/tests/run_unittest.py | 39 ++++++ pipeline/tests/test_private_graph_contract.py | 43 ++++++ pipeline/tests/verify-docker-context.ps1 | 16 +++ src/graph_private.rs | 4 +- src/lib.rs | 6 +- 14 files changed, 393 insertions(+), 12 deletions(-) create mode 100644 Dockerfile.context-test create mode 100644 pipeline/tests/db_schema_preflight.py create mode 100644 pipeline/tests/e2e/test_private_graph.py create mode 100644 pipeline/tests/run_unittest.py create mode 100644 pipeline/tests/test_private_graph_contract.py create mode 100644 pipeline/tests/verify-docker-context.ps1 diff --git a/.dockerignore b/.dockerignore index 72322bb..d916335 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,4 +1,17 @@ target +data/ +private/ +staging/ +.tmp/ +tmp/ +**/.env +**/.env.* +*.env +*.secret +*.secrets +**/*secret* +**/*credential* +**/*token* Old scripts Old CSVs .git diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 6049bba..6d6c358 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -13,7 +13,7 @@ jobs: - uses: actions/setup-python@v5 with: python-version: '3.11' - - run: pip install psycopg[binary] + - run: pip install -r pipeline/requirements.txt - name: Run standard tests in the canonical clean environment shell: pwsh run: ./pipeline/tests/run-standard.ps1 @@ -26,16 +26,23 @@ jobs: - uses: actions/setup-python@v5 with: python-version: '3.11' - - run: pip install psycopg[binary] - - name: Run Docker-backed API E2E tests + - run: pip install -r pipeline/requirements.txt + - name: Verify private Docker build context shell: pwsh - run: ./pipeline/tests/e2e/run-suite.ps1 + run: ./pipeline/tests/verify-docker-context.ps1 + - name: Run Docker-backed API E2E tests (core and extended) + shell: pwsh + run: ./pipeline/tests/e2e/run-suite.ps1 -Suite full backup-restore: # The PostGIS image is Linux-only; Ubuntu includes PowerShell Core for the drill. runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - run: pip install -r pipeline/requirements.txt - name: Run synthetic backup/restore gate shell: pwsh env: diff --git a/Dockerfile.context-test b/Dockerfile.context-test new file mode 100644 index 0000000..632fa0f --- /dev/null +++ b/Dockerfile.context-test @@ -0,0 +1,8 @@ +FROM alpine:3.20 + +WORKDIR /context +COPY . . + +# The CI harness creates this synthetic private marker under data/. The +# assertion fails if the Docker ignore rules allow it into the build context. +RUN test ! -e data/.docker-context-sentinel diff --git a/docker-compose.pipeline.yml b/docker-compose.pipeline.yml index 0cad874..fa6cf91 100644 --- a/docker-compose.pipeline.yml +++ b/docker-compose.pipeline.yml @@ -18,7 +18,10 @@ services: geocode-worker: profiles: [geocoding] - build: . + build: + context: . + dockerfile: Dockerfile.worker + restart: "no" command: - python - pipeline/scripts/stages/geocode-worker.py diff --git a/pipeline/tests/db_schema_preflight.py b/pipeline/tests/db_schema_preflight.py new file mode 100644 index 0000000..7c37bf0 --- /dev/null +++ b/pipeline/tests/db_schema_preflight.py @@ -0,0 +1,50 @@ +"""Fail-fast connection and schema preflight for required database jobs.""" + +import os +import sys +from pathlib import Path + +import psycopg + + +REQUIRED_TABLES = { + "schema_migrations", + "facilities", + "organizations", + "organization_relationship_observations", + "claim_current", + "source_entity_crosswalks", + "source_records", +} + + +def main(): + database_url = os.environ.get("UEC_DATABASE_URL") + if not database_url: + print("UEC_DATABASE_URL is required for the database/schema preflight", file=sys.stderr) + return 2 + try: + with psycopg.connect(database_url) as connection: + rows = connection.execute( + "SELECT c.relname FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace " + "WHERE n.nspname='uec' AND c.relkind IN ('r','p','v','m','f')" + ).fetchall() + tables = {row[0] for row in rows} + missing = sorted(REQUIRED_TABLES - tables) + if missing: + print(f"database schema preflight failed; missing tables: {', '.join(missing)}", file=sys.stderr) + return 1 + expected = len(list((Path(__file__).parents[1] / "migrations").glob("*.sql"))) + applied = connection.execute("SELECT count(*) FROM uec.schema_migrations").fetchone()[0] + if applied != expected: + print(f"database schema preflight failed; applied {applied} migrations, expected {expected}", file=sys.stderr) + return 1 + except psycopg.Error as error: + print(f"database/schema preflight failed: {error.__class__.__name__}", file=sys.stderr) + return 1 + print(f"database/schema preflight passed ({applied} migrations; required graph tables present)") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/pipeline/tests/e2e/fixture.py b/pipeline/tests/e2e/fixture.py index 5af9076..9bd697e 100644 --- a/pipeline/tests/e2e/fixture.py +++ b/pipeline/tests/e2e/fixture.py @@ -89,6 +89,7 @@ def __init__(self): self.start_attempts = 0 # Synthetic only: this token is scoped to the disposable test server. self.dev_preview_token = "uec-e2e-preview-token" + self.private_graph_token = "uec-e2e-private-graph-token" self.test_release_id = None def _ensure_build_temp(self): @@ -228,11 +229,32 @@ def _start_once(self, files, wait_for_ready): if is_retryable_database_failure(output): raise _RetryableStartupFailure("Postgres became unavailable after migrations") raise RuntimeError(f"PostGIS database failed post-migration readiness\n{_sanitize_diagnostics(output)}") + required = { + "facilities", + "organizations", + "organization_relationship_observations", + "claim_current", + "source_entity_crosswalks", + "source_records", + } + with psycopg.connect(self.database_url) as db: + names = { + row[0] + for row in db.execute( + "SELECT c.relname FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace " + "WHERE n.nspname='uec' AND c.relkind IN ('r','p','v','m','f')" + ).fetchall() + } + missing = sorted(required - names) + if missing: + raise RuntimeError( + f"database/schema preflight failed; missing={missing}" + ) print("[e2e] building backend", flush=True) build_env = os.environ.copy() build_env["CARGO_TARGET_DIR"] = str(self.cargo_cache_dir) subprocess.run(["cargo", "build", "--quiet"], cwd=ROOT, check=True, timeout=180, env=build_env) - env = os.environ.copy(); env.update({"UEC_DATABASE_URL": self.database_url, "PORT": str(self.api_port), "UEC_RUNTIME_MODE": "development", "UEC_BIND_HOST": "127.0.0.1", "UEC_DEV_PREVIEW": "true", "UEC_DEV_PREVIEW_TOKEN": self.dev_preview_token}) + env = os.environ.copy(); env.update({"UEC_DATABASE_URL": self.database_url, "PORT": str(self.api_port), "UEC_RUNTIME_MODE": "development", "UEC_BIND_HOST": "127.0.0.1", "UEC_DEV_PREVIEW": "true", "UEC_DEV_PREVIEW_TOKEN": self.dev_preview_token, "UEC_PRIVATE_GRAPH_TOKEN": self.private_graph_token}) if self.test_release_id: env.update({"UEC_TEST_RELEASE_ID": self.test_release_id, "UEC_TEST_RELEASE_TOKEN": self.dev_preview_token}) cached_binary = self.cargo_cache_dir / "debug/uec-api.exe" @@ -418,6 +440,56 @@ def seed_private_candidate_scenario(self): db.execute("INSERT INTO uec.publication_review_events (source_record_id,release_id,factual_review_status,privacy_screening_status,maintainer_approval,publication_eligible,reviewer_role) VALUES (%s,'e2e-private-candidate','reviewed','passed','approved',true,'maintainer')", (pending_record,)) self.private_candidate_facility_id = facility + def seed_private_graph_scenario(self): + """Seed a bounded synthetic graph for authenticated HTTP contract tests.""" + now = datetime.now(timezone.utc) + source_id = f"e2e.private-graph.{uuid.uuid4().hex}" + artifact_id = uuid.uuid4() + record_id = uuid.uuid4() + organization_a, organization_b, organization_c = uuid.uuid4(), uuid.uuid4(), uuid.uuid4() + facility_one, facility_two = uuid.uuid4(), uuid.uuid4() + with psycopg.connect(self.database_url) as db: + with db.transaction(): + db.execute( + "INSERT INTO uec.sources (source_id,country_code,name,official_url,access_method) VALUES (%s,'US','Synthetic private graph source','https://example.invalid/private-graph','fixture')", + (source_id,), + ) + db.execute( + "INSERT INTO uec.raw_artifacts (artifact_id,storage_key,sha256,byte_size,retrieved_at) VALUES (%s,%s,%s,1,%s)", + (artifact_id, f"e2e/private-graph/{artifact_id}", uuid.uuid4().hex * 2, now), + ) + db.execute( + "INSERT INTO uec.source_records (source_record_id,source_id,source_record_key,artifact_id,raw_fields,parsed_at,source_state) VALUES (%s,%s,'private-graph',%s,'{}',%s,'rejected')", + (record_id, source_id, artifact_id, now), + ) + db.execute( + "INSERT INTO uec.organizations (organization_id,canonical_name,country_code) VALUES (%s,'Synthetic graph A','US'),(%s,'Synthetic graph B','US'),(%s,'Synthetic graph C','US')", + (organization_a, organization_b, organization_c), + ) + db.execute( + "INSERT INTO uec.facilities (facility_id,canonical_name,country_code) VALUES (%s,'Synthetic graph facility one','US'),(%s,'Synthetic graph facility two','US')", + (facility_one, facility_two), + ) + edges = [ + (organization_a, None, organization_b, "operator", 0.8750, now), + (organization_b, None, organization_a, "owner", 0.6250, now.replace(microsecond=max(0, now.microsecond - 1))), + (organization_b, facility_one, None, "supplier", None, now.replace(microsecond=max(0, now.microsecond - 2))), + (organization_c, None, organization_b, "parent", 1.0, now.replace(microsecond=max(0, now.microsecond - 3))), + (organization_a, facility_two, None, "customer", 0.5, now.replace(microsecond=max(0, now.microsecond - 4))), + ] + for from_id, target_facility, target_org, relationship_type, confidence, observed_at in edges: + db.execute( + "INSERT INTO uec.organization_relationship_observations (source_id,source_record_id,from_organization_id,target_facility_id,target_organization_id,relationship_type,observed_at,confidence,review_state,storage_state,privacy_status,publication_status,note) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,'review_required','private','suppressed','not_eligible','synthetic private note')", + (source_id, record_id, from_id, target_facility, target_org, relationship_type, observed_at, confidence), + ) + self.private_graph_ids = { + "organization_a": str(organization_a), + "organization_b": str(organization_b), + "organization_c": str(organization_c), + "facility_one": str(facility_one), + "facility_two": str(facility_two), + } + def restore_restricted_record(self): """Append a restoration event; the original evidence is unchanged.""" with psycopg.connect(self.database_url) as db: diff --git a/pipeline/tests/e2e/run-suite.ps1 b/pipeline/tests/e2e/run-suite.ps1 index b02540b..2235d94 100644 --- a/pipeline/tests/e2e/run-suite.ps1 +++ b/pipeline/tests/e2e/run-suite.ps1 @@ -11,6 +11,7 @@ $core = @( 'pipeline.tests.e2e.test_seeded_api', 'pipeline.tests.e2e.test_public_surface_safety', 'pipeline.tests.e2e.test_candidate_import', + 'pipeline.tests.e2e.test_private_graph', 'pipeline.tests.e2e.test_readiness' ) $extended = @( diff --git a/pipeline/tests/e2e/test_private_graph.py b/pipeline/tests/e2e/test_private_graph.py new file mode 100644 index 0000000..57cf1cb --- /dev/null +++ b/pipeline/tests/e2e/test_private_graph.py @@ -0,0 +1,127 @@ +"""Authenticated populated-database tests for both private graph APIs.""" + +import json +import os +import unittest +import urllib.error +import urllib.request + +from .fixture import E2EEnvironment + + +@unittest.skipUnless( + os.environ.get("UEC_RUN_E2E") == "1", + "set UEC_RUN_E2E=1 to run Docker-backed E2E tests", +) +class PrivateGraphE2ETests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.env = E2EEnvironment().start() + cls.env.seed_private_graph_scenario() + cls.base = f"http://127.0.0.1:{cls.env.api_port}" + cls.token = cls.env.private_graph_token + cls.ids = cls.env.private_graph_ids + + @classmethod + def tearDownClass(cls): + cls.env.stop() + + def request(self, path, token=None): + headers = {} + if token is not None: + headers["X-UEC-Private-Graph-Token"] = token + request = urllib.request.Request(self.base + path, headers=headers) + with urllib.request.urlopen(request, timeout=10) as response: + return response.status, json.loads(response.read()) + + def assert_denied(self, path): + for token in (None, "wrong-token"): + request = urllib.request.Request( + self.base + path, + headers={} if token is None else {"X-UEC-Private-Graph-Token": token}, + ) + with self.subTest(path=path, token=token), self.assertRaises(urllib.error.HTTPError) as error: + urllib.request.urlopen(request, timeout=10) + self.assertEqual(error.exception.code, 404) + + def test_all_private_routes_fail_closed_without_valid_authentication(self): + paths = [ + "/api/private/graph/entities", + "/api/private/graph/search?q=Synthetic", + f"/api/private/graph/entities/{self.ids['organization_a']}/neighborhood", + "/api/private/graph/queues/statistics", + "/api/private/graph/traverse?entity_type=organization&entity_id=" + f"{self.ids['organization_a']}", + ] + for path in paths: + self.assert_denied(path) + + def test_populated_neighborhood_supports_both_directions_depth_two_and_numeric_confidence(self): + path = ( + f"/api/private/graph/entities/{self.ids['organization_a']}/neighborhood" + "?direction=both&depth=2&limit=100" + ) + _, body = self.request(path, self.token) + self.assertEqual(body["meta"]["direction"], "both") + self.assertEqual(body["meta"]["depth"], 2) + self.assertEqual(len(body["data"]), 5) + relationships = {row["relationship_type"] for row in body["data"]} + self.assertEqual(relationships, {"operator", "owner", "supplier", "parent", "customer"}) + self.assertIn(0.875, [row["confidence"] for row in body["data"]]) + self.assertIn(None, [row["confidence"] for row in body["data"]]) + self.assertTrue(all(row["storage_state"] == "private" for row in body["data"])) + self.assertTrue(all(row["privacy_status"] == "suppressed" for row in body["data"])) + self.assertTrue(all(row["publication_status"] == "not_eligible" for row in body["data"])) + + def test_entity_search_is_authenticated_bounded_and_deterministic(self): + _, first = self.request("/api/private/graph/search?q=Synthetic%20graph&limit=100", self.token) + _, second = self.request("/api/private/graph/search?q=Synthetic%20graph&limit=100", self.token) + self.assertEqual(first["data"], second["data"]) + self.assertEqual(len(first["data"]), 5) + self.assertTrue(all("note" not in row for row in first["data"])) + + def test_both_directions_are_applied_in_the_legacy_traverse_endpoint(self): + path = ( + "/api/private/graph/traverse?entity_type=organization&" + f"entity_id={self.ids['organization_a']}&direction=both&depth=2" + ) + _, body = self.request(path, self.token) + self.assertEqual(body["meta"]["direction"], "both") + self.assertEqual(body["meta"]["depth"], 2) + self.assertEqual(len(body["data"]), 5) + self.assertNotIn("note", body["data"][0]) + self.assertIn(0.625, [row["confidence"] for row in body["data"]]) + + def test_direction_filters_and_cycle_are_bounded(self): + base = f"/api/private/graph/entities/{self.ids['organization_a']}/neighborhood" + _, outgoing = self.request(base + "?direction=out&depth=2&limit=100", self.token) + _, incoming = self.request(base + "?direction=in&depth=2&limit=100", self.token) + self.assertEqual(len(outgoing["data"]), 4) + self.assertEqual(len(incoming["data"]), 3) + # A->B and B->A form a cycle; recursive traversal must still terminate. + self.assertEqual(len({row["relationship_observation_id"] for row in outgoing["data"]}), 4) + + def test_queue_allowlist_uses_existing_schema_and_deterministic_limit(self): + kinds = ( + "contradictions", + "unresolved-identities", + "quarantine", + "claims", + "rejected-candidates", + "suppression", + "statistics", + ) + for kind in kinds: + with self.subTest(kind=kind): + _, first = self.request(f"/api/private/graph/queues/{kind}?limit=1", self.token) + _, second = self.request(f"/api/private/graph/queues/{kind}?limit=1", self.token) + self.assertEqual(first["meta"]["queue"], kind) + self.assertLessEqual(len(first["data"]), 1) + self.assertEqual(first["data"], second["data"]) + _, quarantine = self.request("/api/private/graph/queues/quarantine?limit=1", self.token) + self.assertEqual(len(quarantine["data"]), 1) + self.assertEqual(quarantine["data"][0][2], "rejected") + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/tests/run-standard.ps1 b/pipeline/tests/run-standard.ps1 index 53b5348..0bb2bab 100644 --- a/pipeline/tests/run-standard.ps1 +++ b/pipeline/tests/run-standard.ps1 @@ -18,6 +18,8 @@ try { $env:UEC_DATABASE_URL = "${databaseUrl}?sslmode=disable" python pipeline/scripts/maintenance/apply-migrations.py if ($LASTEXITCODE -ne 0) { throw "Migration application failed (exit $LASTEXITCODE)." } + python pipeline/tests/db_schema_preflight.py + if ($LASTEXITCODE -ne 0) { throw "Database/schema preflight failed (exit $LASTEXITCODE)." } Remove-Item Env:UEC_RUN_E2E -ErrorAction SilentlyContinue cargo test --locked @@ -27,7 +29,7 @@ try { & docker compose -p $project -f $compose exec -T postgres psql -v ON_ERROR_STOP=1 -U uec -d uec if ($LASTEXITCODE -ne 0) { throw "Synthetic fixture seeding failed (exit $LASTEXITCODE)." } - python -m unittest discover -s pipeline/tests -q + python pipeline/tests/run_unittest.py --start-directory pipeline/tests if ($LASTEXITCODE -ne 0) { throw "Python tests failed (exit $LASTEXITCODE)." } python -m unittest -q pipeline.germany.test_adapter pipeline.germany.test_orchestrator pipeline.common.test_delta pipeline.common.test_orchestrator pipeline.common.test_registry pipeline.contracts.test_adapter_contract pipeline.contracts.test_candidate_handoff pipeline.sources.denmark.test_adapter pipeline.sources.uk.fsa_approved.test_adapter pipeline.sources.uk.fsa_approved.test_handoff pipeline.sources.uk.fss_approved.test_adapter pipeline.sources.uk.fss_approved.test_handoff pipeline.sources.uk.approved.test_compose diff --git a/pipeline/tests/run_unittest.py b/pipeline/tests/run_unittest.py new file mode 100644 index 0000000..d31441b --- /dev/null +++ b/pipeline/tests/run_unittest.py @@ -0,0 +1,39 @@ +"""Run a unittest suite and reject skips outside the documented allowlist.""" + +import argparse +import sys +import unittest +from pathlib import Path + + +sys.path.insert(0, str(Path(__file__).parents[2])) + + +ALLOWED_SKIP_REASONS = { + "set UEC_RUN_E2E=1 to run Docker-backed E2E tests", + "database is older than migration 018; Docker E2E applies the current schema", +} + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--start-directory", default="pipeline/tests") + parser.add_argument("--module", action="append", default=[]) + args = parser.parse_args() + if args.module: + suite = unittest.TestSuite( + unittest.defaultTestLoader.loadTestsFromName(module) for module in args.module + ) + else: + suite = unittest.defaultTestLoader.discover(args.start_directory) + result = unittest.TextTestRunner(verbosity=2).run(suite) + unexpected = [(test.id(), reason) for test, reason in result.skipped if reason not in ALLOWED_SKIP_REASONS] + if unexpected: + for test_id, reason in unexpected: + print(f"unexpected skipped test: {test_id}: {reason}", file=sys.stderr) + return 2 + return 0 if result.wasSuccessful() else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/pipeline/tests/test_private_graph_contract.py b/pipeline/tests/test_private_graph_contract.py new file mode 100644 index 0000000..5c0c52c --- /dev/null +++ b/pipeline/tests/test_private_graph_contract.py @@ -0,0 +1,43 @@ +"""Static guards for the private graph SQL/API contract.""" + +import unittest +from pathlib import Path + + +ROOT = Path(__file__).parents[2] + + +class PrivateGraphContractTests(unittest.TestCase): + def test_both_graph_handlers_cast_numeric_confidence_and_expand_each_direction(self): + for path in (ROOT / "src" / "graph_private.rs", ROOT / "src" / "lib.rs"): + text = path.read_text(encoding="utf-8") + self.assertIn("confidence::double precision AS confidence", text) + self.assertIn("directions(step_direction)", text) + self.assertIn("CASE WHEN d.step_direction='out' THEN e.to_type ELSE e.from_type END", text) + self.assertIn("CASE WHEN d.step_direction='out' THEN e.to_id ELSE e.from_id END", text) + self.assertIn("ORDER BY o.observed_at DESC, o.relationship_observation_id DESC", text) + + def test_queue_uses_existing_source_record_timestamp(self): + text = (ROOT / "src" / "graph_private.rs").read_text(encoding="utf-8") + self.assertIn("parsed_at::text FROM uec.source_records", text) + self.assertNotIn("received_at::text FROM uec.source_records", text) + + def test_graph_response_does_not_return_private_notes(self): + text = (ROOT / "src" / "lib.rs").read_text(encoding="utf-8") + traverse = text[text.index("pub async fn get_private_graph_traverse_handler"):] + self.assertNotIn('"note":r.get', traverse) + + def test_docker_context_excludes_private_staging_and_secret_patterns(self): + ignore = (ROOT / ".dockerignore").read_text(encoding="utf-8") + for pattern in ("data/", "private/", "staging/", "**/*secret*", "**/*credential*", "**/*token*"): + self.assertIn(pattern, ignore) + self.assertTrue((ROOT / "Dockerfile.context-test").exists()) + + def test_pipeline_worker_uses_dedicated_image_target(self): + compose = (ROOT / "docker-compose.pipeline.yml").read_text(encoding="utf-8") + self.assertIn("dockerfile: Dockerfile.worker", compose) + self.assertIn('restart: "no"', compose) + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/tests/verify-docker-context.ps1 b/pipeline/tests/verify-docker-context.ps1 new file mode 100644 index 0000000..1f04f89 --- /dev/null +++ b/pipeline/tests/verify-docker-context.ps1 @@ -0,0 +1,16 @@ +param() + +$ErrorActionPreference = 'Stop' +$root = (Resolve-Path (Join-Path $PSScriptRoot '..\..')).Path +$sentinel = Join-Path $root 'data\.docker-context-sentinel' +$tag = "uec-docker-context-test-$([Guid]::NewGuid().ToString('N'))" + +New-Item -ItemType File -Path $sentinel -Force | Out-Null +try { + & docker build --file (Join-Path $root 'Dockerfile.context-test') --tag $tag $root + if ($LASTEXITCODE -ne 0) { throw "Docker context sentinel proof failed (exit $LASTEXITCODE)." } + Write-Host 'PASS: synthetic private Docker context sentinel was excluded.' +} finally { + if (Test-Path -LiteralPath $sentinel) { Remove-Item -LiteralPath $sentinel -Force } + & docker image rm --force $tag *> $null +} diff --git a/src/graph_private.rs b/src/graph_private.rs index 6578a59..d0bca33 100644 --- a/src/graph_private.rs +++ b/src/graph_private.rs @@ -134,7 +134,7 @@ pub async fn neighborhood( "private graph database unavailable", ); }; - let rows = match client.query("WITH RECURSIVE observations AS (SELECT relationship_observation_id, from_organization_id, target_facility_id, target_organization_id, relationship_type, assertion_status, observed_at, confidence, review_state, storage_state, privacy_status, publication_status, source_id, source_record_id FROM uec.organization_relationship_observations WHERE ($4::text IS NULL OR relationship_type=$4) AND ($5::text IS NULL OR source_id=$5) AND ($6::timestamptz IS NULL OR observed_at >= $6) AND ($7::timestamptz IS NULL OR observed_at < $7)), edges AS (SELECT o.*, 'organization'::text AS from_type, o.from_organization_id AS from_id, CASE WHEN o.target_facility_id IS NOT NULL THEN 'facility'::text ELSE 'organization'::text END AS to_type, COALESCE(o.target_facility_id, o.target_organization_id) AS to_id FROM observations o), seed AS (SELECT 'organization'::text AS node_type, $1::uuid AS node_id, 0 AS hop WHERE EXISTS (SELECT 1 FROM uec.organizations WHERE organization_id=$1) UNION ALL SELECT 'facility'::text, $1::uuid, 0 WHERE EXISTS (SELECT 1 FROM uec.facilities WHERE facility_id=$1)), walk(node_type, node_id, hop) AS (SELECT node_type, node_id, hop FROM seed UNION SELECT CASE WHEN $2 IN ('out','both') THEN e.to_type ELSE e.from_type END, CASE WHEN $2 IN ('out','both') THEN e.to_id ELSE e.from_id END, w.hop + 1 FROM walk w JOIN edges e ON (($2 IN ('out','both') AND e.from_type=w.node_type AND e.from_id=w.node_id) OR ($2 IN ('in','both') AND e.to_type=w.node_type AND e.to_id=w.node_id)) WHERE w.hop < $3), matched AS (SELECT DISTINCT e.relationship_observation_id FROM edges e JOIN walk w ON w.hop < $3 AND (($2 IN ('out','both') AND e.from_type=w.node_type AND e.from_id=w.node_id) OR ($2 IN ('in','both') AND e.to_type=w.node_type AND e.to_id=w.node_id))) SELECT o.relationship_observation_id, o.from_organization_id, o.target_facility_id, o.target_organization_id, o.relationship_type, o.assertion_status, o.observed_at, o.confidence, o.review_state, o.storage_state, o.privacy_status, o.publication_status, o.source_id, o.source_record_id FROM observations o JOIN matched m USING (relationship_observation_id) ORDER BY o.observed_at DESC, o.relationship_observation_id DESC LIMIT $8", &[&entity_id, &direction, &depth, &p.relationship_type, &p.source_id, &p.from.map(|d| d.and_hms_opt(0,0,0).unwrap().and_utc()), &p.to.map(|d| d.and_hms_opt(0,0,0).unwrap().and_utc()), &limit]).await { Ok(v) => v, Err(_) => return v2_error(StatusCode::SERVICE_UNAVAILABLE, "private_graph_query_failed", "private graph query failed") }; + let rows = match client.query("WITH RECURSIVE observations AS (SELECT relationship_observation_id, from_organization_id, target_facility_id, target_organization_id, relationship_type, assertion_status, observed_at, confidence::double precision AS confidence, review_state, storage_state, privacy_status, publication_status, source_id, source_record_id FROM uec.organization_relationship_observations WHERE ($4::text IS NULL OR relationship_type=$4) AND ($5::text IS NULL OR source_id=$5) AND ($6::timestamptz IS NULL OR observed_at >= $6) AND ($7::timestamptz IS NULL OR observed_at < $7)), edges AS (SELECT o.*, 'organization'::text AS from_type, o.from_organization_id AS from_id, CASE WHEN o.target_facility_id IS NOT NULL THEN 'facility'::text ELSE 'organization'::text END AS to_type, COALESCE(o.target_facility_id, o.target_organization_id) AS to_id FROM observations o), directions(step_direction) AS (SELECT 'out'::text WHERE $2 IN ('out','both') UNION ALL SELECT 'in'::text WHERE $2 IN ('in','both')), seed AS (SELECT 'organization'::text AS node_type, $1::uuid AS node_id, 0 AS hop WHERE EXISTS (SELECT 1 FROM uec.organizations WHERE organization_id=$1) UNION ALL SELECT 'facility'::text, $1::uuid, 0 WHERE EXISTS (SELECT 1 FROM uec.facilities WHERE facility_id=$1)), walk(node_type, node_id, hop) AS (SELECT node_type, node_id, hop FROM seed UNION SELECT CASE WHEN d.step_direction='out' THEN e.to_type ELSE e.from_type END, CASE WHEN d.step_direction='out' THEN e.to_id ELSE e.from_id END, w.hop + 1 FROM walk w JOIN directions d ON true JOIN edges e ON ((d.step_direction='out' AND e.from_type=w.node_type AND e.from_id=w.node_id) OR (d.step_direction='in' AND e.to_type=w.node_type AND e.to_id=w.node_id)) WHERE w.hop < $3), matched AS (SELECT DISTINCT e.relationship_observation_id FROM edges e JOIN walk w ON w.hop < $3 AND (($2 IN ('out','both') AND e.from_type=w.node_type AND e.from_id=w.node_id) OR ($2 IN ('in','both') AND e.to_type=w.node_type AND e.to_id=w.node_id))) SELECT o.relationship_observation_id, o.from_organization_id, o.target_facility_id, o.target_organization_id, o.relationship_type, o.assertion_status, o.observed_at, o.confidence, o.review_state, o.storage_state, o.privacy_status, o.publication_status, o.source_id, o.source_record_id FROM observations o JOIN matched m USING (relationship_observation_id) ORDER BY o.observed_at DESC, o.relationship_observation_id DESC LIMIT $8", &[&entity_id, &direction, &depth, &p.relationship_type, &p.source_id, &p.from.map(|d| d.and_hms_opt(0,0,0).unwrap().and_utc()), &p.to.map(|d| d.and_hms_opt(0,0,0).unwrap().and_utc()), &limit]).await { Ok(v) => v, Err(_) => return v2_error(StatusCode::SERVICE_UNAVAILABLE, "private_graph_query_failed", "private graph query failed") }; let data: Vec<_> = rows.into_iter().map(|r| json!({"relationship_observation_id":r.get::<_,Uuid>(0),"from_organization_id":r.get::<_,Option>(1),"target_facility_id":r.get::<_,Option>(2),"target_organization_id":r.get::<_,Option>(3),"relationship_type":r.get::<_,Option>(4),"assertion_status":r.get::<_,String>(5),"observed_at":r.get::<_,chrono::DateTime>(6),"confidence":r.get::<_,Option>(7),"review_state":r.get::<_,String>(8),"storage_state":r.get::<_,String>(9),"privacy_status":r.get::<_,String>(10),"publication_status":r.get::<_,String>(11),"source_id":r.get::<_,String>(12),"source_record_id":r.get::<_,Uuid>(13)})).collect(); Json(json!({"api_version":"private-graph-v1","data":data,"meta":{"private":true,"entity_id":entity_id,"limit":limit,"direction":direction,"depth":depth,"bounded":true}})).into_response() } @@ -190,7 +190,7 @@ pub async fn queue( "SELECT crosswalk_id::text, left_identifier_id::text, right_identifier_id::text, assertion_status, confidence::text, observed_at::text, source_id, source_record_id::text, review_state, privacy_status, publication_status FROM uec.source_entity_crosswalks WHERE assertion_status IN ('candidate','review_required','disputed') ORDER BY observed_at DESC, crosswalk_id DESC LIMIT $1" } "quarantine" => { - "SELECT source_record_id::text, source_id, source_state, received_at::text FROM uec.source_records WHERE source_state IN ('quarantined','rejected') ORDER BY received_at DESC, source_record_id DESC LIMIT $1" + "SELECT source_record_id::text, source_id, source_state, parsed_at::text FROM uec.source_records WHERE source_state IN ('quarantined','rejected') ORDER BY parsed_at DESC, source_record_id DESC LIMIT $1" } "claims" => { "SELECT c.claim_id::text, c.source_id, c.claim_domain, c.claim_kind, c.value_state, c.unknown_reason, c.observed_at::text, c.confidence::text, c.review_state, c.storage_state, c.privacy_status, c.publication_status, COUNT(s.claim_support_id)::text AS support_count, COUNT(s.claim_support_id) FILTER (WHERE s.support_role = 'contradicting')::text AS contradicting_support_count FROM uec.claim_current c LEFT JOIN uec.claim_support s ON s.claim_id = c.claim_id GROUP BY c.claim_id, c.source_id, c.claim_domain, c.claim_kind, c.value_state, c.unknown_reason, c.observed_at, c.confidence, c.review_state, c.storage_state, c.privacy_status, c.publication_status ORDER BY c.observed_at DESC, c.claim_id DESC LIMIT $1" diff --git a/src/lib.rs b/src/lib.rs index 7f3b62a..5479b23 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -84,7 +84,7 @@ pub async fn get_private_graph_search_handler( ); } }; - let rows = match client.query("SELECT entity_type, entity_id, display_name, source_id, source_identifier, observed_at FROM (SELECT 'facility' AS entity_type, f.facility_id AS entity_id, COALESCE(f.canonical_name, '[unnamed facility]') AS display_name, sei.source_id, sei.source_identifier, sei.observed_at FROM uec.facilities f LEFT JOIN uec.source_entity_identifiers sei ON sei.facility_id=f.facility_id UNION ALL SELECT 'organization', o.organization_id, COALESCE(o.canonical_name, '[unnamed organization]'), sei.source_id, sei.source_identifier, sei.observed_at FROM uec.organizations o LEFT JOIN uec.source_entity_identifiers sei ON sei.organization_id=o.organization_id) entities WHERE ($1='' OR display_name ILIKE '%' || $1 || '%' OR source_identifier ILIKE '%' || $1 || '%') ORDER BY display_name, observed_at DESC NULLS LAST LIMIT $2", &[&q, &limit]).await { Ok(r) => r, Err(_) => return v2_error(StatusCode::SERVICE_UNAVAILABLE, "private_graph_query_failed", "private graph search failed") }; + let rows = match client.query("SELECT entity_type, entity_id, display_name, source_id, source_identifier, observed_at FROM (SELECT 'facility' AS entity_type, f.facility_id AS entity_id, COALESCE(f.canonical_name, '[unnamed facility]') AS display_name, sei.source_id, sei.source_identifier, sei.observed_at FROM uec.facilities f LEFT JOIN uec.source_entity_identifiers sei ON sei.facility_id=f.facility_id UNION ALL SELECT 'organization', o.organization_id, COALESCE(o.canonical_name, '[unnamed organization]'), sei.source_id, sei.source_identifier, sei.observed_at FROM uec.organizations o LEFT JOIN uec.source_entity_identifiers sei ON sei.organization_id=o.organization_id) entities WHERE ($1='' OR display_name ILIKE '%' || $1 || '%' OR source_identifier ILIKE '%' || $1 || '%') ORDER BY display_name, observed_at DESC NULLS LAST, entity_id, entity_type LIMIT $2", &[&q, &limit]).await { Ok(r) => r, Err(_) => return v2_error(StatusCode::SERVICE_UNAVAILABLE, "private_graph_query_failed", "private graph search failed") }; let data: Vec = rows.into_iter().map(|r| json!({"entity_type":r.get::<_,String>(0),"entity_id":r.get::<_,uuid::Uuid>(1),"display_name":r.get::<_,String>(2),"source_id":r.get::<_,Option>(3),"source_identifier":r.get::<_,Option>(4),"observed_at":r.get::<_,Option>>(5),"review_state":"unknown","privacy_status":"unknown","publication_status":"unknown"})).collect(); Json(json!({"api_version":"private-graph-v1","data":data,"meta":{"scope":"private_evidence_only","bounded":true,"public_projection":false}})).into_response() } @@ -137,8 +137,8 @@ pub async fn get_private_graph_traverse_handler( ); } }; - let rows = match client.query("WITH RECURSIVE observations AS (SELECT relationship_observation_id, source_id, source_record_id, from_organization_id, target_facility_id, target_organization_id, relationship_type, assertion_status, unknown_reason, valid_from, valid_to, observed_at, confidence, review_state, storage_state, privacy_status, publication_status, note FROM uec.organization_relationship_observations), edges AS (SELECT o.*, 'organization'::text AS from_type, o.from_organization_id AS from_id, CASE WHEN o.target_facility_id IS NOT NULL THEN 'facility'::text ELSE 'organization'::text END AS to_type, COALESCE(o.target_facility_id, o.target_organization_id) AS to_id FROM observations o), walk(node_type, node_id, hop) AS (SELECT $1::text, $2::uuid, 0 UNION SELECT CASE WHEN $3 IN ('out','both') THEN e.to_type ELSE e.from_type END, CASE WHEN $3 IN ('out','both') THEN e.to_id ELSE e.from_id END, w.hop + 1 FROM walk w JOIN edges e ON (($3 IN ('out','both') AND e.from_type=w.node_type AND e.from_id=w.node_id) OR ($3 IN ('in','both') AND e.to_type=w.node_type AND e.to_id=w.node_id)) WHERE w.hop < $4), matched AS (SELECT DISTINCT e.relationship_observation_id FROM edges e JOIN walk w ON w.hop < $4 AND (($3 IN ('out','both') AND e.from_type=w.node_type AND e.from_id=w.node_id) OR ($3 IN ('in','both') AND e.to_type=w.node_type AND e.to_id=w.node_id))) SELECT o.relationship_observation_id, o.source_id, o.source_record_id, o.from_organization_id, o.target_facility_id, o.target_organization_id, o.relationship_type, o.assertion_status, o.unknown_reason, o.valid_from, o.valid_to, o.observed_at, o.confidence, o.review_state, o.storage_state, o.privacy_status, o.publication_status, o.note FROM observations o JOIN matched m USING (relationship_observation_id) ORDER BY o.observed_at DESC LIMIT 200", &[¶ms.entity_type, ¶ms.entity_id, &direction, &depth]).await { Ok(r) => r, Err(_) => return v2_error(StatusCode::SERVICE_UNAVAILABLE, "private_graph_query_failed", "private graph traversal failed") }; - let data: Vec = rows.into_iter().map(|r| json!({"relationship_observation_id":r.get::<_,uuid::Uuid>(0),"source_id":r.get::<_,String>(1),"source_record_id":r.get::<_,uuid::Uuid>(2),"from_organization_id":r.get::<_,Option>(3),"target_facility_id":r.get::<_,Option>(4),"target_organization_id":r.get::<_,Option>(5),"relationship_type":r.get::<_,Option>(6),"assertion_status":r.get::<_,String>(7),"unknown_reason":r.get::<_,Option>(8),"valid_from":r.get::<_,Option>(9),"valid_to":r.get::<_,Option>(10),"observed_at":r.get::<_,chrono::DateTime>(11),"confidence":r.get::<_,Option>(12),"review_state":r.get::<_,String>(13),"storage_state":r.get::<_,String>(14),"privacy_status":r.get::<_,String>(15),"publication_status":r.get::<_,String>(16),"note":r.get::<_,Option>(17)})).collect(); + let rows = match client.query("WITH RECURSIVE observations AS (SELECT relationship_observation_id, source_id, source_record_id, from_organization_id, target_facility_id, target_organization_id, relationship_type, assertion_status, unknown_reason, valid_from, valid_to, observed_at, confidence::double precision AS confidence, review_state, storage_state, privacy_status, publication_status FROM uec.organization_relationship_observations), edges AS (SELECT o.*, 'organization'::text AS from_type, o.from_organization_id AS from_id, CASE WHEN o.target_facility_id IS NOT NULL THEN 'facility'::text ELSE 'organization'::text END AS to_type, COALESCE(o.target_facility_id, o.target_organization_id) AS to_id FROM observations o), directions(step_direction) AS (SELECT 'out'::text WHERE $3 IN ('out','both') UNION ALL SELECT 'in'::text WHERE $3 IN ('in','both')), walk(node_type, node_id, hop) AS (SELECT $1::text, $2::uuid, 0 UNION SELECT CASE WHEN d.step_direction='out' THEN e.to_type ELSE e.from_type END, CASE WHEN d.step_direction='out' THEN e.to_id ELSE e.from_id END, w.hop + 1 FROM walk w JOIN directions d ON true JOIN edges e ON ((d.step_direction='out' AND e.from_type=w.node_type AND e.from_id=w.node_id) OR (d.step_direction='in' AND e.to_type=w.node_type AND e.to_id=w.node_id)) WHERE w.hop < $4), matched AS (SELECT DISTINCT e.relationship_observation_id FROM edges e JOIN walk w ON w.hop < $4 AND (($3 IN ('out','both') AND e.from_type=w.node_type AND e.from_id=w.node_id) OR ($3 IN ('in','both') AND e.to_type=w.node_type AND e.to_id=w.node_id))) SELECT o.relationship_observation_id, o.source_id, o.source_record_id, o.from_organization_id, o.target_facility_id, o.target_organization_id, o.relationship_type, o.assertion_status, o.unknown_reason, o.valid_from, o.valid_to, o.observed_at, o.confidence, o.review_state, o.storage_state, o.privacy_status, o.publication_status FROM observations o JOIN matched m USING (relationship_observation_id) ORDER BY o.observed_at DESC, o.relationship_observation_id DESC LIMIT 200", &[¶ms.entity_type, ¶ms.entity_id, &direction, &depth]).await { Ok(r) => r, Err(_) => return v2_error(StatusCode::SERVICE_UNAVAILABLE, "private_graph_query_failed", "private graph traversal failed") }; + let data: Vec = rows.into_iter().map(|r| json!({"relationship_observation_id":r.get::<_,uuid::Uuid>(0),"source_id":r.get::<_,String>(1),"source_record_id":r.get::<_,uuid::Uuid>(2),"from_organization_id":r.get::<_,Option>(3),"target_facility_id":r.get::<_,Option>(4),"target_organization_id":r.get::<_,Option>(5),"relationship_type":r.get::<_,Option>(6),"assertion_status":r.get::<_,String>(7),"unknown_reason":r.get::<_,Option>(8),"valid_from":r.get::<_,Option>(9),"valid_to":r.get::<_,Option>(10),"observed_at":r.get::<_,chrono::DateTime>(11),"confidence":r.get::<_,Option>(12),"review_state":r.get::<_,String>(13),"storage_state":r.get::<_,String>(14),"privacy_status":r.get::<_,String>(15),"publication_status":r.get::<_,String>(16)})).collect(); Json(json!({"api_version":"private-graph-v1","data":data,"meta":{"scope":"private_evidence_only","bounded":true,"depth":depth,"direction":direction,"contradictions_preserved":true,"public_projection":false}})).into_response() } From 0aa0d11e7ddcf286d175f694d680723950ebe495 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Fri, 18 Sep 2026 19:48:00 -0700 Subject: [PATCH 264/311] Allow documented offline suite exclusions --- pipeline/tests/run_unittest.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pipeline/tests/run_unittest.py b/pipeline/tests/run_unittest.py index d31441b..4c9d8be 100644 --- a/pipeline/tests/run_unittest.py +++ b/pipeline/tests/run_unittest.py @@ -11,6 +11,9 @@ ALLOWED_SKIP_REASONS = { "set UEC_RUN_E2E=1 to run Docker-backed E2E tests", + "set UEC_RUN_E2E=1", + "set UEC_RUN_E2E=1 to run Docker-backed worker lifecycle tests", + "set UEC_RUN_E2E=1 and UEC_RUN_CORPUS_RESILIENCE=1 to run the Docker-backed corpus rehearsal", "database is older than migration 018; Docker E2E applies the current schema", } From 23add6333d7af67e07991ed085d35f0cdfe89b8a Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Fri, 18 Sep 2026 19:28:18 -0700 Subject: [PATCH 265/311] Harden geocoding worker durability and packaging --- Dockerfile.worker | 13 + docs/geocoding-worker.md | 56 ++++ .../040_geocode_worker_durability.sql | 57 ++++ pipeline/scripts/stages/geocode-worker.py | 270 ++++++++++++++---- .../tests/test_geocode_worker_durability.py | 95 ++++++ pipeline/tests/test_graph_migrations.py | 3 +- pipeline/worker-requirements.txt | 2 + 7 files changed, 444 insertions(+), 52 deletions(-) create mode 100644 Dockerfile.worker create mode 100644 docs/geocoding-worker.md create mode 100644 pipeline/migrations/040_geocode_worker_durability.sql create mode 100644 pipeline/tests/test_geocode_worker_durability.py create mode 100644 pipeline/worker-requirements.txt diff --git a/Dockerfile.worker b/Dockerfile.worker new file mode 100644 index 0000000..8d44be0 --- /dev/null +++ b/Dockerfile.worker @@ -0,0 +1,13 @@ +# Dedicated private geocoding worker image. It contains no Rust server or data +# directories; Compose supplies the database URL and provider credentials. +FROM python:3.12.8-slim-bookworm + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 + +WORKDIR /app +COPY pipeline/worker-requirements.txt /app/pipeline/worker-requirements.txt +RUN pip install --no-cache-dir -r /app/pipeline/worker-requirements.txt +COPY pipeline /app/pipeline + +ENTRYPOINT ["python", "-u", "pipeline/scripts/stages/geocode-worker.py"] diff --git a/docs/geocoding-worker.md b/docs/geocoding-worker.md new file mode 100644 index 0000000..c31af1c --- /dev/null +++ b/docs/geocoding-worker.md @@ -0,0 +1,56 @@ +# Private geocoding worker + +The worker drains already queued `uec.geocode_jobs` rows for one provider. It is +an operator tool for bounded private runs; it does not publish a release and it +does not make provider terms, privacy, or human review decisions. + +## Durable lifecycle + +Each job claim commits a `started` event and a random lease token before any +provider work. The provider call happens outside a database transaction. Every +attempt, including retries, first commits a row in +`uec.geocode_request_reservations` and increments the shared +`uec.geocode_provider_budgets` counter for the UTC provider day. The reservation +is therefore conservative if a process dies after sending a request. A result +transaction rechecks the restriction table and the latest lease token before +writing the result and terminal event. An expired worker can finish its network +call, but its stale result is discarded when another worker has taken the lease. + +Previous result rows are append-only and remain available when a worker is +interrupted. A restarted worker can reclaim a `started` event after +`--lease-timeout`; an external request is not claimed to be exactly once. + +## Bounded private run + +Apply migrations in the disposable database, enqueue jobs through the existing +enqueue stage, then run a finite image/container with explicit values. The +worker logs provider and aggregate status only; it must not log a query, +source-record ID, address, provider response, or credential. + +```text +UEC_DATABASE_URL=postgresql://... \ +docker compose -f docker-compose.pipeline.yml run --rm \ + -e UEC_DATABASE_URL \ + worker --provider dawa --limit 100 --daily-budget 100 \ + --retries 3 --lease-timeout 900 --provider-interval 1 +``` + +The worker image is built from `Dockerfile.worker` and installs only the pinned +`pipeline/worker-requirements.txt`. The shared Compose service and build-context +exclusions are owned by the integration lane; keep synthetic queues and fake +providers in disposable test projects. Never point this command at retained +production-like volumes during testing. + +## Recovery checks + +For an operator rehearsal, pause a synthetic provider and query +`geocode_job_events` from a second database connection: the `started` row must +be visible before the call returns. Kill the worker and confirm earlier result +rows and their events remain. Start a second worker with a short lease timeout, +then confirm its terminal event has a different lease token and a late first +worker cannot append a result. Run two workers against one remaining daily +reservation and inspect the reservation ledger: at most one provider call is +allowed. Retryable outcomes create one reservation per retry. + +Rows in the reservation ledger and worker events are private operational +evidence. Reports and logs should contain aggregate counts and status only. diff --git a/pipeline/migrations/040_geocode_worker_durability.sql b/pipeline/migrations/040_geocode_worker_durability.sql new file mode 100644 index 0000000..595ef40 --- /dev/null +++ b/pipeline/migrations/040_geocode_worker_durability.sql @@ -0,0 +1,57 @@ +-- Durable worker leases and conservative provider-request accounting. +-- This migration only adds tables/columns; existing event and result history is +-- retained. A request reservation is committed before the provider call. + +ALTER TABLE uec.geocode_job_events + ADD COLUMN IF NOT EXISTS lease_token UUID; + +CREATE TABLE IF NOT EXISTS uec.geocode_provider_budgets ( + provider_id TEXT NOT NULL, + budget_date DATE NOT NULL, + daily_limit INTEGER NOT NULL CHECK (daily_limit >= 1), + reserved_requests INTEGER NOT NULL DEFAULT 0 + CHECK (reserved_requests >= 0 AND reserved_requests <= daily_limit), + last_reserved_at TIMESTAMPTZ, + PRIMARY KEY (provider_id, budget_date) +); + +-- One row represents one provider request, including a retry. It is deliberately +-- append-only so an operator can reconcile the counter with outbound attempts. +CREATE TABLE IF NOT EXISTS uec.geocode_request_reservations ( + reservation_id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + provider_id TEXT NOT NULL, + budget_date DATE NOT NULL, + job_id UUID NOT NULL REFERENCES uec.geocode_jobs(job_id), + attempt_number INTEGER NOT NULL CHECK (attempt_number >= 1), + retry_number INTEGER NOT NULL CHECK (retry_number >= 1), + reserved_at TIMESTAMPTZ NOT NULL DEFAULT now(), + UNIQUE (job_id, attempt_number, retry_number), + FOREIGN KEY (provider_id, budget_date) + REFERENCES uec.geocode_provider_budgets(provider_id, budget_date) +); + +CREATE INDEX IF NOT EXISTS geocode_request_reservations_provider_day_idx + ON uec.geocode_request_reservations (provider_id, budget_date, reserved_at); + +CREATE OR REPLACE VIEW uec.geocode_job_current AS +SELECT DISTINCT ON (job.job_id) + job.job_id, job.source_record_id, job.provider_id, job.query, + event.event_type, event.attempt_number, event.retryable, event.details, + event.worker_id, event.occurred_at, event.lease_token +FROM uec.geocode_jobs AS job +JOIN uec.geocode_job_events AS event ON event.job_id = job.job_id +ORDER BY job.job_id, event.occurred_at DESC, event.event_id DESC; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_trigger + WHERE tgname = 'geocode_request_reservations_append_only' + AND tgrelid = 'uec.geocode_request_reservations'::regclass + ) THEN + CREATE TRIGGER geocode_request_reservations_append_only + BEFORE UPDATE OR DELETE ON uec.geocode_request_reservations + FOR EACH ROW EXECUTE FUNCTION uec.reject_evidence_mutation(); + END IF; +END; +$$; diff --git a/pipeline/scripts/stages/geocode-worker.py b/pipeline/scripts/stages/geocode-worker.py index f041cdc..8767193 100644 --- a/pipeline/scripts/stages/geocode-worker.py +++ b/pipeline/scripts/stages/geocode-worker.py @@ -1,5 +1,9 @@ #!/usr/bin/env python3 -"""Run a provider-independent, append-only database geocoding worker.""" +"""Run a provider-independent, append-only database geocoding worker. + +Claims, request reservations, and outcomes are separate short transactions. +The worker never keeps a database transaction open while calling a provider. +""" import argparse import json @@ -15,17 +19,7 @@ ROOT = Path(__file__).resolve().parents[3] sys.path.insert(0, str(ROOT)) from pipeline.geocoding.registry import get_adapter - - -def _daily_started(connection, provider_id: str) -> int: - return connection.execute(""" - SELECT count(*) - FROM uec.geocode_job_events event - JOIN uec.geocode_jobs job ON job.job_id = event.job_id - WHERE job.provider_id = %s - AND event.event_type = 'started' - AND event.occurred_at >= date_trunc('day', now() AT TIME ZONE 'UTC') AT TIME ZONE 'UTC' - """, (provider_id,)).fetchone()[0] +from pipeline.geocoding.base import GeocodeOutcome def _claim_job(connection, provider_id: str, worker_id: str, max_attempts: int, lease_timeout: int): @@ -56,25 +50,173 @@ def _claim_job(connection, provider_id: str, worker_id: str, max_attempts: int, return None job_id, source_record_id, query, prior_attempt = job attempt = prior_attempt + 1 + lease_token = uuid.uuid4() connection.execute(""" INSERT INTO uec.geocode_job_events - (job_id, event_type, attempt_number, worker_id, details, occurred_at) - VALUES (%s, 'started', %s, %s, %s, %s) + (job_id, event_type, attempt_number, worker_id, lease_token, details, occurred_at) + VALUES (%s, 'started', %s, %s, %s, %s, %s) """, ( - job_id, attempt, worker_id, + job_id, attempt, worker_id, lease_token, json.dumps({"lease_timeout_seconds": lease_timeout}), datetime.now(timezone.utc), )) - return job_id, source_record_id, query, attempt + return job_id, source_record_id, query, attempt, lease_token def _is_restricted(connection, source_record_id) -> bool: + with connection.transaction(): + return connection.execute(""" + SELECT EXISTS ( + SELECT 1 FROM uec.public_access_restricted + WHERE source_record_id = %s + ) + """, (source_record_id,)).fetchone()[0] + + +def _reserve_request( + connection, + provider_id: str, + job_id, + attempt: int, + retry_number: int, + daily_budget: int, + provider_interval: float, +): + """Commit one shared provider/day reservation before outbound work. + + The budget row is locked by the UPDATE. A failed UPDATE means either the + allowance or the shared rate interval is exhausted; no provider call may + happen in that case. + """ + with connection.transaction(): + today = connection.execute("SELECT (now() AT TIME ZONE 'UTC')::date").fetchone()[0] + connection.execute(""" + INSERT INTO uec.geocode_provider_budgets + (provider_id, budget_date, daily_limit) + VALUES (%s, %s, %s) + ON CONFLICT (provider_id, budget_date) DO NOTHING + """, (provider_id, today, daily_budget)) + row = connection.execute(""" + UPDATE uec.geocode_provider_budgets + SET reserved_requests = reserved_requests + 1, + last_reserved_at = clock_timestamp() + WHERE provider_id = %s + AND budget_date = %s + AND reserved_requests < daily_limit + AND ( + %s <= 0 + OR last_reserved_at IS NULL + OR last_reserved_at <= clock_timestamp() - (%s * interval '1 second') + ) + RETURNING budget_date + """, (provider_id, today, provider_interval, provider_interval)).fetchone() + if not row: + state = connection.execute(""" + SELECT reserved_requests >= daily_limit + FROM uec.geocode_provider_budgets + WHERE provider_id = %s AND budget_date = %s + """, (provider_id, today)).fetchone() + return None, ("budget" if state and state[0] else "rate_limited") + reservation_id = uuid.uuid4() + connection.execute(""" + INSERT INTO uec.geocode_request_reservations + (reservation_id, provider_id, budget_date, job_id, + attempt_number, retry_number) + VALUES (%s, %s, %s, %s, %s, %s) + """, (reservation_id, provider_id, today, job_id, attempt, retry_number)) + return reservation_id, None + + +def _current_lease(connection, job_id): + # Claim and completion transactions serialize on the immutable job row. + # Locking only the latest event would allow a superseding claim to race + # between the lease check and result insertion. + connection.execute( + "SELECT job_id FROM uec.geocode_jobs WHERE job_id = %s FOR UPDATE", + (job_id,), + ) return connection.execute(""" - SELECT EXISTS ( - SELECT 1 FROM uec.public_access_restricted - WHERE source_record_id = %s - ) - """, (source_record_id,)).fetchone()[0] + SELECT event_type, worker_id, lease_token + FROM uec.geocode_job_events + WHERE job_id = %s + ORDER BY occurred_at DESC, event_id DESC + LIMIT 1 + FOR UPDATE + """, (job_id,)).fetchone() + + +def _finish_restricted(connection, job_id, attempt, worker_id, lease_token) -> str: + with connection.transaction(): + current = _current_lease(connection, job_id) + if not current or current[0] != "started" or current[1] != worker_id or current[2] != lease_token: + return "stale_lease" + connection.execute(""" + INSERT INTO uec.geocode_job_events + (job_id, event_type, attempt_number, retryable, worker_id, + lease_token, details, occurred_at) + VALUES (%s, 'cancelled', %s, false, %s, %s, %s, %s) + """, (job_id, attempt, worker_id, lease_token, + json.dumps({"reason": "restricted_before_provider"}), datetime.now(timezone.utc))) + return "cancelled_restricted" + + +def _persist_outcome( + connection, + job_id, + source_record_id, + provider_id: str, + query: str, + attempt: int, + worker_id: str, + lease_token, + outcome: GeocodeOutcome, +) -> str: + """Persist one result only while this worker still owns the lease.""" + with connection.transaction(): + current = _current_lease(connection, job_id) + if not current or current[0] != "started" or current[1] != worker_id or current[2] != lease_token: + return "stale_lease" + if connection.execute(""" + SELECT EXISTS ( + SELECT 1 FROM uec.public_access_restricted + WHERE source_record_id = %s + ) + """, (source_record_id,)).fetchone()[0]: + connection.execute(""" + INSERT INTO uec.geocode_job_events + (job_id, event_type, attempt_number, retryable, worker_id, + lease_token, details, occurred_at) + VALUES (%s, 'cancelled', %s, false, %s, %s, %s, %s) + """, (job_id, attempt, worker_id, lease_token, + json.dumps({"reason": "restricted_during_attempt"}), datetime.now(timezone.utc))) + return "cancelled_restricted" + queried_at = datetime.now(timezone.utc) + result_id = uuid.uuid5(uuid.NAMESPACE_URL, f"urn:uec:geocode:{job_id}:{attempt}") + connection.execute(""" + INSERT INTO uec.geocode_results + (geocode_result_id, source_record_id, provider_id, query, + provider_address_id, result, precision, match_method, status, + attempt_number, retryable, response, queried_at) + VALUES (%s, %s, %s, %s, %s, + CASE WHEN %s IS NULL OR %s IS NULL THEN NULL + ELSE ST_SetSRID(ST_MakePoint(%s, %s), 4326)::geography END, + %s, %s, %s, %s, %s, %s, %s) + ON CONFLICT (geocode_result_id) DO NOTHING + """, ( + result_id, source_record_id, provider_id, query, + outcome.provider_address_id, outcome.longitude, outcome.latitude, + outcome.longitude, outcome.latitude, outcome.precision, + outcome.match_method, outcome.status, attempt, outcome.retryable, + json.dumps(outcome.response, ensure_ascii=False), queried_at, + )) + connection.execute(""" + INSERT INTO uec.geocode_job_events + (job_id, event_type, attempt_number, retryable, worker_id, + lease_token, details, occurred_at) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s) + """, (job_id, outcome.status, attempt, outcome.retryable, worker_id, + lease_token, json.dumps({"acceptance": outcome.acceptance}, ensure_ascii=False), queried_at)) + return outcome.status def run( @@ -87,49 +229,73 @@ def run( daily_budget: int = 2800, max_attempts: int = 5, lease_timeout: int = 900, + provider_interval: float = 1.0, worker_id: str | None = None, ) -> int: - if daily_budget < 1 or max_attempts < 1 or lease_timeout < 1 or retries < 1: - raise ValueError("budgets, attempts, lease timeout, and retries must be positive") + if daily_budget < 1 or max_attempts < 1 or lease_timeout < 1 or retries < 1 or provider_interval < 0: + raise ValueError("budgets, attempts, lease timeout, retries, and provider interval must be valid") adapter = get_adapter(provider_id) worker_id = worker_id or f"geocoder-{uuid.uuid4().hex[:12]}" processed = 0 with psycopg.connect(database_url) as connection: while limit is None or processed < limit: - if _daily_started(connection, provider_id) >= daily_budget: - print(f"provider={provider_id} status=daily_budget_reached processed={processed}", flush=True) - break job = _claim_job(connection, provider_id, worker_id, max_attempts, lease_timeout) if not job: break - job_id, source_record_id, query, attempt = job + job_id, source_record_id, query, attempt, lease_token = job + if _is_restricted(connection, source_record_id): + status = _finish_restricted(connection, job_id, attempt, worker_id, lease_token) + if status != "stale_lease": + processed += 1 + print(f"provider={provider_id} status={status} processed={processed}", flush=True) + continue outcome = None - for retry in range(retries): - outcome = adapter.geocode(query) - if not outcome.retryable or retry == retries - 1: + status = "running" + budget_blocked = False + for retry_number in range(1, retries + 1): + if _is_restricted(connection, source_record_id): + outcome = None + status = _finish_restricted(connection, job_id, attempt, worker_id, lease_token) + break + reservation_id, reservation_reason = _reserve_request( + connection, provider_id, job_id, attempt, retry_number, + daily_budget, provider_interval, + ) + if reservation_id is None: + if reservation_reason == "rate_limited": + time.sleep(max(provider_interval, 0.01)) + continue + status = "daily_budget_reached" + budget_blocked = True break - time.sleep(delay * (retry + 1)) - assert outcome is not None - queried_at = datetime.now(timezone.utc) - result_id = uuid.uuid5(uuid.NAMESPACE_URL, f"urn:uec:geocode:{job_id}:{attempt}") - with connection.transaction(): + # Restrictions can be added while the reservation transaction is + # committing; recheck immediately before the external call. if _is_restricted(connection, source_record_id): - connection.execute(""" - INSERT INTO uec.geocode_job_events - (job_id, event_type, attempt_number, retryable, worker_id, details, occurred_at) - VALUES (%s, 'cancelled', %s, false, %s, %s, %s) - """, (job_id, attempt, worker_id, json.dumps({"reason": "restricted_during_attempt"}), queried_at)) - processed += 1 - print(f"provider={provider_id} status=cancelled_restricted processed={processed}", flush=True) - continue - connection.execute(""" - INSERT INTO uec.geocode_results (geocode_result_id, source_record_id, provider_id, query, provider_address_id, result, precision, match_method, status, attempt_number, retryable, response, queried_at) - VALUES (%s, %s, %s, %s, %s, ST_SetSRID(ST_MakePoint(%s, %s), 4326)::geography, %s, %s, %s, %s, %s, %s, %s) - ON CONFLICT (geocode_result_id) DO NOTHING - """, (result_id, source_record_id, provider_id, query, outcome.provider_address_id, outcome.longitude, outcome.latitude, outcome.precision, outcome.match_method, outcome.status, attempt, outcome.retryable, json.dumps(outcome.response, ensure_ascii=False), queried_at)) - connection.execute("INSERT INTO uec.geocode_job_events (job_id, event_type, attempt_number, retryable, worker_id, details, occurred_at) VALUES (%s, %s, %s, %s, %s, %s, %s)", (job_id, outcome.status, attempt, outcome.retryable, worker_id, json.dumps({"acceptance": outcome.acceptance}, ensure_ascii=False), queried_at)) + status = _finish_restricted(connection, job_id, attempt, worker_id, lease_token) + break + try: + outcome = adapter.geocode(query) + except Exception as error: # provider details remain redacted + outcome = GeocodeOutcome( + "failed", "provider_exception", None, None, None, None, + "provider_exception", True, {"error": type(error).__name__}, + ) + if not outcome.retryable or retry_number == retries: + break + time.sleep(max(delay * retry_number, provider_interval)) + if budget_blocked: + print(f"provider={provider_id} status={status} processed={processed}", flush=True) + break + if outcome is not None and status not in ("stale_lease", "cancelled_restricted"): + status = _persist_outcome( + connection, job_id, source_record_id, provider_id, query, + attempt, worker_id, lease_token, outcome, + ) + if status == "stale_lease": + print(f"provider={provider_id} status=stale_lease processed={processed}", flush=True) + continue processed += 1 - print(f"provider={provider_id} status={outcome.status} processed={processed}", flush=True) + print(f"provider={provider_id} status={status} processed={processed}", flush=True) if limit is None or processed < limit: time.sleep(delay) return processed @@ -145,11 +311,13 @@ def run( parser.add_argument("--daily-budget", type=int, default=2800) parser.add_argument("--max-attempts", type=int, default=5) parser.add_argument("--lease-timeout", type=int, default=900, help="Seconds before an abandoned started event can be reclaimed") + parser.add_argument("--provider-interval", type=float, default=1.0, help="Minimum shared seconds between provider reservations") parser.add_argument("--worker-id", default=os.environ.get("UEC_GEOCODE_WORKER_ID")) args = parser.parse_args() count = run( args.database_url, args.provider, args.limit, args.delay, args.retries, daily_budget=args.daily_budget, max_attempts=args.max_attempts, - lease_timeout=args.lease_timeout, worker_id=args.worker_id, + lease_timeout=args.lease_timeout, provider_interval=args.provider_interval, + worker_id=args.worker_id, ) print(f"provider={args.provider} status=complete processed={count}") diff --git a/pipeline/tests/test_geocode_worker_durability.py b/pipeline/tests/test_geocode_worker_durability.py new file mode 100644 index 0000000..c8c1ae4 --- /dev/null +++ b/pipeline/tests/test_geocode_worker_durability.py @@ -0,0 +1,95 @@ +import importlib.util +import unittest +import uuid +from pathlib import Path +from unittest.mock import Mock, patch + +from pipeline.geocoding.base import GeocodeOutcome + + +ROOT = Path(__file__).parents[1] +SPEC = importlib.util.spec_from_file_location( + "geocode_worker_durability", ROOT / "scripts/stages/geocode-worker.py" +) +WORKER = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(WORKER) + + +class GeocodeWorkerDurabilityTests(unittest.TestCase): + def test_migration_adds_lease_fence_and_atomic_request_ledger(self): + migration = (ROOT / "migrations/040_geocode_worker_durability.sql").read_text() + for required in ( + "lease_token UUID", + "geocode_provider_budgets", + "reserved_requests", + "geocode_request_reservations", + "geocode_request_reservations_append_only", + ): + self.assertIn(required, migration) + + def test_retryable_provider_attempts_each_consume_a_reservation(self): + connection = Mock() + connection.__enter__ = Mock(return_value=connection) + connection.__exit__ = Mock(return_value=False) + job = (uuid.uuid4(), uuid.uuid4(), "private synthetic query", 1, uuid.uuid4()) + reservations = [(uuid.uuid4(), None), (uuid.uuid4(), None)] + adapter = Mock() + adapter.geocode.side_effect = [ + GeocodeOutcome("failed", "temporary", None, None, None, None, "fixture", True, {"error": "temporary"}), + GeocodeOutcome("unresolved", "unresolved", None, None, None, None, "fixture", False, {}), + ] + with patch.object(WORKER.psycopg, "connect", return_value=connection), \ + patch.object(WORKER, "get_adapter", return_value=adapter), \ + patch.object(WORKER, "_claim_job", side_effect=[job, None]), \ + patch.object(WORKER, "_is_restricted", return_value=False), \ + patch.object(WORKER, "_reserve_request", side_effect=reservations) as reserve, \ + patch.object(WORKER, "_persist_outcome", return_value="unresolved") as persist: + processed = WORKER.run( + "postgresql://synthetic", "fixture", 1, 0, 2, + daily_budget=2, provider_interval=0, worker_id="synthetic-worker", + ) + self.assertEqual(processed, 1) + self.assertEqual(adapter.geocode.call_count, 2) + self.assertEqual(reserve.call_count, 2) + persist.assert_called_once() + + def test_stale_result_is_counted_without_persisting_payload(self): + connection = Mock() + connection.__enter__ = Mock(return_value=connection) + connection.__exit__ = Mock(return_value=False) + job = (uuid.uuid4(), uuid.uuid4(), "private synthetic query", 1, uuid.uuid4()) + adapter = Mock() + adapter.geocode.return_value = GeocodeOutcome( + "accepted", "accepted_single_point", 55.0, 12.0, "fixture", "point", "fixture", False, {} + ) + with patch.object(WORKER.psycopg, "connect", return_value=connection), \ + patch.object(WORKER, "get_adapter", return_value=adapter), \ + patch.object(WORKER, "_claim_job", side_effect=[job, None]), \ + patch.object(WORKER, "_is_restricted", return_value=False), \ + patch.object(WORKER, "_reserve_request", return_value=(uuid.uuid4(), None)), \ + patch.object(WORKER, "_persist_outcome", return_value="stale_lease") as persist: + processed = WORKER.run( + "postgresql://synthetic", "fixture", 1, 0, 1, + provider_interval=0, worker_id="synthetic-worker", + ) + self.assertEqual(processed, 0) + self.assertEqual(adapter.geocode.call_count, 1) + persist.assert_called_once() + + def test_worker_output_and_provider_errors_are_redacted(self): + source = (ROOT / "scripts/stages/geocode-worker.py").read_text() + self.assertNotIn("query={", source) + self.assertNotIn("source_record_id=", source) + self.assertIn("type(error).__name__", source) + self.assertNotIn("print(query", source) + + def test_worker_image_is_separate_and_pinned(self): + dockerfile = (ROOT.parent / "Dockerfile.worker").read_text() + requirements = (ROOT.parent / "pipeline/worker-requirements.txt").read_text() + self.assertIn("FROM python:3.12.8-slim-bookworm", dockerfile) + self.assertIn("COPY pipeline /app/pipeline", dockerfile) + self.assertIn("psycopg[binary]==3.2.9", requirements) + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/tests/test_graph_migrations.py b/pipeline/tests/test_graph_migrations.py index a0e1c48..a04fbd9 100644 --- a/pipeline/tests/test_graph_migrations.py +++ b/pipeline/tests/test_graph_migrations.py @@ -20,7 +20,7 @@ def test_reserved_migrations_are_present_and_ordered(self): positions = [migrations.index(name) for name in graph_migrations] self.assertEqual(positions, sorted(positions)) self.assertEqual([migrations[position] for position in positions], graph_migrations) - self.assertEqual(migrations[-14:], [ + self.assertEqual(migrations[-15:], [ "026_graph_entities_crosswalks.sql", "027_graph_relationship_observations.sql", "028_graph_claims_support.sql", @@ -35,6 +35,7 @@ def test_reserved_migrations_are_present_and_ordered(self): "037_public_discovery_read_model.sql", "038_graph_regulatory_authority_relationship.sql", "039_private_graph_query_indexes.sql", + "040_geocode_worker_durability.sql", ]) def test_entities_are_distinct_and_crosswalk_is_scoped(self): diff --git a/pipeline/worker-requirements.txt b/pipeline/worker-requirements.txt new file mode 100644 index 0000000..f10b2ed --- /dev/null +++ b/pipeline/worker-requirements.txt @@ -0,0 +1,2 @@ +# Keep the worker dependency surface separate from source-adapter tooling. +psycopg[binary]==3.2.9 From 7738e4c4d74cdf8808df862d02faf65a4e073e9c Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Fri, 18 Sep 2026 19:30:24 -0700 Subject: [PATCH 266/311] Add disposable worker lifecycle acceptance tests --- pipeline/tests/e2e/test_worker_durability.py | 192 +++++++++++++++++++ 1 file changed, 192 insertions(+) create mode 100644 pipeline/tests/e2e/test_worker_durability.py diff --git a/pipeline/tests/e2e/test_worker_durability.py b/pipeline/tests/e2e/test_worker_durability.py new file mode 100644 index 0000000..88da326 --- /dev/null +++ b/pipeline/tests/e2e/test_worker_durability.py @@ -0,0 +1,192 @@ +"""Disposable database lifecycle tests for the durable geocoding worker.""" + +import importlib.util +import os +import threading +import time +import unittest +import uuid +from datetime import datetime, timezone +from pathlib import Path +from unittest.mock import patch + +import psycopg +from pipeline.geocoding.base import GeocodeOutcome + +try: + from .fixture import E2EEnvironment +except ImportError: + from fixture import E2EEnvironment + + +ROOT = Path(__file__).parents[2] +SPEC = importlib.util.spec_from_file_location("durable_geocode_worker", ROOT / "scripts/stages/geocode-worker.py") +WORKER = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(WORKER) + + +class WorkerDurabilityE2ETests(unittest.TestCase): + @classmethod + def setUpClass(cls): + if os.environ.get("UEC_RUN_E2E") != "1": + raise unittest.SkipTest("set UEC_RUN_E2E=1 to run Docker-backed worker lifecycle tests") + cls.env = E2EEnvironment().start(wait_for_ready=False) + cls.source_id = "e2e.worker.durability" + with psycopg.connect(cls.env.database_url) as db: + with db.transaction(): + db.execute( + "INSERT INTO uec.sources (source_id,country_code,name,official_url,access_method) " + "VALUES (%s,'US','Synthetic worker source','https://example.invalid/worker','fixture') " + "ON CONFLICT (source_id) DO NOTHING", + (cls.source_id,), + ) + + @classmethod + def tearDownClass(cls): + cls.env.stop() + + def _queue(self, provider="synthetic-worker"): + now = datetime.now(timezone.utc) + record_id, job_id, artifact_id = uuid.uuid4(), uuid.uuid4(), uuid.uuid4() + with psycopg.connect(self.env.database_url) as db: + with db.transaction(): + db.execute( + "INSERT INTO uec.raw_artifacts (artifact_id,storage_key,sha256,byte_size,retrieved_at) " + "VALUES (%s,%s,%s,1,%s)", + (artifact_id, f"e2e/worker/{artifact_id}", uuid.uuid4().hex * 2, now), + ) + db.execute( + "INSERT INTO uec.source_records " + "(source_record_id,source_id,source_record_key,artifact_id,raw_fields,parsed_at) " + "VALUES (%s,%s,%s,%s,'{}',%s)", + (record_id, self.source_id, str(record_id), artifact_id, now), + ) + db.execute( + "INSERT INTO uec.geocode_jobs (job_id,source_record_id,provider_id,query) " + "VALUES (%s,%s,%s,'synthetic query')", + (job_id, record_id, provider), + ) + db.execute( + "INSERT INTO uec.geocode_job_events (job_id,event_type,attempt_number) " + "VALUES (%s,'queued',1)", + (job_id,), + ) + return job_id, record_id + + def test_claim_is_visible_before_paused_provider_returns(self): + job_id, _ = self._queue() + provider_started = threading.Event() + release_provider = threading.Event() + + class PausingAdapter: + def geocode(self, _query): + provider_started.set() + if not release_provider.wait(10): + raise AssertionError("synthetic provider was not released") + return GeocodeOutcome("unresolved", "fixture", None, None, None, None, "fixture", False, {}) + + adapter = PausingAdapter() + with patch.object(WORKER, "get_adapter", return_value=adapter): + thread = threading.Thread( + target=WORKER.run, + args=(self.env.database_url, "synthetic-worker", 1, 0, 1), + kwargs={"daily_budget": 5, "provider_interval": 0, "worker_id": "paused-worker"}, + ) + thread.start() + self.assertTrue(provider_started.wait(10)) + with psycopg.connect(self.env.database_url) as db: + event = db.execute( + "SELECT event_type,worker_id FROM uec.geocode_job_current WHERE job_id=%s", + (job_id,), + ).fetchone() + self.assertEqual(event, ("started", "paused-worker")) + release_provider.set() + thread.join(10) + self.assertFalse(thread.is_alive()) + + def test_reclaimed_lease_fences_late_first_worker_result(self): + job_id, record_id = self._queue(provider="synthetic-fence") + first_started = threading.Event() + release_first = threading.Event() + + class FencingAdapter: + calls = 0 + + def geocode(self, _query): + self.calls += 1 + if self.calls == 1: + first_started.set() + if not release_first.wait(10): + raise AssertionError("synthetic first provider call was not released") + return GeocodeOutcome("accepted", "fixture", 55.0, 12.0, "fixture", "point", "fixture", False, {}) + + adapter = FencingAdapter() + with patch.object(WORKER, "get_adapter", return_value=adapter): + first = threading.Thread( + target=WORKER.run, + args=(self.env.database_url, "synthetic-fence", 1, 0, 1), + kwargs={"daily_budget": 5, "provider_interval": 0, "lease_timeout": 1, "worker_id": "first-worker"}, + ) + first.start() + self.assertTrue(first_started.wait(10)) + time.sleep(1.2) + second = threading.Thread( + target=WORKER.run, + args=(self.env.database_url, "synthetic-fence", 1, 0, 1), + kwargs={"daily_budget": 5, "provider_interval": 0, "lease_timeout": 1, "worker_id": "second-worker"}, + ) + second.start() + second.join(10) + self.assertFalse(second.is_alive()) + release_first.set() + first.join(10) + self.assertFalse(first.is_alive()) + with psycopg.connect(self.env.database_url) as db: + result_count = db.execute( + "SELECT count(*) FROM uec.geocode_results WHERE source_record_id=%s AND provider_id='synthetic-fence'", + (record_id,), + ).fetchone()[0] + events = db.execute( + "SELECT event_type,worker_id FROM uec.geocode_job_events WHERE job_id=%s ORDER BY occurred_at,event_id", + (job_id,), + ).fetchall() + self.assertEqual(result_count, 1) + self.assertEqual(events[-1], ("accepted", "second-worker")) + + def test_two_workers_share_one_remaining_request_allowance(self): + self._queue(provider="synthetic-budget") + self._queue(provider="synthetic-budget") + calls = 0 + calls_lock = threading.Lock() + + class CountingAdapter: + def geocode(self, _query): + nonlocal calls + with calls_lock: + calls += 1 + return GeocodeOutcome("unresolved", "fixture", None, None, None, None, "fixture", False, {}) + + def run_worker(worker_id): + WORKER.run( + self.env.database_url, "synthetic-budget", 1, 0, 1, + daily_budget=1, provider_interval=0, worker_id=worker_id, + ) + + with patch.object(WORKER, "get_adapter", return_value=CountingAdapter()): + workers = [threading.Thread(target=run_worker, args=(f"budget-{n}",)) for n in (1, 2)] + for worker in workers: + worker.start() + for worker in workers: + worker.join(10) + self.assertFalse(worker.is_alive()) + with psycopg.connect(self.env.database_url) as db: + reservations = db.execute( + "SELECT count(*) FROM uec.geocode_request_reservations " + "WHERE provider_id='synthetic-budget' AND budget_date=(now() AT TIME ZONE 'UTC')::date" + ).fetchone()[0] + self.assertEqual(calls, 1) + self.assertEqual(reservations, 1) + + +if __name__ == "__main__": + unittest.main() From 415cf52d870200c42797e2e0702cd499b2580691 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Fri, 18 Sep 2026 19:44:58 -0700 Subject: [PATCH 267/311] Defer rate-limited worker leases durably --- .../040_geocode_worker_durability.sql | 5 +- pipeline/scripts/stages/geocode-worker.py | 45 ++++++++++++--- pipeline/tests/e2e/test_worker_durability.py | 55 ++++++++++++------- .../tests/test_geocode_worker_durability.py | 24 ++++++++ 4 files changed, 100 insertions(+), 29 deletions(-) diff --git a/pipeline/migrations/040_geocode_worker_durability.sql b/pipeline/migrations/040_geocode_worker_durability.sql index 595ef40..759a57f 100644 --- a/pipeline/migrations/040_geocode_worker_durability.sql +++ b/pipeline/migrations/040_geocode_worker_durability.sql @@ -3,7 +3,8 @@ -- retained. A request reservation is committed before the provider call. ALTER TABLE uec.geocode_job_events - ADD COLUMN IF NOT EXISTS lease_token UUID; + ADD COLUMN IF NOT EXISTS lease_token UUID, + ADD COLUMN IF NOT EXISTS next_attempt_at TIMESTAMPTZ; CREATE TABLE IF NOT EXISTS uec.geocode_provider_budgets ( provider_id TEXT NOT NULL, @@ -37,7 +38,7 @@ CREATE OR REPLACE VIEW uec.geocode_job_current AS SELECT DISTINCT ON (job.job_id) job.job_id, job.source_record_id, job.provider_id, job.query, event.event_type, event.attempt_number, event.retryable, event.details, - event.worker_id, event.occurred_at, event.lease_token + event.worker_id, event.occurred_at, event.lease_token, event.next_attempt_at FROM uec.geocode_jobs AS job JOIN uec.geocode_job_events AS event ON event.job_id = job.job_id ORDER BY job.job_id, event.occurred_at DESC, event.event_id DESC; diff --git a/pipeline/scripts/stages/geocode-worker.py b/pipeline/scripts/stages/geocode-worker.py index 8767193..dbfad9e 100644 --- a/pipeline/scripts/stages/geocode-worker.py +++ b/pipeline/scripts/stages/geocode-worker.py @@ -34,7 +34,8 @@ def _claim_job(connection, provider_id: str, worker_id: str, max_attempts: int, AND ( current.event_type IS NULL OR current.event_type = 'queued' - OR (current.event_type = 'failed' AND current.retryable) + OR (current.event_type = 'failed' AND current.retryable + AND (current.next_attempt_at IS NULL OR current.next_attempt_at <= now())) OR (current.event_type = 'started' AND current.occurred_at < now() - (%s * interval '1 second')) ) @@ -160,6 +161,26 @@ def _finish_restricted(connection, job_id, attempt, worker_id, lease_token) -> s return "cancelled_restricted" +def _defer_job(connection, job_id, attempt, worker_id, lease_token, reason: str, delay_seconds: float) -> str: + """Record a retryable deferred terminal event without making a provider call.""" + delay_seconds = max(delay_seconds, 0.1) + with connection.transaction(): + current = _current_lease(connection, job_id) + if not current or current[0] != "started" or current[1] != worker_id or current[2] != lease_token: + return "stale_lease" + connection.execute(""" + INSERT INTO uec.geocode_job_events + (job_id, event_type, attempt_number, retryable, worker_id, + lease_token, details, next_attempt_at, occurred_at) + VALUES (%s, 'failed', %s, true, %s, %s, %s, + now() + (%s * interval '1 second'), %s) + """, ( + job_id, attempt, worker_id, lease_token, + json.dumps({"reason": reason}), delay_seconds, datetime.now(timezone.utc), + )) + return "deferred" + + def _persist_outcome( connection, job_id, @@ -198,8 +219,8 @@ def _persist_outcome( provider_address_id, result, precision, match_method, status, attempt_number, retryable, response, queried_at) VALUES (%s, %s, %s, %s, %s, - CASE WHEN %s IS NULL OR %s IS NULL THEN NULL - ELSE ST_SetSRID(ST_MakePoint(%s, %s), 4326)::geography END, + CASE WHEN %s::double precision IS NULL OR %s::double precision IS NULL THEN NULL + ELSE ST_SetSRID(ST_MakePoint(%s::double precision, %s::double precision), 4326)::geography END, %s, %s, %s, %s, %s, %s, %s) ON CONFLICT (geocode_result_id) DO NOTHING """, ( @@ -252,7 +273,8 @@ def run( outcome = None status = "running" budget_blocked = False - for retry_number in range(1, retries + 1): + retry_number = 1 + while retry_number <= retries: if _is_restricted(connection, source_record_id): outcome = None status = _finish_restricted(connection, job_id, attempt, worker_id, lease_token) @@ -263,10 +285,16 @@ def run( ) if reservation_id is None: if reservation_reason == "rate_limited": - time.sleep(max(provider_interval, 0.01)) - continue - status = "daily_budget_reached" - budget_blocked = True + status = _defer_job( + connection, job_id, attempt, worker_id, lease_token, + "provider_rate_limited", max(provider_interval, 0.1), + ) + else: + status = _defer_job( + connection, job_id, attempt, worker_id, lease_token, + "request_budget_exhausted", 86400, + ) + budget_blocked = True break # Restrictions can be added while the reservation transaction is # committing; recheck immediately before the external call. @@ -283,6 +311,7 @@ def run( if not outcome.retryable or retry_number == retries: break time.sleep(max(delay * retry_number, provider_interval)) + retry_number += 1 if budget_blocked: print(f"provider={provider_id} status={status} processed={processed}", flush=True) break diff --git a/pipeline/tests/e2e/test_worker_durability.py b/pipeline/tests/e2e/test_worker_durability.py index 88da326..c2e10c4 100644 --- a/pipeline/tests/e2e/test_worker_durability.py +++ b/pipeline/tests/e2e/test_worker_durability.py @@ -86,12 +86,19 @@ def geocode(self, _query): return GeocodeOutcome("unresolved", "fixture", None, None, None, None, "fixture", False, {}) adapter = PausingAdapter() + errors = [] + + def run_worker(): + try: + WORKER.run( + self.env.database_url, "synthetic-worker", 1, 0, 1, + daily_budget=5, provider_interval=0, worker_id="paused-worker", + ) + except BaseException as error: # surface thread failures to unittest + errors.append(error) + with patch.object(WORKER, "get_adapter", return_value=adapter): - thread = threading.Thread( - target=WORKER.run, - args=(self.env.database_url, "synthetic-worker", 1, 0, 1), - kwargs={"daily_budget": 5, "provider_interval": 0, "worker_id": "paused-worker"}, - ) + thread = threading.Thread(target=run_worker) thread.start() self.assertTrue(provider_started.wait(10)) with psycopg.connect(self.env.database_url) as db: @@ -103,6 +110,7 @@ def geocode(self, _query): release_provider.set() thread.join(10) self.assertFalse(thread.is_alive()) + self.assertEqual(errors, []) def test_reclaimed_lease_fences_late_first_worker_result(self): job_id, record_id = self._queue(provider="synthetic-fence") @@ -121,26 +129,30 @@ def geocode(self, _query): return GeocodeOutcome("accepted", "fixture", 55.0, 12.0, "fixture", "point", "fixture", False, {}) adapter = FencingAdapter() + errors = [] + + def run_worker(worker_id): + try: + WORKER.run( + self.env.database_url, "synthetic-fence", 1, 0, 1, + daily_budget=5, provider_interval=0, lease_timeout=1, worker_id=worker_id, + ) + except BaseException as error: + errors.append((worker_id, error)) + with patch.object(WORKER, "get_adapter", return_value=adapter): - first = threading.Thread( - target=WORKER.run, - args=(self.env.database_url, "synthetic-fence", 1, 0, 1), - kwargs={"daily_budget": 5, "provider_interval": 0, "lease_timeout": 1, "worker_id": "first-worker"}, - ) + first = threading.Thread(target=run_worker, args=("first-worker",)) first.start() self.assertTrue(first_started.wait(10)) time.sleep(1.2) - second = threading.Thread( - target=WORKER.run, - args=(self.env.database_url, "synthetic-fence", 1, 0, 1), - kwargs={"daily_budget": 5, "provider_interval": 0, "lease_timeout": 1, "worker_id": "second-worker"}, - ) + second = threading.Thread(target=run_worker, args=("second-worker",)) second.start() second.join(10) self.assertFalse(second.is_alive()) release_first.set() first.join(10) self.assertFalse(first.is_alive()) + self.assertEqual(errors, []) with psycopg.connect(self.env.database_url) as db: result_count = db.execute( "SELECT count(*) FROM uec.geocode_results WHERE source_record_id=%s AND provider_id='synthetic-fence'", @@ -167,11 +179,15 @@ def geocode(self, _query): return GeocodeOutcome("unresolved", "fixture", None, None, None, None, "fixture", False, {}) def run_worker(worker_id): - WORKER.run( - self.env.database_url, "synthetic-budget", 1, 0, 1, - daily_budget=1, provider_interval=0, worker_id=worker_id, - ) + try: + WORKER.run( + self.env.database_url, "synthetic-budget", 1, 0, 1, + daily_budget=1, provider_interval=0, worker_id=worker_id, + ) + except BaseException as error: + errors.append((worker_id, error)) + errors = [] with patch.object(WORKER, "get_adapter", return_value=CountingAdapter()): workers = [threading.Thread(target=run_worker, args=(f"budget-{n}",)) for n in (1, 2)] for worker in workers: @@ -179,6 +195,7 @@ def run_worker(worker_id): for worker in workers: worker.join(10) self.assertFalse(worker.is_alive()) + self.assertEqual(errors, []) with psycopg.connect(self.env.database_url) as db: reservations = db.execute( "SELECT count(*) FROM uec.geocode_request_reservations " diff --git a/pipeline/tests/test_geocode_worker_durability.py b/pipeline/tests/test_geocode_worker_durability.py index c8c1ae4..0be8f1b 100644 --- a/pipeline/tests/test_geocode_worker_durability.py +++ b/pipeline/tests/test_geocode_worker_durability.py @@ -20,6 +20,7 @@ def test_migration_adds_lease_fence_and_atomic_request_ledger(self): migration = (ROOT / "migrations/040_geocode_worker_durability.sql").read_text() for required in ( "lease_token UUID", + "next_attempt_at TIMESTAMPTZ", "geocode_provider_budgets", "reserved_requests", "geocode_request_reservations", @@ -76,6 +77,29 @@ def test_stale_result_is_counted_without_persisting_payload(self): self.assertEqual(adapter.geocode.call_count, 1) persist.assert_called_once() + def test_rate_contention_is_deferred_without_provider_retry_or_busy_wait(self): + connection = Mock() + connection.__enter__ = Mock(return_value=connection) + connection.__exit__ = Mock(return_value=False) + job = (uuid.uuid4(), uuid.uuid4(), "private synthetic query", 1, uuid.uuid4()) + adapter = Mock() + with patch.object(WORKER.psycopg, "connect", return_value=connection), \ + patch.object(WORKER, "get_adapter", return_value=adapter), \ + patch.object(WORKER, "_claim_job", side_effect=[job, None]), \ + patch.object(WORKER, "_is_restricted", return_value=False), \ + patch.object(WORKER, "_reserve_request", return_value=(None, "rate_limited")) as reserve, \ + patch.object(WORKER, "_defer_job", return_value="deferred") as defer: + processed = WORKER.run( + "postgresql://synthetic", "fixture", 1, 0, 3, + daily_budget=2, provider_interval=2, worker_id="synthetic-worker", + ) + self.assertEqual(processed, 1) + adapter.geocode.assert_not_called() + reserve.assert_called_once() + self.assertEqual(reserve.call_args.args[4], 1) + defer.assert_called_once() + self.assertEqual(defer.call_args.args[5], "provider_rate_limited") + def test_worker_output_and_provider_errors_are_redacted(self): source = (ROOT / "scripts/stages/geocode-worker.py").read_text() self.assertNotIn("query={", source) From 20aaf26734d46e478659d4f15438654d79397b92 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Fri, 18 Sep 2026 19:45:25 -0700 Subject: [PATCH 268/311] Document deferred worker outcomes --- docs/geocoding-worker.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/geocoding-worker.md b/docs/geocoding-worker.md index c31af1c..84500cf 100644 --- a/docs/geocoding-worker.md +++ b/docs/geocoding-worker.md @@ -20,6 +20,13 @@ Previous result rows are append-only and remain available when a worker is interrupted. A restarted worker can reclaim a `started` event after `--lease-timeout`; an external request is not claimed to be exactly once. +If the shared provider interval is occupied, the worker does not spin or spend +one of the provider retry slots. It appends a retryable `failed` event with a +`provider_rate_limited` reason and a future `next_attempt_at`, then moves on. +Daily allowance exhaustion is recorded the same way with a next-day retry time. +The latest event is therefore always an inspectable deferred outcome rather +than an unbounded `started` lease. + ## Bounded private run Apply migrations in the disposable database, enqueue jobs through the existing From 313fc779c2258d35875e3846a2245908668fea94 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Fri, 18 Sep 2026 19:26:09 -0700 Subject: [PATCH 269/311] Add private APHIS investigation packet --- .../us/aphis-investigation-packet.md | 60 +++ .../us/accountability/aphis_evidence.py | 409 ++++++++++++++++++ .../us/accountability/test_aphis_evidence.py | 109 +++++ 3 files changed, 578 insertions(+) create mode 100644 docs/countries/us/aphis-investigation-packet.md create mode 100644 pipeline/sources/us/accountability/aphis_evidence.py create mode 100644 pipeline/sources/us/accountability/test_aphis_evidence.py diff --git a/docs/countries/us/aphis-investigation-packet.md b/docs/countries/us/aphis-investigation-packet.md new file mode 100644 index 0000000..d0239a8 --- /dev/null +++ b/docs/countries/us/aphis-investigation-packet.md @@ -0,0 +1,60 @@ +# Private APHIS investigation packet + +This packet is a private, bounded evidence product over the existing APHIS +source-local adapter and Wave 2 identity candidate ledger. It is intended for +an investigator who needs to answer which captured annual-report and +inspection observations attach to a registration, which periods were covered, +and why a link or document remains unresolved. + +The builder is `pipeline.sources.us.accountability.aphis_evidence`. It accepts +an explicit manifest and named artifact root; it never scans for files or +performs upstream acquisition. Each manifest-named artifact must carry a +SHA-256 and byte size. A missing artifact, invalid hash, or size mismatch is a +visible `failed` input. A replay keeps the source retrieval timestamp from its +manifest and is not a fresh source observation. + +Example private run after an authorized handoff: + +```powershell +python -m pipeline.sources.us.accountability.aphis_evidence ` + --manifest C:\restricted\aphis\input-manifest.json ` + --artifact-root C:\restricted\aphis\artifacts ` + --run-dir C:\restricted\aphis\runs\investigation- +``` + +The packet writes `private/` JSONL files for authorized review and a separate +`row-free-summary.json`. The row-free report contains counts, periods, +artifact verification outcomes, link exclusion reasons, coverage boundaries, +and unknowns. It never contains names, addresses, raw rows, document URLs, or +signed URLs. The companion editorial note records what the evidence makes +visible, why it matters to activists, what it cannot establish, and useful +next research questions. + +Each timeline item has one of these states: + +* `observed`: an accepted source-local record with its source key, period and + profile provenance; +* `not_observed`: an expected row or period was outside the captured input; +* `quarantined`: duplicate, suppressed, conflicting, or otherwise unresolved + evidence remains visible for review; +* `failed`: an input artifact or processing step could not be verified; and +* `document_not_captured`: a source record explicitly references a document + key whose retained document is unavailable. + +Annual report years are represented as `start`, `end`, and +`precision: "year"`. The builder does not turn a year into a claimed event +date. Inspection dates retain day precision when supplied. Document links +require an explicit source document key; a URL alone is never copied into a +packet. + +Links are copied from the existing source-native candidate ledger. Exact APHIS +certificate/customer identifier matches retain matched identifier types and +review state. Conflicts and duplicate evidence remain quarantined. Names and +addresses are not identity evidence, and no candidate establishes ownership, +current operation, approval, wrongdoing, or a complete animal-use total. + +All output remains private, `not_eligible`, and `release_state: not-created`. +Raw and parsed inputs remain outside Git under the retention and removal rules +in `docs/ETHICS.md`. A synthetic test pass demonstrates engineering behavior +only; the real retained-input rehearsal requires a permitted artifact handoff +and a second deterministic replay with aggregate count reconciliation. diff --git a/pipeline/sources/us/accountability/aphis_evidence.py b/pipeline/sources/us/accountability/aphis_evidence.py new file mode 100644 index 0000000..79d36fe --- /dev/null +++ b/pipeline/sources/us/accountability/aphis_evidence.py @@ -0,0 +1,409 @@ +"""Build a private, profile-aware APHIS investigation packet. + +The Wave 2 runner produces source-local rows and identity candidates. This +module adds the operator-facing evidence boundary on top of those artifacts: +it verifies explicitly supplied retained files, keeps periods (including +year-only annual reports) precise, and writes a row-bearing private packet +alongside a row-free report. It intentionally does not acquire data, +geocode, publish, or infer ownership/current operation. +""" +from __future__ import annotations + +import argparse +import hashlib +import json +import re +from collections import Counter, defaultdict +from datetime import date +from pathlib import Path +from typing import Any, Iterable, Mapping + +from pipeline.contracts.source_lifecycle import atomic_json, atomic_jsonl, read_jsonl + + +PACKET_VERSION = "us-aphis-investigation-packet-v1" +PROFILES = ("registrations", "annual_reports", "inspections") +STATES = frozenset({"observed", "not_observed", "quarantined", "failed", "document_not_captured"}) +_SHA256 = re.compile(r"^[0-9a-f]{64}$", re.IGNORECASE) + + +class AphisEvidenceError(ValueError): + """The retained-input or packet contract cannot be satisfied.""" + + +def _text(value: Any) -> str | None: + if value is None: + return None + value = str(value).strip() + return value or None + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _json(path: Path) -> Mapping[str, Any]: + try: + value = json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc: + raise AphisEvidenceError(f"cannot read manifest {path}: {exc}") from exc + if not isinstance(value, Mapping): + raise AphisEvidenceError(f"manifest {path} must be a JSON object") + return value + + +def _artifact_entries(manifest: Mapping[str, Any]) -> list[dict[str, Any]]: + """Normalize Wave 1/Wave 2 artifact metadata without reading rows.""" + profiles = manifest.get("profiles") + entries: list[dict[str, Any]] = [] + if isinstance(profiles, Mapping): + for profile, value in profiles.items(): + if not isinstance(value, Mapping): + continue + artifacts = value.get("artifacts") + if isinstance(artifacts, list): + for item in artifacts: + if isinstance(item, Mapping): + entry = dict(item) + entry.setdefault("profile", profile) + entries.append(entry) + retrieval = manifest.get("retrieval") + # The tracked Wave 2 aggregate has no paths, so retain its profile-level + # metadata as explicit unavailable entries instead of guessing locations. + if isinstance(retrieval, Mapping) and not entries: + for profile, value in retrieval.items(): + if isinstance(value, Mapping): + entry = dict(value) + entry["profile"] = profile + entry["artifact"] = entry.get("artifact") or None + entries.append(entry) + return entries + + +def verify_retained_artifacts( + manifest_path: str | Path, + *, + artifact_root: str | Path | None = None, +) -> dict[str, Any]: + """Resolve only manifest-named artifacts and verify bytes/hash/size. + + A missing path or a bad digest is a visible ``failed`` input. The caller + may still build a bounded packet from other profiles, but cannot label the + failed profile observed. No directory scan or credential lookup occurs. + """ + path = Path(manifest_path).resolve() + manifest = _json(path) + root = Path(artifact_root).resolve() if artifact_root else path.parent + checked: list[dict[str, Any]] = [] + for item in _artifact_entries(manifest): + profile = _text(item.get("profile")) or "unknown" + artifact_name = _text(item.get("artifact") or item.get("name")) + declared_path = _text(item.get("path_private") or item.get("artifact_path") or item.get("path")) + candidate = Path(declared_path) if declared_path else (root / artifact_name if artifact_name else None) + result = { + "profile": profile, + "artifact": artifact_name, + "declared_sha256": _text(item.get("sha256")), + "declared_byte_size": item.get("byte_size") if item.get("byte_size") is not None else item.get("bytes"), + "path": str(candidate.resolve()) if candidate else None, + "state": "failed", + "failure": None, + } + if candidate is None: + result["failure"] = "artifact_path_not_declared" + elif not candidate.is_file(): + result["failure"] = "artifact_missing" + else: + actual_size = candidate.stat().st_size + actual_hash = _sha256(candidate) + result.update({"actual_sha256": actual_hash, "actual_byte_size": actual_size}) + expected_hash = result["declared_sha256"] + expected_size = result["declared_byte_size"] + if not expected_hash or not _SHA256.fullmatch(str(expected_hash)): + result["failure"] = "artifact_hash_not_declared_or_invalid" + elif actual_hash.lower() != str(expected_hash).lower(): + result["failure"] = "artifact_hash_mismatch" + elif expected_size is not None and int(expected_size) != actual_size: + result["failure"] = "artifact_size_mismatch" + else: + result["state"] = "verified" + checked.append(result) + by_profile: dict[str, dict[str, int]] = {} + for item in checked: + stats = by_profile.setdefault(item["profile"], {"verified": 0, "failed": 0}) + stats["verified" if item["state"] == "verified" else "failed"] += 1 + return { + "manifest_path": str(path), + "manifest_sha256": _sha256(path), + "artifacts": checked, + "by_profile": by_profile, + "all_verified": bool(checked) and all(item["state"] == "verified" for item in checked), + } + + +def _normalized(record: Mapping[str, Any]) -> Mapping[str, Any]: + value = record.get("normalized") + return value if isinstance(value, Mapping) else {} + + +def _profile(record: Mapping[str, Any]) -> str: + return str(_normalized(record).get("evidence_type") or record.get("profile") or "unknown") + + +def _record_key(record: Mapping[str, Any]) -> str: + key = _text(record.get("source_record_key")) + if not key: + raise AphisEvidenceError("record lacks source_record_key") + return key + + +def _period(record: Mapping[str, Any]) -> dict[str, str] | None: + normalized = _normalized(record) + year = _text(normalized.get("report_year")) + if year and re.fullmatch(r"\d{4}", year): + return {"start": f"{year}-01-01", "end": f"{year}-12-31", "precision": "year"} + observed = _text(normalized.get("status_date") or normalized.get("observed_at")) + if observed and re.fullmatch(r"\d{4}-\d{2}-\d{2}", observed): + return {"start": observed, "end": observed, "precision": "day"} + if observed and re.fullmatch(r"\d{4}-\d{2}", observed): + year_num, month_num = (int(part) for part in observed.split("-")) + last = date(year_num + (month_num == 12), 1 if month_num == 12 else month_num + 1, 1).toordinal() - 1 + return {"start": f"{observed}-01", "end": date.fromordinal(last).isoformat(), "precision": "month"} + return None + + +def _provenance_for(provenance: Mapping[Any, Any], record: Mapping[str, Any], profile: str) -> dict[str, Any] | None: + source_id = _text(record.get("source_id")) or "us.aphis" + value = provenance.get((source_id, profile)) or provenance.get(f"{source_id}:{profile}") or provenance.get(source_id) + if not isinstance(value, Mapping): + return None + digest = _text(value.get("artifact_sha256") or value.get("sha256")) + url = _text(value.get("source_url") or value.get("final_url")) + retrieved = _text(value.get("retrieved_at_utc")) + if not digest or not _SHA256.fullmatch(digest) or not url or not retrieved: + return None + return {"artifact_sha256": digest.lower(), "source_url": url, "retrieved_at_utc": retrieved} + + +def _safe_record(record: Mapping[str, Any]) -> dict[str, Any]: + """Private packet row; no generated URLs or external links are added.""" + return json.loads(json.dumps(record, ensure_ascii=False, default=list)) + + +def _document_events(record_key: str, refs: Iterable[Mapping[str, Any]]) -> list[dict[str, Any]]: + events: list[dict[str, Any]] = [] + for ref in refs: + if not isinstance(ref, Mapping) or not _text(ref.get("document_key")): + events.append({"state": "document_not_captured", "source_record_key": record_key, "reason": "document_key_missing"}) + continue + # Document URLs are intentionally ignored. Only an explicitly named + # source document key may associate a document with an observation. + event = {"state": "observed" if ref.get("captured") else "document_not_captured", "source_record_key": record_key, + "document_key": _text(ref.get("document_key"))} + if ref.get("captured") and _text(ref.get("artifact_sha256")): + event["artifact_sha256"] = _text(ref.get("artifact_sha256")) + elif not ref.get("captured"): + event["reason"] = "referenced_document_unavailable" + events.append(event) + return events + + +def build_packet( + *, + records_by_profile: Mapping[str, Iterable[Mapping[str, Any]]], + provenance: Mapping[Any, Any], + graph: Mapping[str, Any] | None = None, + registration_key: str | None = None, + expected_rows: Mapping[str, int] | None = None, + input_failures: Iterable[Mapping[str, Any]] = (), + artifact_verification: Mapping[str, Any] | None = None, + document_refs: Mapping[str, Iterable[Mapping[str, Any]]] | None = None, + previous_packet: Mapping[str, Any] | None = None, +) -> dict[str, Any]: + """Build deterministic private packet and row-free summary in memory.""" + failures = [dict(item) for item in input_failures] + expected_rows = dict(expected_rows or {}) + document_refs = document_refs or {} + accepted: dict[str, list[Mapping[str, Any]]] = {profile: list(records_by_profile.get(profile, ())) for profile in PROFILES} + occurrences: Counter[str] = Counter(_record_key(record) for rows in accepted.values() for record in rows) + timeline: list[dict[str, Any]] = [] + private_rows: list[dict[str, Any]] = [] + profile_summary: dict[str, dict[str, Any]] = {} + for profile in PROFILES: + rows = accepted[profile] + duplicate_keys = {key for key, count in occurrences.items() if count > 1} + profile_failures = [item for item in failures if _text(item.get("profile")) == profile] + observed = quarantined = 0 + periods: set[str] = set() + for record in sorted(rows, key=lambda item: _record_key(item)): + key = _record_key(record) + record_provenance = _provenance_for(provenance, record, profile) + state = "quarantined" if key in duplicate_keys or _normalized(record).get("privacy_gate") in {"suppressed", "restricted"} or record_provenance is None else "observed" + reason = ( + "duplicate_source_record_key" if key in duplicate_keys else + "suppressed_or_restricted" if _normalized(record).get("privacy_gate") in {"suppressed", "restricted"} else + "missing_or_invalid_provenance" if record_provenance is None else None + ) + if state == "observed": + observed += 1 + else: + quarantined += 1 + period = _period(record) + if period: + periods.add(json.dumps(period, sort_keys=True)) + evidence = { + "state": state, + "source_id": _text(record.get("source_id")) or "us.aphis", + "profile": profile, + "source_record_key": key, + "period": period, + "provenance": record_provenance, + "review_state": "review_required" if state == "observed" else "quarantined", + } + if reason: + evidence["reason"] = reason + timeline.append(evidence) + private_rows.append({"evidence": evidence, "record": _safe_record(record)}) + timeline.extend(_document_events(key, document_refs.get(key, ()))) + expected = expected_rows.get(profile) + if expected is not None and observed < int(expected): + timeline.append({"state": "not_observed", "profile": profile, "period": None, + "missing_count": int(expected) - observed, + "reason": "expected_source_rows_not_captured"}) + if profile_failures: + timeline.extend({"state": "failed", "profile": profile, "period": None, "failure": dict(item)} for item in profile_failures) + profile_summary[profile] = { + "input_rows": len(rows), "observed_rows": observed, "quarantined_rows": quarantined, + "failed_inputs": len(profile_failures), "periods": [json.loads(value) for value in sorted(periods)], + "coverage_state": "failed" if profile_failures else ("incomplete" if expected is not None and observed < int(expected) else "bounded"), + } + + links: list[dict[str, Any]] = [] + link_quarantine: list[dict[str, Any]] = [] + if graph: + for candidate in graph.get("candidates", ()) if isinstance(graph.get("candidates", ()), Iterable) else (): + if not isinstance(candidate, Mapping): + continue + left = candidate.get("left") if isinstance(candidate.get("left"), Mapping) else {} + right = candidate.get("right") if isinstance(candidate.get("right"), Mapping) else {} + if registration_key and left.get("source_record_key") != registration_key: + continue + links.append({"state": "observed", "candidate_id": candidate.get("candidate_id"), + "left": dict(left), "right": dict(right), "matched_identifiers": dict(candidate.get("matched_identifiers") or {}), + "review_state": candidate.get("review_state"), "assertion_status": candidate.get("assertion_status"), + "provenance": (candidate.get("evidence") or {}).get("provenance")}) + for candidate in graph.get("quarantined", ()) if isinstance(graph.get("quarantined", ()), Iterable) else (): + if not isinstance(candidate, Mapping): + continue + if registration_key and candidate.get("left_source_record_key") != registration_key and (candidate.get("left") or {}).get("source_record_key") != registration_key: + continue + link_quarantine.append({"state": "quarantined", "reason": candidate.get("reason") or candidate.get("quarantine_reason") or "identity_review_required", + "left_source_record_key": candidate.get("left_source_record_key") or (candidate.get("left") or {}).get("source_record_key"), + "right_source_record_key": candidate.get("right_source_record_key") or (candidate.get("right") or {}).get("source_record_key"), + "candidate_id": candidate.get("candidate_id")}) + timeline.sort(key=lambda item: (str(item.get("period") or ""), str(item.get("profile") or ""), str(item.get("source_record_key") or ""), str(item.get("state") or ""))) + summary = { + "schema_version": PACKET_VERSION, "storage_state": "private", "publication_status": "not_eligible", "release_state": "not-created", + "registration_source_record_key": registration_key, "profiles": profile_summary, + "timeline_state_counts": dict(sorted(Counter(str(item.get("state")) for item in timeline).items())), + "timeline_periods": sorted({json.dumps(item["period"], sort_keys=True) for item in timeline if isinstance(item.get("period"), Mapping)}), + "links": {"candidate_count": len(links), "quarantined_count": len(link_quarantine), "excluded_reasons": dict(sorted(Counter(item["reason"] for item in link_quarantine).items()))}, + "input_failures": failures, "artifact_verification": dict(artifact_verification or {}), + "coverage_boundary": "bounded retained APHIS source profiles; not a facility master, national census, current-operation, ownership, or animal-use total", + "unknowns": ["location/current operation", "ownership/control", "unobserved source rows and reporting periods", "animal-use coverage outside the captured APHIS profiles"], + "publication": {"api": False, "map": False, "export": False, "cache": False, "history": False}, + } + private = {"schema_version": PACKET_VERSION, "summary": summary, "timeline": timeline, "links": links, "link_quarantine": link_quarantine, "rows": private_rows, + "previous_packet_current": False if previous_packet else None} + return {"private_packet": private, "row_free_summary": summary, "timeline": timeline, "links": links, "link_quarantine": link_quarantine} + + +def write_packet(run_dir: str | Path, packet: Mapping[str, Any]) -> dict[str, Any]: + """Atomically write row-bearing private files and a separate row-free report.""" + root = Path(run_dir) + private = packet["private_packet"] + summary = packet["row_free_summary"] + atomic_json(root / "row-free-summary.json", dict(summary)) + atomic_json(root / "editorial-note.json", { + "what_it_makes_visible": "A bounded, source-native view of registrations and their captured annual-report and inspection observations.", + "why_it_matters": "It gives activists a traceable way to investigate regulatory records without turning partial evidence into a facility census or allegation.", + "what_it_does_not_establish": "It does not establish current operation, ownership, wrongdoing, animal-use totals, closure, or absence.", + "next_questions": ["Which missing periods or documents can be obtained through an authorized route?", "Which quarantined identities need human review?", "What source evidence can establish location or operation separately?"], + }) + atomic_jsonl(root / "private" / "timeline.jsonl", list(private["timeline"])) + atomic_jsonl(root / "private" / "rows.jsonl", list(private["rows"])) + atomic_jsonl(root / "private" / "links.jsonl", list(private["links"])) + atomic_jsonl(root / "private" / "link-quarantine.jsonl", list(private["link_quarantine"])) + manifest = {"schema_version": PACKET_VERSION, "row_free_summary_sha256": hashlib.sha256(json.dumps(summary, sort_keys=True).encode()).hexdigest(), + "private_packet_sha256": hashlib.sha256(json.dumps(private, sort_keys=True, default=list).encode()).hexdigest(), + "storage_state": "private", "publication_status": "not_eligible", "release_state": "not-created", "test_only": True} + atomic_json(root / "packet-manifest.json", manifest) + return manifest + + +def run_from_wave2( + wave2_dir: str | Path, + packet_dir: str | Path, + *, + registration_key: str | None = None, + expected_rows: Mapping[str, int] | None = None, + document_refs: Mapping[str, Iterable[Mapping[str, Any]]] | None = None, +) -> dict[str, Any]: + """Reuse an existing Wave 2 private run as the packet input. + + This is deliberately a file boundary: the function does not reacquire or + reinterpret source rows. A Wave 2 run with a failed input keeps that + failure in the packet and never promotes an earlier packet to current. + """ + root = Path(wave2_dir) + report_path = root / "wave2-report.json" + report = _json(report_path) + records = { + profile: read_jsonl(root / "private-rows" / f"{profile}-accepted.jsonl") + if (root / "private-rows" / f"{profile}-accepted.jsonl").is_file() else [] + for profile in PROFILES + } + graph_dir = root / "identity-graph" + graph = { + "candidates": read_jsonl(graph_dir / "candidate" / "identity-links.jsonl") + if (graph_dir / "candidate" / "identity-links.jsonl").is_file() else [], + "quarantined": read_jsonl(graph_dir / "quarantined" / "identity-links.jsonl") + if (graph_dir / "quarantined" / "identity-links.jsonl").is_file() else [], + } + provenance = {} + for profile, value in (report.get("profiles") or {}).items(): + if isinstance(value, Mapping): + provenance[("us.aphis", profile)] = { + "artifact_sha256": value.get("aggregate_artifact_sha256"), + "source_url": value.get("source_url"), + "retrieved_at_utc": value.get("retrieved_at_utc"), + } + packet = build_packet( + records_by_profile=records, provenance=provenance, graph=graph, + registration_key=registration_key, expected_rows=expected_rows, + input_failures=report.get("input_failures") or [], document_refs=document_refs, + ) + manifest = write_packet(packet_dir, packet) + return {"packet": packet, "manifest": manifest} + + +def _cli() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--manifest", required=True, type=Path) + parser.add_argument("--artifact-root", type=Path) + parser.add_argument("--run-dir", required=True, type=Path) + args = parser.parse_args() + verification = verify_retained_artifacts(args.manifest, artifact_root=args.artifact_root) + summary = {"schema_version": PACKET_VERSION, "artifact_verification": verification, "input_failures": verification["artifacts"]} + failures = [item for item in verification["artifacts"] if item.get("state") != "verified"] + write_packet(args.run_dir, build_packet(records_by_profile={}, provenance={}, artifact_verification=verification, input_failures=failures)) + print(json.dumps(summary, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(_cli()) diff --git a/pipeline/sources/us/accountability/test_aphis_evidence.py b/pipeline/sources/us/accountability/test_aphis_evidence.py new file mode 100644 index 0000000..ed9caf2 --- /dev/null +++ b/pipeline/sources/us/accountability/test_aphis_evidence.py @@ -0,0 +1,109 @@ +import hashlib +import json +import tempfile +import unittest +from pathlib import Path + +from pipeline.sources.us.aphis.adapter import AphisPublicSearchAdapter + +from .aphis_evidence import ( + PACKET_VERSION, + build_packet, + verify_retained_artifacts, + write_packet, +) + + +ROOT = Path(__file__).parents[1] / "aphis" + + +class AphisEvidencePacketTests(unittest.TestCase): + def _records(self): + adapter = AphisPublicSearchAdapter() + records = {} + for profile in ("registrations", "annual_reports", "inspections"): + result = adapter.parse_bytes((ROOT / "fixtures" / f"{profile}.csv").read_bytes()) + records[profile] = result["accepted"] + return records + + def test_year_is_a_year_period_and_dates_are_not_synthetic_events(self): + packet = build_packet( + records_by_profile=self._records(), + provenance={ + ("us.aphis", profile): { + "artifact_sha256": "a" * 64, + "source_url": "https://example.invalid/" + profile, + "retrieved_at_utc": "2026-09-18T00:00:00Z", + } + for profile in ("registrations", "annual_reports", "inspections") + }, + ) + annual = next(item for item in packet["timeline"] if item.get("profile") == "annual_reports") + inspection = next(item for item in packet["timeline"] if item.get("profile") == "inspections") + self.assertEqual(annual["period"], {"start": "2025-01-01", "end": "2025-12-31", "precision": "year"}) + self.assertEqual(inspection["period"]["precision"], "day") + self.assertNotEqual(annual["period"]["precision"], "day") + + def test_expected_gap_failure_and_document_unavailable_remain_visible(self): + records = self._records() + key = records["annual_reports"][0]["source_record_key"] + packet = build_packet( + records_by_profile=records, + provenance={}, + expected_rows={"inspections": 4}, + input_failures=[{"profile": "annual_reports", "state": "failed", "failure": "artifact_missing"}], + document_refs={key: [{"document_key": "annual-report-2025", "captured": False, "url": "https://signed.invalid/secret"}]}, + ) + states = {item["state"] for item in packet["timeline"]} + self.assertTrue({"not_observed", "failed", "document_not_captured"}.issubset(states)) + serialized = json.dumps(packet["row_free_summary"]) + self.assertNotIn("signed.invalid", serialized) + self.assertEqual(packet["row_free_summary"]["publication_status"], "not_eligible") + + def test_duplicate_source_keys_are_quarantined_and_links_preserve_conflicts(self): + records = self._records() + duplicate = dict(records["inspections"][0]) + records["inspections"].append(duplicate) + key = records["registrations"][0]["source_record_key"] + graph = { + "candidates": [{"candidate_id": "candidate-1", "left": {"source_record_key": key}, + "right": {"source_record_key": records["annual_reports"][0]["source_record_key"]}, + "matched_identifiers": {"aphis_certificate_number": "87-R-0001"}, + "review_state": "review_required", "assertion_status": "candidate"}], + "quarantined": [{"candidate_id": "candidate-2", "left_source_record_key": key, + "right_source_record_key": "inspections:conflict", "reason": "conflicting_official_identifiers"}], + } + packet = build_packet(records_by_profile=records, provenance={}, graph=graph, registration_key=key) + self.assertTrue(any(item["state"] == "quarantined" and item.get("reason") == "duplicate_source_record_key" for item in packet["timeline"])) + self.assertEqual(packet["links"][0]["matched_identifiers"]["aphis_certificate_number"], "87-R-0001") + self.assertEqual(packet["link_quarantine"][0]["reason"], "conflicting_official_identifiers") + + def test_manifest_hash_and_size_fail_visibly(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + artifact = root / "annual.csv" + artifact.write_bytes(b"retained") + manifest = root / "manifest.json" + digest = hashlib.sha256(artifact.read_bytes()).hexdigest() + manifest.write_text(json.dumps({"profiles": {"annual_reports": {"artifacts": [{ + "artifact": artifact.name, "path": str(artifact), "sha256": digest, "byte_size": 99, + }]}}}), encoding="utf-8") + result = verify_retained_artifacts(manifest) + self.assertFalse(result["all_verified"]) + self.assertEqual(result["artifacts"][0]["failure"], "artifact_size_mismatch") + artifact.write_bytes(b"changed") + result = verify_retained_artifacts(manifest) + self.assertEqual(result["artifacts"][0]["failure"], "artifact_hash_mismatch") + + def test_write_packet_has_separate_row_free_summary(self): + with tempfile.TemporaryDirectory() as directory: + packet = build_packet(records_by_profile=self._records(), provenance={}) + manifest = write_packet(directory, packet) + self.assertEqual(manifest["schema_version"], PACKET_VERSION) + self.assertTrue((Path(directory) / "row-free-summary.json").exists()) + self.assertTrue((Path(directory) / "private" / "rows.jsonl").exists()) + self.assertNotIn("source_values", (Path(directory) / "row-free-summary.json").read_text(encoding="utf-8")) + + +if __name__ == "__main__": + unittest.main() From ba1b02b85e44a9121c20d78c2acf0471cc9a4b8f Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Fri, 18 Sep 2026 19:42:19 -0700 Subject: [PATCH 270/311] Harden APHIS packet provenance and replay --- .../us/accountability/aphis_evidence.py | 168 ++++++++++++++++-- .../sources/us/accountability/aphis_wave2.py | 15 +- .../us/accountability/test_aphis_evidence.py | 81 ++++++++- 3 files changed, 243 insertions(+), 21 deletions(-) diff --git a/pipeline/sources/us/accountability/aphis_evidence.py b/pipeline/sources/us/accountability/aphis_evidence.py index 79d36fe..e9175c4 100644 --- a/pipeline/sources/us/accountability/aphis_evidence.py +++ b/pipeline/sources/us/accountability/aphis_evidence.py @@ -19,6 +19,7 @@ from typing import Any, Iterable, Mapping from pipeline.contracts.source_lifecycle import atomic_json, atomic_jsonl, read_jsonl +from pipeline.sources.us.aphis.adapter import AphisContractError, AphisPublicSearchAdapter PACKET_VERSION = "us-aphis-investigation-packet-v1" @@ -104,11 +105,15 @@ def verify_retained_artifacts( artifact_name = _text(item.get("artifact") or item.get("name")) declared_path = _text(item.get("path_private") or item.get("artifact_path") or item.get("path")) candidate = Path(declared_path) if declared_path else (root / artifact_name if artifact_name else None) + if candidate is not None and not candidate.is_absolute(): + candidate = root / candidate result = { "profile": profile, "artifact": artifact_name, "declared_sha256": _text(item.get("sha256")), "declared_byte_size": item.get("byte_size") if item.get("byte_size") is not None else item.get("bytes"), + "source_url": _text(item.get("source_url") or item.get("requested_url") or item.get("final_url")), + "retrieved_at_utc": _text(item.get("retrieved_at_utc")), "path": str(candidate.resolve()) if candidate else None, "state": "failed", "failure": None, @@ -127,10 +132,17 @@ def verify_retained_artifacts( result["failure"] = "artifact_hash_not_declared_or_invalid" elif actual_hash.lower() != str(expected_hash).lower(): result["failure"] = "artifact_hash_mismatch" - elif expected_size is not None and int(expected_size) != actual_size: - result["failure"] = "artifact_size_mismatch" + elif expected_size is None: + result["failure"] = "artifact_size_not_declared" else: - result["state"] = "verified" + try: + size_matches = int(expected_size) == actual_size + except (TypeError, ValueError): + size_matches = False + if not size_matches: + result["failure"] = "artifact_size_mismatch" + else: + result["state"] = "verified" checked.append(result) by_profile: dict[str, dict[str, int]] = {} for item in checked: @@ -176,9 +188,20 @@ def _period(record: Mapping[str, Any]) -> dict[str, str] | None: return None -def _provenance_for(provenance: Mapping[Any, Any], record: Mapping[str, Any], profile: str) -> dict[str, Any] | None: +def _provenance_for( + provenance: Mapping[Any, Any], + record: Mapping[str, Any], + profile: str, + *, + verified_artifact_hashes: set[str] | None = None, +) -> dict[str, Any] | None: source_id = _text(record.get("source_id")) or "us.aphis" - value = provenance.get((source_id, profile)) or provenance.get(f"{source_id}:{profile}") or provenance.get(source_id) + retained = record.get("_retained_artifact") + profile_value = provenance.get((source_id, profile)) or provenance.get(f"{source_id}:{profile}") or provenance.get(source_id) + if isinstance(retained, Mapping) and isinstance(profile_value, Mapping): + value = {**profile_value, **retained} + else: + value = retained if isinstance(retained, Mapping) else profile_value if not isinstance(value, Mapping): return None digest = _text(value.get("artifact_sha256") or value.get("sha256")) @@ -186,6 +209,8 @@ def _provenance_for(provenance: Mapping[Any, Any], record: Mapping[str, Any], pr retrieved = _text(value.get("retrieved_at_utc")) if not digest or not _SHA256.fullmatch(digest) or not url or not retrieved: return None + if verified_artifact_hashes is not None and digest.lower() not in verified_artifact_hashes: + return None return {"artifact_sha256": digest.lower(), "source_url": url, "retrieved_at_utc": retrieved} @@ -223,15 +248,24 @@ def build_packet( artifact_verification: Mapping[str, Any] | None = None, document_refs: Mapping[str, Iterable[Mapping[str, Any]]] | None = None, previous_packet: Mapping[str, Any] | None = None, + profile_input_rows: Mapping[str, int] | None = None, + quarantine_rows_by_profile: Mapping[str, Iterable[Mapping[str, Any]]] | None = None, ) -> dict[str, Any]: """Build deterministic private packet and row-free summary in memory.""" failures = [dict(item) for item in input_failures] expected_rows = dict(expected_rows or {}) + profile_input_rows = dict(profile_input_rows or {}) + quarantine_rows_by_profile = quarantine_rows_by_profile or {} document_refs = document_refs or {} accepted: dict[str, list[Mapping[str, Any]]] = {profile: list(records_by_profile.get(profile, ())) for profile in PROFILES} occurrences: Counter[str] = Counter(_record_key(record) for rows in accepted.values() for record in rows) timeline: list[dict[str, Any]] = [] private_rows: list[dict[str, Any]] = [] + verified_hashes = { + str(item.get("actual_sha256")).lower() + for item in (artifact_verification or {}).get("artifacts", ()) + if isinstance(item, Mapping) and item.get("state") == "verified" and item.get("actual_sha256") + } if artifact_verification else None profile_summary: dict[str, dict[str, Any]] = {} for profile in PROFILES: rows = accepted[profile] @@ -241,7 +275,10 @@ def build_packet( periods: set[str] = set() for record in sorted(rows, key=lambda item: _record_key(item)): key = _record_key(record) - record_provenance = _provenance_for(provenance, record, profile) + record_provenance = _provenance_for( + provenance, record, profile, + verified_artifact_hashes=verified_hashes, + ) state = "quarantined" if key in duplicate_keys or _normalized(record).get("privacy_gate") in {"suppressed", "restricted"} or record_provenance is None else "observed" reason = ( "duplicate_source_record_key" if key in duplicate_keys else @@ -269,15 +306,22 @@ def build_packet( timeline.append(evidence) private_rows.append({"evidence": evidence, "record": _safe_record(record)}) timeline.extend(_document_events(key, document_refs.get(key, ()))) + adapter_quarantine = [dict(item) for item in quarantine_rows_by_profile.get(profile, ())] + for item in adapter_quarantine: + record = item.get("record") if isinstance(item.get("record"), Mapping) else {} + timeline.append({"state": "quarantined", "profile": profile, + "source_record_key": record.get("source_record_key"), + "reason": "adapter_quarantine", "reasons": list(item.get("reasons", ()))}) expected = expected_rows.get(profile) - if expected is not None and observed < int(expected): + captured_rows = int(profile_input_rows.get(profile, len(rows) + len(adapter_quarantine))) + if expected is not None and captured_rows < int(expected): timeline.append({"state": "not_observed", "profile": profile, "period": None, - "missing_count": int(expected) - observed, + "missing_count": int(expected) - captured_rows, "reason": "expected_source_rows_not_captured"}) if profile_failures: timeline.extend({"state": "failed", "profile": profile, "period": None, "failure": dict(item)} for item in profile_failures) profile_summary[profile] = { - "input_rows": len(rows), "observed_rows": observed, "quarantined_rows": quarantined, + "input_rows": captured_rows, "observed_rows": observed, "quarantined_rows": quarantined + len(adapter_quarantine), "failed_inputs": len(profile_failures), "periods": [json.loads(value) for value in sorted(periods)], "coverage_state": "failed" if profile_failures else ("incomplete" if expected is not None and observed < int(expected) else "bounded"), } @@ -292,10 +336,31 @@ def build_packet( right = candidate.get("right") if isinstance(candidate.get("right"), Mapping) else {} if registration_key and left.get("source_record_key") != registration_key: continue - links.append({"state": "observed", "candidate_id": candidate.get("candidate_id"), - "left": dict(left), "right": dict(right), "matched_identifiers": dict(candidate.get("matched_identifiers") or {}), - "review_state": candidate.get("review_state"), "assertion_status": candidate.get("assertion_status"), - "provenance": (candidate.get("evidence") or {}).get("provenance")}) + candidate_provenance = (candidate.get("evidence") or {}).get("provenance") + has_provenance = isinstance(candidate_provenance, Mapping) and all( + isinstance(value, Mapping) + and _text(value.get("artifact_sha256")) + and _text(value.get("source_url")) + and _text(value.get("retrieved_at_utc")) + and (verified_hashes is None or _text(value.get("artifact_sha256")).lower() in verified_hashes) + for value in candidate_provenance.values() + ) + review_state = _text(candidate.get("review_state")) + assertion_status = _text(candidate.get("assertion_status")) + conflict = bool(candidate.get("conflicting_official_identifiers") or candidate.get("identifier_conflicts")) + reason = ( + "conflicting_official_identifiers" if conflict else + "link_missing_or_invalid_provenance" if not has_provenance else + "link_review_required" if review_state == "review_required" or assertion_status in {"candidate", "quarantined"} else None + ) + item = {"candidate_id": candidate.get("candidate_id"), + "left": dict(left), "right": dict(right), "matched_identifiers": dict(candidate.get("matched_identifiers") or {}), + "review_state": review_state, "assertion_status": assertion_status, + "provenance": candidate_provenance} + if reason: + link_quarantine.append({"state": "quarantined", "reason": reason, **item}) + else: + links.append({"state": "observed", **item}) for candidate in graph.get("quarantined", ()) if isinstance(graph.get("quarantined", ()), Iterable) else (): if not isinstance(candidate, Mapping): continue @@ -362,6 +427,10 @@ def run_from_wave2( root = Path(wave2_dir) report_path = root / "wave2-report.json" report = _json(report_path) + input_manifest = root / "input-manifest.json" + if not input_manifest.is_file(): + raise AphisEvidenceError(f"Wave 2 input manifest is missing: {input_manifest}") + verification = verify_retained_artifacts(input_manifest) records = { profile: read_jsonl(root / "private-rows" / f"{profile}-accepted.jsonl") if (root / "private-rows" / f"{profile}-accepted.jsonl").is_file() else [] @@ -378,29 +447,94 @@ def run_from_wave2( for profile, value in (report.get("profiles") or {}).items(): if isinstance(value, Mapping): provenance[("us.aphis", profile)] = { - "artifact_sha256": value.get("aggregate_artifact_sha256"), + # Wave 2's aggregate hash covers metadata strings, not source + # bytes. Row-level hashes come from _retained_artifact tags. "source_url": value.get("source_url"), "retrieved_at_utc": value.get("retrieved_at_utc"), } + completeness = report.get("completeness") if isinstance(report.get("completeness"), Mapping) else {} + expected = dict(expected_rows or {}) + input_rows: dict[str, int] = {} + for profile, value in completeness.items(): + if not isinstance(value, Mapping): + continue + if value.get("expected_displayed_rows") is not None and profile not in expected: + expected[profile] = int(value["expected_displayed_rows"]) + if value.get("observed_input_rows") is not None: + input_rows[profile] = int(value["observed_input_rows"]) + quarantine_rows = { + profile: read_jsonl(root / "private-rows" / f"{profile}-adapter-quarantine.jsonl") + if (root / "private-rows" / f"{profile}-adapter-quarantine.jsonl").is_file() else [] + for profile in PROFILES + } packet = build_packet( records_by_profile=records, provenance=provenance, graph=graph, - registration_key=registration_key, expected_rows=expected_rows, + registration_key=registration_key, expected_rows=expected, input_failures=report.get("input_failures") or [], document_refs=document_refs, + artifact_verification=verification, profile_input_rows=input_rows, + quarantine_rows_by_profile=quarantine_rows, ) manifest = write_packet(packet_dir, packet) return {"packet": packet, "manifest": manifest} +def _load_verified_exports( + verification: Mapping[str, Any], + manifest: Mapping[str, Any], +) -> tuple[dict[str, list[dict[str, Any]]], dict[str, list[dict[str, Any]]], dict[tuple[str, str], dict[str, Any]], dict[str, int]]: + """Parse only verified manifest-named exports for the standalone CLI.""" + records: dict[str, list[dict[str, Any]]] = defaultdict(list) + quarantined: dict[str, list[dict[str, Any]]] = defaultdict(list) + provenance: dict[tuple[str, str], dict[str, Any]] = {} + input_rows: dict[str, int] = defaultdict(int) + adapter = AphisPublicSearchAdapter() + report_profiles = manifest.get("profile_metadata") if isinstance(manifest.get("profile_metadata"), Mapping) else {} + for item in verification.get("artifacts", ()): + if not isinstance(item, Mapping) or item.get("state") != "verified": + continue + profile = _text(item.get("profile")) + path = _text(item.get("path")) + if not profile or profile not in PROFILES or not path: + continue + try: + result = adapter.parse_bytes(Path(path).read_bytes()) + except (OSError, AphisContractError) as exc: + quarantined[profile].append({"state": "failed", "reasons": [f"adapter_error:{type(exc).__name__}"]}) + continue + artifact = {"artifact_sha256": item.get("actual_sha256"), "source_url": item.get("source_url"), + "retrieved_at_utc": item.get("retrieved_at_utc")} + profile_meta = report_profiles.get(profile) if isinstance(report_profiles, Mapping) else None + if isinstance(profile_meta, Mapping): + artifact["source_url"] = artifact.get("source_url") or profile_meta.get("source_url") + artifact["retrieved_at_utc"] = artifact.get("retrieved_at_utc") or profile_meta.get("retrieved_at_utc") + for record in result["accepted"]: + records[profile].append({**record, "_retained_artifact": artifact}) + quarantined[profile].extend({**row, "record": {**row["record"], "_retained_artifact": artifact}} for row in result["quarantined"]) + input_rows[profile] += int(result["input_rows"]) + provenance[("us.aphis", profile)] = artifact + expected = manifest.get("expected_rows") if isinstance(manifest.get("expected_rows"), Mapping) else {} + return dict(records), dict(quarantined), provenance, dict(input_rows) + + def _cli() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--manifest", required=True, type=Path) parser.add_argument("--artifact-root", type=Path) parser.add_argument("--run-dir", required=True, type=Path) args = parser.parse_args() + manifest = _json(args.manifest) verification = verify_retained_artifacts(args.manifest, artifact_root=args.artifact_root) - summary = {"schema_version": PACKET_VERSION, "artifact_verification": verification, "input_failures": verification["artifacts"]} + records, quarantined, provenance, input_rows = _load_verified_exports(verification, manifest) failures = [item for item in verification["artifacts"] if item.get("state") != "verified"] - write_packet(args.run_dir, build_packet(records_by_profile={}, provenance={}, artifact_verification=verification, input_failures=failures)) + packet = build_packet( + records_by_profile=records, provenance=provenance, + expected_rows=manifest.get("expected_rows") if isinstance(manifest.get("expected_rows"), Mapping) else {}, + profile_input_rows=input_rows, quarantine_rows_by_profile=quarantined, + artifact_verification=verification, input_failures=failures, + ) + write_packet(args.run_dir, packet) + summary = {"schema_version": PACKET_VERSION, "artifact_verification": verification, + "input_failures": failures, "row_free_summary": packet["row_free_summary"]} print(json.dumps(summary, sort_keys=True)) return 0 diff --git a/pipeline/sources/us/accountability/aphis_wave2.py b/pipeline/sources/us/accountability/aphis_wave2.py index a1efcfa..6e66d78 100644 --- a/pipeline/sources/us/accountability/aphis_wave2.py +++ b/pipeline/sources/us/accountability/aphis_wave2.py @@ -144,8 +144,19 @@ def _load_profile(paths: list[Path], profile: str) -> tuple[list[dict[str, Any]] raise ValueError(f"{path.name} parsed as {result['profile']}, expected {profile}") _, metadata = _classify(path, adapter) artifacts.append(metadata) - records.extend(result["accepted"]) - quarantined.extend(result["quarantined"]) + # Keep the retained-artifact edge with every private row. The + # aggregate profile hash below is metadata accounting, not a source + # artifact hash and must never be used as row provenance. + row_artifact = { + "artifact": metadata["artifact"], + "artifact_sha256": metadata["sha256"], + "byte_size": metadata["byte_size"], + } + records.extend([{**record, "_retained_artifact": row_artifact} for record in result["accepted"]]) + quarantined.extend([ + {**item, "record": {**item["record"], "_retained_artifact": row_artifact}} + for item in result["quarantined"] + ]) return records, quarantined, artifacts diff --git a/pipeline/sources/us/accountability/test_aphis_evidence.py b/pipeline/sources/us/accountability/test_aphis_evidence.py index ed9caf2..1b655f7 100644 --- a/pipeline/sources/us/accountability/test_aphis_evidence.py +++ b/pipeline/sources/us/accountability/test_aphis_evidence.py @@ -1,5 +1,8 @@ import hashlib import json +import shutil +import subprocess +import sys import tempfile import unittest from pathlib import Path @@ -10,8 +13,10 @@ PACKET_VERSION, build_packet, verify_retained_artifacts, + run_from_wave2, write_packet, ) +from .aphis_wave2 import run_wave2 ROOT = Path(__file__).parents[1] / "aphis" @@ -75,8 +80,10 @@ def test_duplicate_source_keys_are_quarantined_and_links_preserve_conflicts(self } packet = build_packet(records_by_profile=records, provenance={}, graph=graph, registration_key=key) self.assertTrue(any(item["state"] == "quarantined" and item.get("reason") == "duplicate_source_record_key" for item in packet["timeline"])) - self.assertEqual(packet["links"][0]["matched_identifiers"]["aphis_certificate_number"], "87-R-0001") - self.assertEqual(packet["link_quarantine"][0]["reason"], "conflicting_official_identifiers") + self.assertFalse(packet["links"]) + reasons = {item["reason"] for item in packet["link_quarantine"]} + self.assertIn("link_missing_or_invalid_provenance", reasons) + self.assertIn("conflicting_official_identifiers", reasons) def test_manifest_hash_and_size_fail_visibly(self): with tempfile.TemporaryDirectory() as directory: @@ -91,6 +98,11 @@ def test_manifest_hash_and_size_fail_visibly(self): result = verify_retained_artifacts(manifest) self.assertFalse(result["all_verified"]) self.assertEqual(result["artifacts"][0]["failure"], "artifact_size_mismatch") + manifest.write_text(json.dumps({"profiles": {"annual_reports": {"artifacts": [{ + "artifact": artifact.name, "path": str(artifact), "sha256": digest, + }]}}}), encoding="utf-8") + result = verify_retained_artifacts(manifest) + self.assertEqual(result["artifacts"][0]["failure"], "artifact_size_not_declared") artifact.write_bytes(b"changed") result = verify_retained_artifacts(manifest) self.assertEqual(result["artifacts"][0]["failure"], "artifact_hash_mismatch") @@ -104,6 +116,71 @@ def test_write_packet_has_separate_row_free_summary(self): self.assertTrue((Path(directory) / "private" / "rows.jsonl").exists()) self.assertNotIn("source_values", (Path(directory) / "row-free-summary.json").read_text(encoding="utf-8")) + def test_aggregate_metadata_hash_cannot_be_link_provenance(self): + records = self._records() + registration_key = records["registrations"][0]["source_record_key"] + graph = {"candidates": [{ + "candidate_id": "candidate-reviewed", "left": {"source_record_key": registration_key}, + "right": {"source_record_key": records["annual_reports"][0]["source_record_key"]}, + "matched_identifiers": {"aphis_certificate_number": "87-R-0001"}, + "review_state": "reviewed", "assertion_status": "candidate", + "evidence": {"provenance": { + "us.aphis:registrations": {"artifact_sha256": "a" * 64, "source_url": "https://example.invalid/r", "retrieved_at_utc": "2026-09-18T00:00:00Z"}, + "us.aphis:annual_reports": {"artifact_sha256": "b" * 64, "source_url": "https://example.invalid/a", "retrieved_at_utc": "2026-09-18T00:00:00Z"}, + }}, + }], "quarantined": []} + packet = build_packet( + records_by_profile=records, provenance={}, graph=graph, registration_key=registration_key, + artifact_verification={"artifacts": [{"state": "verified", "actual_sha256": "c" * 64}]}, + ) + self.assertFalse(packet["links"]) + self.assertEqual(packet["link_quarantine"][0]["reason"], "link_missing_or_invalid_provenance") + + def test_documented_cli_parses_verified_exports_and_preserves_coverage_gap(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + artifacts = {} + for profile in ("registrations", "annual_reports", "inspections"): + source = ROOT / "fixtures" / f"{profile}.csv" + target = root / f"ExportData_{profile}.csv" + shutil.copyfile(source, target) + artifacts[profile] = {"artifact": target.name, "path": str(target), + "sha256": hashlib.sha256(target.read_bytes()).hexdigest(), + "byte_size": target.stat().st_size, + "source_url": f"https://example.invalid/{profile}", + "retrieved_at_utc": "2026-09-18T00:00:00Z"} + manifest = root / "input-manifest.json" + manifest.write_text(json.dumps({"profiles": {profile: {"artifacts": [item]} for profile, item in artifacts.items()}, + "expected_rows": {"inspections": 2}}), encoding="utf-8") + run_dir = root / "run" + completed = subprocess.run([ + sys.executable, "-m", "pipeline.sources.us.accountability.aphis_evidence", + "--manifest", str(manifest), "--artifact-root", str(root), "--run-dir", str(run_dir), + ], cwd=Path(__file__).parents[4], capture_output=True, text=True, check=True) + output = json.loads(completed.stdout) + summary = json.loads((run_dir / "row-free-summary.json").read_text(encoding="utf-8")) + self.assertGreater(summary["profiles"]["annual_reports"]["observed_rows"], 0) + self.assertIn("not_observed", summary["timeline_state_counts"]) + self.assertEqual(output["artifact_verification"]["artifacts"][0]["actual_sha256"], artifacts[output["artifact_verification"]["artifacts"][0]["profile"]]["sha256"]) + self.assertNotEqual(output["artifact_verification"]["artifacts"][0]["actual_sha256"], "") + + def test_wave2_replay_verifies_input_manifest_and_uses_row_hashes(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + input_root = root / "input" + input_root.mkdir() + for profile in ("registrations", "annual_reports", "inspections"): + shutil.copyfile(ROOT / "fixtures" / f"{profile}.csv", input_root / f"ExportData_{profile}.csv") + wave2_dir = root / "wave2" + run_wave2(input_root=input_root, run_dir=wave2_dir) + result = run_from_wave2(wave2_dir, root / "packet") + summary = result["packet"]["row_free_summary"] + self.assertGreater(summary["profiles"]["annual_reports"]["observed_rows"], 0) + self.assertIn("not_observed", summary["timeline_state_counts"]) + hashes = [item.get("actual_sha256") for item in summary["artifact_verification"]["artifacts"]] + self.assertTrue(all(hashes)) + self.assertNotIn("aggregate_artifact_sha256", json.dumps(summary)) + if __name__ == "__main__": unittest.main() From 70f5a7c32f6ac270aad2e3c02400be42e4a9bb5e Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Fri, 18 Sep 2026 19:41:21 -0700 Subject: [PATCH 271/311] Add exact source rights release gate --- .../release-activation-defects.md | 33 ++++ .../release-manifest-verification.md | 2 +- docs/architecture/source-rights-decisions.md | 30 +++ docs/data-product.md | 2 +- pipeline/common/source_rights.py | 172 ++++++++++++++++++ .../041_source_rights_decisions.sql | 72 ++++++++ .../build_public_discovery_read_model.py | 14 +- pipeline/scripts/stages/export-release.py | 8 +- pipeline/scripts/stages/promote-release.py | 19 +- pipeline/scripts/stages/validate-release.py | 13 +- pipeline/tests/test_graph_migrations.py | 8 +- pipeline/tests/test_source_rights.py | 102 +++++++++++ .../tests/test_source_rights_migration.py | 50 +++++ 13 files changed, 512 insertions(+), 13 deletions(-) create mode 100644 docs/architecture/release-activation-defects.md create mode 100644 docs/architecture/source-rights-decisions.md create mode 100644 pipeline/common/source_rights.py create mode 100644 pipeline/migrations/041_source_rights_decisions.sql create mode 100644 pipeline/tests/test_source_rights.py create mode 100644 pipeline/tests/test_source_rights_migration.py diff --git a/docs/architecture/release-activation-defects.md b/docs/architecture/release-activation-defects.md new file mode 100644 index 0000000..f4fcdeb --- /dev/null +++ b/docs/architecture/release-activation-defects.md @@ -0,0 +1,33 @@ +# Deferred release activation reproductions + +Sprint 01 keeps the known activation defects visible while the source-rights +gate is added. These are synthetic, private launch blockers; this lane does +not redesign activation or rollback. + +## Failed replacement build can remove the active release + +Create a promoted release `synthetic-a`, a validated release `synthetic-b`, +and exact cleared source-rights decisions for each artifact in `synthetic-b`. +Run `promote-release.py synthetic-b --no-distributed-artifacts`, then inject a +failure before `build_public_discovery_read_model.py synthetic-b` commits its +model (the builder's `fail_after_rows` hook is the existing injection point). +The current promotion transaction demotes `synthetic-a` before the model is +built. A public read therefore has no usable promoted model after the failed +build. The existing `pipeline/tests/e2e/test_public_discovery_read_model.py` +interruption test proves row/model rollback inside the builder, but does not +claim cross-stage active-release safety. + +## Re-promoting an old release collides with its immutable manifest + +Promote `synthetic-a`, promote `synthetic-b`, then set `synthetic-a` back to +`validated` in the disposable database without changing its existing +`uec.release_manifests` row. Running `promote-release.py synthetic-a` reaches +the unconditional manifest insert and fails on the manifest primary key. The +immutable manifest is preserved, but the old release cannot be selected again +through the current command. The following sprint must stage validation and +active-release selection so A→B→A and an injected build failure preserve a +usable active service. + +These reproductions contain no real records or approvals. Source-rights +validation is still required before either promotion attempt; the fixture +decisions are synthetic and do not authorize publication. diff --git a/docs/architecture/release-manifest-verification.md b/docs/architecture/release-manifest-verification.md index f75d064..995a2ed 100644 --- a/docs/architecture/release-manifest-verification.md +++ b/docs/architecture/release-manifest-verification.md @@ -2,7 +2,7 @@ Promotion stores an immutable machine-readable `uec-release-manifest-v2` with the release/profile IDs, ruleset and projection schema versions, generated/retrieved timestamps, source coverage, eligible row counts, independent review/publication state, limitations, supersession, and an inventory of declared distributed files. Supply each file with a repeated `--artifact ` option; promotion hashes the file bytes and records its basename, size, and SHA-256. If there really are no distributed files, the operator must explicitly pass `--no-distributed-artifacts`. `--manifest ` exports the canonical JSON whose SHA-256 is stored in `uec.release_manifests`; the CLI result printed to stdout is a separate operation receipt. Obtain the manifest from a trusted project channel, verify its SHA-256 against the stored digest, then hash each listed artifact locally. -The workflow does not discover distributed files or prove that the operator supplied a complete inventory. An empty declared inventory is not evidence that no files were distributed. `source_coverage` is aggregate coverage metadata; record-level provenance remains in the public projection rows. Source `rights_status` is recorded as `cleared`, `attribution_required`, or `unknown`; the bulk packager refuses unknown/restricted rows. Hashing at promotion does not freeze later distribution bytes. The database promotion and writing `--manifest` to disk are separate operations; if the file write fails, the database manifest remains stored and an operator must recover and verify it before distribution. These limits require operational review before making a full release-integrity claim. +The workflow does not discover distributed files or prove that the operator supplied a complete inventory. An empty declared inventory is not evidence that no files were distributed. `source_coverage` is aggregate coverage metadata; record-level provenance remains in the public projection rows. A release must have an exact `cleared` entry in `uec.source_rights_decisions` for every source artifact contributing visible output. Coverage can still report `attribution_required` when source metadata carries an attribution obligation; attribution is not the rights decision. The bulk packager refuses missing, unknown, restricted, out-of-scope, or ambiguous decisions. Hashing at promotion does not freeze later distribution bytes. The database promotion and writing `--manifest` to disk are separate operations; if the file write fails, the database manifest remains stored and an operator must recover and verify it before distribution. These limits require operational review before making a full release-integrity claim. Migration 022 binds publication review events to releases. After importing the candidate, use the actual `release_id` recorded in `uec.releases` and `uec.release_members` when an authorized maintainer records a reviewed decision. For example, a maintainer can parameterize the following SQL with a source record ID from the candidate and the candidate's real release ID; the values and decision must be chosen by that maintainer: diff --git a/docs/architecture/source-rights-decisions.md b/docs/architecture/source-rights-decisions.md new file mode 100644 index 0000000..ee2aa94 --- /dev/null +++ b/docs/architecture/source-rights-decisions.md @@ -0,0 +1,30 @@ +# Source redistribution decisions + +The release gates require a recorded redistribution decision for every +immutable source artifact that contributes a default-visible row to the +selected release. The decision key is the canonical `source_id`, release +`profile`, `release_id`, and the exact `raw_artifacts.artifact_id` plus its +immutable SHA-256 digest. A release that combines two source snapshots needs +two decisions even when both snapshots use the same source ID. + +`uec.source_rights_decisions` is append-only. Each entry records +`cleared`, `unknown`, or `restricted`, an attributable decision actor, a +decision reference, and the decision time. The actor value is an audit +reference; it does not establish that the actor was authorized. The trusted +operator and approval boundary must enforce that separately. No pipeline gate +creates a decision or grants source rights. + +The shared gate evaluates all exact requirements. It uses the newest decision +time for each source/artifact/release scope. Multiple decisions at that time +are acceptable only when they agree; conflicting newest decisions are +ambiguous and block. Missing, `unknown`, `restricted`, profile-mismatched, +release-mismatched, and artifact-mismatched decisions block closed-world. +Attribution text remains source metadata and may describe an attribution +obligation, but it never substitutes for a redistribution decision. + +Acquisition permission is a separate source or run decision. A permitted +acquisition does not imply redistribution permission, and a cleared +redistribution decision does not authorize a new acquisition. This sprint +checks the decision at validation, promotion, discovery read-model build, and +package export time. It does not implement ongoing revocation or expiry of +already-served releases; those remain launch blockers. diff --git a/docs/data-product.md b/docs/data-product.md index 85668f3..2f67fe4 100644 --- a/docs/data-product.md +++ b/docs/data-product.md @@ -20,7 +20,7 @@ python pipeline/scripts/stages/export-release.py RELEASE_ID ` --output-dir data/releases/RELEASE_ID-official ``` -The database query uses a repeatable-read snapshot and repeats the release/profile, publication review, privacy screening, and current suppression gates. It does not validate, promote, deploy, or publish a release. Source `attribution` is treated as `attribution_required`; missing attribution is `unknown` and blocks packaging until reuse status is reviewed. This is a conservative source-rights gate, not a claim that attribution alone grants redistribution rights. +The database query uses a repeatable-read snapshot and repeats the release/profile, publication review, privacy screening, current suppression, and exact source-rights gates. It does not validate, promote, deploy, or publish a release. Source `attribution` remains presentation metadata; it is not a redistribution decision. Packaging requires an attributable, release/profile/source/artifact-scoped `cleared` decision for every contributing immutable artifact. Missing, unknown, restricted, or out-of-scope decisions block closed-world. This build-time check does not implement ongoing revocation or expiry enforcement for an already-served release. ## Release metadata diff --git a/pipeline/common/source_rights.py b/pipeline/common/source_rights.py new file mode 100644 index 0000000..7a3455e --- /dev/null +++ b/pipeline/common/source_rights.py @@ -0,0 +1,172 @@ +"""Exact, release-scoped source redistribution rights gates. + +The ledger is append-only and records attributable owner decisions. This +module only verifies that a trusted operator has recorded a matching decision; +it does not grant authority, make a legal determination, or perform approval +writes. Acquisition permission and source attribution remain separate. +""" + +from __future__ import annotations + +from collections import defaultdict +from dataclasses import dataclass +from typing import Any + + +class SourceRightsBlocked(ValueError): + """A release has no unambiguous cleared decision for every input artifact.""" + + +@dataclass(frozen=True) +class RightsRequirement: + source_id: str + profile: str + release_id: str + artifact_id: str + artifact_sha256: str + status: str + decision_ids: tuple[str, ...] + + +# The requirement set is deliberately based on every default-visible release +# member, deduplicated by immutable artifact digest. A release made from two +# source snapshots therefore needs two exact decisions. Current suppression +# removes a record from the output and is already independently enforced by +# every projection/package query. +REQUIRED_ARTIFACTS_SQL = """ +SELECT DISTINCT + source.source_id, + release.profile, + release.release_id, + artifact.artifact_id, + artifact.sha256 +FROM uec.release_members member +JOIN uec.releases release ON release.release_id = member.release_id +JOIN uec.observations observation ON observation.observation_id = member.observation_id +JOIN uec.source_records record ON record.source_record_id = observation.source_record_id +JOIN uec.sources source ON source.source_id = record.source_id +JOIN uec.raw_artifacts artifact ON artifact.artifact_id = record.artifact_id +WHERE member.release_id = %s + AND member.default_visible = true + AND NOT EXISTS ( + SELECT 1 + FROM uec.public_access_restricted restricted + WHERE restricted.source_record_id = record.source_record_id + ) +ORDER BY source.source_id, release.profile, release.release_id, artifact.artifact_id +""" + +DECISIONS_SQL = """ +SELECT source_rights_decision_id::text, + source_id, + profile, + release_id, + artifact_id::text, + artifact_sha256, + redistribution_status, + decision_actor, + decision_reference, + decided_at +FROM uec.source_rights_decisions +WHERE source_id = ANY(%s) + AND profile = %s + AND release_id = %s + AND artifact_id = ANY(%s::uuid[]) +ORDER BY source_id, artifact_id, decided_at DESC, source_rights_decision_id DESC +""" + + +def _field(row: Any, name: str, index: int) -> Any: + if isinstance(row, dict): + return row[name] + return row[index] + + +def evaluate(connection: Any, release_id: str) -> dict[str, Any]: + """Return exact rights status for all artifact versions in a release. + + Multiple decisions at the newest timestamp are allowed only when they + agree. Conflicting newest decisions are ambiguous and block the release; + an older cleared decision cannot override a newer unknown/restricted one. + """ + + required_rows = connection.execute(REQUIRED_ARTIFACTS_SQL, (release_id,)).fetchall() + if not required_rows: + return {"requirements": [], "blockers": [], "status": "cleared"} + + required = [ + { + "source_id": _field(row, "source_id", 0), + "profile": _field(row, "profile", 1), + "release_id": _field(row, "release_id", 2), + "artifact_id": str(_field(row, "artifact_id", 3)), + "artifact_sha256": str(_field(row, "sha256", 4)), + } + for row in required_rows + ] + source_ids = sorted({item["source_id"] for item in required}) + profiles = {item["profile"] for item in required} + release_ids = {item["release_id"] for item in required} + if len(profiles) != 1 or release_ids != {release_id}: + return { + "requirements": required, + "blockers": [{**item, "reason": "release identity is ambiguous"} for item in required], + "status": "blocked", + } + artifact_ids = [item["artifact_id"] for item in required] + decision_rows = connection.execute( + DECISIONS_SQL, + (source_ids, next(iter(profiles)), release_id, artifact_ids), + ).fetchall() + by_scope: dict[tuple[str, str], list[dict[str, Any]]] = defaultdict(list) + for row in decision_rows: + by_scope[(str(_field(row, "source_id", 1)), str(_field(row, "artifact_id", 4)))].append( + { + "decision_id": str(_field(row, "source_rights_decision_id", 0)), + "status": _field(row, "redistribution_status", 6), + "actor": _field(row, "decision_actor", 7), + "reference": _field(row, "decision_reference", 8), + "decided_at": _field(row, "decided_at", 9), + } + ) + + evaluated: list[dict[str, Any]] = [] + blockers: list[dict[str, Any]] = [] + for item in required: + decisions = by_scope.get((item["source_id"], item["artifact_id"]), []) + latest_at = max((decision["decided_at"] for decision in decisions), default=None) + latest = [decision for decision in decisions if decision["decided_at"] == latest_at] if latest_at is not None else [] + statuses = {decision["status"] for decision in latest} + status = next(iter(statuses)) if len(statuses) == 1 else "ambiguous" + if not latest: + reason = "missing exact decision" + elif len(statuses) > 1: + reason = "conflicting decisions at the latest decision time" + elif status != "cleared": + reason = f"redistribution decision is {status}" + else: + reason = None + result = { + **item, + "status": status if latest else "unknown", + "decision_ids": tuple(decision["decision_id"] for decision in latest), + } + evaluated.append(result) + if reason: + blockers.append({**item, "reason": reason, "decision_ids": result["decision_ids"]}) + return { + "requirements": evaluated, + "blockers": blockers, + "status": "cleared" if not blockers else "blocked", + } + + +def require_cleared(connection: Any, release_id: str) -> dict[str, Any]: + result = evaluate(connection, release_id) + if result["blockers"]: + details = "; ".join( + f"{item['source_id']}:{item['artifact_sha256']} ({item['reason']})" + for item in result["blockers"] + ) + raise SourceRightsBlocked(f"source redistribution rights gate failed: {details}") + return result diff --git a/pipeline/migrations/041_source_rights_decisions.sql b/pipeline/migrations/041_source_rights_decisions.sql new file mode 100644 index 0000000..a02bfe4 --- /dev/null +++ b/pipeline/migrations/041_source_rights_decisions.sql @@ -0,0 +1,72 @@ +-- Release-scoped source redistribution decisions. +-- +-- This is deliberately separate from acquisition metadata and from the +-- source.attribution presentation field. A decision is attributable evidence +-- that an authorized owner/maintainer reviewed one immutable source artifact +-- for one canonical source/profile/release scope. The actor string records +-- attribution; application authorization must be enforced by the operator +-- boundary and is not inferred from this value. +CREATE TABLE uec.source_rights_decisions ( + source_rights_decision_id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + source_id TEXT NOT NULL REFERENCES uec.sources(source_id), + profile TEXT NOT NULL CHECK (profile IN ('official', 'secondary', 'community')), + release_id TEXT NOT NULL REFERENCES uec.releases(release_id), + artifact_id UUID NOT NULL REFERENCES uec.raw_artifacts(artifact_id), + artifact_sha256 CHAR(64) NOT NULL CHECK (artifact_sha256 ~ '^[0-9a-f]{64}$'), + redistribution_status TEXT NOT NULL CHECK (redistribution_status IN ('cleared', 'unknown', 'restricted')), + decision_actor TEXT NOT NULL CHECK (btrim(decision_actor) <> ''), + decision_reference TEXT NOT NULL CHECK (btrim(decision_reference) <> ''), + decided_at TIMESTAMPTZ NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE INDEX source_rights_decisions_scope_idx + ON uec.source_rights_decisions + (source_id, profile, release_id, artifact_id, decided_at DESC, + source_rights_decision_id DESC); + +CREATE FUNCTION uec.validate_source_rights_decision() +RETURNS trigger +LANGUAGE plpgsql +AS $$ +DECLARE + release_profile TEXT; + artifact_digest CHAR(64); +BEGIN + SELECT profile INTO release_profile + FROM uec.releases + WHERE release_id = NEW.release_id; + IF release_profile IS NULL THEN + RAISE EXCEPTION 'rights decision release does not exist: %', NEW.release_id; + END IF; + IF release_profile IS DISTINCT FROM NEW.profile THEN + RAISE EXCEPTION 'rights decision profile does not match release profile'; + END IF; + + SELECT sha256 INTO artifact_digest + FROM uec.raw_artifacts + WHERE artifact_id = NEW.artifact_id; + IF artifact_digest IS NULL THEN + RAISE EXCEPTION 'rights decision artifact does not exist: %', NEW.artifact_id; + END IF; + IF artifact_digest IS DISTINCT FROM NEW.artifact_sha256 THEN + RAISE EXCEPTION 'rights decision artifact digest does not match immutable artifact'; + END IF; + RETURN NEW; +END; +$$; + +CREATE TRIGGER source_rights_decisions_scope_guard + BEFORE INSERT ON uec.source_rights_decisions + FOR EACH ROW EXECUTE FUNCTION uec.validate_source_rights_decision(); + +CREATE TRIGGER source_rights_decisions_append_only + BEFORE UPDATE OR DELETE ON uec.source_rights_decisions + FOR EACH ROW EXECUTE FUNCTION uec.reject_evidence_mutation(); + +COMMENT ON TABLE uec.source_rights_decisions IS + 'Append-only attributable redistribution decisions; acquisition permission, attribution, and operator authorization remain separate.'; +COMMENT ON COLUMN uec.source_rights_decisions.artifact_sha256 IS + 'Immutable source-version digest. Every contributing artifact needs its own exact decision.'; +COMMENT ON COLUMN uec.source_rights_decisions.decision_actor IS + 'Attributable actor reference only; trusted operator authorization is enforced outside this ledger.'; diff --git a/pipeline/scripts/maintenance/build_public_discovery_read_model.py b/pipeline/scripts/maintenance/build_public_discovery_read_model.py index 2d6b829..06d56dc 100644 --- a/pipeline/scripts/maintenance/build_public_discovery_read_model.py +++ b/pipeline/scripts/maintenance/build_public_discovery_read_model.py @@ -12,6 +12,17 @@ import psycopg +# Keep the documented ``python pipeline/scripts/...`` invocation independent +# of the caller's PYTHONPATH. +from pathlib import Path + +REPOSITORY_ROOT = Path(__file__).resolve().parents[3] +import sys +if str(REPOSITORY_ROOT) not in sys.path: + sys.path.insert(0, str(REPOSITORY_ROOT)) + +from pipeline.common.source_rights import require_cleared + class ReadModelBlocked(ValueError): """The read model cannot safely be activated.""" @@ -193,7 +204,7 @@ def _release_manifest(connection: Any, release_id: str) -> str: eligible.provenance_source_url, eligible.provenance_retrieved_at, CASE WHEN source.attribution IS NULL OR btrim(source.attribution) = '' - THEN 'unknown' ELSE 'attribution_required' END AS source_rights_status + THEN 'cleared' ELSE 'attribution_required' END AS source_rights_status FROM eligible JOIN uec.sources source ON source.source_id = eligible.provenance_source_id LEFT JOIN LATERAL ( @@ -258,6 +269,7 @@ def build(database_url: str, release_id: str, fail_after_rows: int | None = None with psycopg.connect(database_url) as connection: with connection.transaction(): manifest_sha256 = _release_manifest(connection, release_id) + require_cleared(connection, release_id) rows = connection.execute(SELECT_ROWS, (release_id,)).fetchall() content_sha256 = content_digest(rows) existing = connection.execute( diff --git a/pipeline/scripts/stages/export-release.py b/pipeline/scripts/stages/export-release.py index ee5072b..0e0849f 100644 --- a/pipeline/scripts/stages/export-release.py +++ b/pipeline/scripts/stages/export-release.py @@ -24,6 +24,7 @@ sys.path.insert(0, str(REPOSITORY_ROOT)) from pipeline.common.data_product import SUPPORTED_PROFILES, write_package +from pipeline.common.source_rights import require_cleared def _utc(value) -> str: @@ -56,6 +57,7 @@ def export_release(database_url: str, release_id: str, profile: str, output_dir: raise ValueError("release/profile not found") if release[2] != "promoted" or release[3]: raise ValueError("only a non-test promoted release can be packaged") + require_cleared(connection, release_id) rows = connection.execute( """ SELECT h.facility_id, h.canonical_name, h.country_code, h.city, @@ -92,7 +94,9 @@ def export_release(database_url: str, release_id: str, profile: str, output_dir: projection = [] for row in rows: - rights = _rights(row[21]) + # require_cleared above proves the decision; attribution remains an + # independent presentation obligation and does not grant clearance. + rights = "attribution_required" if row[21] and row[21].strip() else "cleared" projection.append( { "facility_id": str(row[0]), "canonical_name": row[1], "country_code": row[2], @@ -112,6 +116,7 @@ def export_release(database_url: str, release_id: str, profile: str, output_dir: "release_ruleset_version": release[4], "provenance_source_id": row[17], "provenance_source_name": row[18], "provenance_source_url": row[19], "provenance_retrieved_at": _utc(row[20]), "source_rights_status": rights, + "source_attribution": row[21], "publication_eligible": True, } ) @@ -124,6 +129,7 @@ def export_release(database_url: str, release_id: str, profile: str, output_dir: "row_count": len(source_rows), "retrieved_at": {"first": min(r["provenance_retrieved_at"] for r in source_rows), "last": max(r["provenance_retrieved_at"] for r in source_rows)}, "rights_status": sorted({r["source_rights_status"] for r in source_rows}), + "attribution": sorted({r["source_attribution"] for r in source_rows if r.get("source_attribution")}), } for source_id, source_rows in sorted(by_source.items()) ] diff --git a/pipeline/scripts/stages/promote-release.py b/pipeline/scripts/stages/promote-release.py index e70b4a1..9f14012 100644 --- a/pipeline/scripts/stages/promote-release.py +++ b/pipeline/scripts/stages/promote-release.py @@ -11,6 +11,12 @@ import psycopg +REPOSITORY_ROOT = Path(__file__).resolve().parents[3] +if str(REPOSITORY_ROOT) not in sys.path: + sys.path.insert(0, str(REPOSITORY_ROOT)) + +from pipeline.common.source_rights import require_cleared + def can_promote(status: str, test_only: bool = False) -> bool: return status == "validated" and not test_only @@ -66,6 +72,7 @@ def promote(database_url: str, release_id: str, artifacts: list[dict]) -> dict: if target[3]: raise ValueError("test-only releases cannot be validated or promoted") raise ValueError(f"release must be validated before promotion; current status is {target[0]}") + rights_gate = require_cleared(connection, release_id) unsafe = connection.execute(""" SELECT count(*) FILTER (WHERE m.default_visible AND (g.status IS DISTINCT FROM 'accepted' OR g.result IS NULL)), @@ -82,8 +89,8 @@ def promote(database_url: str, release_id: str, artifacts: list[dict]) -> dict: LEFT JOIN uec.public_access_restricted s ON s.source_record_id=o.source_record_id WHERE m.release_id=%s """, (release_id,)).fetchone() - if any(unsafe): - raise ValueError(f"release safety gates failed: coordinate_not_ready={unsafe[0]}, review_required={unsafe[1]}, publication_not_approved={unsafe[2]}, active_suppression={unsafe[3]}, rights_not_cleared={unsafe[4]}") + if any(unsafe) or rights_gate["blockers"]: + raise ValueError(f"release safety gates failed: coordinate_not_ready={unsafe[0]}, review_required={unsafe[1]}, publication_not_approved={unsafe[2]}, active_suppression={unsafe[3]}, rights_not_cleared={unsafe[4] + len(rights_gate['blockers'])}") demonstration = target[4].get("demonstration") if isinstance(target[4], dict) else None if demonstration is not None and demonstration.get("review_status") != "approved": raise ValueError("demonstration release requires an explicit recorded review") @@ -96,8 +103,7 @@ def promote(database_url: str, release_id: str, artifacts: list[dict]) -> dict: AND NOT EXISTS (SELECT 1 FROM uec.public_access_restricted x WHERE x.source_record_id=sr.source_record_id) """, (release_id,)).fetchone() coverage_rows = connection.execute(""" - SELECT sr.source_id, count(*)::int, min(artifact.retrieved_at), max(artifact.retrieved_at), - CASE WHEN source.attribution IS NULL OR btrim(source.attribution) = '' THEN 'unknown' ELSE 'attribution_required' END + SELECT sr.source_id, count(*)::int, min(artifact.retrieved_at), max(artifact.retrieved_at), source.attribution FROM uec.release_members m JOIN uec.observations o ON o.observation_id=m.observation_id JOIN uec.source_records sr ON sr.source_record_id=o.source_record_id @@ -111,8 +117,8 @@ def promote(database_url: str, release_id: str, artifacts: list[dict]) -> dict: if created_at.tzinfo is None: raise ValueError("database manifest creation time must include a timezone") source_coverage = [ - {"source_id": source_id, "row_count": row_count, "retrieved_at": {"first": utc_iso(first), "last": utc_iso(last)}, "rights_status": rights_status} - for source_id, row_count, first, last, rights_status in coverage_rows + {"source_id": source_id, "row_count": row_count, "retrieved_at": {"first": utc_iso(first), "last": utc_iso(last)}, "rights_status": "cleared", "attribution": attribution} + for source_id, row_count, first, last, attribution in coverage_rows ] retrieved_at = min((item["retrieved_at"]["first"] for item in source_coverage), default=utc_iso(created_at)) manifest = { @@ -141,6 +147,7 @@ def promote(database_url: str, release_id: str, artifacts: list[dict]) -> dict: ], "supersedes": previous[0] if previous else None, "rights_review": (demonstration or {}).get("rights_status") if demonstration else "not-recorded", + "source_rights_gate": "cleared", "created_at": utc_iso(created_at), "distributed_artifacts": artifacts, } diff --git a/pipeline/scripts/stages/validate-release.py b/pipeline/scripts/stages/validate-release.py index 5aa01d6..8d15410 100644 --- a/pipeline/scripts/stages/validate-release.py +++ b/pipeline/scripts/stages/validate-release.py @@ -10,6 +10,12 @@ import psycopg +REPOSITORY_ROOT = Path(__file__).resolve().parents[3] +if str(REPOSITORY_ROOT) not in sys.path: + sys.path.insert(0, str(REPOSITORY_ROOT)) + +from pipeline.common.source_rights import evaluate as evaluate_source_rights + def evaluate(metrics: dict, expected_records: int | None = None) -> dict: findings = [] @@ -79,7 +85,12 @@ def validate(database_url: str, release_id: str, expected_records: int | None, m WHERE release_member.release_id = %s """, (release_id, release_id)).fetchone() names = ["release_records", "distinct_observations", "duplicate_observations", "review_visible", "exact_display_ready", "city_display_ready", "unmapped_display", "coordinate_not_ready", "publication_not_approved", "active_suppression", "rights_not_cleared", "validation_errors"] - metrics_dict = dict(zip(names, metrics)); metrics_dict["test_only"] = bool(release[1]) + metrics_dict = dict(zip(names, metrics)) + rights_gate = evaluate_source_rights(connection, release_id) + metrics_dict["rights_not_cleared"] = len(rights_gate["blockers"]) + metrics_dict["rights_gate"] = rights_gate["status"] + metrics_dict["rights_requirements"] = len(rights_gate["requirements"]) + metrics_dict["test_only"] = bool(release[1]) result = evaluate(metrics_dict, expected_records) result.update({"release_id": release_id, "release_status_before": release[0], "marked_validated": False}) if result["status"] == "passed" and mark_validated: diff --git a/pipeline/tests/test_graph_migrations.py b/pipeline/tests/test_graph_migrations.py index a04fbd9..9185b08 100644 --- a/pipeline/tests/test_graph_migrations.py +++ b/pipeline/tests/test_graph_migrations.py @@ -20,7 +20,7 @@ def test_reserved_migrations_are_present_and_ordered(self): positions = [migrations.index(name) for name in graph_migrations] self.assertEqual(positions, sorted(positions)) self.assertEqual([migrations[position] for position in positions], graph_migrations) - self.assertEqual(migrations[-15:], [ + expected_graph_suffix = [ "026_graph_entities_crosswalks.sql", "027_graph_relationship_observations.sql", "028_graph_claims_support.sql", @@ -36,7 +36,11 @@ def test_reserved_migrations_are_present_and_ordered(self): "038_graph_regulatory_authority_relationship.sql", "039_private_graph_query_indexes.sql", "040_geocode_worker_durability.sql", - ]) + "041_source_rights_decisions.sql", + ] + graph_start = migrations.index(expected_graph_suffix[0]) + graph_end = migrations.index(expected_graph_suffix[-1]) + 1 + self.assertEqual(migrations[graph_start:graph_end], expected_graph_suffix) def test_entities_are_distinct_and_crosswalk_is_scoped(self): sql = self.read("026_graph_entities_crosswalks.sql") diff --git a/pipeline/tests/test_source_rights.py b/pipeline/tests/test_source_rights.py new file mode 100644 index 0000000..0ab94d1 --- /dev/null +++ b/pipeline/tests/test_source_rights.py @@ -0,0 +1,102 @@ +import importlib +import unittest +from datetime import datetime, timezone + + +MODULE = importlib.import_module("pipeline.common.source_rights") + + +class _Result: + def __init__(self, rows): + self.rows = rows + + def fetchall(self): + return self.rows + + +class _Connection: + def __init__(self, required, decisions=()): + self.required = required + self.decisions = decisions + + def execute(self, query, params): + if "FROM uec.release_members" in query: + return _Result(self.required) + return _Result(self.decisions) + + +def required(source="source-a", artifact="00000000-0000-0000-0000-000000000001", digest="a" * 64): + return (source, "official", "release-a", artifact, digest) + + +def decision(source, artifact, digest, status, when, decision_id="10000000-0000-0000-0000-000000000001"): + return (decision_id, source, "official", "release-a", artifact, digest, status, "synthetic-owner", "synthetic-reference", when) + + +class SourceRightsTests(unittest.TestCase): + def setUp(self): + self.when = datetime(2026, 9, 18, tzinfo=timezone.utc) + self.artifact = "00000000-0000-0000-0000-000000000001" + self.digest = "a" * 64 + + def test_missing_decision_blocks_even_when_source_has_attribution(self): + result = MODULE.evaluate(_Connection([required()]), "release-a") + self.assertEqual(result["status"], "blocked") + self.assertEqual(result["blockers"][0]["reason"], "missing exact decision") + + def test_every_distinct_artifact_needs_its_own_clearance(self): + second_artifact = "00000000-0000-0000-0000-000000000002" + result = MODULE.evaluate( + _Connection( + [required(), required(artifact=second_artifact, digest="b" * 64)], + [decision("source-a", self.artifact, self.digest, "cleared", self.when)], + ), + "release-a", + ) + self.assertEqual(result["status"], "blocked") + self.assertEqual(len(result["blockers"]), 1) + self.assertEqual(result["blockers"][0]["artifact_id"], second_artifact) + + def test_newer_restricted_decision_overrides_older_clearance(self): + result = MODULE.evaluate( + _Connection( + [required()], + [ + decision("source-a", self.artifact, self.digest, "restricted", self.when, "20000000-0000-0000-0000-000000000001"), + decision("source-a", self.artifact, self.digest, "cleared", datetime(2026, 9, 17, tzinfo=timezone.utc)), + ], + ), + "release-a", + ) + self.assertEqual(result["blockers"][0]["reason"], "redistribution decision is restricted") + + def test_conflicting_latest_decisions_fail_closed(self): + result = MODULE.evaluate( + _Connection( + [required()], + [ + decision("source-a", self.artifact, self.digest, "cleared", self.when, "30000000-0000-0000-0000-000000000001"), + decision("source-a", self.artifact, self.digest, "unknown", self.when, "40000000-0000-0000-0000-000000000001"), + ], + ), + "release-a", + ) + self.assertEqual(result["blockers"][0]["reason"], "conflicting decisions at the latest decision time") + + def test_same_status_latest_duplicates_are_deterministically_clear(self): + result = MODULE.evaluate( + _Connection( + [required()], + [ + decision("source-a", self.artifact, self.digest, "cleared", self.when, "50000000-0000-0000-0000-000000000001"), + decision("source-a", self.artifact, self.digest, "cleared", self.when, "60000000-0000-0000-0000-000000000001"), + ], + ), + "release-a", + ) + self.assertEqual(result["status"], "cleared") + self.assertEqual(len(result["requirements"][0]["decision_ids"]), 2) + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/tests/test_source_rights_migration.py b/pipeline/tests/test_source_rights_migration.py new file mode 100644 index 0000000..eb11881 --- /dev/null +++ b/pipeline/tests/test_source_rights_migration.py @@ -0,0 +1,50 @@ +import unittest +from pathlib import Path + + +ROOT = Path(__file__).parents[1] + + +class SourceRightsMigrationContractTests(unittest.TestCase): + def test_reserved_migration_is_append_only_and_exactly_scoped(self): + migration = (ROOT / "migrations" / "041_source_rights_decisions.sql").read_text(encoding="utf-8").lower() + for token in ( + "source_rights_decisions", + "source_id", + "profile", + "release_id", + "artifact_id", + "artifact_sha256", + "redistribution_status", + "decision_actor", + "decision_reference", + "append_only", + "validate_source_rights_decision", + ): + self.assertIn(token, migration) + self.assertIn("before update or delete", migration) + self.assertIn("release_profile is distinct from new.profile", migration) + self.assertIn("artifact_digest is distinct from new.artifact_sha256", migration) + self.assertNotIn("attribution", migration.split("create table", 1)[1].split("create index", 1)[0]) + + def test_contract_documents_acquisition_and_authorization_boundaries(self): + docs = (ROOT.parents[0] / "docs" / "architecture" / "source-rights-decisions.md").read_text(encoding="utf-8") + self.assertIn("Acquisition permission is a separate", docs) + self.assertIn("does not establish that the actor was authorized", docs) + self.assertIn("ongoing revocation or expiry", docs) + + def test_all_four_boundaries_use_the_shared_gate(self): + paths = ( + ROOT / "scripts" / "stages" / "validate-release.py", + ROOT / "scripts" / "stages" / "promote-release.py", + ROOT / "scripts" / "stages" / "export-release.py", + ROOT / "scripts" / "maintenance" / "build_public_discovery_read_model.py", + ) + for path in paths: + source = path.read_text(encoding="utf-8") + self.assertIn("source_rights", source, path.name) + self.assertIn("release_id", source, path.name) + + +if __name__ == "__main__": + unittest.main() From b9fc9554eee6309a84ce2ea93d98214f21cb5aed Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Fri, 18 Sep 2026 19:57:14 -0700 Subject: [PATCH 272/311] test: seed exact rights for synthetic release fixtures --- pipeline/tests/e2e/fixture.py | 56 +++++++++++++++++++ pipeline/tests/e2e/run-suite.ps1 | 1 + .../tests/test_publication_scoped_stages.py | 18 ++++++ 3 files changed, 75 insertions(+) diff --git a/pipeline/tests/e2e/fixture.py b/pipeline/tests/e2e/fixture.py index 9bd697e..2070694 100644 --- a/pipeline/tests/e2e/fixture.py +++ b/pipeline/tests/e2e/fixture.py @@ -399,8 +399,63 @@ def seed_official_scenario(self): def build_public_read_model(self, release_id): """Activate a synthetic model through the same atomic operator flow.""" + self._seed_synthetic_rights_decisions(release_id) return _read_model_builder().build(self.database_url, release_id) + def _seed_synthetic_rights_decisions(self, release_id): + """Record exact rights decisions for this disposable release. + + Migration 041 scopes each decision to the immutable source artifact + and release profile. Synthetic E2E data can provide an attributable + cleared decision for every release member before a model build or + export gate is exercised. + """ + now = datetime.now(timezone.utc) + with psycopg.connect(self.database_url) as db: + with db.transaction(): + db.execute( + """ + INSERT INTO uec.source_rights_decisions ( + source_id, profile, release_id, artifact_id, + artifact_sha256, redistribution_status, decision_actor, + decision_reference, decided_at + ) + SELECT DISTINCT + source.source_id, + release.profile, + release.release_id, + artifact.artifact_id, + artifact.sha256, + 'cleared', + 'synthetic-e2e-fixture', + 'synthetic rights fixture', + %s + FROM uec.release_members member + JOIN uec.releases release + ON release.release_id = member.release_id + JOIN uec.observations observation + ON observation.observation_id = member.observation_id + JOIN uec.source_records record + ON record.source_record_id = observation.source_record_id + JOIN uec.sources source + ON source.source_id = record.source_id + JOIN uec.raw_artifacts artifact + ON artifact.artifact_id = record.artifact_id + WHERE member.release_id = %s + AND member.default_visible = true + AND NOT EXISTS ( + SELECT 1 + FROM uec.source_rights_decisions existing + WHERE existing.source_id = source.source_id + AND existing.profile = release.profile + AND existing.release_id = release.release_id + AND existing.artifact_id = artifact.artifact_id + AND existing.artifact_sha256 = artifact.sha256 + ) + """, + (now, release_id), + ) + def create_failed_candidate(self): """Create an invalid candidate without touching the promoted release.""" with psycopg.connect(self.database_url) as db: @@ -439,6 +494,7 @@ def seed_private_candidate_scenario(self): db.execute("INSERT INTO uec.geocode_results (source_record_id,provider_id,query,match_method,status,attempt_number,result,queried_at) VALUES (%s,'e2e','synthetic candidate','fixture','accepted',1,ST_SetSRID(ST_MakePoint(12,56),4326)::geography,%s)", (pending_record, now)) db.execute("INSERT INTO uec.publication_review_events (source_record_id,release_id,factual_review_status,privacy_screening_status,maintainer_approval,publication_eligible,reviewer_role) VALUES (%s,'e2e-private-candidate','reviewed','passed','approved',true,'maintainer')", (pending_record,)) self.private_candidate_facility_id = facility + self._seed_synthetic_rights_decisions('e2e-private-candidate') def seed_private_graph_scenario(self): """Seed a bounded synthetic graph for authenticated HTTP contract tests.""" diff --git a/pipeline/tests/e2e/run-suite.ps1 b/pipeline/tests/e2e/run-suite.ps1 index 2235d94..c272ab6 100644 --- a/pipeline/tests/e2e/run-suite.ps1 +++ b/pipeline/tests/e2e/run-suite.ps1 @@ -12,6 +12,7 @@ $core = @( 'pipeline.tests.e2e.test_public_surface_safety', 'pipeline.tests.e2e.test_candidate_import', 'pipeline.tests.e2e.test_private_graph', + 'pipeline.tests.e2e.test_worker_durability', 'pipeline.tests.e2e.test_readiness' ) $extended = @( diff --git a/pipeline/tests/test_publication_scoped_stages.py b/pipeline/tests/test_publication_scoped_stages.py index 2455b1f..b495b32 100644 --- a/pipeline/tests/test_publication_scoped_stages.py +++ b/pipeline/tests/test_publication_scoped_stages.py @@ -61,6 +61,24 @@ def test_legacy_ambiguity_and_explicit_scope(self): for release_id in (release_a, release_b): db.execute("INSERT INTO uec.releases (release_id,status,ruleset_version,profile,summary) VALUES (%s,'candidate','synthetic-v1','secondary','{}')", (release_id,)) db.execute("INSERT INTO uec.release_members (release_id,facility_id,observation_id,default_visible) VALUES (%s,%s,%s,true)", (release_id, facility_id, observation_id)) + # Migration 041 adds an exact source/profile/release/artifact + # gate to promotion. Seed both synthetic releases so this test + # continues to exercise publication scope independently. + if db.execute("SELECT to_regclass('uec.source_rights_decisions')").fetchone()[0] is not None: + for release_id in (release_a, release_b): + db.execute( + """ + INSERT INTO uec.source_rights_decisions + (source_id,profile,release_id,artifact_id,artifact_sha256, + redistribution_status,decision_actor,decision_reference,decided_at) + SELECT %s,'secondary',%s,%s,sha256,'cleared', + 'synthetic-publication-scope-test', + 'synthetic publication scope fixture',now() + FROM uec.raw_artifacts + WHERE artifact_id=%s + """, + (prefix, release_id, artifact_id, artifact_id), + ) db.execute("INSERT INTO uec.geocode_results (source_record_id,provider_id,query,match_method,status,result,queried_at) VALUES (%s,'synthetic','synthetic','test','accepted',ST_SetSRID(ST_MakePoint(10,55),4326)::geography,now())", (record_id,)) has_scope = db.execute("SELECT to_regclass('uec.publication_review_release_scopes')").fetchone()[0] is not None if not has_scope: From ba787ced0e0a4a8527271fdacde152749f52468e Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Fri, 18 Sep 2026 19:56:55 -0700 Subject: [PATCH 273/311] Fail closed on APHIS profile mismatch --- .../us/accountability/aphis_evidence.py | 20 ++++++++++++----- .../us/accountability/test_aphis_evidence.py | 22 +++++++++++++++++++ 2 files changed, 37 insertions(+), 5 deletions(-) diff --git a/pipeline/sources/us/accountability/aphis_evidence.py b/pipeline/sources/us/accountability/aphis_evidence.py index e9175c4..7ec8c93 100644 --- a/pipeline/sources/us/accountability/aphis_evidence.py +++ b/pipeline/sources/us/accountability/aphis_evidence.py @@ -481,12 +481,13 @@ def run_from_wave2( def _load_verified_exports( verification: Mapping[str, Any], manifest: Mapping[str, Any], -) -> tuple[dict[str, list[dict[str, Any]]], dict[str, list[dict[str, Any]]], dict[tuple[str, str], dict[str, Any]], dict[str, int]]: +) -> tuple[dict[str, list[dict[str, Any]]], dict[str, list[dict[str, Any]]], dict[tuple[str, str], dict[str, Any]], dict[str, int], list[dict[str, Any]]]: """Parse only verified manifest-named exports for the standalone CLI.""" records: dict[str, list[dict[str, Any]]] = defaultdict(list) quarantined: dict[str, list[dict[str, Any]]] = defaultdict(list) provenance: dict[tuple[str, str], dict[str, Any]] = {} input_rows: dict[str, int] = defaultdict(int) + failures: list[dict[str, Any]] = [] adapter = AphisPublicSearchAdapter() report_profiles = manifest.get("profile_metadata") if isinstance(manifest.get("profile_metadata"), Mapping) else {} for item in verification.get("artifacts", ()): @@ -499,7 +500,16 @@ def _load_verified_exports( try: result = adapter.parse_bytes(Path(path).read_bytes()) except (OSError, AphisContractError) as exc: - quarantined[profile].append({"state": "failed", "reasons": [f"adapter_error:{type(exc).__name__}"]}) + failure = {"profile": profile, "artifact": item.get("artifact"), "state": "failed", "failure": f"adapter_error:{type(exc).__name__}"} + failures.append(failure) + quarantined[profile].append({"state": "failed", "reasons": [failure["failure"]]}) + continue + if result.get("profile") != profile: + failure = {"profile": profile, "artifact": item.get("artifact"), "state": "failed", + "failure": "manifest_profile_mismatch", "parsed_profile": result.get("profile")} + failures.append(failure) + quarantined[profile].append({"state": "failed", "reasons": [failure["failure"]], + "parsed_profile": result.get("profile")}) continue artifact = {"artifact_sha256": item.get("actual_sha256"), "source_url": item.get("source_url"), "retrieved_at_utc": item.get("retrieved_at_utc")} @@ -513,7 +523,7 @@ def _load_verified_exports( input_rows[profile] += int(result["input_rows"]) provenance[("us.aphis", profile)] = artifact expected = manifest.get("expected_rows") if isinstance(manifest.get("expected_rows"), Mapping) else {} - return dict(records), dict(quarantined), provenance, dict(input_rows) + return dict(records), dict(quarantined), provenance, dict(input_rows), failures def _cli() -> int: @@ -524,8 +534,8 @@ def _cli() -> int: args = parser.parse_args() manifest = _json(args.manifest) verification = verify_retained_artifacts(args.manifest, artifact_root=args.artifact_root) - records, quarantined, provenance, input_rows = _load_verified_exports(verification, manifest) - failures = [item for item in verification["artifacts"] if item.get("state") != "verified"] + records, quarantined, provenance, input_rows, parse_failures = _load_verified_exports(verification, manifest) + failures = [item for item in verification["artifacts"] if item.get("state") != "verified"] + parse_failures packet = build_packet( records_by_profile=records, provenance=provenance, expected_rows=manifest.get("expected_rows") if isinstance(manifest.get("expected_rows"), Mapping) else {}, diff --git a/pipeline/sources/us/accountability/test_aphis_evidence.py b/pipeline/sources/us/accountability/test_aphis_evidence.py index 1b655f7..2b0d75f 100644 --- a/pipeline/sources/us/accountability/test_aphis_evidence.py +++ b/pipeline/sources/us/accountability/test_aphis_evidence.py @@ -136,6 +136,28 @@ def test_aggregate_metadata_hash_cannot_be_link_provenance(self): self.assertFalse(packet["links"]) self.assertEqual(packet["link_quarantine"][0]["reason"], "link_missing_or_invalid_provenance") + def test_cli_fails_closed_when_manifest_profile_disagrees_with_csv(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + artifact = root / "annual.csv" + shutil.copyfile(ROOT / "fixtures" / "annual_reports.csv", artifact) + manifest = root / "input-manifest.json" + manifest.write_text(json.dumps({"profiles": {"registrations": {"artifacts": [{ + "artifact": artifact.name, "path": str(artifact), + "sha256": hashlib.sha256(artifact.read_bytes()).hexdigest(), "byte_size": artifact.stat().st_size, + "source_url": "https://example.invalid/registrations", "retrieved_at_utc": "2026-09-18T00:00:00Z", + }]}}}), encoding="utf-8") + run_dir = root / "run" + completed = subprocess.run([ + sys.executable, "-m", "pipeline.sources.us.accountability.aphis_evidence", + "--manifest", str(manifest), "--run-dir", str(run_dir), + ], cwd=Path(__file__).parents[4], capture_output=True, text=True, check=True) + output = json.loads(completed.stdout) + summary = json.loads((run_dir / "row-free-summary.json").read_text(encoding="utf-8")) + self.assertEqual(output["input_failures"][0]["failure"], "manifest_profile_mismatch") + self.assertEqual(summary["profiles"]["registrations"]["observed_rows"], 0) + self.assertEqual(summary["profiles"]["registrations"]["coverage_state"], "failed") + def test_documented_cli_parses_verified_exports_and_preserves_coverage_gap(self): with tempfile.TemporaryDirectory() as directory: root = Path(directory) From 9111a273bf7a33350862efca7e17dc051cd3e859 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Fri, 18 Sep 2026 20:02:01 -0700 Subject: [PATCH 274/311] fix: apply private graph confidence filters --- pipeline/tests/e2e/test_private_graph.py | 25 +++++++++++++++++++ pipeline/tests/test_private_graph_contract.py | 3 +++ src/graph_private.rs | 19 +++++++++++++- 3 files changed, 46 insertions(+), 1 deletion(-) diff --git a/pipeline/tests/e2e/test_private_graph.py b/pipeline/tests/e2e/test_private_graph.py index 57cf1cb..d99a21e 100644 --- a/pipeline/tests/e2e/test_private_graph.py +++ b/pipeline/tests/e2e/test_private_graph.py @@ -72,6 +72,31 @@ def test_populated_neighborhood_supports_both_directions_depth_two_and_numeric_c self.assertTrue(all(row["storage_state"] == "private" for row in body["data"])) self.assertTrue(all(row["privacy_status"] == "suppressed" for row in body["data"])) self.assertTrue(all(row["publication_status"] == "not_eligible" for row in body["data"])) + _, high_confidence = self.request( + f"/api/private/graph/entities/{self.ids['organization_a']}/neighborhood" + "?direction=out&depth=1&min_confidence=0.8", + self.token, + ) + self.assertEqual({row["relationship_type"] for row in high_confidence["data"]}, {"operator"}) + self.assertTrue(all(row["confidence"] is not None and row["confidence"] >= 0.8 for row in high_confidence["data"])) + _, reviewed = self.request( + f"/api/private/graph/entities/{self.ids['organization_a']}/neighborhood" + "?direction=out&depth=1&state=review_required", + self.token, + ) + self.assertEqual(len(reviewed["data"]), 2) + _, no_review_match = self.request( + f"/api/private/graph/entities/{self.ids['organization_a']}/neighborhood" + "?direction=out&depth=1&state=accepted", + self.token, + ) + self.assertEqual(no_review_match["data"], []) + _, no_match = self.request( + f"/api/private/graph/entities/{self.ids['organization_a']}/neighborhood" + "?direction=out&depth=1&min_confidence=0.9", + self.token, + ) + self.assertEqual(no_match["data"], []) def test_entity_search_is_authenticated_bounded_and_deterministic(self): _, first = self.request("/api/private/graph/search?q=Synthetic%20graph&limit=100", self.token) diff --git a/pipeline/tests/test_private_graph_contract.py b/pipeline/tests/test_private_graph_contract.py index 5c0c52c..fce8395 100644 --- a/pipeline/tests/test_private_graph_contract.py +++ b/pipeline/tests/test_private_graph_contract.py @@ -16,6 +16,9 @@ def test_both_graph_handlers_cast_numeric_confidence_and_expand_each_direction(s self.assertIn("CASE WHEN d.step_direction='out' THEN e.to_type ELSE e.from_type END", text) self.assertIn("CASE WHEN d.step_direction='out' THEN e.to_id ELSE e.from_id END", text) self.assertIn("ORDER BY o.observed_at DESC, o.relationship_observation_id DESC", text) + graph = (ROOT / "src" / "graph_private.rs").read_text(encoding="utf-8") + self.assertIn("review_state=$8", graph) + self.assertIn("confidence >= $9", graph) def test_queue_uses_existing_source_record_timestamp(self): text = (ROOT / "src" / "graph_private.rs").read_text(encoding="utf-8") diff --git a/src/graph_private.rs b/src/graph_private.rs index d0bca33..a79b05d 100644 --- a/src/graph_private.rs +++ b/src/graph_private.rs @@ -120,6 +120,23 @@ pub async fn neighborhood( }) { return bad("unsupported relationship type"); } + if p.state.as_deref().is_some_and(|v| { + ![ + "unreviewed", + "review_required", + "reviewed", + "accepted", + "rejected", + ] + .contains(&v) + }) { + return bad("unsupported review state"); + } + if p.min_confidence + .is_some_and(|v| !v.is_finite() || !(0.0..=1.0).contains(&v)) + { + return bad("min_confidence must be between 0 and 1"); + } let Some(pool) = state.database else { return v2_error( StatusCode::SERVICE_UNAVAILABLE, @@ -134,7 +151,7 @@ pub async fn neighborhood( "private graph database unavailable", ); }; - let rows = match client.query("WITH RECURSIVE observations AS (SELECT relationship_observation_id, from_organization_id, target_facility_id, target_organization_id, relationship_type, assertion_status, observed_at, confidence::double precision AS confidence, review_state, storage_state, privacy_status, publication_status, source_id, source_record_id FROM uec.organization_relationship_observations WHERE ($4::text IS NULL OR relationship_type=$4) AND ($5::text IS NULL OR source_id=$5) AND ($6::timestamptz IS NULL OR observed_at >= $6) AND ($7::timestamptz IS NULL OR observed_at < $7)), edges AS (SELECT o.*, 'organization'::text AS from_type, o.from_organization_id AS from_id, CASE WHEN o.target_facility_id IS NOT NULL THEN 'facility'::text ELSE 'organization'::text END AS to_type, COALESCE(o.target_facility_id, o.target_organization_id) AS to_id FROM observations o), directions(step_direction) AS (SELECT 'out'::text WHERE $2 IN ('out','both') UNION ALL SELECT 'in'::text WHERE $2 IN ('in','both')), seed AS (SELECT 'organization'::text AS node_type, $1::uuid AS node_id, 0 AS hop WHERE EXISTS (SELECT 1 FROM uec.organizations WHERE organization_id=$1) UNION ALL SELECT 'facility'::text, $1::uuid, 0 WHERE EXISTS (SELECT 1 FROM uec.facilities WHERE facility_id=$1)), walk(node_type, node_id, hop) AS (SELECT node_type, node_id, hop FROM seed UNION SELECT CASE WHEN d.step_direction='out' THEN e.to_type ELSE e.from_type END, CASE WHEN d.step_direction='out' THEN e.to_id ELSE e.from_id END, w.hop + 1 FROM walk w JOIN directions d ON true JOIN edges e ON ((d.step_direction='out' AND e.from_type=w.node_type AND e.from_id=w.node_id) OR (d.step_direction='in' AND e.to_type=w.node_type AND e.to_id=w.node_id)) WHERE w.hop < $3), matched AS (SELECT DISTINCT e.relationship_observation_id FROM edges e JOIN walk w ON w.hop < $3 AND (($2 IN ('out','both') AND e.from_type=w.node_type AND e.from_id=w.node_id) OR ($2 IN ('in','both') AND e.to_type=w.node_type AND e.to_id=w.node_id))) SELECT o.relationship_observation_id, o.from_organization_id, o.target_facility_id, o.target_organization_id, o.relationship_type, o.assertion_status, o.observed_at, o.confidence, o.review_state, o.storage_state, o.privacy_status, o.publication_status, o.source_id, o.source_record_id FROM observations o JOIN matched m USING (relationship_observation_id) ORDER BY o.observed_at DESC, o.relationship_observation_id DESC LIMIT $8", &[&entity_id, &direction, &depth, &p.relationship_type, &p.source_id, &p.from.map(|d| d.and_hms_opt(0,0,0).unwrap().and_utc()), &p.to.map(|d| d.and_hms_opt(0,0,0).unwrap().and_utc()), &limit]).await { Ok(v) => v, Err(_) => return v2_error(StatusCode::SERVICE_UNAVAILABLE, "private_graph_query_failed", "private graph query failed") }; + let rows = match client.query("WITH RECURSIVE observations AS (SELECT relationship_observation_id, from_organization_id, target_facility_id, target_organization_id, relationship_type, assertion_status, observed_at, confidence::double precision AS confidence, review_state, storage_state, privacy_status, publication_status, source_id, source_record_id FROM uec.organization_relationship_observations WHERE ($4::text IS NULL OR relationship_type=$4) AND ($5::text IS NULL OR source_id=$5) AND ($6::timestamptz IS NULL OR observed_at >= $6) AND ($7::timestamptz IS NULL OR observed_at < $7) AND ($8::text IS NULL OR review_state=$8) AND ($9::double precision IS NULL OR confidence >= $9)), edges AS (SELECT o.*, 'organization'::text AS from_type, o.from_organization_id AS from_id, CASE WHEN o.target_facility_id IS NOT NULL THEN 'facility'::text ELSE 'organization'::text END AS to_type, COALESCE(o.target_facility_id, o.target_organization_id) AS to_id FROM observations o), directions(step_direction) AS (SELECT 'out'::text WHERE $2 IN ('out','both') UNION ALL SELECT 'in'::text WHERE $2 IN ('in','both')), seed AS (SELECT 'organization'::text AS node_type, $1::uuid AS node_id, 0 AS hop WHERE EXISTS (SELECT 1 FROM uec.organizations WHERE organization_id=$1) UNION ALL SELECT 'facility'::text, $1::uuid, 0 WHERE EXISTS (SELECT 1 FROM uec.facilities WHERE facility_id=$1)), walk(node_type, node_id, hop) AS (SELECT node_type, node_id, hop FROM seed UNION SELECT CASE WHEN d.step_direction='out' THEN e.to_type ELSE e.from_type END, CASE WHEN d.step_direction='out' THEN e.to_id ELSE e.from_id END, w.hop + 1 FROM walk w JOIN directions d ON true JOIN edges e ON ((d.step_direction='out' AND e.from_type=w.node_type AND e.from_id=w.node_id) OR (d.step_direction='in' AND e.to_type=w.node_type AND e.to_id=w.node_id)) WHERE w.hop < $3), matched AS (SELECT DISTINCT e.relationship_observation_id FROM edges e JOIN walk w ON w.hop < $3 AND (($2 IN ('out','both') AND e.from_type=w.node_type AND e.from_id=w.node_id) OR ($2 IN ('in','both') AND e.to_type=w.node_type AND e.to_id=w.node_id))) SELECT o.relationship_observation_id, o.from_organization_id, o.target_facility_id, o.target_organization_id, o.relationship_type, o.assertion_status, o.observed_at, o.confidence, o.review_state, o.storage_state, o.privacy_status, o.publication_status, o.source_id, o.source_record_id FROM observations o JOIN matched m USING (relationship_observation_id) ORDER BY o.observed_at DESC, o.relationship_observation_id DESC LIMIT $10", &[&entity_id, &direction, &depth, &p.relationship_type, &p.source_id, &p.from.map(|d| d.and_hms_opt(0,0,0).unwrap().and_utc()), &p.to.map(|d| d.and_hms_opt(0,0,0).unwrap().and_utc()), &p.state, &p.min_confidence, &limit]).await { Ok(v) => v, Err(_) => return v2_error(StatusCode::SERVICE_UNAVAILABLE, "private_graph_query_failed", "private graph query failed") }; let data: Vec<_> = rows.into_iter().map(|r| json!({"relationship_observation_id":r.get::<_,Uuid>(0),"from_organization_id":r.get::<_,Option>(1),"target_facility_id":r.get::<_,Option>(2),"target_organization_id":r.get::<_,Option>(3),"relationship_type":r.get::<_,Option>(4),"assertion_status":r.get::<_,String>(5),"observed_at":r.get::<_,chrono::DateTime>(6),"confidence":r.get::<_,Option>(7),"review_state":r.get::<_,String>(8),"storage_state":r.get::<_,String>(9),"privacy_status":r.get::<_,String>(10),"publication_status":r.get::<_,String>(11),"source_id":r.get::<_,String>(12),"source_record_id":r.get::<_,Uuid>(13)})).collect(); Json(json!({"api_version":"private-graph-v1","data":data,"meta":{"private":true,"entity_id":entity_id,"limit":limit,"direction":direction,"depth":depth,"bounded":true}})).into_response() } From d45edcfd8acb2f09a5419d044bd21d484c788fab Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Fri, 18 Sep 2026 20:03:04 -0700 Subject: [PATCH 275/311] Prove worker image and crash recovery lifecycle --- .../tests/e2e/docker_worker_sitecustomize.py | 35 +++ pipeline/tests/e2e/test_worker_durability.py | 251 ++++++++++++++++++ 2 files changed, 286 insertions(+) create mode 100644 pipeline/tests/e2e/docker_worker_sitecustomize.py diff --git a/pipeline/tests/e2e/docker_worker_sitecustomize.py b/pipeline/tests/e2e/docker_worker_sitecustomize.py new file mode 100644 index 0000000..c9bbe47 --- /dev/null +++ b/pipeline/tests/e2e/docker_worker_sitecustomize.py @@ -0,0 +1,35 @@ +"""Test-only adapter injection for the real worker image. + +This module is mounted as ``sitecustomize.py`` only by the disposable Docker +worker acceptance test. The production image has no synthetic provider entry. +""" + +import json +import os +import urllib.parse +import urllib.request + +from pipeline.geocoding.base import GeocodeOutcome +from pipeline.geocoding import registry + + +class SyntheticDockerAdapter: + provider_id = "synthetic-docker" + + def geocode(self, query: str) -> GeocodeOutcome: + base_url = os.environ["UEC_SYNTHETIC_PROVIDER_URL"] + request_url = f"{base_url}?{urllib.parse.urlencode({'q': query})}" + with urllib.request.urlopen(request_url, timeout=10) as response: + payload = json.loads(response.read().decode("utf-8")) + if payload.get("status") != "ok": + return GeocodeOutcome( + "failed", "synthetic_failure", None, None, None, None, + "synthetic_http", False, {"error": "synthetic_failure"}, + ) + return GeocodeOutcome( + "accepted", "synthetic_point", 55.0, 12.0, "synthetic-point", + "address_point", "synthetic_http", False, {"source": "synthetic"}, + ) + + +registry.ADAPTER_FACTORIES["synthetic-docker"] = SyntheticDockerAdapter diff --git a/pipeline/tests/e2e/test_worker_durability.py b/pipeline/tests/e2e/test_worker_durability.py index c2e10c4..c29dc76 100644 --- a/pipeline/tests/e2e/test_worker_durability.py +++ b/pipeline/tests/e2e/test_worker_durability.py @@ -1,12 +1,16 @@ """Disposable database lifecycle tests for the durable geocoding worker.""" import importlib.util +import json import os +import shutil +import subprocess import threading import time import unittest import uuid from datetime import datetime, timezone +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path from unittest.mock import patch @@ -26,6 +30,8 @@ class WorkerDurabilityE2ETests(unittest.TestCase): + worker_image = None + @classmethod def setUpClass(cls): if os.environ.get("UEC_RUN_E2E") != "1": @@ -43,8 +49,93 @@ def setUpClass(cls): @classmethod def tearDownClass(cls): + if cls.worker_image: + subprocess.run(["docker", "rmi", "--force", cls.worker_image], capture_output=True, text=True, check=False) cls.env.stop() + @classmethod + def _build_worker_image(cls): + if cls.worker_image: + return cls.worker_image + if shutil.which("docker") is None: + raise unittest.SkipTest("docker CLI is unavailable") + cls.worker_image = f"uec-worker-e2e-{uuid.uuid4().hex[:10]}" + result = subprocess.run( + ["docker", "build", "--file", "Dockerfile.worker", "--tag", cls.worker_image, "."], + cwd=ROOT.parent, + capture_output=True, + text=True, + timeout=240, + check=False, + ) + if result.returncode: + raise AssertionError(f"worker image build failed:\n{result.stdout[-4000:]}\n{result.stderr[-4000:]}") + return cls.worker_image + + @staticmethod + def _provider_server(*, hold_second=False): + state = {"calls": 0, "lock": threading.Lock(), "second_started": threading.Event(), "release_second": threading.Event()} + + class Handler(BaseHTTPRequestHandler): + def do_GET(self): # noqa: N802 - BaseHTTPRequestHandler API + with state["lock"]: + state["calls"] += 1 + call_number = state["calls"] + if hold_second and call_number == 2: + state["second_started"].set() + state["release_second"].wait(30) + body = json.dumps({"status": "ok"}).encode("utf-8") + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + try: + self.wfile.write(body) + except OSError: + pass + + def log_message(self, _format, *_args): + return + + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, state + + @staticmethod + def _wait_for(predicate, timeout=20): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if predicate(): + return True + time.sleep(0.1) + return False + + def _container_database_url(self): + return self.env.database_url.replace("localhost", "host.docker.internal") + + def _result_count(self, record_id, provider): + with psycopg.connect(self.env.database_url) as db: + return db.execute( + "SELECT count(*) FROM uec.geocode_results WHERE source_record_id=%s AND provider_id=%s", + (record_id, provider), + ).fetchone()[0] + + def _docker_worker_command(self, image, database_url, provider_url, *, limit=1, lease_timeout=900): + harness = (ROOT / "tests" / "e2e" / "docker_worker_sitecustomize.py").resolve() + volume = f"{str(harness).replace(chr(92), '/') }:/app/sitecustomize.py:ro" + return [ + "docker", "run", "--rm", "--add-host", "host.docker.internal:host-gateway", + "--volume", volume, + "--env", f"UEC_DATABASE_URL={database_url}", + "--env", f"UEC_SYNTHETIC_PROVIDER_URL={provider_url}", + "--env", "PYTHONPATH=/app", + image, "--provider", "synthetic-docker", "--limit", str(limit), + "--daily-budget", "10", "--retries", "3", "--delay", "0", + "--provider-interval", "0", "--lease-timeout", str(lease_timeout), + "--worker-id", "docker-worker", + ] + def _queue(self, provider="synthetic-worker"): now = datetime.now(timezone.utc) record_id, job_id, artifact_id = uuid.uuid4(), uuid.uuid4(), uuid.uuid4() @@ -204,6 +295,166 @@ def run_worker(worker_id): self.assertEqual(calls, 1) self.assertEqual(reservations, 1) + def test_retryable_provider_attempts_are_each_charged_in_database(self): + job_id, record_id = self._queue(provider="synthetic-retry") + calls = [] + + class RetryAdapter: + def geocode(self, _query): + calls.append(len(calls) + 1) + if len(calls) == 1: + return GeocodeOutcome("failed", "temporary", None, None, None, None, "fixture", True, {"error": "temporary"}) + return GeocodeOutcome("unresolved", "fixture", None, None, None, None, "fixture", False, {}) + + with patch.object(WORKER, "get_adapter", return_value=RetryAdapter()): + WORKER.run( + self.env.database_url, "synthetic-retry", 1, 0, 3, + daily_budget=3, provider_interval=0, worker_id="retry-worker", + ) + with psycopg.connect(self.env.database_url) as db: + reservation_count = db.execute( + "SELECT count(*) FROM uec.geocode_request_reservations WHERE job_id=%s", + (job_id,), + ).fetchone()[0] + event = db.execute( + "SELECT event_type FROM uec.geocode_job_current WHERE job_id=%s", + (job_id,), + ).fetchone()[0] + result_count = db.execute( + "SELECT count(*) FROM uec.geocode_results WHERE source_record_id=%s AND provider_id='synthetic-retry'", + (record_id,), + ).fetchone()[0] + self.assertEqual(calls, [1, 2]) + self.assertEqual(reservation_count, 2) + self.assertEqual(event, "unresolved") + self.assertEqual(result_count, 1) + + def test_restriction_added_while_provider_paused_discards_final_result(self): + job_id, record_id = self._queue(provider="synthetic-restriction") + provider_started = threading.Event() + release_provider = threading.Event() + errors = [] + + class PausingAdapter: + calls = 0 + + def geocode(self, _query): + self.calls += 1 + provider_started.set() + if not release_provider.wait(10): + raise AssertionError("synthetic provider was not released") + return GeocodeOutcome("accepted", "fixture", 55.0, 12.0, "fixture", "point", "fixture", False, {}) + + def run_worker(): + try: + WORKER.run( + self.env.database_url, "synthetic-restriction", 1, 0, 1, + daily_budget=3, provider_interval=0, worker_id="restriction-worker", + ) + except BaseException as error: + errors.append(error) + + with patch.object(WORKER, "get_adapter", return_value=PausingAdapter()): + thread = threading.Thread(target=run_worker) + thread.start() + self.assertTrue(provider_started.wait(10)) + with psycopg.connect(self.env.database_url) as db: + db.execute( + "INSERT INTO uec.record_access_events " + "(source_record_id,action,reason_category,policy_version,maintainer) " + "VALUES (%s,'public_access_revoked','privacy','e2e-worker','synthetic-test')", + (record_id,), + ) + release_provider.set() + thread.join(10) + self.assertFalse(thread.is_alive()) + self.assertEqual(errors, []) + with psycopg.connect(self.env.database_url) as db: + event = db.execute( + "SELECT event_type FROM uec.geocode_job_current WHERE job_id=%s", + (job_id,), + ).fetchone()[0] + result_count = db.execute( + "SELECT count(*) FROM uec.geocode_results WHERE source_record_id=%s AND provider_id='synthetic-restriction'", + (record_id,), + ).fetchone()[0] + self.assertEqual(event, "cancelled") + self.assertEqual(result_count, 0) + + def test_real_worker_image_drains_synthetic_queue_and_exits_without_private_logs(self): + job_id, record_id = self._queue(provider="synthetic-docker") + server, server_thread, state = self._provider_server() + image = self._build_worker_image() + provider_url = f"http://host.docker.internal:{server.server_port}/geocode" + try: + result = subprocess.run( + self._docker_worker_command(image, self._container_database_url(), provider_url), + cwd=ROOT.parent, + capture_output=True, + text=True, + timeout=90, + check=False, + ) + finally: + server.shutdown() + server_thread.join(10) + server.server_close() + self.assertEqual(result.returncode, 0, result.stderr[-4000:]) + self.assertEqual(state["calls"], 1) + self.assertNotIn("synthetic query", result.stdout) + self.assertNotIn("GEOAPIFY_API_KEY", result.stdout + result.stderr) + with psycopg.connect(self.env.database_url) as db: + self.assertEqual( + db.execute("SELECT event_type FROM uec.geocode_job_current WHERE job_id=%s", (job_id,)).fetchone()[0], + "accepted", + ) + self.assertEqual( + db.execute("SELECT count(*) FROM uec.geocode_results WHERE source_record_id=%s", (record_id,)).fetchone()[0], + 1, + ) + + def test_process_kill_preserves_completed_result_and_restart_reclaims_second_job(self): + first_job, first_record = self._queue(provider="synthetic-docker") + second_job, second_record = self._queue(provider="synthetic-docker") + server, server_thread, state = self._provider_server(hold_second=True) + image = self._build_worker_image() + provider_url = f"http://host.docker.internal:{server.server_port}/geocode" + container_name = f"uec-worker-kill-{uuid.uuid4().hex[:8]}" + command = self._docker_worker_command(image, self._container_database_url(), provider_url, limit=2, lease_timeout=1) + command[2:2] = ["--detach", "--name", container_name] + try: + started = subprocess.run(command, cwd=ROOT.parent, capture_output=True, text=True, timeout=30, check=False) + self.assertEqual(started.returncode, 0, started.stderr[-2000:]) + self.assertTrue(self._wait_for(lambda: self._result_count(first_record, "synthetic-docker") == 1, 20)) + self.assertTrue(state["second_started"].wait(20)) + killed = subprocess.run(["docker", "kill", container_name], capture_output=True, text=True, check=False) + self.assertEqual(killed.returncode, 0, killed.stderr[-2000:]) + with psycopg.connect(self.env.database_url) as db: + second_event = db.execute( + "SELECT event_type FROM uec.geocode_job_current WHERE job_id=%s", (second_job,) + ).fetchone()[0] + self.assertEqual(self._result_count(first_record, "synthetic-docker"), 1) + self.assertEqual(second_event, "started") + time.sleep(1.2) + state["release_second"].set() + restart = subprocess.run( + self._docker_worker_command(image, self._container_database_url(), provider_url, limit=1, lease_timeout=1), + cwd=ROOT.parent, + capture_output=True, + text=True, + timeout=90, + check=False, + ) + self.assertEqual(restart.returncode, 0, restart.stderr[-4000:]) + self.assertEqual(self._result_count(first_record, "synthetic-docker"), 1) + self.assertEqual(self._result_count(second_record, "synthetic-docker"), 1) + finally: + state["release_second"].set() + subprocess.run(["docker", "rm", "--force", container_name], capture_output=True, text=True, check=False) + server.shutdown() + server_thread.join(10) + server.server_close() + if __name__ == "__main__": unittest.main() From 611166c595a048165d7bf835742ad725fa5bc811 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Fri, 18 Sep 2026 20:03:16 -0700 Subject: [PATCH 276/311] Prove source rights gates against disposable PostGIS --- pipeline/tests/test_source_rights_db.py | 260 ++++++++++++++++++++++++ 1 file changed, 260 insertions(+) create mode 100644 pipeline/tests/test_source_rights_db.py diff --git a/pipeline/tests/test_source_rights_db.py b/pipeline/tests/test_source_rights_db.py new file mode 100644 index 0000000..92a4105 --- /dev/null +++ b/pipeline/tests/test_source_rights_db.py @@ -0,0 +1,260 @@ +"""Disposable PostGIS proof for the source-rights ledger and all four gates.""" + +from __future__ import annotations + +import hashlib +import importlib.util +import json +import os +import socket +import subprocess +import tempfile +import unittest +import uuid +from datetime import datetime, timezone +from pathlib import Path + +import psycopg + +from pipeline.common import source_rights + + +REPOSITORY_ROOT = Path(__file__).parents[1].parent +COMPOSE_FILE = REPOSITORY_ROOT / "docker-compose.e2e.yml" + + +def _load_script(name: str, relative_path: str): + path = REPOSITORY_ROOT / relative_path + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + assert spec.loader + spec.loader.exec_module(module) + return module + + +VALIDATE = _load_script("validate_release_rights_db", "pipeline/scripts/stages/validate-release.py") +PROMOTE = _load_script("promote_release_rights_db", "pipeline/scripts/stages/promote-release.py") +EXPORT = _load_script("export_release_rights_db", "pipeline/scripts/stages/export-release.py") +READ_MODEL = _load_script("read_model_rights_db", "pipeline/scripts/maintenance/build_public_discovery_read_model.py") + + +def _free_port() -> int: + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + return int(sock.getsockname()[1]) + + +class SourceRightsPostgresTests(unittest.TestCase): + """Run only with UEC_RUN_RIGHTS_DB=1; all records are synthetic.""" + + @classmethod + def setUpClass(cls): + if os.environ.get("UEC_RUN_RIGHTS_DB") != "1": + raise unittest.SkipTest("set UEC_RUN_RIGHTS_DB=1 for disposable PostGIS source-rights proof") + cls.project = f"uec-rights-{uuid.uuid4().hex[:8]}" + cls.port = _free_port() + cls.database_url = f"postgresql://uec:uec-e2e@localhost:{cls.port}/uec" + cls.compose_env = {**os.environ, "UEC_E2E_DB_PORT": str(cls.port)} + cls._compose("up", "-d", "--wait", "postgres", check=True) + for migration in sorted((REPOSITORY_ROOT / "pipeline" / "migrations").glob("*.sql")): + cls._compose( + "exec", "-T", "postgres", "psql", "-v", "ON_ERROR_STOP=1", "-U", "uec", "-d", "uec", + input_text=migration.read_text(encoding="utf-8"), + ) + + @classmethod + def tearDownClass(cls): + if hasattr(cls, "project"): + cls._compose("down", "-v", "--remove-orphans", check=False) + + @classmethod + def _compose(cls, *args, input_text=None, check=False): + command = ["docker", "compose", "-p", cls.project, "-f", str(COMPOSE_FILE), *args] + result = subprocess.run( + command, + cwd=REPOSITORY_ROOT, + env=cls.compose_env, + input=input_text, + text=True, + capture_output=True, + timeout=180, + check=False, + ) + if check and result.returncode: + raise RuntimeError(f"Docker Compose failed: {' '.join(command)}\n{result.stdout}\n{result.stderr}") + return result + + @staticmethod + def _ids(label: str): + suffix = uuid.uuid4().hex[:10] + return f"rights.{label}.{suffix}", f"rights-{label}-{suffix}" + + def seed_release(self, label: str, *, status="candidate", profile="official", second_artifact=False): + source_id, release_id = self._ids(label) + artifact_id = uuid.uuid4() + digest = hashlib.sha256(f"{label}-artifact".encode()).hexdigest() + now = datetime.now(timezone.utc) + record_id, facility_id, observation_id = uuid.uuid4(), uuid.uuid4(), uuid.uuid4() + with psycopg.connect(self.database_url) as db, db.transaction(): + db.execute( + "INSERT INTO uec.sources (source_id,country_code,name,official_url,access_method,attribution) VALUES (%s,'US',%s,'https://example.invalid/rights','fixture','Synthetic attribution')", + (source_id, f"Synthetic rights {label}"), + ) + db.execute( + "INSERT INTO uec.releases (release_id,status,ruleset_version,profile,test_only,summary) VALUES (%s,%s,'rights-v1',%s,false,'{}')", + (release_id, status, profile), + ) + db.execute( + "INSERT INTO uec.raw_artifacts (artifact_id,storage_key,sha256,byte_size,retrieved_at) VALUES (%s,%s,%s,1,%s)", + (artifact_id, f"synthetic/{label}/{artifact_id}", digest, now), + ) + db.execute( + "INSERT INTO uec.source_records (source_record_id,source_id,source_record_key,artifact_id,raw_fields,parsed_at) VALUES (%s,%s,%s,%s,'{}',%s)", + (record_id, source_id, f"record-{label}", artifact_id, now), + ) + db.execute( + "INSERT INTO uec.facilities (facility_id,canonical_name,country_code,city) VALUES (%s,%s,'US','Syntheticville')", + (facility_id, f"Synthetic facility {label}"), + ) + db.execute( + """INSERT INTO uec.observations + (observation_id,facility_id,source_record_id,observed_at,observation, + classification,ruleset_id,rule_id,classification_category, + classification_review_status,default_visible,first_observed_at) + VALUES (%s,%s,%s,%s,'{}','{}','rights-v1','rights','slaughter','approved',true,%s)""", + (observation_id, facility_id, record_id, now, now), + ) + db.execute( + "INSERT INTO uec.release_members (release_id,facility_id,observation_id,default_visible) VALUES (%s,%s,%s,true)", + (release_id, facility_id, observation_id), + ) + db.execute( + "INSERT INTO uec.geocode_results (source_record_id,provider_id,query,match_method,status,result,queried_at) VALUES (%s,'synthetic','Syntheticville','fixture','accepted',ST_SetSRID(ST_MakePoint(-100,40),4326)::geography,%s)", + (record_id, now), + ) + db.execute( + "INSERT INTO uec.publication_review_events (source_record_id,release_id,factual_review_status,privacy_screening_status,maintainer_approval,publication_eligible,reviewer_role) VALUES (%s,%s,'reviewed','passed','approved',true,'synthetic')", + (record_id, release_id), + ) + if second_artifact: + second_id = uuid.uuid4() + second_digest = hashlib.sha256(f"{label}-second".encode()).hexdigest() + db.execute( + "INSERT INTO uec.raw_artifacts (artifact_id,storage_key,sha256,byte_size,retrieved_at) VALUES (%s,%s,%s,1,%s)", + (second_id, f"synthetic/{label}/{second_id}", second_digest, now), + ) + second_record = uuid.uuid4() + second_observation = uuid.uuid4() + second_facility = uuid.uuid4() + db.execute( + "INSERT INTO uec.source_records (source_record_id,source_id,source_record_key,artifact_id,raw_fields,parsed_at) VALUES (%s,%s,%s,%s,'{}',%s)", + (second_record, source_id, f"record-{label}-second", second_id, now), + ) + db.execute( + "INSERT INTO uec.facilities (facility_id,canonical_name,country_code,city) VALUES (%s,%s,'US','Syntheticville')", + (second_facility, f"Synthetic facility {label} second"), + ) + db.execute( + """INSERT INTO uec.observations + (observation_id,facility_id,source_record_id,observed_at,observation, + classification,ruleset_id,rule_id,classification_category, + classification_review_status,default_visible,first_observed_at) + VALUES (%s,%s,%s,%s,'{}','{}','rights-v1','rights','slaughter','approved',true,%s)""", + (second_observation, second_facility, second_record, now, now), + ) + db.execute( + "INSERT INTO uec.release_members (release_id,facility_id,observation_id,default_visible) VALUES (%s,%s,%s,true)", + (release_id, second_facility, second_observation), + ) + return { + "source_id": source_id, + "release_id": release_id, + "artifact_id": str(artifact_id), + "digest": digest, + "now": now, + } + + def add_decision(self, seeded, status="cleared", *, release_id=None, profile="official", artifact_id=None, digest=None, decided_at=None, decision_id=None): + with psycopg.connect(self.database_url) as db, db.transaction(): + db.execute( + """INSERT INTO uec.source_rights_decisions + (source_rights_decision_id,source_id,profile,release_id,artifact_id, + artifact_sha256,redistribution_status,decision_actor,decision_reference,decided_at) + VALUES (%s,%s,%s,%s,%s,%s,%s,'synthetic-owner','synthetic-rights-case',%s)""", + ( + decision_id or uuid.uuid4(), seeded["source_id"], profile, + release_id or seeded["release_id"], artifact_id or seeded["artifact_id"], + digest or seeded["digest"], status, decided_at or seeded["now"], + ), + ) + + def test_exact_statuses_and_closed_world_scope(self): + for status in ("cleared", "unknown", "restricted"): + seeded = self.seed_release(status) + if status != "unknown": + self.add_decision(seeded, status) + with psycopg.connect(self.database_url) as db: + result = source_rights.evaluate(db, seeded["release_id"]) + self.assertEqual(result["status"], "cleared" if status == "cleared" else "blocked") + self.assertEqual(result["requirements"][0]["status"], status if status != "unknown" else "unknown") + + def test_changed_source_version_and_out_of_scope_release_do_not_inherit(self): + changed = self.seed_release("changed", second_artifact=True) + self.add_decision(changed, "cleared") + with psycopg.connect(self.database_url) as db: + result = source_rights.evaluate(db, changed["release_id"]) + self.assertEqual(len(result["blockers"]), 1) + self.assertEqual(result["blockers"][0]["reason"], "missing exact decision") + + scoped = self.seed_release("outofscope") + other = self.seed_release("other-release") + self.add_decision(scoped, "cleared", release_id=other["release_id"]) + with psycopg.connect(self.database_url) as db: + result = source_rights.evaluate(db, scoped["release_id"]) + self.assertEqual(result["blockers"][0]["reason"], "missing exact decision") + + def test_latest_conflict_and_digest_profile_trigger_fail_closed(self): + seeded = self.seed_release("conflict") + timestamp = seeded["now"] + self.add_decision(seeded, "cleared", decided_at=timestamp) + self.add_decision(seeded, "unknown", decided_at=timestamp) + with psycopg.connect(self.database_url) as db: + result = source_rights.evaluate(db, seeded["release_id"]) + self.assertEqual(result["blockers"][0]["reason"], "conflicting decisions at the latest decision time") + + mismatch = self.seed_release("digest-mismatch") + with self.assertRaises(psycopg.errors.RaiseException): + self.add_decision(mismatch, artifact_id=mismatch["artifact_id"], digest="b" * 64) + profile_mismatch = self.seed_release("profile-mismatch") + with self.assertRaises(psycopg.errors.RaiseException): + self.add_decision(profile_mismatch, profile="secondary") + + def test_all_four_runtime_boundaries_block_unknown_rights(self): + seeded = self.seed_release("boundaries", status="candidate") + release_id = seeded["release_id"] + report = VALIDATE.validate(self.database_url, release_id, None, False) + self.assertEqual(report["status"], "blocked") + self.assertGreater(report["metrics"]["rights_not_cleared"], 0) + + with psycopg.connect(self.database_url) as db, db.transaction(): + db.execute("UPDATE uec.releases SET status='validated' WHERE release_id=%s", (release_id,)) + with self.assertRaises(source_rights.SourceRightsBlocked): + PROMOTE.promote(self.database_url, release_id, []) + + manifest = {"manifest_version": "synthetic-rights-v1", "profile": "official", "release_id": release_id, "ruleset_version": "rights-v1"} + encoded = json.dumps(manifest, sort_keys=True, separators=(",", ":")) + with psycopg.connect(self.database_url) as db, db.transaction(): + db.execute("UPDATE uec.releases SET status='promoted' WHERE release_id=%s", (release_id,)) + db.execute( + "INSERT INTO uec.release_manifests (release_id,manifest,manifest_sha256) VALUES (%s,%s::jsonb,%s)", + (release_id, encoded, hashlib.sha256(encoded.encode()).hexdigest()), + ) + with self.assertRaises(source_rights.SourceRightsBlocked): + READ_MODEL.build(self.database_url, release_id) + with tempfile.TemporaryDirectory() as output_dir: + with self.assertRaises(source_rights.SourceRightsBlocked): + EXPORT.export_release(self.database_url, release_id, "official", Path(output_dir)) + + +if __name__ == "__main__": + unittest.main() From dae9f27636cca3bb1aceac243d7e00f6787489e2 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Fri, 18 Sep 2026 20:05:09 -0700 Subject: [PATCH 277/311] ci: require disposable source rights proof --- pipeline/tests/run-standard.ps1 | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pipeline/tests/run-standard.ps1 b/pipeline/tests/run-standard.ps1 index 0bb2bab..a82e091 100644 --- a/pipeline/tests/run-standard.ps1 +++ b/pipeline/tests/run-standard.ps1 @@ -29,6 +29,9 @@ try { & docker compose -p $project -f $compose exec -T postgres psql -v ON_ERROR_STOP=1 -U uec -d uec if ($LASTEXITCODE -ne 0) { throw "Synthetic fixture seeding failed (exit $LASTEXITCODE)." } + # The rights proof owns its own disposable PostGIS stack and is mandatory + # in the canonical standard run; it must never become an accidental skip. + $env:UEC_RUN_RIGHTS_DB = '1' python pipeline/tests/run_unittest.py --start-directory pipeline/tests if ($LASTEXITCODE -ne 0) { throw "Python tests failed (exit $LASTEXITCODE)." } @@ -36,6 +39,7 @@ try { if ($LASTEXITCODE -ne 0) { throw "Country adapter tests failed (exit $LASTEXITCODE)." } } finally { + Remove-Item Env:UEC_RUN_RIGHTS_DB -ErrorAction SilentlyContinue $savedPreference = $ErrorActionPreference $ErrorActionPreference = 'Continue' & docker compose -p $project -f $compose down -v --remove-orphans *> $null From a7f558017c021e4479f337a4cad22401d479b3c3 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Fri, 18 Sep 2026 20:15:01 -0700 Subject: [PATCH 278/311] test: preserve partial schema readiness fixture --- pipeline/tests/e2e/fixture.py | 47 ++++++++++++++-------------- pipeline/tests/e2e/test_readiness.py | 5 ++- 2 files changed, 28 insertions(+), 24 deletions(-) diff --git a/pipeline/tests/e2e/fixture.py b/pipeline/tests/e2e/fixture.py index 2070694..a8877f0 100644 --- a/pipeline/tests/e2e/fixture.py +++ b/pipeline/tests/e2e/fixture.py @@ -149,7 +149,7 @@ def _database_ready(self): return result return result - def _start_once(self, files, wait_for_ready): + def _start_once(self, files, wait_for_ready, schema_preflight): self._ensure_build_temp() print(f"[e2e] starting {self.project}", flush=True) startup = subprocess.run(self.command("up", "-d", "--wait"), cwd=ROOT, capture_output=True, text=True, env=self.compose_env()) @@ -229,27 +229,28 @@ def _start_once(self, files, wait_for_ready): if is_retryable_database_failure(output): raise _RetryableStartupFailure("Postgres became unavailable after migrations") raise RuntimeError(f"PostGIS database failed post-migration readiness\n{_sanitize_diagnostics(output)}") - required = { - "facilities", - "organizations", - "organization_relationship_observations", - "claim_current", - "source_entity_crosswalks", - "source_records", - } - with psycopg.connect(self.database_url) as db: - names = { - row[0] - for row in db.execute( - "SELECT c.relname FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace " - "WHERE n.nspname='uec' AND c.relkind IN ('r','p','v','m','f')" - ).fetchall() + if schema_preflight: + required = { + "facilities", + "organizations", + "organization_relationship_observations", + "claim_current", + "source_entity_crosswalks", + "source_records", } - missing = sorted(required - names) - if missing: - raise RuntimeError( - f"database/schema preflight failed; missing={missing}" - ) + with psycopg.connect(self.database_url) as db: + names = { + row[0] + for row in db.execute( + "SELECT c.relname FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace " + "WHERE n.nspname='uec' AND c.relkind IN ('r','p','v','m','f')" + ).fetchall() + } + missing = sorted(required - names) + if missing: + raise RuntimeError( + f"database/schema preflight failed; missing={missing}" + ) print("[e2e] building backend", flush=True) build_env = os.environ.copy() build_env["CARGO_TARGET_DIR"] = str(self.cargo_cache_dir) @@ -294,12 +295,12 @@ def _start_once(self, files, wait_for_ready): f"exit_code={exit_code}; log_path={log_path}\n{log_text}" ) - def start(self, migration_files=None, wait_for_ready=True): + def start(self, migration_files=None, wait_for_ready=True, schema_preflight=True): files = tuple(migration_files if migration_files is not None else sorted((ROOT / "pipeline/migrations").glob("*.sql"))) for attempt in range(MAX_START_ATTEMPTS): self.start_attempts = attempt + 1 try: - return self._start_once(files, wait_for_ready) + return self._start_once(files, wait_for_ready, schema_preflight) except _RetryableStartupFailure as exc: diagnostics = self._container_diagnostics() self.stop() diff --git a/pipeline/tests/e2e/test_readiness.py b/pipeline/tests/e2e/test_readiness.py index 107f63c..d2bda26 100644 --- a/pipeline/tests/e2e/test_readiness.py +++ b/pipeline/tests/e2e/test_readiness.py @@ -13,7 +13,10 @@ def test_partial_schema_is_not_ready(self): env = E2EEnvironment() try: migrations = sorted((ROOT / "pipeline/migrations").glob("*.sql")) - env.start(migration_files=migrations[:1], wait_for_ready=False) + # This deliberately exercises the backend's own readiness + # response against a partial schema, so skip the normal full + # schema preflight for this one negative-path fixture. + env.start(migration_files=migrations[:1], wait_for_ready=False, schema_preflight=False) env.wait_for_listening() self.assertIsNone(env.backend.poll(), "backend must stay alive to report schema readiness") try: From 41d3d3e027c53d542ff307d9132cdf6b70575622 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Fri, 18 Sep 2026 20:22:59 -0700 Subject: [PATCH 279/311] test: align disposable fixtures with durable gates --- pipeline/tests/e2e/test_public_discovery_read_model.py | 1 + pipeline/tests/e2e/test_suppression_lifecycle.py | 9 +++++++++ 2 files changed, 10 insertions(+) diff --git a/pipeline/tests/e2e/test_public_discovery_read_model.py b/pipeline/tests/e2e/test_public_discovery_read_model.py index 39aca8a..e115d67 100644 --- a/pipeline/tests/e2e/test_public_discovery_read_model.py +++ b/pipeline/tests/e2e/test_public_discovery_read_model.py @@ -100,6 +100,7 @@ def test_interrupted_activation_leaves_no_rows_or_metadata(self): WHERE release_id='e2e-promoted'""", (release_id,), ) + self.env._seed_synthetic_rights_decisions(release_id) with self.assertRaisesRegex(RuntimeError, "interrupted"): BUILDER.build(self.env.database_url, release_id, fail_after_rows=1) with psycopg.connect(self.env.database_url) as db: diff --git a/pipeline/tests/e2e/test_suppression_lifecycle.py b/pipeline/tests/e2e/test_suppression_lifecycle.py index fc96ab9..1879fda 100644 --- a/pipeline/tests/e2e/test_suppression_lifecycle.py +++ b/pipeline/tests/e2e/test_suppression_lifecycle.py @@ -182,6 +182,15 @@ def geocode(self, _query): def test_c_daily_budget_stops_before_provider_call(self): self._queue_geocode("synthetic-budget", "already counted", started_at=datetime.now(timezone.utc)) self._queue_geocode("synthetic-budget", "must remain queued") + with psycopg.connect(self.env.database_url) as db: + with db.transaction(): + db.execute( + """ + INSERT INTO uec.geocode_provider_budgets + (provider_id,budget_date,daily_limit,reserved_requests,last_reserved_at) + VALUES ('synthetic-budget',(now() AT TIME ZONE 'UTC')::date,1,1,clock_timestamp()) + """ + ) class Adapter: calls = 0 From 2a1103f5d6866a656f072e01c5b175c8bd3e917c Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Fri, 18 Sep 2026 20:08:54 -0700 Subject: [PATCH 280/311] Prove preclaim restriction blocks worker work --- pipeline/tests/e2e/test_worker_durability.py | 34 ++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/pipeline/tests/e2e/test_worker_durability.py b/pipeline/tests/e2e/test_worker_durability.py index c29dc76..392960b 100644 --- a/pipeline/tests/e2e/test_worker_durability.py +++ b/pipeline/tests/e2e/test_worker_durability.py @@ -381,6 +381,40 @@ def run_worker(): self.assertEqual(event, "cancelled") self.assertEqual(result_count, 0) + def test_active_restriction_before_claim_blocks_provider_and_budget(self): + job_id, record_id = self._queue(provider="synthetic-preblocked") + calls = [] + + with psycopg.connect(self.env.database_url) as db: + db.execute( + "INSERT INTO uec.record_access_events " + "(source_record_id,action,reason_category,policy_version,maintainer) " + "VALUES (%s,'public_access_revoked','privacy','e2e-worker','synthetic-test')", + (record_id,), + ) + + class MustNotCallAdapter: + def geocode(self, _query): + calls.append(True) + raise AssertionError("an actively restricted queued job must not call a provider") + + with patch.object(WORKER, "get_adapter", return_value=MustNotCallAdapter()): + processed = WORKER.run( + self.env.database_url, "synthetic-preblocked", 1, 0, 1, + daily_budget=1, provider_interval=0, worker_id="preblocked-worker", + ) + with psycopg.connect(self.env.database_url) as db: + event = db.execute( + "SELECT event_type FROM uec.geocode_job_current WHERE job_id=%s", (job_id,) + ).fetchone()[0] + reservation_count = db.execute( + "SELECT count(*) FROM uec.geocode_request_reservations WHERE job_id=%s", (job_id,) + ).fetchone()[0] + self.assertEqual(processed, 0) + self.assertEqual(calls, []) + self.assertEqual(event, "queued") + self.assertEqual(reservation_count, 0) + def test_real_worker_image_drains_synthetic_queue_and_exits_without_private_logs(self): job_id, record_id = self._queue(provider="synthetic-docker") server, server_thread, state = self._provider_server() From 713eb3ae11136910df54ed82ced234792402373c Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Fri, 18 Sep 2026 20:29:31 -0700 Subject: [PATCH 281/311] Document Sprint 01 integration evidence --- docs/sprint01-integration-execution.md | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 docs/sprint01-integration-execution.md diff --git a/docs/sprint01-integration-execution.md b/docs/sprint01-integration-execution.md new file mode 100644 index 0000000..ec0984b --- /dev/null +++ b/docs/sprint01-integration-execution.md @@ -0,0 +1,10 @@ +# Sprint 01 integration execution + +Status: active engineering integration evidence (2026-09-18) + +- Baseline: `a45ebe55f7291378db1f1b43d9cc7240372664cc` +- Revision: `2a1103f5` (`codex/sprint01-integration`) +- Scope: private graph correctness, authenticated populated graph coverage, rights and worker acceptance integration, CI/Compose/build-context safeguards. +- Evidence: Rust `cargo test --locked` passed (83 tests); standard Python run passed (288 tests, 22 documented E2E skips); graph E2E passed 6/6; public API E2E passed 11/11; worker E2E passed 8/8 with the real image; rights DB proof passed 4/4; backup/restore passed; Docker context sentinel passed; root Jest passed 25/25. +- Extended E2E modules were run serially against disposable databases. Readiness, suppression-budget, and interrupted-discovery fixtures each received a narrow fixture correction and their targeted reruns passed. No live credentials, private rows, provider calls, publication, or deployment were used. +- Remaining milestone boundary: retained authorized APHIS inputs were unavailable in this checkout, so no real-input rehearsal is claimed. Production publication, ongoing rights revocation, and deployment review remain deferred launch work. From 1e462b2efd6760806d76b6459683ee64cbf36e8f Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Fri, 18 Sep 2026 20:30:19 -0700 Subject: [PATCH 282/311] Clarify Sprint 01 gate evidence --- docs/sprint01-integration-execution.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/sprint01-integration-execution.md b/docs/sprint01-integration-execution.md index ec0984b..d8df5fd 100644 --- a/docs/sprint01-integration-execution.md +++ b/docs/sprint01-integration-execution.md @@ -3,8 +3,8 @@ Status: active engineering integration evidence (2026-09-18) - Baseline: `a45ebe55f7291378db1f1b43d9cc7240372664cc` -- Revision: `2a1103f5` (`codex/sprint01-integration`) +- Final checkpoint: `713eb3ae` (`codex/sprint01-integration`), with implementation through `2a1103f5` and the final commit documentation-only. - Scope: private graph correctness, authenticated populated graph coverage, rights and worker acceptance integration, CI/Compose/build-context safeguards. -- Evidence: Rust `cargo test --locked` passed (83 tests); standard Python run passed (288 tests, 22 documented E2E skips); graph E2E passed 6/6; public API E2E passed 11/11; worker E2E passed 8/8 with the real image; rights DB proof passed 4/4; backup/restore passed; Docker context sentinel passed; root Jest passed 25/25. +- Evidence: Rust `cargo test --locked` passed (83 tests); standard Python run passed (288 tests, 22 documented E2E skips); graph E2E passed 6/6; public API E2E passed 11/11; worker E2E passed 8/8 with the real image; rights DB proof passed 4/4 with `UEC_RUN_RIGHTS_DB=1`; backup/restore passed; Docker context sentinel passed; root Jest passed 25/25. The final standard runner wires that rights proof as mandatory. - Extended E2E modules were run serially against disposable databases. Readiness, suppression-budget, and interrupted-discovery fixtures each received a narrow fixture correction and their targeted reruns passed. No live credentials, private rows, provider calls, publication, or deployment were used. - Remaining milestone boundary: retained authorized APHIS inputs were unavailable in this checkout, so no real-input rehearsal is claimed. Production publication, ongoing rights revocation, and deployment review remain deferred launch work. From 5570b6ab42e74dc5227e08c2cd7c1ac7f08aab96 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Fri, 18 Sep 2026 22:52:17 -0700 Subject: [PATCH 283/311] Make synthetic worker provider reachable from Linux containers --- pipeline/tests/e2e/test_worker_durability.py | 36 +++++++++++++++++--- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/pipeline/tests/e2e/test_worker_durability.py b/pipeline/tests/e2e/test_worker_durability.py index 392960b..148f276 100644 --- a/pipeline/tests/e2e/test_worker_durability.py +++ b/pipeline/tests/e2e/test_worker_durability.py @@ -3,6 +3,7 @@ import importlib.util import json import os +import re import shutil import subprocess import threading @@ -97,7 +98,11 @@ def do_GET(self): # noqa: N802 - BaseHTTPRequestHandler API def log_message(self, _format, *_args): return - server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + # Linux Docker's host-gateway reaches the host interface rather than + # loopback. This server is disposable and returns only synthetic data; + # bind on all interfaces so both Docker Desktop and Linux Engine can + # reach it through the test-only host-gateway route. + server = ThreadingHTTPServer(("0.0.0.0", 0), Handler) thread = threading.Thread(target=server.serve_forever, daemon=True) thread.start() return server, thread, state @@ -121,6 +126,22 @@ def _result_count(self, record_id, provider): (record_id, provider), ).fetchone()[0] + @staticmethod + def _redacted_diagnostics(text): + text = text or "" + text = re.sub(r"(?i)postgres(?:ql)?://[^\s]+", "postgresql://[redacted]", text) + text = text.replace("synthetic query", "[query-redacted]") + return text[-2000:] + + def _container_diagnostics(self, container_name): + result = subprocess.run( + ["docker", "logs", "--tail", "40", container_name], + capture_output=True, + text=True, + check=False, + ) + return self._redacted_diagnostics(result.stdout + result.stderr) + def _docker_worker_command(self, image, database_url, provider_url, *, limit=1, lease_timeout=900): harness = (ROOT / "tests" / "e2e" / "docker_worker_sitecustomize.py").resolve() volume = f"{str(harness).replace(chr(92), '/') }:/app/sitecustomize.py:ro" @@ -434,7 +455,11 @@ def test_real_worker_image_drains_synthetic_queue_and_exits_without_private_logs server_thread.join(10) server.server_close() self.assertEqual(result.returncode, 0, result.stderr[-4000:]) - self.assertEqual(state["calls"], 1) + self.assertEqual( + state["calls"], 1, + "synthetic provider request missing; " + f"worker_exit={result.returncode} diagnostics={self._redacted_diagnostics(result.stdout + result.stderr)}", + ) self.assertNotIn("synthetic query", result.stdout) self.assertNotIn("GEOAPIFY_API_KEY", result.stdout + result.stderr) with psycopg.connect(self.env.database_url) as db: @@ -459,8 +484,11 @@ def test_process_kill_preserves_completed_result_and_restart_reclaims_second_job try: started = subprocess.run(command, cwd=ROOT.parent, capture_output=True, text=True, timeout=30, check=False) self.assertEqual(started.returncode, 0, started.stderr[-2000:]) - self.assertTrue(self._wait_for(lambda: self._result_count(first_record, "synthetic-docker") == 1, 20)) - self.assertTrue(state["second_started"].wait(20)) + self.assertTrue( + self._wait_for(lambda: self._result_count(first_record, "synthetic-docker") == 1, 20), + self._container_diagnostics(container_name), + ) + self.assertTrue(state["second_started"].wait(20), self._container_diagnostics(container_name)) killed = subprocess.run(["docker", "kill", container_name], capture_output=True, text=True, check=False) self.assertEqual(killed.returncode, 0, killed.stderr[-2000:]) with psycopg.connect(self.env.database_url) as db: From a5358ccfc0d57f0d298ef27a07f4e316c2071a5c Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sat, 19 Sep 2026 11:08:23 -0700 Subject: [PATCH 284/311] chore: guard sprint02 private evidence boundary --- .dockerignore | 2 + .gitignore | 1 + Dockerfile.context-test | 6 ++- docs/SPRINT-02-CONTRACT.md | 57 ++++++++++++++++++++++++ docs/sprint02-integration-ledger.md | 45 +++++++++++++++++++ pipeline/tests/verify-docker-context.ps1 | 4 ++ 6 files changed, 113 insertions(+), 2 deletions(-) create mode 100644 docs/SPRINT-02-CONTRACT.md create mode 100644 docs/sprint02-integration-ledger.md diff --git a/.dockerignore b/.dockerignore index d916335..9ad72e8 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,6 +1,8 @@ target data/ private/ +.private/ +**/.private/ staging/ .tmp/ tmp/ diff --git a/.gitignore b/.gitignore index 64c6bfe..4b28efa 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ /node_modules/ /snapshots/ /static/v2-preview/ +/.private/ /\.shuttle* /Secrets*.toml __pycache__/ diff --git a/Dockerfile.context-test b/Dockerfile.context-test index 632fa0f..2f97108 100644 --- a/Dockerfile.context-test +++ b/Dockerfile.context-test @@ -3,6 +3,8 @@ FROM alpine:3.20 WORKDIR /context COPY . . -# The CI harness creates this synthetic private marker under data/. The -# assertion fails if the Docker ignore rules allow it into the build context. +# The CI harness creates synthetic private markers under data/ and .private/. +# These assertions fail if Docker ignore rules allow either boundary into the +# build context. RUN test ! -e data/.docker-context-sentinel +RUN test ! -e .private/.docker-context-sentinel diff --git a/docs/SPRINT-02-CONTRACT.md b/docs/SPRINT-02-CONTRACT.md new file mode 100644 index 0000000..249b233 --- /dev/null +++ b/docs/SPRINT-02-CONTRACT.md @@ -0,0 +1,57 @@ +# Sprint 02 — real evidence activation + +Status: authorized and active, 2026-09-19. Owner approved all eight lanes before dispatch. + +## Goal and baseline + +Deliver a reproducible private data release covering current US laboratory and slaughterhouse evidence plus France and Italy, with useful evidence links, explicit coverage limits, and independently verified provenance. This is private candidate delivery, not public publication or factual approval. + +Base: `eli/front-end-overhaul`, `5570b6ab42e74dc5227e08c2cd7c1ac7f08aab96`. The saved project root is legacy master; all task code belongs in isolated V2 worktrees starting from the named branch. Final integration/push target: `origin/eli/front-end-overhaul`. + +The project manager coordinates scope, interfaces, architecture, and acceptance. Luna tasks implement, integrate, and independently review. No major frontend work, new general framework, automatic ownership inference, or country substitution. + +## Authority and operating rules + +Live public-source acquisition, private processing, necessary implementation fixes, testing, and normal integration/push are authorized. Follow AGENTS.md and docs/ETHICS.md; ethics governs conflicts. Do not ask again for ordinary downloads or private pipeline runs. Actual paid access, access-control bypass, exceptional destructive retention changes, and publication are outside this sprint. A blocked endpoint requires investigation of permitted alternatives, not an immediate permission question or silent synthetic substitution. + +Use separate tasks and worktrees. Never overwrite another task's edits, existing user data, or applied migrations. No force pushes. Raw records, identifiers, private URLs and payloads must not enter Git, public logs, or task reports. Reports are row-free. Prior manifests claiming captures do not prove file availability. + +## Shared interfaces and storage + +Each acquisition lane owns a separate directory under `C:/New Projects/UntilEveryCage/.private/sprint02-20260919/`: `aphis-core`, `aphis-inspections`, `fsis`, `france`, `italy`. Integration initializes and verifies Git exclusion before retained writes; each lane must independently check exclusion before writing raw bytes. No Docker build context may include this shared private root. Existing authorized evidence is preserved; new retrievals get new run directories. + +Preserve original response/export bytes, authoritative source URL, actual retrieval timestamp, content hash, byte size, source publication/effective date when known, acquisition method/parameters, code revision, parsing configuration, and completeness limits using existing contracts. Separate raw, parsed, normalized, quarantine, candidate, and review outputs. Downloaded attachments require explicit association and integrity metadata; do not fetch arbitrary links or expose signed URLs. + +Each lane records a private handoff manifest with exact local artifact paths and verification commands. A row-free tracked report names the handoff location, source IDs, hashes/counts where safe, source dates, eligibility/rights unknowns, attempted/captured/failed partitions, and commands. Acquisition is distinct from redistribution clearance. Never fabricate owner approvals to make real candidates pass publication gates. + +## Eight lanes + +1. **APHIS registrations and annual reports.** Acquire the currently exposed research-facility registration population and latest complete annual-report year exposed by APHIS. FY2025 is the expected year, but verify rather than assume; distinguish a complete reporting period from complete submission coverage. Exhaust permitted enumerated pages/partitions for those queries, reconcile provider totals/page counts/deduplication, retain original bytes, and produce source-ID-compatible registration/report observations. Own APHIS acquisition/adapter core paths; coordinate inspection changes through lane 1 if files overlap. +2. **APHIS inspections.** Acquire research-facility inspections with inspection dates in calendar 2025 (`2025-01-01` through `2025-12-31`), retaining actual query semantics and unknown dates separately. This fixed completed-year window supports cross-source analysis without pretending inspection year equals annual-report coverage. Enumerate all exposed permitted pages for the window; identify caps/unavailable partitions explicitly. Retain explicitly referenced documents available through permitted public routes and inventory unavailable documents. Own inspection-specific code/tests; lane 1 owns shared APHIS adapter files. +3. **FSIS modernization.** Acquire current official MPI establishment directory and currently available supplemental demographic files. Reuse existing adapters. Produce private normalized/candidate outputs and a legacy comparison distinguishing source rows, source-native establishments, duplicates, category coverage, additions and not-observed records. No inferred closure. +4. **France activation.** Refresh the existing French slaughterhouse and approved-establishment source profiles already supported by the repository. Freeze their exact registry source IDs in the lane kickoff report before download. Preserve both source scopes, reconcile overlap without automatic merges, build a real private candidate with provenance, quarantine, identity and coordinate coverage. +5. **Italy activation.** Refresh the existing supported Italian establishment source/profile, recording its exact registry ID at kickoff. Produce a reproducible private candidate preserving establishment categories and address/location precision. No new category invention, guessed coordinates, paid geocoding, or public promotion. +6. **Evidence integration.** Consume lane 1–3 handoffs with existing APHIS packet/identity/graph contracts. Deliver real registration-to-report and registration-to-inspection paths with dates, uncertainties, missing documents/coverage, and artifact traceability. Keep FSIS identities separate unless explicit source-native evidence supports a link. Use existing graph projections only when semantics fit. Own packet/accountability integration paths and a private investigator runbook; no frontend redesign or new universal graph importer. +7. **Independent QA.** Independently inspect source-to-output samples stratified by profile/category, valid/quarantined/conflicting/missing-location states and available partitions; verify hashes and replay from retained files in a fresh output directory. Verify deterministic content (excluding documented volatile metadata), reconciliation and privacy boundaries. Review authored changes and send findings back to authors. Do not implement and approve your own fixes. No manifest-only or synthetic real-data acceptance. +8. **Integration/release engineering.** Verify baseline and storage exclusions, maintain source/task handoff ledger, reserve any necessary migration IDs before edits, own shared CI/build/runner files and final integration. Assemble reviewed commits, run appropriate local plus native Linux CI checks, and push normal updates to `eli/front-end-overhaul`. Observe CI for exact SHA, fix failures via owners/review, and produce final evidence report. No public release or deployment. + +## Cohesion and execution + +Lanes 1–5 start concurrently. Lane 6 prepares existing-contract consumption and begins real replay as soon as first handoffs arrive. QA reviews incremental code and evidence throughout, then performs independent reproduction. Lane 8 integrates continuously without letting unreviewed changes reach the shared target. All tasks report concise checkpoints: actual artifacts/capability delivered, exact tests, blockers, next handoff. Keep network acquisition respectful of source limits. Serialize destructive disposable-DB suites; never use retained/private production-like databases as test fixtures. + +Exact source IDs come from the existing registry, not invented new source names. Source/profile/year choices above are frozen; agents may solve technical acquisition problems autonomously but cannot replace a blocked source, change the inspection window, or call a partial capture complete without the project manager recording the limitation and obtaining owner agreement for a material scope change. + +## Completion conditions + +- Actual retained original artifacts exist and match hashes/byte sizes for every committed source; provider totals, enumerated partitions, accepted rows, duplicates, quarantine, failed/missing captures reconcile. Source limitations are quantified; source observations are never summed as unique facilities without a justified identity rule. +- APHIS current registration/latest complete-year report capture and 2025 inspection capture have reproducible existing-pipeline outputs; real investigator paths trace every displayed observation and link to retained evidence. A packet explains what it establishes and what it does not establish. +- Current FSIS outputs and the legacy comparison are reproducible and dated. +- France and Italy each build a genuine private candidate, with unresolved identity, location, privacy and rights states retained; publication permission is not required or fabricated for a private candidate. +- Independent QA rebuilds all handoffs and verifies stratified evidence, deterministic outputs and privacy/provenance boundaries; no unresolved critical/high defect in changed workflows. +- All reviewed code and safe reports are integrated; required local checks and Linux CI pass for the final implementation revision; origin/eli/front-end-overhaul contains it. Final report identifies exact SHA, artifact handoff locations, commands/results/skips, data coverage and remaining public-launch blockers. + +## Escalation and stopping + +Continue unaffected lanes when one source is blocked. Investigate official downloads, documented APIs and ordinary browser exports without bypassing controls. Escalate only a concrete unresolved need for owner access, payment, retention/legal judgment, destructive change, publication, or material scope substitution. Report endpoint/operation, attempted permitted alternatives and the specific decision needed, without raw evidence or credentials. Transient failures, difficult parsers, CI failures, missing tools and ordinary implementation choices are work to solve, not permission gates. + +The sprint is not complete merely because code/tests pass. Missing real artifacts or unfinished source deliverables keep it incomplete. The project manager records the goal as complete only after requirement-by-requirement evidence audit. User stop/pause requests stop all tasks promptly. diff --git a/docs/sprint02-integration-ledger.md b/docs/sprint02-integration-ledger.md new file mode 100644 index 0000000..e14c004 --- /dev/null +++ b/docs/sprint02-integration-ledger.md @@ -0,0 +1,45 @@ +# Sprint 02 integration and storage ledger + +Status: lane 8 kickoff checkpoint, 2026-09-19. This is a row-free engineering handoff; it is not a release approval or a claim that source acquisition is complete. + +## Ownership and baseline + +- Contract: [`SPRINT-02-CONTRACT.md`](SPRINT-02-CONTRACT.md) +- Required baseline: `5570b6ab42e74dc5227e08c2cd7c1ac7f08aab96` +- Lane worktree: `C:\Users\pnael\.codex\worktrees\8a23\UntilEveryCage` +- Lane branch: `codex/sprint02-integration-lane8` +- Integration target: `origin/eli/front-end-overhaul` +- Raw evidence boundary: `C:\New Projects\UntilEveryCage\.private\sprint02-20260919\` + +## Storage checkpoint + +The shared private root exists with separate directories for `aphis-core`, `aphis-inspections`, `fsis`, `france`, `italy`, `integration`, and `qa`. The repository-level `/.private/` rule is active in the shared checkout and this lane worktree. Raw records, URLs, identifiers, payloads, and derived rows remain outside Git and task reports. + +Verification commands: + +```powershell +git -C 'C:\New Projects\UntilEveryCage' check-ignore -v --no-index '.private/sprint02-20260919/aphis-core/probe.bin' +git check-ignore -v --no-index '.private/sprint02-20260919/probe.bin' +powershell -ExecutionPolicy Bypass -File pipeline/tests/verify-docker-context.ps1 +``` + +The Docker context test creates synthetic sentinels under both `data/` and `.private/`; the context image fails if either sentinel is copied. The shared Docker ignore rules exclude `.private/` and `**/.private/`, and the tracked worktree rules carry the same boundary. + +Kickoff validation: `npm ci` completed with no vulnerabilities; `python -m unittest scripts.test_dev` passed (7 tests); `python scripts/dev.py --json doctor` passed (database not probed because `UEC_DATABASE_URL` is unset); `git diff --check` passed; both shared-root and lane-worktree `git check-ignore` probes matched `/.private/`. `powershell -ExecutionPolicy Bypass -File pipeline/tests/verify-docker-context.ps1` was attempted with and without escalation but remains blocked because Docker Desktop's Linux engine pipe is unavailable: the client is installed, Docker Desktop processes are running, `com.docker.service` is stopped, and starting that service returns `Cannot open ... service on computer '.'`. No private data or database was used. + +## Integration ledger + +| Area | Owner/interface | Acceptance state | +| --- | --- | --- | +| APHIS registrations/reports | Lane 1 handoff under private storage | Pending source handoff and review | +| APHIS inspections | Lane 2 handoff under private storage | Pending source handoff and review | +| FSIS current parity | Lane 3 handoff under private storage | Pending source handoff and review | +| France candidate | Lane 4 handoff under private storage | Pending source handoff and review | +| Italy candidate | Lane 5 handoff under private storage | Pending source handoff and review | +| Evidence integration | Lane 6 existing APHIS/FSIS contracts | Pending reviewed handoffs | +| Independent QA | Lane 7 replay and review | Pending reviewed handoffs | +| CI/build/release engineering | Lane 8 | Storage and context boundary implemented; native CI and final integration pending | + +## Release gate + +No public release, deployment, or publication approval is implied. Before final integration, lane 8 must verify reviewed commits, migration reservations, private-artifact availability, reproducible commands, native Linux CI for the exact integrated SHA, and the remaining coverage/privacy/rights limitations. Failed acquisition leaves the previous validated release available subject to current restrictions. diff --git a/pipeline/tests/verify-docker-context.ps1 b/pipeline/tests/verify-docker-context.ps1 index 1f04f89..53a8d3e 100644 --- a/pipeline/tests/verify-docker-context.ps1 +++ b/pipeline/tests/verify-docker-context.ps1 @@ -3,14 +3,18 @@ param() $ErrorActionPreference = 'Stop' $root = (Resolve-Path (Join-Path $PSScriptRoot '..\..')).Path $sentinel = Join-Path $root 'data\.docker-context-sentinel' +$privateSentinel = Join-Path $root '.private\.docker-context-sentinel' $tag = "uec-docker-context-test-$([Guid]::NewGuid().ToString('N'))" New-Item -ItemType File -Path $sentinel -Force | Out-Null +New-Item -ItemType Directory -Path (Split-Path -Parent $privateSentinel) -Force | Out-Null +New-Item -ItemType File -Path $privateSentinel -Force | Out-Null try { & docker build --file (Join-Path $root 'Dockerfile.context-test') --tag $tag $root if ($LASTEXITCODE -ne 0) { throw "Docker context sentinel proof failed (exit $LASTEXITCODE)." } Write-Host 'PASS: synthetic private Docker context sentinel was excluded.' } finally { if (Test-Path -LiteralPath $sentinel) { Remove-Item -LiteralPath $sentinel -Force } + if (Test-Path -LiteralPath $privateSentinel) { Remove-Item -LiteralPath $privateSentinel -Force } & docker image rm --force $tag *> $null } From 061e5d766bf18a3adbd271ca6f29a289e5fdfa7b Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sat, 19 Sep 2026 11:08:57 -0700 Subject: [PATCH 285/311] Refresh Italy private 853 candidate handoff --- data/manifests/italy-sprint02-20260919.json | 63 +++++++++++++++++++ docs/reports/italy-sprint02-20260919.md | 58 +++++++++++++++++ docs/review-packet-italy.md | 11 +++- docs/source-status.json | 2 +- pipeline/sources/italy/README.md | 18 ++++++ pipeline/sources/italy/it_853_adapter.py | 12 ++++ pipeline/sources/italy/test_it_853_adapter.py | 2 +- 7 files changed, 163 insertions(+), 3 deletions(-) create mode 100644 data/manifests/italy-sprint02-20260919.json create mode 100644 docs/reports/italy-sprint02-20260919.md diff --git a/data/manifests/italy-sprint02-20260919.json b/data/manifests/italy-sprint02-20260919.json new file mode 100644 index 0000000..e036693 --- /dev/null +++ b/data/manifests/italy-sprint02-20260919.json @@ -0,0 +1,63 @@ +{ + "report_type": "private-source-handoff", + "report_version": "italy-sprint02-v1", + "source_id": "it.853-2004", + "scope": "Italy; Ministry of Health establishments recognized under Regulation (EC) 853/2004; one source row per establishment/activity observation", + "excluded_scope": ["it.1069-2009", "legacy Italy CSVs", "public release or promotion"], + "source_url": "https://www.dati.salute.gov.it/it/dataset/stabilimenti-italiani-gli-alimenti-di-origine-animale/", + "download_url": "https://www.dati.salute.gov.it/sites/default/files/opendata/STAB_POA_8_20260919.csv", + "catalog_last_updated": "2026-09-19", + "filename_publication_date": "2026-09-19", + "retrieved_at_utc": "2026-09-19T18:00:29Z", + "source_sha256": "06c853fea232bd692e101d3c0b1b660d92ff80f56b2421bdd9e9746b01189dd8", + "source_byte_size": 49940074, + "catalog_sha256": "1db2f27a54a1b54a51295da027624baf94f853afb5d05a2d310177b3af2cdcdf", + "schema_fingerprint": "3ba24374ea6412c7240218984fac00b2e0e9f84e382ec72c19bbcef1a73c0826", + "counts": { + "input_rows": 47375, + "normalized_rows": 41849, + "quarantined_rows": 5526, + "ambiguous_repeated_recognition_activity": 5526, + "provisional_facility_groups_in_accepted_rows": 25316 + }, + "normalized_sha256": "661ef0b5f1a9c30b175c239438a22bdef023745d38fa94a871da0343fe09fb7b", + "parsed_sha256": "22f7ebf04255b1d46c1d2828fefb3fbe0e228ce544ac927fae046410ce2b3335", + "categories": { + "source_category_values_preserved": 18, + "source_activity_codes_preserved": 19 + }, + "privacy_and_rights": { + "addresses_tax_identifiers_and_source_coordinates": "restricted-source-values-only", + "registered_location": "not-supplied-by-source", + "recognized_establishment_location": "preserved privately; not operating-proof", + "geocoding": "disabled", + "privacy_gate": "pending-review", + "coordinate_gate": "review-required", + "rights_gate": "review-required", + "publication": "blocked; no release created" + }, + "partitions": { + "attempted": ["official catalog", "catalog-discovered 853/2004 CSV"], + "captured": ["catalog.html", "source.csv", "acquisition-metadata.json", "private lifecycle candidate and review artifacts"], + "failed": ["initial Python urllib transport: TLS handshake failure; ordinary curl HTTPS transport succeeded"], + "not_attempted": ["1069/2009 by-products dataset; excluded by sprint scope"] + }, + "private_handoff": { + "root": "C:\\New Projects\\UntilEveryCage\\.private\\sprint02-20260919\\italy", + "raw_run": "C:\\New Projects\\UntilEveryCage\\.private\\sprint02-20260919\\italy\\raw\\it.853-2004\\20260919T000000Z-live", + "staging_run": "C:\\New Projects\\UntilEveryCage\\.private\\sprint02-20260919\\italy\\staging\\20260919T000000Z-live\\06c853fea232bd69-wiuptilp", + "status": "candidate-ready; private-validated; no public exposure" + }, + "acquisition_commands": [ + "curl.exe -fL --http1.1 --tlsv1.2 -A 'UntilEveryCage/controlled-acquisition' -D \\catalog.headers -o \\catalog.html https://www.dati.salute.gov.it/it/dataset/stabilimenti-italiani-gli-alimenti-di-origine-animale/", + "discover_csv(\\catalog.html, official catalog URL) -> https://www.dati.salute.gov.it/sites/default/files/opendata/STAB_POA_8_20260919.csv", + "curl.exe -fL --http1.1 --tlsv1.2 -A 'UntilEveryCage/controlled-acquisition' -D \\source.headers -o \\source.csv " + ], + "verification_commands": [ + "git -C 'C:\\New Projects\\UntilEveryCage' check-ignore -v --no-index .private/sprint02-20260919/italy/probe", + "python -m unittest pipeline.sources.italy.test_acquire pipeline.sources.italy.test_it_853_adapter", + "python -m pipeline.sources.italy.refresh --raw '\\raw\\it.853-2004\\20260919T000000Z-live\\source.csv' --run-dir '\\staging\\20260919T000000Z-live'" + ], + "code_revision": "worktree baseline 5570b6ab42e74dc5227e08c2cd7c1ac7f08aab96 plus Italy location-semantics change", + "release_state": "not-created" +} diff --git a/docs/reports/italy-sprint02-20260919.md b/docs/reports/italy-sprint02-20260919.md new file mode 100644 index 0000000..8918b1b --- /dev/null +++ b/docs/reports/italy-sprint02-20260919.md @@ -0,0 +1,58 @@ +# Italy Sprint 02 private handoff + +This row-free report records the lane-5 private candidate refresh completed on +2026-09-19. It is not a release, factual approval, completeness claim, or +redistribution clearance. + +## Frozen source and scope + +- Source ID: `it.853-2004`. +- Official boundary: Italian Ministry of Health catalog for establishments + recognized under Regulation (EC) 853/2004. +- Catalog: `https://www.dati.salute.gov.it/it/dataset/stabilimenti-italiani-gli-alimenti-di-origine-animale/`. +- Download observed: dated `STAB_POA_8_20260919.csv`. +- Separate `it.1069-2009` by-products data was not acquired or unioned. + +## Acquisition evidence + +The retained source is 49,940,074 bytes with SHA-256 +`06c853fea232bd692e101d3c0b1b660d92ff80f56b2421bdd9e9746b01189dd8`. +The catalog artifact hash is +`1db2f27a54a1b54a51295da027624baf94f853afb5d05a2d310177b3af2cdcdf`. +The catalog supplied `2026-09-19` as its last-updated date and the filename +supplied the same publication-date signal. Retrieval was recorded at +`2026-09-19T18:00:29Z`. + +The repository's Python urllib route encountered a TLS handshake failure. The +same official catalog-discovered URL was acquired with ordinary `curl.exe` +HTTPS (`--http1.1 --tlsv1.2`), preserving catalog and source response headers. +No challenge bypass, paid service, or alternate source was used. + +## Candidate and review boundary + +The private lifecycle reconciles 47,375 source observations into 41,849 +normalized observations and 5,526 quarantined rows. Every quarantine reason +was `ambiguous_repeated_recognition_activity`; repeated recognition/activity +observations remain occurrence-qualified and are not merged. Accepted rows +form 25,316 provisional source-recognition groups, not canonical facilities. + +The source supplies a recognized-establishment location, not a legal +registered office. The adapter now carries that distinction explicitly: +registered location is `not-supplied-by-source`, and the establishment address +remains restricted source evidence rather than proof of current operation. +Source categories and activity codes remain preserved diagnostics. Geocoding is +disabled; coordinates remain source-value-pending-review. Address, tax, +coordinate, privacy, rights, project-approval, and publication gates remain +closed. + +## Private handoff and replay + +Exact restricted paths, safe hashes/counts, and replay commands are recorded in +`data/manifests/italy-sprint02-20260919.json`. The private handoff is under +`C:\New Projects\UntilEveryCage\.private\sprint02-20260919\italy`; no raw or +row-level artifact is tracked in Git or included in this report. + +Checks run: `python -m unittest pipeline.sources.italy.test_acquire +pipeline.sources.italy.test_it_853_adapter` (15 tests, pass), live catalog and +CSV hash verification, private lifecycle replay, candidate handoff, row-free +review packet, private health report, and graph-candidate manifest. diff --git a/docs/review-packet-italy.md b/docs/review-packet-italy.md index cbe83a5..d9b03c4 100644 --- a/docs/review-packet-italy.md +++ b/docs/review-packet-italy.md @@ -1,6 +1,15 @@ # Italy private review packet -As of 2026-09-15, `it.853-2004` has catalog-linked private acquisition and candidate lifecycle support. `it.1069-2009` remains a separate, unimplemented candidate. Publication is blocked. +As of 2026-09-19, `it.853-2004` has a fresh catalog-linked private acquisition +and candidate lifecycle run. `it.1069-2009` remains a separate, unimplemented +candidate. Publication is blocked. + +The live handoff contains 47,375 source observations, 41,849 normalized rows, +and 5,526 quarantined repeated recognition/activity observations. The source +artifact is 49,940,074 bytes with SHA-256 +`06c853fea232bd692e101d3c0b1b660d92ff80f56b2421bdd9e9746b01189dd8`; see the +row-free [Sprint 02 handoff](reports/italy-sprint02-20260919.md) for the exact +private path and replay evidence. - Terms/licensing: the Ministry catalogue indicates Italian Open Data Licence v2.0; attribution and project redistribution review remain open. - Privacy: addresses, tax identifiers, and precise source coordinates remain in restricted evidence; normalized/API-shaped rows suppress them pending review. diff --git a/docs/source-status.json b/docs/source-status.json index 1e73b15..e2b8971 100644 --- a/docs/source-status.json +++ b/docs/source-status.json @@ -17,7 +17,7 @@ {"source_id":"be.locations","metadata":"verified","acquisition":"artifact_private_only","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-be.md","data/manifests/de-be-private-candidates-2026-09-17.json","pipeline/sources/belgium/adapter.py","pipeline/sources/belgium/refresh.py","docs/review-packet-belgium.md","pipeline/source_registry.json"],"next_action":"Review the private current pair: 310,660 input, 310,640 normalized, 20 quarantined; names are not supplied by this snapshot. Keep privacy, attribution, classification, terms, and project approval gates closed."}, {"source_id":"fr.dgal.section-i","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-fr.md","docs/countries/france/dgal-853-pipeline.md","pipeline/sources/france/adapter.py","pipeline/sources/france/acquire.py","pipeline/sources/france/refresh.py","pipeline/sources/france/fixtures/section_i.csv","pipeline/common/review_packet.py","pipeline/source_registry.json"],"next_action":"Run the bounded private Section I refresh with an approved terms record or authorized capture; review category semantics, address privacy, schema drift, and release approval."}, {"source_id":"fr.dgal.section-ii","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-fr.md","docs/countries/france/dgal-853-pipeline.md","pipeline/sources/france/adapter.py","pipeline/sources/france/acquire.py","pipeline/sources/france/refresh.py","pipeline/sources/france/fixtures/section_ii.csv","pipeline/common/review_packet.py","pipeline/source_registry.json"],"next_action":"Run the bounded private Section II refresh separately from Section I; review category/species semantics, address privacy, schema drift, and release approval."}, - {"source_id":"it.853-2004","metadata":"verified","acquisition":"artifact_private_only","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-it.md","pipeline/source_registry.json","pipeline/sources/italy/acquire.py","pipeline/sources/italy/it_853_adapter.py","pipeline/sources/italy/README.md","pipeline/common/review_packet.py","docs/review-packet-italy.md","pipeline/tests/e2e/test_italy_candidate_import.py"],"next_action":"Keep catalog acquisition, lifecycle, candidate import, and guarded API evidence private; use source-category/activity diagnostics to resolve repeated identity, coordinate/address privacy, coverage, and project approval before release review."}, + {"source_id":"it.853-2004","metadata":"verified","acquisition":"artifact_private_only","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-it.md","data/manifests/italy-sprint02-20260919.json","docs/reports/italy-sprint02-20260919.md","pipeline/source_registry.json","pipeline/sources/italy/acquire.py","pipeline/sources/italy/it_853_adapter.py","pipeline/sources/italy/README.md","pipeline/common/review_packet.py","docs/review-packet-italy.md","pipeline/tests/e2e/test_italy_candidate_import.py"],"next_action":"Private 2026-09-19 catalog-linked refresh is candidate-ready: 47,375 input, 41,849 normalized, 5,526 quarantined repeated recognition/activity observations. Keep the handoff private; review source categories, provisional identity, coordinate/address privacy, terms, coverage, and project approval before release review."}, {"source_id":"it.1069-2009","metadata":"verified","acquisition":"not_run","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-it.md","pipeline/source_registry.json","pipeline/sources/italy/README.md"],"next_action":"Keep the by-products dataset separate; assess scope, schema, terms, identity links, privacy, and a dedicated adapter before any acquisition or integration."}, {"source_id":"mx.locations","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-mx.md","pipeline/source_registry.json"],"next_action":"Resolve DENUE token/terms and SENASICA current directory/schema before bounded acquisition."}, {"source_id":"nz.locations","metadata":"verified","acquisition":"blocked","runtime_health":"not_run","publication_eligibility":"blocked","evidence":["docs/country-recon-nz.md","pipeline/source_registry.json"],"next_action":"Confirm MPI permitted acquisition route, list-specific terms, and schema before private staging."}, diff --git a/pipeline/sources/italy/README.md b/pipeline/sources/italy/README.md index a7cceed..3d02701 100644 --- a/pipeline/sources/italy/README.md +++ b/pipeline/sources/italy/README.md @@ -46,3 +46,21 @@ This is private staging evidence, not a completeness, accuracy, project- approval, or publication claim. The catalog notes that some coordinates came from OpenStreetMap contributors; that provenance does not itself authorize precise-coordinate publication. + +## Sprint 02 live handoff + +The 2026-09-19 private refresh is recorded in +`data/manifests/italy-sprint02-20260919.json` and +`docs/reports/italy-sprint02-20260919.md`. The exact raw and staging paths are +restricted under the sprint's excluded private root; no row-level artifact is +tracked. The source run had 47,375 observations, 41,849 normalized rows, and +5,526 quarantined repeated recognition/activity observations. A registered +office is not supplied by this source: the source address is a recognized- +establishment location and is not treated as proof of current operation. + +The preferred replay command is the `acquire.py --fetch` command above. If the +local Python TLS stack cannot negotiate the Ministry host, use the ordinary +HTTPS `curl.exe --http1.1 --tlsv1.2` catalog-then-discovered-CSV capture route +documented in the private handoff, preserving both response headers and the +catalog-discovered URL before running `refresh.py`. This is a transport +fallback only; it does not bypass access controls or change source scope. diff --git a/pipeline/sources/italy/it_853_adapter.py b/pipeline/sources/italy/it_853_adapter.py index ad44aa4..2b0c5d2 100644 --- a/pipeline/sources/italy/it_853_adapter.py +++ b/pipeline/sources/italy/it_853_adapter.py @@ -140,6 +140,17 @@ def parse_bytes(self, content: bytes) -> dict[str, Any]: "name": clean(row.get("ragione_sociale")), "trading_name": clean(row.get("ragione_sociale")), "address": None, + # The Ministry feed supplies the address of the recognized + # establishment, not a legal entity's registered office. Keep + # both concepts explicit and do not turn the source address + # into an operating-site claim without review. + "location_role": "recognized-establishment-location", + "location_semantics": "source-recognized-establishment-address; not-registered-office; not-operating-proof", + "registered_location": None, + "registered_location_state": "not-supplied-by-source", + "operating_location": None, + "operating_location_state": "source-location-not-operating-proof", + "source_location_state": "source-address-private-pending-review" if clean(row.get("indirizzo")) else "unknown", "municipality": clean(row.get("comune")), "city": clean(row.get("comune")), "province": clean(row.get("provincia")), @@ -164,6 +175,7 @@ def parse_bytes(self, content: bytes) -> dict[str, Any]: "coordinate_precision": "source-precision-unknown" if clean(row.get("longitudine")) or clean(row.get("latitudine")) else "unresolved", "privacy_gate": "pending-review", "coordinate_gate": "review_required", + "rights_gate": "review_required", "publication_gate": "blocked", } record = { diff --git a/pipeline/sources/italy/test_it_853_adapter.py b/pipeline/sources/italy/test_it_853_adapter.py index 7ec7f5b..e142c4b 100644 --- a/pipeline/sources/italy/test_it_853_adapter.py +++ b/pipeline/sources/italy/test_it_853_adapter.py @@ -7,7 +7,7 @@ def row(n="A",a="10",s="Autorizzata"): return f";{n};Name;;Town;;010;Piemonte;X;{a};Activity;P;S;IT;12;45;1;tax;vat;001001;;;{s};2026-09-13;\n" class Test(unittest.TestCase): def test_safe_mapping(self): - r=Italy853Adapter().parse_bytes((H+"\n"+row()).encode())["accepted"][0]; self.assertIsNone(r["normalized"]["coordinates"]); self.assertIsNone(r["normalized"]["address"]); self.assertIn("p_iva",r["source_values"]) + r=Italy853Adapter().parse_bytes((H+"\n"+row()).encode())["accepted"][0]; self.assertIsNone(r["normalized"]["coordinates"]); self.assertIsNone(r["normalized"]["address"]); self.assertIn("p_iva",r["source_values"]); self.assertEqual(r["normalized"]["location_role"],"recognized-establishment-location"); self.assertEqual(r["normalized"]["registered_location_state"],"not-supplied-by-source"); self.assertEqual(r["normalized"]["operating_location_state"],"source-location-not-operating-proof"); self.assertEqual(r["normalized"]["rights_gate"],"review_required") def test_quarantine(self): r=Italy853Adapter().parse_bytes((H+"\n"+row()+row("A","10","Unknown")).encode()); self.assertEqual(len(r["accepted"]),1); self.assertIn("unknown_status",r["quarantined"][0]["reasons"]) def test_sensitive_and_deterministic_identity(self): From db3761adcd91bd64d46211ac38b16df61641384a Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sat, 19 Sep 2026 11:10:52 -0700 Subject: [PATCH 286/311] Clarify Italy retrieval timestamp audit --- data/manifests/italy-sprint02-20260919.json | 1 + docs/reports/italy-sprint02-20260919.md | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/data/manifests/italy-sprint02-20260919.json b/data/manifests/italy-sprint02-20260919.json index e036693..b708e71 100644 --- a/data/manifests/italy-sprint02-20260919.json +++ b/data/manifests/italy-sprint02-20260919.json @@ -48,6 +48,7 @@ "staging_run": "C:\\New Projects\\UntilEveryCage\\.private\\sprint02-20260919\\italy\\staging\\20260919T000000Z-live\\06c853fea232bd69-wiuptilp", "status": "candidate-ready; private-validated; no public exposure" }, + "run_id_semantics": "20260919T000000Z-live is an operator-assigned path label, not a retrieval timestamp", "acquisition_commands": [ "curl.exe -fL --http1.1 --tlsv1.2 -A 'UntilEveryCage/controlled-acquisition' -D \\catalog.headers -o \\catalog.html https://www.dati.salute.gov.it/it/dataset/stabilimenti-italiani-gli-alimenti-di-origine-animale/", "discover_csv(\\catalog.html, official catalog URL) -> https://www.dati.salute.gov.it/sites/default/files/opendata/STAB_POA_8_20260919.csv", diff --git a/docs/reports/italy-sprint02-20260919.md b/docs/reports/italy-sprint02-20260919.md index 8918b1b..a700e99 100644 --- a/docs/reports/italy-sprint02-20260919.md +++ b/docs/reports/italy-sprint02-20260919.md @@ -23,6 +23,11 @@ The catalog supplied `2026-09-19` as its last-updated date and the filename supplied the same publication-date signal. Retrieval was recorded at `2026-09-19T18:00:29Z`. +The private run directory name `20260919T000000Z-live` is a deterministic +operator label only; it is not used as the retrieval timestamp. The private +acquisition metadata records the catalog response at `18:00:07Z`, the CSV +response at `18:00:29Z`, and an audit event documenting this distinction. + The repository's Python urllib route encountered a TLS handshake failure. The same official catalog-discovered URL was acquired with ordinary `curl.exe` HTTPS (`--http1.1 --tlsv1.2`), preserving catalog and source response headers. From fbf4e6824792078a4c3aa5ac1f730e0629039224 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sat, 19 Sep 2026 11:02:31 -0700 Subject: [PATCH 287/311] Activate France DGAL private candidate handoff --- docs/countries/france/dgal-853-pipeline.md | 12 ++ .../france/sprint02-handoff-20260919.md | 82 +++++++++ .../france/sprint02-kickoff-20260919.md | 43 +++++ pipeline/sources/france/adapter.py | 34 +++- pipeline/sources/france/reconcile.py | 173 ++++++++++++++++++ pipeline/sources/france/test_adapter.py | 12 ++ pipeline/sources/france/test_reconcile.py | 33 ++++ 7 files changed, 382 insertions(+), 7 deletions(-) create mode 100644 docs/countries/france/sprint02-handoff-20260919.md create mode 100644 docs/countries/france/sprint02-kickoff-20260919.md create mode 100644 pipeline/sources/france/reconcile.py create mode 100644 pipeline/sources/france/test_reconcile.py diff --git a/docs/countries/france/dgal-853-pipeline.md b/docs/countries/france/dgal-853-pipeline.md index d30f62e..cefe8b4 100644 --- a/docs/countries/france/dgal-853-pipeline.md +++ b/docs/countries/france/dgal-853-pipeline.md @@ -28,3 +28,15 @@ publication approval; the current site-wide Etalab indication still needs file-specific confirmation. Run with `python -m pipeline.sources.france.refresh --section I --raw --run-dir ` or `--fetch --terms-review `. Raw artifacts belong in ignored private storage only. + +For a two-scope row-free reconciliation after both private lifecycle runs: + +`python -m pipeline.sources.france.reconcile --section-i-run --section-ii-run --output /france-reconciliation.json` + +The reconciliation reports source totals, input/normalized/quarantine +partitions, category and location/privacy gate counts, and approval-number +overlap counts without emitting identifiers. It never treats either section's +observations as unique facilities and never merges overlap automatically. It +also carries shared-SIRET identity-review counts when a source entity spans +distinct approval/category observations; those rows remain source observations +and are not quarantined solely for that unresolved identity signal. diff --git a/docs/countries/france/sprint02-handoff-20260919.md b/docs/countries/france/sprint02-handoff-20260919.md new file mode 100644 index 0000000..a03eea9 --- /dev/null +++ b/docs/countries/france/sprint02-handoff-20260919.md @@ -0,0 +1,82 @@ +# France Sprint 02 private handoff + +This row-free report records the completed private candidate handoff. It does +not contain facility names, addresses, SIRET values, coordinates, or source +rows. The retained source and derived row payloads remain under the ignored +private root below. + +## Handoff location + +`C:\New Projects\UntilEveryCage\.private\sprint02-20260919\france` + +The shared checkout verifies this path as ignored by: + +```powershell +git -C 'C:\New Projects\UntilEveryCage' check-ignore -v --no-index '.private/sprint02-20260919/france/probe' +``` + +The verified rule is `/.private/` in the shared checkout's `.gitignore`, with +an additional local `.private/.gitignore` containing `*`. + +## Captured sources + +| Source ID | Scope | Retrieved UTC | Source effective/last-modified date | Bytes | SHA-256 | Input | Normalized | Quarantined | +| --- | --- | --- | --- | ---: | --- | ---: | ---: | ---: | +| `fr.dgal.section-i` | DGAL 853/2004 Section I; domestic ungulates | 2026-09-19T17:58:11Z | 2026-09-19T02:21:55Z | 204401 | `30375dc6d426ae723496d2a777e99b43328920f7d50a39ebd2e5fb3baf4bf933` | 1449 | 1449 | 0 | +| `fr.dgal.section-ii` | DGAL 853/2004 Section II; poultry and lagomorphs | 2026-09-19T17:58:27Z | 2026-09-19T02:22:19Z | 136766 | `f011341de6fcaf352b625d7dbee4fb9252f5722d7f598d3e35bed0b044dab9e1` | 1068 | 1068 | 0 | + +Both official routes returned one bounded response and were captured. No +failed acquisition partition occurred. Combined input/normalized/quarantine +totals are 2517/2517/0, and the partition check is valid. The two scopes are +not summed as unique facilities. The row-free reconciliation counted 233 +shared provisional approval-number signals; this is an unresolved identity +review signal and no automatic merge was performed. `unique_facility_count` +remains intentionally unknown/null. The corrected v2 replay reports 2 +shared-SIRET groups spanning approval/category observations across 4 Section I +rows and 0 Section II groups/rows. These remain accepted source observations +with an explicit unresolved identity-review state. Source address-state +counts are 1434 present / 15 blank-or-whitespace for Section I and 1037 / 31 +for Section II; normalized address and coordinate fields remain suppressed. + +## Exact private artifacts + +- Section I raw run: `raw\fr.dgal.section-i\section-i-20260919T175808Z\source.txt` +- Section I lifecycle run (v2 replay): `refresh-section-i-replay-v2-20260919T\lifecycle\30375dc6d426ae72-5p9j19m3` +- Section I candidate handoff (v2 replay): `refresh-section-i-replay-v2-20260919T\lifecycle\30375dc6d426ae72-5p9j19m3\candidate-handoff` +- Section II raw run: `raw\fr.dgal.section-ii\section-ii-20260919T175824Z\source.txt` +- Section II lifecycle run (v2 replay): `refresh-section-ii-replay-v2-20260919T\lifecycle\f011341de6fcaf35-mkj1wakx` +- Section II candidate handoff (v2 replay): `refresh-section-ii-replay-v2-20260919T\lifecycle\f011341de6fcaf35-mkj1wakx\candidate-handoff` +- Row-free reconciliation: `france-reconciliation-20260919.json` +- Private acquisition terms record: `terms-review-private-acquisition.json` + +Each lifecycle run contains acquisition metadata, immutable source hash/size, +parsed and normalized JSONL, quarantine JSONL, QA, health, run status, review +packet, release diff, history ledger, and candidate handoff. The normalized +records retain source provenance while suppressing address and coordinates +from normalized location fields; original source values remain restricted. + +## Reproduction and verification + +From the repository checkout, with the shared private root already present: + +```powershell +python -m unittest pipeline.sources.france.test_adapter pipeline.sources.france.test_refresh pipeline.sources.france.test_reconcile -v +python -m pipeline.sources.france.reconcile --section-i-run '\refresh-section-i-replay-v2-20260919T\lifecycle\30375dc6d426ae72-5p9j19m3' --section-ii-run '\refresh-section-ii-replay-v2-20260919T\lifecycle\f011341de6fcaf35-mkj1wakx' --output '\france-reconciliation-20260919.json' +Get-FileHash -Algorithm SHA256 '\raw\fr.dgal.section-i\section-i-20260919T175808Z\source.txt' +Get-FileHash -Algorithm SHA256 '\raw\fr.dgal.section-ii\section-ii-20260919T175824Z\source.txt' +``` + +The corrected v2 replay lifecycle states are `candidate-ready` / +`private-candidate`, +`publication_state=human-gate-required`, `release_state=not-created`, and +`geocoding=disabled` for both sources. The replay uses adapter +`fr-dgal-853-v2` and schema `fr-dgal-853-txt-v2`. No public release, API +promotion, or paid geocoding was performed. + +## Remaining gates + +File-specific reuse/attribution terms, privacy and address screening, +coordinate review, category-code review, cross-section identity review, and +project publication approval remain open. A source disappearance is not +interpreted as closure. This handoff is suitable for restricted review and +replay only, not public redistribution. diff --git a/docs/countries/france/sprint02-kickoff-20260919.md b/docs/countries/france/sprint02-kickoff-20260919.md new file mode 100644 index 0000000..51a37d9 --- /dev/null +++ b/docs/countries/france/sprint02-kickoff-20260919.md @@ -0,0 +1,43 @@ +# France Sprint 02 kickoff + +Recorded before any Sprint 02 retained download: 2026-09-19. + +This is a row-free lane report. Facility names, addresses, SIRET values, +coordinates, raw responses, and normalized records remain in the shared +restricted handoff only. + +## Frozen existing registry sources + +| Source ID | Existing source scope | Official route | +| --- | --- | --- | +| `fr.dgal.section-i` | France DGAL Regulation (EC) 853/2004 Section I; domestic ungulate establishments | `https://fichiers-publics.agriculture.gouv.fr/dgal/ListesOfficielles/SSA1_VIAN_ONG_DOM.txt` | +| `fr.dgal.section-ii` | France DGAL Regulation (EC) 853/2004 Section II; poultry and lagomorph establishments | `https://fichiers-publics.agriculture.gouv.fr/dgal/ListesOfficielles/SSA1_VIAN_COL_LAGO.txt` | + +These IDs and scopes are taken from the existing `pipeline/source_registry.json` +and are not newly invented for this lane. Section I and Section II remain +separate source scopes; overlap is reconciled as observations and is not an +automatic facility merge. + +## Intended private handoff + +The restricted handoff root is: + +`C:\New Projects\UntilEveryCage\.private\sprint02-20260919\france` + +The lane will preserve one new run directory per source retrieval, including +the original response bytes, acquisition metadata, parsed/normalized output, +quarantine, QA/health/review artifacts, and candidate handoff. Geocoding is +disabled. The output is a private candidate only; no release or public API +promotion is performed by this lane. + +## Coverage and unresolved gates + +- Retrieval is a current snapshot of each published DGAL file, not a claim of + historical completeness or operating status. +- A missing later row means `not observed`, never closure. +- Source category/activity labels remain alongside conservative derived labels; + unknown categories, duplicate observations, identity ambiguity, and missing + location precision remain review states. +- File-specific terms/attribution, privacy screening, coordinate review, and + project approval remain human gates. No paid geocoding or guessed coordinate + is used. diff --git a/pipeline/sources/france/adapter.py b/pipeline/sources/france/adapter.py index db29768..b606fe3 100644 --- a/pipeline/sources/france/adapter.py +++ b/pipeline/sources/france/adapter.py @@ -21,7 +21,7 @@ "approval_number": ("approval number", "approval no", "n dagrement", "numero dagrement", "num dagrement", "agrément", "agrement", "numéro agrément/approval number"), "siret": ("siret", "siret number"), "legal_name": ("legal name", "company name", "raison sociale", "nom de letablissement", "nom de l'etablissement", "establishment name", "raison sociale - enseigne commerciale/name"), - "address": ("address", "adresse", "location address"), + "address": ("address", "adresse", "adresse/adress", "address/adresse", "location address"), "postal_code": ("postal code", "code postal", "postcode", "code postal/postal code"), "commune": ("commune", "municipality", "city", "town", "commune/town"), "category": ("category", "categorie", "catégorie", "establishment category", "catégorie/category"), @@ -59,18 +59,36 @@ def __init__(self, source_id: str, section: str, source_url: str) -> None: if section not in {"I", "II"}: raise ValueError("DGAL section must be I or II") self.source_id, self.section, self.source_url = source_id, section, source_url - self.adapter_version = "fr-dgal-853-v1" - self.schema_version = "fr-dgal-853-txt-v1" + self.adapter_version = "fr-dgal-853-v2" + self.schema_version = "fr-dgal-853-txt-v2" def parse_bytes(self, content: bytes) -> dict[str, Any]: headers, rows, delimiter, schema_fingerprint = read_rows(content, ALIASES, required=REQUIRED) mapping = resolve_mapping(headers, ALIASES) + siret_contexts: dict[str, set[tuple[str | None, str | None]]] = {} + for row in rows: + siret = _clean(value(row, mapping, "siret")) + if siret: + siret_contexts.setdefault(siret, set()).add(( + _clean(value(row, mapping, "approval_number")), + _clean(value(row, mapping, "category")), + )) + conflicted_sirets = { + siret for siret, contexts in siret_contexts.items() + if len({approval for approval, _ in contexts}) > 1 + or len({category for _, category in contexts}) > 1 + } occurrences: Counter[tuple[str | None, ...]] = Counter() accepted: list[dict[str, Any]] = [] quarantined: list[dict[str, Any]] = [] + identity_conflicted_rows = 0 for line, row in enumerate(rows, 2): approval, category = _clean(value(row, mapping, "approval_number")), _clean(value(row, mapping, "category")) activities = _clean(value(row, mapping, "associated_activities")) + siret = _clean(value(row, mapping, "siret")) + identity_conflict = bool(siret and siret in conflicted_sirets) + if identity_conflict: + identity_conflicted_rows += 1 key = occurrence_key(row, mapping, ("approval_number", "category", "associated_activities", "species")) occurrences[key] += 1 reasons: list[str] = [] @@ -90,7 +108,8 @@ def parse_bytes(self, content: bytes) -> dict[str, Any]: "establishment_id": approval, "recognition_number": approval, "facility_grouping": "provisional-dgal-approval-number", "identity_review": "required-before-merge", "name": _clean(value(row, mapping, "legal_name")), "trading_name": _clean(value(row, mapping, "legal_name")), - "siret": _clean(value(row, mapping, "siret")), + "siret": siret, + "identity_conflict_state": "shared-siret-across-approval-or-category; unresolved-before-merge" if identity_conflict else "none-observed", "address": None, "address_state": "source-value-present-pending-review" if _clean(value(row, mapping, "address")) else "unknown", "postal_code": _clean(value(row, mapping, "postal_code")), "municipality": _clean(value(row, mapping, "commune")), "city": _clean(value(row, mapping, "commune")), "department_number": _clean(value(row, mapping, "department_number")), @@ -104,7 +123,7 @@ def parse_bytes(self, content: bytes) -> dict[str, Any]: } if reasons: quarantined.append({"reasons": tuple(dict.fromkeys(reasons)), "record": record}) else: accepted.append(record) - return {"accepted": accepted, "quarantined": quarantined, "input_rows": len(rows), "delimiter": delimiter, "headers": headers, "schema_fingerprint": schema_fingerprint, "source_sha256": hashlib.sha256(content).hexdigest()} + return {"accepted": accepted, "quarantined": quarantined, "input_rows": len(rows), "delimiter": delimiter, "headers": headers, "schema_fingerprint": schema_fingerprint, "source_sha256": hashlib.sha256(content).hexdigest(), "identity_conflicted_siret_groups": len(conflicted_sirets), "identity_conflicted_rows": identity_conflicted_rows} def parse_file(self, path: str | Path) -> dict[str, Any]: return self.parse_bytes(Path(path).read_bytes()) @@ -118,10 +137,11 @@ def run(self, raw_path: str | Path, run_dir: str | Path, artifact: SourceArtifac _, normalized_sha256, _ = atomic_jsonl(root / "normalized" / "records.jsonl", accepted) atomic_jsonl(root / "quarantined" / "records.jsonl", quarantined) anomaly_counts = Counter(reason for item in quarantined for reason in item["reasons"]) + anomaly_counts["identity_conflict_siret"] = result["identity_conflicted_rows"] manifest = private_manifest(source_id=self.source_id, adapter_version=self.adapter_version, schema_version=self.schema_version, artifact=artifact, input_rows=result["input_rows"], normalized_rows=len(accepted), quarantined_rows=len(quarantined), normalized_sha256=normalized_sha256, parsed_sha256=parsed_sha256, anomaly_counts=dict(sorted(anomaly_counts.items()))) - manifest.update({"country_code": "FR", "section": self.section, "delimiter": result["delimiter"], "schema_fingerprint": result["schema_fingerprint"], "coverage": f"France DGAL Regulation (EC) 853/2004 Section {self.section}; source rows only; no completeness claim", "geocoding": "disabled"}) + manifest.update({"country_code": "FR", "section": self.section, "delimiter": result["delimiter"], "schema_fingerprint": result["schema_fingerprint"], "coverage": f"France DGAL Regulation (EC) 853/2004 Section {self.section}; source rows only; no completeness claim", "geocoding": "disabled", "identity_conflict_counts": {"siret_groups_with_cross_approval_or_category": result["identity_conflicted_siret_groups"], "rows_flagged": result["identity_conflicted_rows"], "state": "unresolved-before-human-review; no automatic merge"}}) atomic_json(root / "manifest.json", manifest) - write_operator_review_packet(root, manifest, source_scope=manifest["coverage"], checks=("confirm DGAL file terms and attribution", "review residential or mixed-use addresses", "review duplicate approval/activity identities", "confirm category codebook and current-list semantics", "approve any project release separately"), blockers=("publication approval not granted", "privacy and coordinate review pending", "source disappearance means not observed, not closure")) + write_operator_review_packet(root, manifest, source_scope=manifest["coverage"], checks=("confirm DGAL file terms and attribution", "review residential or mixed-use addresses", "review duplicate approval/activity identities", "review shared SIRET groups spanning approval/category observations; do not merge automatically", "confirm category codebook and current-list semantics", "approve any project release separately"), blockers=("publication approval not granted", "privacy and coordinate review pending", "source disappearance means not observed, not closure")) return manifest diff --git a/pipeline/sources/france/reconcile.py b/pipeline/sources/france/reconcile.py new file mode 100644 index 0000000..02a3349 --- /dev/null +++ b/pipeline/sources/france/reconcile.py @@ -0,0 +1,173 @@ +"""Build a row-free reconciliation for the two France DGAL section runs.""" + +from __future__ import annotations + +import argparse +import json +import os +from collections import Counter +from pathlib import Path +from typing import Any + + +def _read_json(path: Path) -> dict[str, Any]: + value = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(value, dict): + raise ValueError(f"expected JSON object: {path}") + return value + + +def _rows(path: Path) -> list[dict[str, Any]]: + rows: list[dict[str, Any]] = [] + for line in path.read_text(encoding="utf-8").splitlines(): + if line.strip(): + value = json.loads(line) + if not isinstance(value, dict): + raise ValueError(f"expected JSON object row: {path}") + rows.append(value) + return rows + + +def _normalized(row: dict[str, Any]) -> dict[str, Any]: + value = row.get("normalized") + return value if isinstance(value, dict) else {} + + +def _count(rows: list[dict[str, Any]], field: str) -> dict[str, int]: + values = Counter() + for row in rows: + value = _normalized(row).get(field) + if isinstance(value, (list, tuple)): + for item in value: + values[str(item)] += 1 + else: + values[str(value) if value is not None else "unknown"] += 1 + return dict(sorted(values.items())) + + +def _approval_keys(rows: list[dict[str, Any]]) -> set[str]: + keys: set[str] = set() + for row in rows: + normalized = _normalized(row) + value = normalized.get("establishment_id") or normalized.get("recognition_number") + if isinstance(value, str) and value.strip(): + keys.add(value.strip()) + return keys + + +def _run_summary(run_dir: Path) -> tuple[dict[str, Any], set[str]]: + manifest = _read_json(run_dir / "manifest.json") + normalized = _rows(run_dir / "normalized" / "records.jsonl") + quarantine_path = run_dir / "quarantined" / "records.jsonl" + quarantined = _rows(quarantine_path) if quarantine_path.is_file() else [] + input_rows = int(manifest["input_rows"]) + normalized_rows = int(manifest["normalized_rows"]) + quarantined_rows = int(manifest["quarantined_rows"]) + if input_rows != normalized_rows + quarantined_rows: + raise ValueError(f"partition mismatch in {run_dir}") + quarantine_reasons = Counter() + for item in quarantined: + for reason in item.get("reasons", ()): # quarantined envelope + quarantine_reasons[str(reason)] += 1 + all_rows = normalized + [item.get("record", {}) for item in quarantined] + summary = { + "source_id": manifest["source_id"], + "section": manifest["section"], + "source_url": manifest["source_url"], + "retrieved_at_utc": manifest["retrieved_at_utc"], + "sha256": manifest["sha256"], + "byte_size": manifest["byte_size"], + "input_rows": input_rows, + "normalized_rows": normalized_rows, + "quarantined_rows": quarantined_rows, + "partition_valid": True, + "anomaly_counts": dict(sorted((str(k), int(v)) for k, v in manifest.get("anomaly_counts", {}).items())), + "quarantine_reason_counts": dict(sorted(quarantine_reasons.items())), + "identity_conflict_counts": manifest.get("identity_conflict_counts", {"state": "not-reported"}), + "activity_category_observation_counts": _count(normalized, "activity_categories"), + "address_state_counts": _count(all_rows, "address_state"), + "coordinate_state_counts": _count(all_rows, "coordinate_state"), + "coordinate_gate_counts": _count(all_rows, "coordinate_gate"), + "privacy_gate_counts": _count(all_rows, "privacy_gate"), + "publication_gate_counts": _count(all_rows, "publication_gate"), + "provenance_and_release": { + "rights_state": "file-specific-terms-pending-human-confirmation", + "privacy_state": manifest.get("privacy_gate", "unknown"), + "coordinate_state": manifest.get("coordinate_gate", "unknown"), + "publication_state": manifest.get("publication_state", "unknown"), + "release_state": manifest.get("release_state", "unknown"), + "geocoding": "disabled", + }, + } + return summary, _approval_keys(normalized) + + +def reconcile(section_i_run: str | Path, section_ii_run: str | Path) -> dict[str, Any]: + summaries: list[dict[str, Any]] = [] + approval_sets: dict[str, set[str]] = {} + for run_dir in (Path(section_i_run), Path(section_ii_run)): + summary, keys = _run_summary(run_dir) + source_id = str(summary["source_id"]) + if source_id in approval_sets: + raise ValueError(f"duplicate source run: {source_id}") + summaries.append(summary) + approval_sets[source_id] = keys + expected = {"fr.dgal.section-i", "fr.dgal.section-ii"} + if set(approval_sets) != expected: + raise ValueError(f"expected exactly the two France DGAL source IDs, found {sorted(approval_sets)}") + section_i_keys = approval_sets["fr.dgal.section-i"] + section_ii_keys = approval_sets["fr.dgal.section-ii"] + totals = {field: sum(int(summary[field]) for summary in summaries) for field in ("input_rows", "normalized_rows", "quarantined_rows")} + return { + "schema_version": "france-dgal-reconciliation-v1", + "source_scopes": [summary["source_id"] for summary in summaries], + "sources": summaries, + "totals": { + **totals, + "partition_valid": totals["input_rows"] == totals["normalized_rows"] + totals["quarantined_rows"], + "unique_facility_count": None, + "identity_semantics": "source observations remain separate; approval-number overlap is a review signal, not an automatic merge", + }, + "cross_section_overlap": { + "shared_provisional_approval_number_count": len(section_i_keys & section_ii_keys), + "section_i_distinct_provisional_approval_numbers": len(section_i_keys), + "section_ii_distinct_provisional_approval_numbers": len(section_ii_keys), + "identity_state": "unresolved-before-human-review", + "merge_policy": "no automatic merge", + }, + "coverage_limits": [ + "Current DGAL snapshots establish listed-at-retrieval observations only; missing later rows are not closure.", + "The two files are separate scopes and are not summed as unique facilities.", + "Address and coordinate values remain restricted source evidence; geocoding is disabled.", + "File-specific terms, privacy/coordinate review, and project publication approval remain open.", + ], + } + + +def write_reconciliation(section_i_run: str | Path, section_ii_run: str | Path, output: str | Path) -> dict[str, Any]: + report = reconcile(section_i_run, section_ii_run) + path = Path(output) + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_name(path.name + ".tmp") + temporary.write_text(json.dumps(report, ensure_ascii=False, sort_keys=True, indent=2) + "\n", encoding="utf-8") + os.replace(temporary, path) + return report + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--section-i-run", type=Path, required=True) + parser.add_argument("--section-ii-run", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + try: + report = write_reconciliation(args.section_i_run, args.section_ii_run, args.output) + except (OSError, ValueError, KeyError, json.JSONDecodeError) as error: + print(json.dumps({"status": "failed", "error": str(error)}, sort_keys=True)) + return 2 + print(json.dumps({"status": "ok", "output": str(args.output), "totals": report["totals"], "cross_section_overlap": report["cross_section_overlap"]}, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/pipeline/sources/france/test_adapter.py b/pipeline/sources/france/test_adapter.py index 96d6020..d994c59 100644 --- a/pipeline/sources/france/test_adapter.py +++ b/pipeline/sources/france/test_adapter.py @@ -22,6 +22,18 @@ def test_bilingual_composite_headers_from_live_dgal_file_are_supported(self): result = FranceDgalSectionIAdapter().parse_bytes(content) self.assertEqual(len(result["accepted"]), 1) self.assertEqual(result["accepted"][0]["normalized"]["activity_categories"], ("slaughter",)) + self.assertEqual(result["accepted"][0]["normalized"]["address_state"], "source-value-present-pending-review") + + def test_shared_siret_across_approval_or_category_is_review_signal_not_quarantine(self): + content = ("approval_number;legal_name;siret;commune;category;associated activities\n" + "FR-1;Shared One;12345678901234;Town;SH;Abattage\n" + "FR-2;Shared Two;12345678901234;Town;CP;Découpe\n").encode() + result = FranceDgalSectionIAdapter().parse_bytes(content) + self.assertEqual(len(result["accepted"]), 2) + self.assertEqual(len(result["quarantined"]), 0) + self.assertEqual(result["identity_conflicted_siret_groups"], 1) + self.assertEqual(result["identity_conflicted_rows"], 2) + self.assertTrue(all(row["normalized"]["identity_conflict_state"].startswith("shared-siret") for row in result["accepted"])) def test_section_i_preserves_source_and_quarantines_duplicate(self): adapter = FranceDgalSectionIAdapter(); result = adapter.parse_file(FIXTURES / "section_i.csv") diff --git a/pipeline/sources/france/test_reconcile.py b/pipeline/sources/france/test_reconcile.py new file mode 100644 index 0000000..e5fd121 --- /dev/null +++ b/pipeline/sources/france/test_reconcile.py @@ -0,0 +1,33 @@ +import tempfile +import unittest +from pathlib import Path + +from .reconcile import reconcile, write_reconciliation +from .refresh import refresh + + +class FranceReconciliationTests(unittest.TestCase): + def test_reconciles_two_scopes_without_claiming_unique_facilities(self): + fixtures = Path(__file__).parent / "fixtures" + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + first = refresh(section="I", raw_path=fixtures / "section_i.csv", run_dir=root / "i", retrieved_at_utc="2026-09-19T00:00:00Z") + second = refresh(section="II", raw_path=fixtures / "section_ii.csv", run_dir=root / "ii", retrieved_at_utc="2026-09-19T00:00:00Z") + report = reconcile(first["report"]["run_dir"], second["report"]["run_dir"]) + self.assertEqual(report["totals"]["input_rows"], 5) + self.assertEqual(report["totals"]["normalized_rows"], 3) + self.assertEqual(report["totals"]["quarantined_rows"], 2) + self.assertTrue(report["totals"]["partition_valid"]) + self.assertIsNone(report["totals"]["unique_facility_count"]) + self.assertEqual(report["cross_section_overlap"]["shared_provisional_approval_number_count"], 0) + self.assertEqual(report["sources"][0]["coordinate_state_counts"], {"not-supplied-by-source": 3}) + self.assertEqual(report["sources"][0]["identity_conflict_counts"]["rows_flagged"], 0) + self.assertEqual(report["sources"][1]["quarantine_reason_counts"], {"missing_approval_number": 1}) + + output = root / "reconciliation.json" + write_reconciliation(first["report"]["run_dir"], second["report"]["run_dir"], output) + self.assertIn('"unique_facility_count": null', output.read_text(encoding="utf-8")) + + +if __name__ == "__main__": + unittest.main() From ce064d3c182562ede0a3f03ce4722a4578115ca2 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sat, 19 Sep 2026 11:30:47 -0700 Subject: [PATCH 288/311] docs: record reviewed country checkpoint --- docs/sprint02-integration-ledger.md | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/docs/sprint02-integration-ledger.md b/docs/sprint02-integration-ledger.md index e14c004..40d88e0 100644 --- a/docs/sprint02-integration-ledger.md +++ b/docs/sprint02-integration-ledger.md @@ -1,6 +1,6 @@ # Sprint 02 integration and storage ledger -Status: lane 8 kickoff checkpoint, 2026-09-19. This is a row-free engineering handoff; it is not a release approval or a claim that source acquisition is complete. +Status: partial reviewed country checkpoint, 2026-09-19. France and Italy are accepted for private candidate/replay integration; this is not a release approval or a claim that Sprint 02 source acquisition is complete. ## Ownership and baseline @@ -27,6 +27,14 @@ The Docker context test creates synthetic sentinels under both `data/` and `.pri Kickoff validation: `npm ci` completed with no vulnerabilities; `python -m unittest scripts.test_dev` passed (7 tests); `python scripts/dev.py --json doctor` passed (database not probed because `UEC_DATABASE_URL` is unset); `git diff --check` passed; both shared-root and lane-worktree `git check-ignore` probes matched `/.private/`. `powershell -ExecutionPolicy Bypass -File pipeline/tests/verify-docker-context.ps1` was attempted with and without escalation but remains blocked because Docker Desktop's Linux engine pipe is unavailable: the client is installed, Docker Desktop processes are running, `com.docker.service` is stopped, and starting that service returns `Cannot open ... service on computer '.'`. No private data or database was used. +## Reviewed country checkpoint + +- France commit `707ac93e` was independently replayed from retained Section I/II raw artifacts and approved by QA; it is integrated as `fbf4e682` on `eli/front-end-overhaul` with unresolved identity signals preserved, no automatic merges, and publication/DB import blocked. +- Italy commits `e04adecf` and `291ad21` were independently replayed byte-for-byte and approved by QA; they are integrated in the same checkpoint with quarantine, location/privacy, rights, and publication gates preserved. +- The combined local validation passed 256 pipeline tests, 25 Jest tests, 7 developer tests, doctor, and diff checks. Native GitHub Actions run [98](https://github.com/eliperez-dev/UntilEveryCage/actions/runs/35461131905) succeeded for exact SHA `fbf4e6824792078a4c3aa5ac1f730e0629039224`. +- FSIS current files remain blocked after bounded ordinary GETs to the three displayed official routes returned HTTP 403; no response body was retained. The row-free evidence is private at `C:\New Projects\UntilEveryCage\.private\sprint02-20260919\fsis\handoff\bounded-get-20260919.json`. +- APHIS registration/report and inspection lanes are still in progress; their real handoffs require independent replay before integration. + ## Integration ledger | Area | Owner/interface | Acceptance state | @@ -34,8 +42,8 @@ Kickoff validation: `npm ci` completed with no vulnerabilities; `python -m unitt | APHIS registrations/reports | Lane 1 handoff under private storage | Pending source handoff and review | | APHIS inspections | Lane 2 handoff under private storage | Pending source handoff and review | | FSIS current parity | Lane 3 handoff under private storage | Pending source handoff and review | -| France candidate | Lane 4 handoff under private storage | Pending source handoff and review | -| Italy candidate | Lane 5 handoff under private storage | Pending source handoff and review | +| France candidate | Lane 4 handoff under private storage | QA-approved and integrated for private candidate/replay | +| Italy candidate | Lane 5 handoff under private storage | QA-approved and integrated for private candidate/replay | | Evidence integration | Lane 6 existing APHIS/FSIS contracts | Pending reviewed handoffs | | Independent QA | Lane 7 replay and review | Pending reviewed handoffs | | CI/build/release engineering | Lane 8 | Storage and context boundary implemented; native CI and final integration pending | From 05eec990441cf5b9cca7f3825c8b1b095db1a039 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sat, 19 Sep 2026 11:18:34 -0700 Subject: [PATCH 289/311] Add APHIS inspection capture provenance helper --- pipeline/sources/us/aphis/capture.py | 180 ++++++++++++++++++++++ pipeline/sources/us/aphis/test_capture.py | 71 +++++++++ 2 files changed, 251 insertions(+) create mode 100644 pipeline/sources/us/aphis/capture.py create mode 100644 pipeline/sources/us/aphis/test_capture.py diff --git a/pipeline/sources/us/aphis/capture.py b/pipeline/sources/us/aphis/capture.py new file mode 100644 index 0000000..0e9e441 --- /dev/null +++ b/pipeline/sources/us/aphis/capture.py @@ -0,0 +1,180 @@ +"""Record provenance for APHIS inspection exports captured one public page at a time.""" + +from __future__ import annotations + +import argparse +import csv +import hashlib +import json +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Sequence + + +CAPTURE_SCHEMA_VERSION = "us-aphis-inspection-page-capture-v1" +DOCUMENT_INVENTORY_SCHEMA_VERSION = "us-aphis-inspection-document-inventory-v1" + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _page_stats(path: Path) -> tuple[list[str], int]: + with path.open("r", encoding="utf-8-sig", newline="") as handle: + reader = csv.reader(handle) + header = next(reader, None) + if not header: + raise ValueError(f"empty CSV page: {path}") + rows = sum(1 for row in reader if any(cell.strip() for cell in row)) + return header, rows + + +def build_capture_manifest( + page_paths: Sequence[Path], + *, + output_path: Path, + source_url: str, + retrieved_at_utc: str, + query_context: dict[str, Any], + excluded_files: Sequence[str] = (), +) -> dict[str, Any]: + """Write a row-free manifest for an ordered set of original page exports.""" + + if not page_paths: + raise ValueError("at least one page export is required") + if not source_url or not retrieved_at_utc: + raise ValueError("source_url and retrieved_at_utc are required") + pages: list[dict[str, Any]] = [] + expected_header: list[str] | None = None + total_rows = 0 + for ordinal, raw_path in enumerate(page_paths, start=1): + path = Path(raw_path) + if not path.is_file(): + raise FileNotFoundError(path) + header, rows = _page_stats(path) + if expected_header is None: + expected_header = header + elif header != expected_header: + raise ValueError(f"page header mismatch: {path}") + pages.append( + { + "ordinal": ordinal, + "file": path.name, + "sha256": _sha256(path), + "byte_size": path.stat().st_size, + "data_rows": rows, + } + ) + total_rows += rows + + manifest = { + "schema_version": CAPTURE_SCHEMA_VERSION, + "source_id": "us.aphis", + "profile": "inspections", + "source_url": source_url, + "retrieved_at_utc": retrieved_at_utc, + "query_context": query_context, + "page_count": len(pages), + "input_rows": total_rows, + "headers": expected_header, + "pages": pages, + "excluded_files": list(excluded_files), + "created_by": "pipeline.sources.us.aphis.capture", + } + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8") + return manifest + + +def write_document_inventory( + output_path: Path, + *, + source_url: str, + query_context: dict[str, Any], + associated_ui_link_rows: int, + document_route: str, + downloaded_documents: int, + status: str, + failure_reason: str, +) -> dict[str, Any]: + """Write a row-free inventory when linked public documents cannot be retained.""" + + inventory = { + "schema_version": DOCUMENT_INVENTORY_SCHEMA_VERSION, + "source_id": "us.aphis", + "profile": "inspections", + "source_url": source_url, + "query_context": query_context, + "associated_ui_link_rows": associated_ui_link_rows, + "document_route_observed": document_route, + "csv_document_reference_rows": 0, + "downloaded_documents": downloaded_documents, + "status": status, + "failure_reason": failure_reason, + "non_ui_endpoint_not_attempted": True, + } + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(json.dumps(inventory, indent=2, sort_keys=True) + "\n", encoding="utf-8") + return inventory + + +def _parse_query_context(value: str) -> dict[str, Any]: + parsed = json.loads(value) + if not isinstance(parsed, dict): + raise ValueError("query context must be a JSON object") + return parsed + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--page-dir", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--source-url", required=True) + parser.add_argument("--retrieved-at-utc", required=True) + parser.add_argument("--query-context", required=True, type=_parse_query_context) + parser.add_argument("--exclude", action="append", default=[]) + parser.add_argument("--document-inventory", type=Path) + parser.add_argument("--associated-ui-link-rows", type=int) + parser.add_argument("--document-route") + parser.add_argument("--downloaded-documents", type=int, default=0) + parser.add_argument("--document-status") + parser.add_argument("--document-failure-reason") + args = parser.parse_args(argv) + pages = sorted(args.page_dir.glob("page-*.csv")) + build_capture_manifest( + pages, + output_path=args.output, + source_url=args.source_url, + retrieved_at_utc=args.retrieved_at_utc, + query_context=args.query_context, + excluded_files=args.exclude, + ) + if args.document_inventory: + required = { + "associated-ui-link-rows": args.associated_ui_link_rows, + "document-route": args.document_route, + "document-status": args.document_status, + "document-failure-reason": args.document_failure_reason, + } + missing = [name for name, value in required.items() if value is None] + if missing: + parser.error("document inventory requires: " + ", ".join(missing)) + write_document_inventory( + args.document_inventory, + source_url=args.source_url, + query_context=args.query_context, + associated_ui_link_rows=args.associated_ui_link_rows, + document_route=args.document_route, + downloaded_documents=args.downloaded_documents, + status=args.document_status, + failure_reason=args.document_failure_reason, + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/pipeline/sources/us/aphis/test_capture.py b/pipeline/sources/us/aphis/test_capture.py new file mode 100644 index 0000000..f5e3958 --- /dev/null +++ b/pipeline/sources/us/aphis/test_capture.py @@ -0,0 +1,71 @@ +import csv +import json +from pathlib import Path +from tempfile import TemporaryDirectory +import unittest + +from .capture import build_capture_manifest, write_document_inventory + + +class CaptureManifestTests(unittest.TestCase): + def test_records_ordered_page_provenance_and_totals(self): + with TemporaryDirectory() as temporary: + root = Path(temporary) + header = ["Customer Number", "Certificate Number", "Inspection Date"] + for name, rows in (("page-01.csv", [["1", "1-R-1", "1/2/2025"]]), ("page-02.csv", [["2", "2-R-2", "1/3/2025"], ["3", "3-R-3", "1/4/2025"]])): + with (root / name).open("w", newline="", encoding="utf-8") as handle: + writer = csv.writer(handle) + writer.writerow(header) + writer.writerows(rows) + output = root / "capture-manifest.json" + manifest = build_capture_manifest( + [root / "page-01.csv", root / "page-02.csv"], + output_path=output, + source_url="https://example.invalid/inspection-reports", + retrieved_at_utc="2026-09-19T18:00:00Z", + query_context={"earliest_inspection_date": "2025-01-01"}, + excluded_files=["unrelated-download.csv"], + ) + + self.assertEqual(manifest["input_rows"], 3) + self.assertEqual(manifest["page_count"], 2) + self.assertEqual([page["ordinal"] for page in manifest["pages"]], [1, 2]) + self.assertEqual(manifest["excluded_files"], ["unrelated-download.csv"]) + self.assertEqual(json.loads(output.read_text(encoding="utf-8"))["input_rows"], 3) + + def test_rejects_header_drift(self): + with TemporaryDirectory() as temporary: + root = Path(temporary) + (root / "page-01.csv").write_text("a,b\n1,2\n", encoding="utf-8") + (root / "page-02.csv").write_text("a,c\n3,4\n", encoding="utf-8") + with self.assertRaisesRegex(ValueError, "header mismatch"): + build_capture_manifest( + [root / "page-01.csv", root / "page-02.csv"], + output_path=root / "manifest.json", + source_url="https://example.invalid", + retrieved_at_utc="2026-09-19T18:00:00Z", + query_context={}, + ) + + def test_document_inventory_keeps_missing_downloads_explicit(self): + with TemporaryDirectory() as temporary: + output = Path(temporary) / "document-inventory.json" + inventory = write_document_inventory( + output, + source_url="https://example.invalid", + query_context={"provider_total": 3}, + associated_ui_link_rows=3, + document_route="public-document-route", + downloaded_documents=0, + status="associated-links-visible-download-not-captured", + failure_reason="browser blocked", + ) + + self.assertEqual(inventory["associated_ui_link_rows"], 3) + self.assertEqual(inventory["downloaded_documents"], 0) + self.assertTrue(inventory["non_ui_endpoint_not_attempted"]) + self.assertEqual(json.loads(output.read_text(encoding="utf-8"))["status"], inventory["status"]) + + +if __name__ == "__main__": + unittest.main() From cd01a8299ae7b5f592a9f99a0f89586054f68482 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sat, 19 Sep 2026 11:25:23 -0700 Subject: [PATCH 290/311] Preserve distinct APHIS inspection observations --- pipeline/sources/us/aphis/adapter.py | 70 ++++++++++++++++++----- pipeline/sources/us/aphis/config.json | 2 +- pipeline/sources/us/aphis/test_adapter.py | 37 ++++++++++++ 3 files changed, 94 insertions(+), 15 deletions(-) diff --git a/pipeline/sources/us/aphis/adapter.py b/pipeline/sources/us/aphis/adapter.py index 94f6dc0..eb68314 100644 --- a/pipeline/sources/us/aphis/adapter.py +++ b/pipeline/sources/us/aphis/adapter.py @@ -40,6 +40,9 @@ "Amendment Number", "Amendment ID", "Amendment Date", "Amended", "Amendment", "Report Version", "Version", ) +INSPECTION_REPORT_ID_COLUMNS = ( + "Inspection Report ID", "Inspection ID", "Report ID", "Report Number", +) NON_ANIMAL_COLUMNS = { "Account Name", "Certificate Number", "Certificate Status", "Status Date", "Registration Type", "License Type", "Year", *CUSTOMER_COLUMNS, @@ -48,6 +51,7 @@ "Geocodio Latitude", "Geocodio Longitude", "Exception Report", "Inspection Date", "Direct NCIs", "Non-Critical NCIs", "Critical NCIs", "Teachable Moments", "Site Name", "Legal Name", "License-Registration Type", + *INSPECTION_REPORT_ID_COLUMNS, } @@ -146,19 +150,17 @@ def _amendment_version(row: dict[str, Any]) -> str | None: return None -def _is_amendment(row: dict[str, Any]) -> bool: - marker = _amendment_version(row) - if not marker: - return False - # A version/date/number is explicit. Boolean-ish flags only count when - # the source says the row was amended; false remains a base report. - column, value = marker.split("=", 1) - if column in {"Amended", "Amendment"}: - return value.casefold() in {"true", "yes", "y", "1", "amended"} - return True +def _native_inspection_id(row: dict[str, Any]) -> str | None: + """Use a report identifier only when the source explicitly supplies it.""" + for column in INSPECTION_REPORT_ID_COLUMNS: + value = _clean(row.get(column)) + if value: + return f"{column}={value}" + return None -def _observation_key(profile: str, row: dict[str, Any]) -> str | None: +def _provisional_event_key(profile: str, row: dict[str, Any]) -> str | None: + """Return the source-native event key retained for review and tracing.""" identity = _certificate_or_customer(row) if not identity: return None @@ -173,13 +175,47 @@ def _observation_key(profile: str, row: dict[str, Any]) -> str | None: parts.append(f"year={_year(row) or 'unknown'}") parts.append(f"version={_amendment_version(row) or 'original'}") elif profile == "inspections": - # A certificate/customer can have multiple inspection observations over - # time. Keep those observations distinct when the source supplies its - # observation date; an undated duplicate remains quarantine-worthy. parts.append(f"status_date={_inspection_date(row) or 'unknown'}") return f"{profile}|" + "|".join(parts) +def _source_row_fingerprint(row: dict[str, Any]) -> str: + """Fingerprint the complete source row without discarding distinguishing fields.""" + canonical = { + str(key): "" if value is None else str(value) + for key, value in sorted(row.items(), key=lambda item: str(item[0])) + } + encoded = json.dumps(canonical, ensure_ascii=False, separators=(",", ":")) + return hashlib.sha256(encoded.encode("utf-8")).hexdigest() + + +def _is_amendment(row: dict[str, Any]) -> bool: + marker = _amendment_version(row) + if not marker: + return False + # A version/date/number is explicit. Boolean-ish flags only count when + # the source says the row was amended; false remains a base report. + column, value = marker.split("=", 1) + if column in {"Amended", "Amendment"}: + return value.casefold() in {"true", "yes", "y", "1", "amended"} + return True + + +def _observation_key(profile: str, row: dict[str, Any]) -> str | None: + provisional = _provisional_event_key(profile, row) + if not provisional: + return None + if profile != "inspections": + return provisional + native_id = _native_inspection_id(row) + if native_id: + return f"inspections|{native_id}" + # The compact live export has no native report ID. A full-row fingerprint + # preserves same-day/site observations without pretending the event identity + # is resolved. Exact source-row duplicates still share this key and quarantine. + return f"inspections|source_row_sha256={_source_row_fingerprint(row)}" + + def _evidence_type(profile: str, row: dict[str, Any]) -> str: return "amendments" if profile == "annual_reports" and _is_amendment(row) else profile @@ -189,6 +225,8 @@ def _record(profile: str, row: dict[str, Any], line: int) -> dict[str, Any]: customers = _customer_values(row) customer = customers["customer_number"] or customers["customer_number_y"] or customers["customer_number_x"] observation_key = _observation_key(profile, row) + provisional_event_key = _provisional_event_key(profile, row) + native_inspection_id = _native_inspection_id(row) if profile == "inspections" else None evidence_type = _evidence_type(profile, row) animal_use_fields = tuple(sorted(key for key, value in row.items() if key not in NON_ANIMAL_COLUMNS and _clean(value))) normalized = { @@ -197,6 +235,10 @@ def _record(profile: str, row: dict[str, Any], line: int) -> dict[str, Any]: # silently reinterpret this evidence. "establishment_id": None, "source_observation_key": observation_key, + "provisional_event_key": provisional_event_key, + "event_identity_unresolved": profile == "inspections" and native_inspection_id is None, + "event_identity_review_state": "review_required" if profile == "inspections" and native_inspection_id is None else "not_applicable", + "review_state": "review_required", "country_code": "US", "evidence_type": evidence_type, "profile": profile, diff --git a/pipeline/sources/us/aphis/config.json b/pipeline/sources/us/aphis/config.json index af3d093..4214caf 100644 --- a/pipeline/sources/us/aphis/config.json +++ b/pipeline/sources/us/aphis/config.json @@ -1,7 +1,7 @@ { "source_id": "us.aphis", "contract_version": "us-aphis-public-search-v2", - "adapter_version": "us-aphis-candidate-v2", + "adapter_version": "us-aphis-candidate-v3", "authority": "USDA Animal and Plant Health Inspection Service, Animal Care", "public_search_url": "https://www.aphis.usda.gov/animal-care/awa-services/usda-animal-care-public-search-tool", "annual_reports_url": "https://direct.aphis.usda.gov/awa/research-facility-report/annual-summary", diff --git a/pipeline/sources/us/aphis/test_adapter.py b/pipeline/sources/us/aphis/test_adapter.py index 42daa09..6909fe1 100644 --- a/pipeline/sources/us/aphis/test_adapter.py +++ b/pipeline/sources/us/aphis/test_adapter.py @@ -71,6 +71,43 @@ def test_inspections_use_status_date_in_observation_identity(self): self.assertEqual(len(result["accepted"]), 2) self.assertNotEqual(result["accepted"][0]["source_record_key"], result["accepted"][1]["source_record_key"]) + def test_inspections_preserve_same_day_distinct_sites_with_provisional_identity(self): + header = "Customer Number,Certificate Number,Inspection Date,Site Name,Legal Name,City,State,Zip,Direct NCIs\n" + raw = ( + header + + '"2","87-R-0002","2026-08-21","North Site","Lab","Austin","TX","78701","0"\n' + + '"2","87-R-0002","2026-08-21","South Site","Lab","Austin","TX","78701","1"\n' + ).encode() + result = AphisPublicSearchAdapter().parse_bytes(raw) + self.assertEqual(len(result["accepted"]), 2) + self.assertEqual(len(result["quarantined"]), 0) + self.assertNotEqual(result["accepted"][0]["source_record_key"], result["accepted"][1]["source_record_key"]) + self.assertEqual(result["accepted"][0]["normalized"]["event_identity_unresolved"], True) + self.assertEqual(result["accepted"][0]["normalized"]["event_identity_review_state"], "review_required") + self.assertEqual(result["accepted"][0]["normalized"]["provisional_event_key"], result["accepted"][1]["normalized"]["provisional_event_key"]) + + def test_inspections_exact_duplicate_rows_remain_explicitly_quarantined(self): + raw = ( + "Customer Number,Certificate Number,Inspection Date,Site Name,Direct NCIs\n" + '"2","87-R-0002","2026-08-21","Same Site","0"\n' + '"2","87-R-0002","2026-08-21","Same Site","0"\n' + ).encode() + result = AphisPublicSearchAdapter().parse_bytes(raw) + self.assertEqual(len(result["accepted"]), 0) + self.assertEqual(len(result["quarantined"]), 2) + self.assertEqual(result["quarantined"][0]["reasons"], ("duplicate_observation_id",)) + + def test_inspections_use_explicit_native_report_id_when_present(self): + raw = ( + "Customer Number,Certificate Number,Inspection Date,Inspection Report ID,Site Name\n" + '"2","87-R-0002","2026-08-21","R-1","North Site"\n' + '"2","87-R-0002","2026-08-21","R-2","South Site"\n' + ).encode() + result = AphisPublicSearchAdapter().parse_bytes(raw) + self.assertEqual(len(result["accepted"]), 2) + self.assertIn("Inspection Report ID=R-1", result["accepted"][0]["source_record_key"]) + self.assertFalse(result["accepted"][0]["normalized"]["event_identity_unresolved"]) + def test_unsupported_profile_fails_closed(self): with self.assertRaises(AphisContractError): AphisPublicSearchAdapter().parse_bytes(b"Name,Value\nA,B\n") From e946d15e1244a047bf11d1f4bda30d64a76d8cda Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sat, 19 Sep 2026 11:33:54 -0700 Subject: [PATCH 291/311] Add APHIS original-page row lineage --- pipeline/sources/us/aphis/adapter.py | 41 +++++++++ pipeline/sources/us/aphis/capture.py | 107 +++++++++++++++++++++- pipeline/sources/us/aphis/handoff.py | 11 +++ pipeline/sources/us/aphis/test_adapter.py | 24 +++++ pipeline/sources/us/aphis/test_capture.py | 10 ++ 5 files changed, 192 insertions(+), 1 deletion(-) diff --git a/pipeline/sources/us/aphis/adapter.py b/pipeline/sources/us/aphis/adapter.py index eb68314..8d3fab9 100644 --- a/pipeline/sources/us/aphis/adapter.py +++ b/pipeline/sources/us/aphis/adapter.py @@ -43,6 +43,10 @@ INSPECTION_REPORT_ID_COLUMNS = ( "Inspection Report ID", "Inspection ID", "Report ID", "Report Number", ) +CAPTURE_LINEAGE_COLUMNS = ( + "__capture_page_ordinal", "__capture_page_sha256", "__capture_page_byte_size", + "__capture_page_row", "__capture_page_retrieved_at_utc", "__capture_source_url", +) NON_ANIMAL_COLUMNS = { "Account Name", "Certificate Number", "Certificate Status", "Status Date", "Registration Type", "License Type", "Year", *CUSTOMER_COLUMNS, @@ -52,6 +56,7 @@ "Inspection Date", "Direct NCIs", "Non-Critical NCIs", "Critical NCIs", "Teachable Moments", "Site Name", "Legal Name", "License-Registration Type", *INSPECTION_REPORT_ID_COLUMNS, + *CAPTURE_LINEAGE_COLUMNS, } @@ -184,11 +189,32 @@ def _source_row_fingerprint(row: dict[str, Any]) -> str: canonical = { str(key): "" if value is None else str(value) for key, value in sorted(row.items(), key=lambda item: str(item[0])) + if key not in CAPTURE_LINEAGE_COLUMNS } encoded = json.dumps(canonical, ensure_ascii=False, separators=(",", ":")) return hashlib.sha256(encoded.encode("utf-8")).hexdigest() +def _capture_lineage(row: dict[str, Any]) -> dict[str, Any] | None: + if not any(_clean(row.get(column)) for column in CAPTURE_LINEAGE_COLUMNS): + return None + lineage: dict[str, Any] = { + "artifact_classification": "derived_staging_with_original_page_lineage", + "page_sha256": _clean(row.get("__capture_page_sha256")), + "page_byte_size": _clean(row.get("__capture_page_byte_size")), + "page_retrieved_at_utc": _clean(row.get("__capture_page_retrieved_at_utc")) or "unknown", + "source_url": _clean(row.get("__capture_source_url")), + } + for output, column in (("page_ordinal", "__capture_page_ordinal"), ("page_row", "__capture_page_row")): + value = _clean(row.get(column)) + if value is not None: + try: + lineage[output] = int(value) + except ValueError: + lineage[output] = value + return lineage + + def _is_amendment(row: dict[str, Any]) -> bool: marker = _amendment_version(row) if not marker: @@ -227,6 +253,7 @@ def _record(profile: str, row: dict[str, Any], line: int) -> dict[str, Any]: observation_key = _observation_key(profile, row) provisional_event_key = _provisional_event_key(profile, row) native_inspection_id = _native_inspection_id(row) if profile == "inspections" else None + capture_lineage = _capture_lineage(row) evidence_type = _evidence_type(profile, row) animal_use_fields = tuple(sorted(key for key, value in row.items() if key not in NON_ANIMAL_COLUMNS and _clean(value))) normalized = { @@ -239,6 +266,7 @@ def _record(profile: str, row: dict[str, Any], line: int) -> dict[str, Any]: "event_identity_unresolved": profile == "inspections" and native_inspection_id is None, "event_identity_review_state": "review_required" if profile == "inspections" and native_inspection_id is None else "not_applicable", "review_state": "review_required", + "source_capture_lineage": capture_lineage, "country_code": "US", "evidence_type": evidence_type, "profile": profile, @@ -315,6 +343,9 @@ def parse_bytes(self, content: bytes) -> dict[str, Any]: "schema_fingerprint": _schema_fingerprint(headers), "source_sha256": digest, "input_rows": len(rows), + "capture_lineage_rows": sum( + 1 for record in records if record["normalized"].get("source_capture_lineage") + ), "evidence_type_counts": dict(sorted(Counter(record["normalized"]["evidence_type"] for record in accepted).items())), } @@ -351,6 +382,16 @@ def run(self, raw_path: str | Path, run_dir: str | Path, artifact: SourceArtifac "geocoding": "disabled", "publication_gate": "blocked", "test_only": True, + "source_artifact_classification": ( + "derived_staging_with_original_page_lineage" + if result["capture_lineage_rows"] + else "preserved_source_artifact" + ), + "source_row_lineage": ( + "normalized.source_capture_lineage links each derived row to original page ordinal, page hash, page row, and bundle retrieval timestamp" + if result["capture_lineage_rows"] + else "not supplied" + ), "coverage": "APHIS Animal Care public-search observations for the captured profile only; AWA coverage, currentness, completeness, and facility equivalence remain unknown; no FSIS/facility merge", "coverage_limitations": [ "The public-search export is not a complete census of animal use or all AWA-regulated entities.", diff --git a/pipeline/sources/us/aphis/capture.py b/pipeline/sources/us/aphis/capture.py index 0e9e441..1bd6a93 100644 --- a/pipeline/sources/us/aphis/capture.py +++ b/pipeline/sources/us/aphis/capture.py @@ -6,13 +6,21 @@ import csv import hashlib import json -from datetime import datetime, timezone from pathlib import Path +import tempfile from typing import Any, Sequence CAPTURE_SCHEMA_VERSION = "us-aphis-inspection-page-capture-v1" DOCUMENT_INVENTORY_SCHEMA_VERSION = "us-aphis-inspection-document-inventory-v1" +LINEAGE_COLUMNS = ( + "__capture_page_ordinal", + "__capture_page_sha256", + "__capture_page_byte_size", + "__capture_page_row", + "__capture_page_retrieved_at_utc", + "__capture_source_url", +) def _sha256(path: Path) -> str: @@ -33,6 +41,85 @@ def _page_stats(path: Path) -> tuple[list[str], int]: return header, rows +def _canonical_row(row: dict[str, Any], headers: Sequence[str]) -> bytes: + values = {str(header): "" if row.get(header) is None else str(row.get(header)) for header in headers} + return (json.dumps(values, ensure_ascii=False, separators=(",", ":")) + "\n").encode("utf-8") + + +def build_lineage_csv( + page_paths: Sequence[Path], + *, + page_metadata: Sequence[dict[str, Any]], + output_path: Path, + source_url: str, +) -> dict[str, Any]: + """Create a derived CSV with explicit original-page row lineage.""" + + if len(page_paths) != len(page_metadata): + raise ValueError("page metadata must match page count") + output_path.parent.mkdir(parents=True, exist_ok=True) + temporary_path: Path | None = None + total_rows = 0 + expected_header: list[str] | None = None + original_digest = hashlib.sha256() + try: + with tempfile.NamedTemporaryFile( + mode="w", encoding="utf-8", newline="", dir=output_path.parent, + prefix=f".{output_path.name}.", delete=False, + ) as handle: + temporary_path = Path(handle.name) + writer: csv.DictWriter | None = None + for ordinal, (page_path, metadata) in enumerate(zip(page_paths, page_metadata), start=1): + with page_path.open("r", encoding="utf-8-sig", newline="") as source: + reader = csv.DictReader(source) + header = list(reader.fieldnames or []) + if expected_header is None: + expected_header = header + writer = csv.DictWriter(handle, fieldnames=header + list(LINEAGE_COLUMNS), lineterminator="\n") + writer.writeheader() + elif header != expected_header: + raise ValueError(f"page header mismatch: {page_path}") + assert writer is not None + for page_row, row in enumerate(reader, start=1): + original_digest.update(_canonical_row(row, expected_header)) + row.update({ + "__capture_page_ordinal": str(ordinal), + "__capture_page_sha256": str(metadata["sha256"]), + "__capture_page_byte_size": str(metadata["byte_size"]), + "__capture_page_row": str(page_row), + "__capture_page_retrieved_at_utc": "unknown", + "__capture_source_url": source_url, + }) + writer.writerow(row) + total_rows += 1 + handle.flush() + derived_digest = hashlib.sha256() + with temporary_path.open("r", encoding="utf-8", newline="") as derived_handle: + derived_reader = csv.DictReader(derived_handle) + derived_header = list(derived_reader.fieldnames or []) + if derived_header[:len(expected_header or [])] != (expected_header or []): + raise ValueError("derived lineage CSV header drift") + for derived_row in derived_reader: + derived_digest.update(_canonical_row(derived_row, expected_header or [])) + if derived_digest.hexdigest() != original_digest.hexdigest(): + raise ValueError("derived lineage CSV changed an original source row") + temporary_path.replace(output_path) + finally: + if temporary_path is not None and temporary_path.exists(): + temporary_path.unlink() + return { + "classification": "derived_staging_with_original_page_lineage", + "file": output_path.name, + "sha256": _sha256(output_path), + "byte_size": output_path.stat().st_size, + "input_rows": total_rows, + "lineage_columns": list(LINEAGE_COLUMNS), + "original_row_sequence_sha256": original_digest.hexdigest(), + "derived_row_sequence_sha256": derived_digest.hexdigest(), + "row_sequence_match": True, + } + + def build_capture_manifest( page_paths: Sequence[Path], *, @@ -41,6 +128,7 @@ def build_capture_manifest( retrieved_at_utc: str, query_context: dict[str, Any], excluded_files: Sequence[str] = (), + lineage_output_path: Path | None = None, ) -> dict[str, Any]: """Write a row-free manifest for an ordered set of original page exports.""" @@ -67,6 +155,8 @@ def build_capture_manifest( "sha256": _sha256(path), "byte_size": path.stat().st_size, "data_rows": rows, + "source_url": source_url, + "page_retrieved_at_utc": "unknown", } ) total_rows += rows @@ -77,6 +167,7 @@ def build_capture_manifest( "profile": "inspections", "source_url": source_url, "retrieved_at_utc": retrieved_at_utc, + "retrieved_at_scope": "bundle_generation; per-page actual retrieval timestamps unknown", "query_context": query_context, "page_count": len(pages), "input_rows": total_rows, @@ -85,6 +176,18 @@ def build_capture_manifest( "excluded_files": list(excluded_files), "created_by": "pipeline.sources.us.aphis.capture", } + if lineage_output_path is not None: + derived = build_lineage_csv( + page_paths, + page_metadata=pages, + output_path=lineage_output_path, + source_url=source_url, + ) + try: + derived["file"] = str(lineage_output_path.relative_to(output_path.parent)) + except ValueError: + pass + manifest["derived_artifact"] = derived output_path.parent.mkdir(parents=True, exist_ok=True) output_path.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8") return manifest @@ -137,6 +240,7 @@ def main(argv: Sequence[str] | None = None) -> int: parser.add_argument("--retrieved-at-utc", required=True) parser.add_argument("--query-context", required=True, type=_parse_query_context) parser.add_argument("--exclude", action="append", default=[]) + parser.add_argument("--lineage-output", type=Path) parser.add_argument("--document-inventory", type=Path) parser.add_argument("--associated-ui-link-rows", type=int) parser.add_argument("--document-route") @@ -152,6 +256,7 @@ def main(argv: Sequence[str] | None = None) -> int: retrieved_at_utc=args.retrieved_at_utc, query_context=args.query_context, excluded_files=args.exclude, + lineage_output_path=args.lineage_output, ) if args.document_inventory: required = { diff --git a/pipeline/sources/us/aphis/handoff.py b/pipeline/sources/us/aphis/handoff.py index f3f14f3..eab5b7e 100644 --- a/pipeline/sources/us/aphis/handoff.py +++ b/pipeline/sources/us/aphis/handoff.py @@ -39,6 +39,9 @@ def write_private_handoff( payload = _jsonl(rows) root = Path(run_dir) atomic_bytes(root / "records.jsonl", payload) + has_page_lineage = any( + row.get("normalized", {}).get("source_capture_lineage") for row in rows + ) manifest = { "contract_version": HANDOFF_VERSION, "source_id": "us.aphis", @@ -47,6 +50,14 @@ def write_private_handoff( "retrieved_at_utc": artifact.retrieved_at_utc, "checksum_sha256": artifact.sha256, "source_artifact_sha256": source_sha256, + "source_artifact_classification": ( + "derived_staging_with_original_page_lineage" + if has_page_lineage else "preserved_source_artifact" + ), + "source_row_lineage": ( + "normalized.source_capture_lineage links each derived row to original page ordinal, page hash, byte size, source row, source URL, and retrieval timestamp scope" + if has_page_lineage else "not supplied" + ), "byte_size": artifact.byte_size, "code_version": artifact.code_version, "config_version": artifact.config_version, diff --git a/pipeline/sources/us/aphis/test_adapter.py b/pipeline/sources/us/aphis/test_adapter.py index 6909fe1..fa7d288 100644 --- a/pipeline/sources/us/aphis/test_adapter.py +++ b/pipeline/sources/us/aphis/test_adapter.py @@ -108,6 +108,30 @@ def test_inspections_use_explicit_native_report_id_when_present(self): self.assertIn("Inspection Report ID=R-1", result["accepted"][0]["source_record_key"]) self.assertFalse(result["accepted"][0]["normalized"]["event_identity_unresolved"]) + def test_inspection_lineage_is_preserved_on_derived_rows(self): + raw = ( + "Customer Number,Certificate Number,Inspection Date,Site Name,__capture_page_ordinal,__capture_page_sha256,__capture_page_byte_size,__capture_page_row,__capture_page_retrieved_at_utc,__capture_source_url\n" + '"2","87-R-0002","2026-08-21","North Site","3","abc","1234","7","unknown","https://example.invalid/search"\n' + ).encode() + result = AphisPublicSearchAdapter().parse_bytes(raw) + lineage = result["accepted"][0]["normalized"]["source_capture_lineage"] + self.assertEqual(lineage["page_ordinal"], 3) + self.assertEqual(lineage["page_row"], 7) + self.assertEqual(lineage["page_sha256"], "abc") + self.assertEqual(lineage["page_byte_size"], "1234") + self.assertEqual(lineage["page_retrieved_at_utc"], "unknown") + self.assertEqual(lineage["source_url"], "https://example.invalid/search") + + def test_lineage_metadata_does_not_change_exact_payload_identity(self): + raw = ( + "Customer Number,Certificate Number,Inspection Date,Site Name,__capture_page_ordinal,__capture_page_sha256,__capture_page_byte_size,__capture_page_row,__capture_page_retrieved_at_utc,__capture_source_url\n" + '"2","87-R-0002","2026-08-21","Same Site","1","abc","1234","7","unknown","https://example.invalid/search"\n' + '"2","87-R-0002","2026-08-21","Same Site","2","def","5678","1","unknown","https://example.invalid/search"\n' + ).encode() + result = AphisPublicSearchAdapter().parse_bytes(raw) + self.assertEqual(len(result["accepted"]), 0) + self.assertEqual(len(result["quarantined"]), 2) + def test_unsupported_profile_fails_closed(self): with self.assertRaises(AphisContractError): AphisPublicSearchAdapter().parse_bytes(b"Name,Value\nA,B\n") diff --git a/pipeline/sources/us/aphis/test_capture.py b/pipeline/sources/us/aphis/test_capture.py index f5e3958..dbe3693 100644 --- a/pipeline/sources/us/aphis/test_capture.py +++ b/pipeline/sources/us/aphis/test_capture.py @@ -18,6 +18,7 @@ def test_records_ordered_page_provenance_and_totals(self): writer.writerow(header) writer.writerows(rows) output = root / "capture-manifest.json" + lineage = root / "staging" / "inspections-with-lineage.csv" manifest = build_capture_manifest( [root / "page-01.csv", root / "page-02.csv"], output_path=output, @@ -25,12 +26,21 @@ def test_records_ordered_page_provenance_and_totals(self): retrieved_at_utc="2026-09-19T18:00:00Z", query_context={"earliest_inspection_date": "2025-01-01"}, excluded_files=["unrelated-download.csv"], + lineage_output_path=lineage, ) self.assertEqual(manifest["input_rows"], 3) self.assertEqual(manifest["page_count"], 2) self.assertEqual([page["ordinal"] for page in manifest["pages"]], [1, 2]) self.assertEqual(manifest["excluded_files"], ["unrelated-download.csv"]) + self.assertEqual(manifest["derived_artifact"]["input_rows"], 3) + self.assertTrue(manifest["derived_artifact"]["row_sequence_match"]) + with lineage.open(encoding="utf-8", newline="") as handle: + lineage_rows = list(csv.DictReader(handle)) + self.assertEqual(lineage_rows[1]["__capture_page_ordinal"], "2") + self.assertEqual(lineage_rows[1]["__capture_page_row"], "1") + self.assertEqual(lineage_rows[1]["__capture_page_retrieved_at_utc"], "unknown") + self.assertEqual(lineage_rows[1]["__capture_source_url"], "https://example.invalid/inspection-reports") self.assertEqual(json.loads(output.read_text(encoding="utf-8"))["input_rows"], 3) def test_rejects_header_drift(self): From 51df7f22fbdaefb9066d045938138a3990b3f0c3 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sat, 19 Sep 2026 11:48:18 -0700 Subject: [PATCH 292/311] docs: record APHIS inspection acceptance --- docs/sprint02-integration-ledger.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/sprint02-integration-ledger.md b/docs/sprint02-integration-ledger.md index 40d88e0..13d74d1 100644 --- a/docs/sprint02-integration-ledger.md +++ b/docs/sprint02-integration-ledger.md @@ -34,13 +34,15 @@ Kickoff validation: `npm ci` completed with no vulnerabilities; `python -m unitt - The combined local validation passed 256 pipeline tests, 25 Jest tests, 7 developer tests, doctor, and diff checks. Native GitHub Actions run [98](https://github.com/eliperez-dev/UntilEveryCage/actions/runs/35461131905) succeeded for exact SHA `fbf4e6824792078a4c3aa5ac1f730e0629039224`. - FSIS current files remain blocked after bounded ordinary GETs to the three displayed official routes returned HTTP 403; no response body was retained. The row-free evidence is private at `C:\New Projects\UntilEveryCage\.private\sprint02-20260919\fsis\handoff\bounded-get-20260919.json`. - APHIS registration/report and inspection lanes are still in progress; their real handoffs require independent replay before integration. +- APHIS inspection code chain `addcef0c` -> `4518bce6` -> `8b4d3c68` is QA-approved and integrated as `e946d15e`. The authoritative replay accepted 1,075 input rows, 1,071 candidates, and 4 exact-duplicate quarantines, with every source row mapped to verified original-page lineage. Public release/import remains blocked. +- APHIS registration/annual-report per-row lineage is not yet accepted; the core/evidence owners are correcting that gap before integration. ## Integration ledger | Area | Owner/interface | Acceptance state | | --- | --- | --- | -| APHIS registrations/reports | Lane 1 handoff under private storage | Pending source handoff and review | -| APHIS inspections | Lane 2 handoff under private storage | Pending source handoff and review | +| APHIS registrations/reports | Lane 1 handoff under private storage | Pending per-row lineage correction and QA | +| APHIS inspections | Lane 2 handoff under private storage | QA-approved and integrated for private candidate/replay | | FSIS current parity | Lane 3 handoff under private storage | Pending source handoff and review | | France candidate | Lane 4 handoff under private storage | QA-approved and integrated for private candidate/replay | | Italy candidate | Lane 5 handoff under private storage | QA-approved and integrated for private candidate/replay | From 03f47460ff6049a845cec4ad0a87103d7d46cb66 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sat, 19 Sep 2026 11:00:56 -0700 Subject: [PATCH 293/311] Integrate APHIS handoffs with exact artifact provenance --- .../us/aphis-investigation-packet.md | 16 +++ .../us/accountability/aphis_evidence.py | 124 ++++++++++++++++++ .../sources/us/accountability/aphis_wave2.py | 15 +++ .../us/accountability/current_identity.py | 42 +++++- .../us/accountability/test_aphis_evidence.py | 55 +++++++- .../accountability/test_current_identity.py | 19 +++ 6 files changed, 262 insertions(+), 9 deletions(-) diff --git a/docs/countries/us/aphis-investigation-packet.md b/docs/countries/us/aphis-investigation-packet.md index d0239a8..3466594 100644 --- a/docs/countries/us/aphis-investigation-packet.md +++ b/docs/countries/us/aphis-investigation-packet.md @@ -13,6 +13,16 @@ SHA-256 and byte size. A missing artifact, invalid hash, or size mismatch is a visible `failed` input. A replay keeps the source retrieval timestamp from its manifest and is not a fresh source observation. +When acquisition lanes hand off normalized observations separately, use the +explicit `run_from_handoffs` boundary with one handoff directory for each of +`registrations`, `annual_reports`, and `inspections`. Each directory must +contain the existing `us-aphis-observation-handoff-v1` `manifest.json` and +`records.jsonl`; the consumer verifies the handoff checksum, row count, source +identity, and blocked/private state before building the same packet and +identity graph. The handoff's source-artifact checksum remains row provenance, +but is not treated as independently verified raw bytes unless a retained raw +artifact is separately replayed through `verify_retained_artifacts`. + Example private run after an authorized handoff: ```powershell @@ -53,6 +63,12 @@ review state. Conflicts and duplicate evidence remain quarantined. Names and addresses are not identity evidence, and no candidate establishes ownership, current operation, approval, wrongdoing, or a complete animal-use total. +For a real capture spanning multiple export pages, graph edges retain the +source-record-specific artifact hash for each side of a link. Profile-level +aggregate hashes remain accounting metadata only and cannot stand in for the +bytes containing an individual observation. This is what allows a replay to +distinguish an exact evidence path from a missing or unverifiable artifact. + All output remains private, `not_eligible`, and `release_state: not-created`. Raw and parsed inputs remain outside Git under the retention and removal rules in `docs/ETHICS.md`. A synthetic test pass demonstrates engineering behavior diff --git a/pipeline/sources/us/accountability/aphis_evidence.py b/pipeline/sources/us/accountability/aphis_evidence.py index 7ec8c93..99bfcf5 100644 --- a/pipeline/sources/us/accountability/aphis_evidence.py +++ b/pipeline/sources/us/accountability/aphis_evidence.py @@ -23,6 +23,7 @@ PACKET_VERSION = "us-aphis-investigation-packet-v1" +HANDOFF_VERSION = "us-aphis-observation-handoff-v1" PROFILES = ("registrations", "annual_reports", "inspections") STATES = frozenset({"observed", "not_observed", "quarantined", "failed", "document_not_captured"}) _SHA256 = re.compile(r"^[0-9a-f]{64}$", re.IGNORECASE) @@ -478,6 +479,129 @@ def run_from_wave2( return {"packet": packet, "manifest": manifest} +def _read_observation_handoff(handoff_dir: str | Path, expected_profile: str) -> tuple[list[dict[str, Any]], dict[str, Any]]: + """Read one source-local APHIS handoff without scanning its parent tree. + + Acquisition lanes hand off normalized rows separately. This boundary + verifies the handoff payload and its private contract, while leaving raw + artifact-byte verification to an optional manifest/artifact replay. + """ + root = Path(handoff_dir).resolve() + manifest_path = root / "manifest.json" + records_path = root / "records.jsonl" + if not manifest_path.is_file() or not records_path.is_file(): + raise AphisEvidenceError(f"APHIS handoff is incomplete: {root}") + manifest = _json(manifest_path) + required = { + "contract_version": HANDOFF_VERSION, + "source_id": "us.aphis", + "profile": expected_profile, + "release_state": "not-created", + "publication_state": "private-candidate", + "review_state": "review_required", + "privacy_gate": "pending", + "coordinate_gate": "review_required", + "graph_candidate_emission": False, + "auto_merge": False, + "test_only": True, + "row_payloads_included": True, + } + for key, value in required.items(): + if manifest.get(key) != value: + raise AphisEvidenceError(f"APHIS handoff {root} violates {key} contract") + payload = records_path.read_bytes() + actual_hash = hashlib.sha256(payload).hexdigest() + if actual_hash != _text(manifest.get("normalized_sha256")): + raise AphisEvidenceError(f"APHIS handoff payload checksum mismatch: {root}") + try: + rows = [json.loads(line) for line in payload.decode("utf-8").splitlines() if line] + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise AphisEvidenceError(f"APHIS handoff payload is malformed: {root}") from exc + if len(rows) != int(manifest.get("normalized_rows", -1)): + raise AphisEvidenceError(f"APHIS handoff row count mismatch: {root}") + for row in rows: + if not isinstance(row, Mapping) or _text(row.get("source_id")) != "us.aphis" or not _text(row.get("source_record_key")): + raise AphisEvidenceError(f"APHIS handoff row lacks source identity: {root}") + evidence_type = _profile(row) + allowed = {expected_profile} + if expected_profile == "annual_reports": + allowed.add("amendments") + if evidence_type not in allowed: + raise AphisEvidenceError(f"APHIS handoff row profile mismatch: {root}") + return rows, dict(manifest) + + +def run_from_handoffs( + handoff_dirs: Mapping[str, str | Path], + packet_dir: str | Path, + *, + registration_key: str | None = None, + expected_rows: Mapping[str, int] | None = None, + document_refs: Mapping[str, Iterable[Mapping[str, Any]]] | None = None, +) -> dict[str, Any]: + """Consume lane-specific APHIS observation handoffs into one packet. + + ``handoff_dirs`` is explicit by profile and never discovered recursively. + The source artifact checksum is retained as row provenance; callers that + also retain the raw files can use the standalone artifact verifier before + treating those bytes as independently verified. + """ + records: dict[str, list[dict[str, Any]]] = {profile: [] for profile in PROFILES} + provenance: dict[tuple[str, str], dict[str, Any]] = {} + failures: list[dict[str, Any]] = [] + profile_input_rows: dict[str, int] = {} + for profile in PROFILES: + handoff = handoff_dirs.get(profile) + if handoff is None: + failures.append({"profile": profile, "state": "failed", "failure": "handoff_missing"}) + continue + try: + rows, manifest = _read_observation_handoff(handoff, profile) + except (OSError, TypeError, ValueError, AphisEvidenceError) as exc: + failures.append({"profile": profile, "state": "failed", "failure": f"handoff_invalid:{type(exc).__name__}"}) + continue + source_url = _text(manifest.get("source_url")) + retrieved = _text(manifest.get("retrieved_at_utc")) + digest = _text(manifest.get("source_artifact_sha256") or manifest.get("checksum_sha256")) + profile_input_rows[profile] = len(rows) + provenance[("us.aphis", profile)] = { + "artifact_sha256": digest, + "source_url": source_url, + "retrieved_at_utc": retrieved, + } + for row in rows: + row_copy = dict(row) + row_copy["_retained_artifact"] = { + "artifact_sha256": digest, + "source_url": source_url, + "retrieved_at_utc": retrieved, + } + records[profile].append(row_copy) + if digest: + provenance[("us.aphis", profile, str(row["source_record_key"]))] = dict(row_copy["_retained_artifact"]) + + from pipeline.sources.us.accountability.current_identity import build_current_identity_graph + + graph = build_current_identity_graph( + aphis_records=records, + fsis_records=(), + fsis_observations=(), + provenance=provenance, + ) + packet = build_packet( + records_by_profile=records, + provenance=provenance, + graph=graph, + registration_key=registration_key, + expected_rows=expected_rows, + input_failures=failures, + profile_input_rows=profile_input_rows, + document_refs=document_refs, + ) + manifest = write_packet(packet_dir, packet) + return {"packet": packet, "manifest": manifest, "graph": graph, "input_failures": failures} + + def _load_verified_exports( verification: Mapping[str, Any], manifest: Mapping[str, Any], diff --git a/pipeline/sources/us/accountability/aphis_wave2.py b/pipeline/sources/us/accountability/aphis_wave2.py index 6e66d78..0e2edef 100644 --- a/pipeline/sources/us/accountability/aphis_wave2.py +++ b/pipeline/sources/us/accountability/aphis_wave2.py @@ -151,6 +151,8 @@ def _load_profile(paths: list[Path], profile: str) -> tuple[list[dict[str, Any]] "artifact": metadata["artifact"], "artifact_sha256": metadata["sha256"], "byte_size": metadata["byte_size"], + "source_url": profile_url(profile), + "retrieved_at_utc": PROFILE_RETRIEVED_AT[profile], } records.extend([{**record, "_retained_artifact": row_artifact} for record in result["accepted"]]) quarantined.extend([ @@ -302,6 +304,19 @@ def run_wave2(*, input_root: str | Path, run_dir: str | Path, expected_rows: Map } for profile in profile_manifests } + # Retain exact source-record -> artifact provenance for graph edges. A + # profile can contain many pages, so its aggregate hash is accounting + # metadata and cannot prove which bytes contain an individual row. + for profile, rows in records_by_profile.items(): + for record in rows: + artifact = record.get("_retained_artifact") + source_key = record.get("source_record_key") + if isinstance(artifact, Mapping) and source_key: + provenance[(SOURCE_ID, profile, str(source_key))] = { + key: artifact[key] + for key in ("artifact_sha256", "source_url", "retrieved_at_utc") + if artifact.get(key) + } graph = build_current_identity_graph( aphis_records=graph_records, fsis_records=(), diff --git a/pipeline/sources/us/accountability/current_identity.py b/pipeline/sources/us/accountability/current_identity.py index e455351..89871ac 100644 --- a/pipeline/sources/us/accountability/current_identity.py +++ b/pipeline/sources/us/accountability/current_identity.py @@ -131,19 +131,32 @@ def _provenance( *, source_id: str, profile: str, + source_record_key: str | None = None, ) -> dict[str, str] | None: - """Resolve per-profile provenance, falling back to source-wide metadata.""" + """Resolve row provenance, then profile/source fallback metadata. + + A real capture can span multiple retained source artifacts. Prefer the + source-record-specific key when available so graph edges point to the + exact bytes containing each observation; profile/source metadata remains a + compatibility fallback for older handoffs and synthetic fixtures. + """ profile_keys = [profile] # Amendments are versioned rows in the annual-report capture unless a # separate amendment artifact was explicitly supplied. if profile == "amendments": profile_keys.append("annual_reports") values = None - for profile_key in profile_keys: + if source_record_key: values = ( - provenance.get((source_id, profile_key)) - or provenance.get(f"{source_id}:{profile_key}") + provenance.get((source_id, profile, source_record_key)) + or provenance.get(f"{source_id}:{profile}:{source_record_key}") ) + for profile_key in profile_keys: + if not values: + values = ( + provenance.get((source_id, profile_key)) + or provenance.get(f"{source_id}:{profile_key}") + ) if values: break values = values or provenance.get(source_id) @@ -249,8 +262,18 @@ def _candidate( right_source = _text(right.get("source_id")) or "unknown" left_key = _source_key(left) right_key = _source_key(right) - left_provenance = _provenance(provenance, source_id=left_source, profile=left_profile) - right_provenance = _provenance(provenance, source_id=right_source, profile=right_profile) + left_provenance = _provenance( + provenance, + source_id=left_source, + profile=left_profile, + source_record_key=left_key, + ) + right_provenance = _provenance( + provenance, + source_id=right_source, + profile=right_profile, + source_record_key=right_key, + ) provenance_ok = left_provenance is not None and right_provenance is not None evidence = { "source_record_keys": [left_key, right_key], @@ -482,7 +505,12 @@ def build_current_identity_graph( all_records.extend((record, "fsis_observations") for record in observations) nodes = [] for record, profile in all_records: - nodes.append(_node(record, profile, _provenance(provenance, source_id=_text(record.get("source_id")) or "unknown", profile=profile))) + nodes.append(_node(record, profile, _provenance( + provenance, + source_id=_text(record.get("source_id")) or "unknown", + profile=profile, + source_record_key=_source_key(record), + ))) nodes.sort(key=lambda node: node["entity_id"]) candidates: list[dict[str, Any]] = [] diff --git a/pipeline/sources/us/accountability/test_aphis_evidence.py b/pipeline/sources/us/accountability/test_aphis_evidence.py index 2b0d75f..daf5395 100644 --- a/pipeline/sources/us/accountability/test_aphis_evidence.py +++ b/pipeline/sources/us/accountability/test_aphis_evidence.py @@ -14,6 +14,7 @@ build_packet, verify_retained_artifacts, run_from_wave2, + run_from_handoffs, write_packet, ) from .aphis_wave2 import run_wave2 @@ -116,6 +117,41 @@ def test_write_packet_has_separate_row_free_summary(self): self.assertTrue((Path(directory) / "private" / "rows.jsonl").exists()) self.assertNotIn("source_values", (Path(directory) / "row-free-summary.json").read_text(encoding="utf-8")) + def test_source_local_handoffs_build_exact_private_paths(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + handoffs = {} + records = json.loads((Path(__file__).parent / "fixtures" / "current_identity.json").read_text(encoding="utf-8"))["aphis"] + for profile in ("registrations", "annual_reports", "inspections"): + handoff = root / profile + handoff.mkdir() + payload = b"".join( + (json.dumps(row, ensure_ascii=False, sort_keys=True) + "\n").encode("utf-8") + for row in records[profile] + ) + digest = hashlib.sha256(payload).hexdigest() + (handoff / "records.jsonl").write_bytes(payload) + (handoff / "manifest.json").write_text(json.dumps({ + "contract_version": "us-aphis-observation-handoff-v1", + "source_id": "us.aphis", "profile": profile, + "source_url": f"https://example.invalid/{profile}", + "retrieved_at_utc": "2026-09-19T00:00:00Z", + "source_artifact_sha256": "a" * 64, + "normalized_rows": len(records[profile]), "normalized_sha256": digest, + "graph_candidate_emission": False, "auto_merge": False, + "release_state": "not-created", "publication_state": "private-candidate", + "review_state": "review_required", "privacy_gate": "pending", + "coordinate_gate": "review_required", "test_only": True, + "row_payloads_included": True, + }), encoding="utf-8") + handoffs[profile] = handoff + result = run_from_handoffs(handoffs, root / "packet") + summary = result["packet"]["row_free_summary"] + self.assertEqual(result["input_failures"], []) + self.assertGreater(summary["links"]["quarantined_count"], 0) + self.assertNotIn("link_missing_or_invalid_provenance", summary["links"]["excluded_reasons"]) + self.assertEqual(summary["publication_status"], "not_eligible") + def test_aggregate_metadata_hash_cannot_be_link_provenance(self): records = self._records() registration_key = records["registrations"][0]["source_record_key"] @@ -191,8 +227,21 @@ def test_wave2_replay_verifies_input_manifest_and_uses_row_hashes(self): root = Path(directory) input_root = root / "input" input_root.mkdir() - for profile in ("registrations", "annual_reports", "inspections"): - shutil.copyfile(ROOT / "fixtures" / f"{profile}.csv", input_root / f"ExportData_{profile}.csv") + (input_root / "ExportData_registrations.csv").write_text( + "Account Name,Customer Number,Certificate Number,Registration Type,Certificate Status,Status Date\n" + '"Synthetic Registrant","2","00-R-0002","Class R - Research Facility","Active","2026-01-01"\n', + encoding="utf-8", + ) + (input_root / "ExportData_annual_reports.csv").write_text( + "Customer Number,Certificate Number,Year,Dogs,Cats\n" + '"2","00-R-0002","2025","","1"\n', + encoding="utf-8", + ) + (input_root / "ExportData_inspections.csv").write_text( + "Customer Number,Certificate Number,Inspection Date,Direct NCIs,Non-Critical NCIs,Critical NCIs,Teachable Moments,Site Name,Legal Name,License-Registration Type,City,State,Zip\n" + '"2","00-R-0002","2026-02-01","","","","","Synthetic Site","Synthetic Registrant","Class R - Research Facility","Testville","TX","75001"\n', + encoding="utf-8", + ) wave2_dir = root / "wave2" run_wave2(input_root=input_root, run_dir=wave2_dir) result = run_from_wave2(wave2_dir, root / "packet") @@ -202,6 +251,8 @@ def test_wave2_replay_verifies_input_manifest_and_uses_row_hashes(self): hashes = [item.get("actual_sha256") for item in summary["artifact_verification"]["artifacts"]] self.assertTrue(all(hashes)) self.assertNotIn("aggregate_artifact_sha256", json.dumps(summary)) + self.assertGreater(summary["links"]["quarantined_count"], 0) + self.assertNotIn("link_missing_or_invalid_provenance", summary["links"]["excluded_reasons"]) if __name__ == "__main__": diff --git a/pipeline/sources/us/accountability/test_current_identity.py b/pipeline/sources/us/accountability/test_current_identity.py index 67ca709..bce66fe 100644 --- a/pipeline/sources/us/accountability/test_current_identity.py +++ b/pipeline/sources/us/accountability/test_current_identity.py @@ -38,6 +38,25 @@ def test_exact_official_ids_link_each_observation_family(self): self.assertEqual(len(graph["entities"]), 5) self.assertTrue(all(candidate["confidence_explanation"] for candidate in graph["candidates"])) + def test_row_specific_provenance_overrides_profile_fallback(self): + payload = load_fixture() + registration = payload["aphis"]["registrations"][0] + report = payload["aphis"]["annual_reports"][0] + payload["provenance"][("us.aphis", "registrations", registration["source_record_key"])] = { + "artifact_sha256": "c" * 64, + "source_url": "https://example.invalid/registration-page.csv", + "retrieved_at_utc": "2026-09-19T00:00:00Z", + } + payload["provenance"][("us.aphis", "annual_reports", report["source_record_key"])] = { + "artifact_sha256": "d" * 64, + "source_url": "https://example.invalid/annual-page.csv", + "retrieved_at_utc": "2026-09-19T00:01:00Z", + } + graph = self.build(payload) + link = next(item for item in graph["candidates"] if item["right"]["profile"] == "annual_reports") + self.assertEqual(link["evidence"]["provenance"]["us.aphis:registrations"]["artifact_sha256"], "c" * 64) + self.assertEqual(link["evidence"]["provenance"]["us.aphis:annual_reports"]["artifact_sha256"], "d" * 64) + def test_amended_annual_reports_remain_source_versioned_and_linkable(self): payload = load_fixture() amendment = copy.deepcopy(payload["aphis"]["annual_reports"][0]) From 4e9b8861ec23467e92edac9296990d01b719e415 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sat, 19 Sep 2026 11:11:14 -0700 Subject: [PATCH 294/311] Preserve APHIS handoff quarantine accounting --- .../us/accountability/aphis_evidence.py | 14 ++++++-- .../us/accountability/test_aphis_evidence.py | 32 +++++++++++++++++++ 2 files changed, 43 insertions(+), 3 deletions(-) diff --git a/pipeline/sources/us/accountability/aphis_evidence.py b/pipeline/sources/us/accountability/aphis_evidence.py index 99bfcf5..520e673 100644 --- a/pipeline/sources/us/accountability/aphis_evidence.py +++ b/pipeline/sources/us/accountability/aphis_evidence.py @@ -538,6 +538,8 @@ def run_from_handoffs( registration_key: str | None = None, expected_rows: Mapping[str, int] | None = None, document_refs: Mapping[str, Iterable[Mapping[str, Any]]] | None = None, + profile_input_rows: Mapping[str, int] | None = None, + quarantine_rows_by_profile: Mapping[str, Iterable[Mapping[str, Any]]] | None = None, ) -> dict[str, Any]: """Consume lane-specific APHIS observation handoffs into one packet. @@ -545,11 +547,16 @@ def run_from_handoffs( The source artifact checksum is retained as row provenance; callers that also retain the raw files can use the standalone artifact verifier before treating those bytes as independently verified. + + ``profile_input_rows`` and ``quarantine_rows_by_profile`` are explicit + acquisition-lane accounting inputs. They allow an accepted-only handoff + to retain adapter quarantine and source capture totals without guessing + that excluded rows were absent. """ records: dict[str, list[dict[str, Any]]] = {profile: [] for profile in PROFILES} provenance: dict[tuple[str, str], dict[str, Any]] = {} failures: list[dict[str, Any]] = [] - profile_input_rows: dict[str, int] = {} + observed_input_rows: dict[str, int] = {} for profile in PROFILES: handoff = handoff_dirs.get(profile) if handoff is None: @@ -563,7 +570,7 @@ def run_from_handoffs( source_url = _text(manifest.get("source_url")) retrieved = _text(manifest.get("retrieved_at_utc")) digest = _text(manifest.get("source_artifact_sha256") or manifest.get("checksum_sha256")) - profile_input_rows[profile] = len(rows) + observed_input_rows[profile] = len(rows) provenance[("us.aphis", profile)] = { "artifact_sha256": digest, "source_url": source_url, @@ -595,8 +602,9 @@ def run_from_handoffs( registration_key=registration_key, expected_rows=expected_rows, input_failures=failures, - profile_input_rows=profile_input_rows, + profile_input_rows=profile_input_rows or observed_input_rows, document_refs=document_refs, + quarantine_rows_by_profile=quarantine_rows_by_profile, ) manifest = write_packet(packet_dir, packet) return {"packet": packet, "manifest": manifest, "graph": graph, "input_failures": failures} diff --git a/pipeline/sources/us/accountability/test_aphis_evidence.py b/pipeline/sources/us/accountability/test_aphis_evidence.py index daf5395..87ade0b 100644 --- a/pipeline/sources/us/accountability/test_aphis_evidence.py +++ b/pipeline/sources/us/accountability/test_aphis_evidence.py @@ -152,6 +152,38 @@ def test_source_local_handoffs_build_exact_private_paths(self): self.assertNotIn("link_missing_or_invalid_provenance", summary["links"]["excluded_reasons"]) self.assertEqual(summary["publication_status"], "not_eligible") + def test_handoff_accounting_keeps_adapter_quarantine_distinct_from_missing_rows(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + records = json.loads((Path(__file__).parent / "fixtures" / "current_identity.json").read_text(encoding="utf-8"))["aphis"] + handoffs = {} + for profile in ("registrations", "annual_reports", "inspections"): + handoff = root / profile + handoff.mkdir() + payload = b"".join((json.dumps(row, sort_keys=True) + "\n").encode() for row in records[profile]) + (handoff / "records.jsonl").write_bytes(payload) + (handoff / "manifest.json").write_text(json.dumps({ + "contract_version": "us-aphis-observation-handoff-v1", "source_id": "us.aphis", + "profile": profile, "source_url": "https://example.invalid/" + profile, + "retrieved_at_utc": "2026-09-19T00:00:00Z", "source_artifact_sha256": "a" * 64, + "normalized_rows": len(records[profile]), "normalized_sha256": hashlib.sha256(payload).hexdigest(), + "graph_candidate_emission": False, "auto_merge": False, "release_state": "not-created", + "publication_state": "private-candidate", "review_state": "review_required", + "privacy_gate": "pending", "coordinate_gate": "review_required", "test_only": True, + "row_payloads_included": True, + }), encoding="utf-8") + handoffs[profile] = handoff + quarantine = {"reasons": ["duplicate_observation_id"], "record": {"source_record_key": "inspections:quarantined"}} + result = run_from_handoffs( + handoffs, root / "packet", profile_input_rows={"inspections": len(records["inspections"]) + 1}, + quarantine_rows_by_profile={"inspections": [quarantine]}, + ) + summary = result["packet"]["row_free_summary"] + inspections = summary["profiles"]["inspections"] + self.assertEqual(inspections["input_rows"], len(records["inspections"]) + 1) + self.assertEqual(inspections["quarantined_rows"], 1) + self.assertNotIn("not_observed", summary["timeline_state_counts"]) + def test_aggregate_metadata_hash_cannot_be_link_provenance(self): records = self._records() registration_key = records["registrations"][0]["source_record_key"] From 98ac203ab6cd7a7495358c5579a8f8b7be1ad5e4 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sat, 19 Sep 2026 11:18:01 -0700 Subject: [PATCH 295/311] Clarify APHIS collision and evidence origin states --- docs/countries/us/aphis-investigation-packet.md | 8 ++++++++ pipeline/sources/us/accountability/aphis_evidence.py | 8 ++++++++ pipeline/sources/us/accountability/test_aphis_evidence.py | 2 ++ 3 files changed, 18 insertions(+) diff --git a/docs/countries/us/aphis-investigation-packet.md b/docs/countries/us/aphis-investigation-packet.md index 3466594..9cb3233 100644 --- a/docs/countries/us/aphis-investigation-packet.md +++ b/docs/countries/us/aphis-investigation-packet.md @@ -40,6 +40,14 @@ signed URLs. The companion editorial note records what the evidence makes visible, why it matters to activists, what it cannot establish, and useful next research questions. +Adapter quarantine labels such as `duplicate_observation_id` describe a +provisional source-identity collision. They do not establish that the +retained rows are factually duplicate observations; collision groups and full +row equality remain separate review questions. For real source handoffs, +`test_only` means private/no-publication gating, while the packet separately +labels the evidence origin as government-sourced retained evidence rather +than a synthetic fixture. + Each timeline item has one of these states: * `observed`: an accepted source-local record with its source key, period and diff --git a/pipeline/sources/us/accountability/aphis_evidence.py b/pipeline/sources/us/accountability/aphis_evidence.py index 520e673..36e43b1 100644 --- a/pipeline/sources/us/accountability/aphis_evidence.py +++ b/pipeline/sources/us/accountability/aphis_evidence.py @@ -380,6 +380,7 @@ def build_packet( "links": {"candidate_count": len(links), "quarantined_count": len(link_quarantine), "excluded_reasons": dict(sorted(Counter(item["reason"] for item in link_quarantine).items()))}, "input_failures": failures, "artifact_verification": dict(artifact_verification or {}), "coverage_boundary": "bounded retained APHIS source profiles; not a facility master, national census, current-operation, ownership, or animal-use total", + "quarantine_semantics": "adapter quarantine reasons such as duplicate_observation_id are provisional source-identity collisions for review, not claims that retained rows are factually duplicate observations", "unknowns": ["location/current operation", "ownership/control", "unobserved source rows and reporting periods", "animal-use coverage outside the captured APHIS profiles"], "publication": {"api": False, "map": False, "export": False, "cache": False, "history": False}, } @@ -606,6 +607,13 @@ def run_from_handoffs( document_refs=document_refs, quarantine_rows_by_profile=quarantine_rows_by_profile, ) + # ``test_only`` is the existing private/no-publication gate on this + # handoff contract. It does not mean the source is synthetic. Keep source + # origin explicit so real government evidence is not mislabeled as a test + # fixture merely because publication is blocked. + for value in (packet["row_free_summary"], packet["private_packet"]): + value["evidence_origin"] = "government-sourced" + value["capture_classification"] = "real-retained-source-handoff" manifest = write_packet(packet_dir, packet) return {"packet": packet, "manifest": manifest, "graph": graph, "input_failures": failures} diff --git a/pipeline/sources/us/accountability/test_aphis_evidence.py b/pipeline/sources/us/accountability/test_aphis_evidence.py index 87ade0b..18b84db 100644 --- a/pipeline/sources/us/accountability/test_aphis_evidence.py +++ b/pipeline/sources/us/accountability/test_aphis_evidence.py @@ -151,6 +151,8 @@ def test_source_local_handoffs_build_exact_private_paths(self): self.assertGreater(summary["links"]["quarantined_count"], 0) self.assertNotIn("link_missing_or_invalid_provenance", summary["links"]["excluded_reasons"]) self.assertEqual(summary["publication_status"], "not_eligible") + self.assertEqual(summary["evidence_origin"], "government-sourced") + self.assertEqual(summary["capture_classification"], "real-retained-source-handoff") def test_handoff_accounting_keeps_adapter_quarantine_distinct_from_missing_rows(self): with tempfile.TemporaryDirectory() as directory: From 0870cebc7cbc869ff7da857b2575554095ecba33 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sat, 19 Sep 2026 11:23:16 -0700 Subject: [PATCH 296/311] Fail closed on APHIS handoff accounting and origin --- .../us/accountability/aphis_evidence.py | 42 +++++++++++++++---- .../us/accountability/current_identity.py | 18 ++++++-- .../us/accountability/test_aphis_evidence.py | 25 ++++++++++- .../accountability/test_current_identity.py | 15 +++++++ 4 files changed, 86 insertions(+), 14 deletions(-) diff --git a/pipeline/sources/us/accountability/aphis_evidence.py b/pipeline/sources/us/accountability/aphis_evidence.py index 36e43b1..1d1aed2 100644 --- a/pipeline/sources/us/accountability/aphis_evidence.py +++ b/pipeline/sources/us/accountability/aphis_evidence.py @@ -255,6 +255,7 @@ def build_packet( """Build deterministic private packet and row-free summary in memory.""" failures = [dict(item) for item in input_failures] expected_rows = dict(expected_rows or {}) + explicit_input_rows = profile_input_rows is not None profile_input_rows = dict(profile_input_rows or {}) quarantine_rows_by_profile = quarantine_rows_by_profile or {} document_refs = document_refs or {} @@ -270,6 +271,22 @@ def build_packet( profile_summary: dict[str, dict[str, Any]] = {} for profile in PROFILES: rows = accepted[profile] + adapter_quarantine = [dict(item) for item in quarantine_rows_by_profile.get(profile, ())] + captured_rows = int(profile_input_rows.get(profile, len(rows) + len(adapter_quarantine))) + if captured_rows < 0: + raise AphisEvidenceError(f"negative input row count for {profile}") + if explicit_input_rows and profile in profile_input_rows and captured_rows != len(rows) + len(adapter_quarantine): + raise AphisEvidenceError( + f"input row reconciliation failed for {profile}: " + f"declared={captured_rows}, accepted={len(rows)}, adapter_quarantined={len(adapter_quarantine)}" + ) + expected = expected_rows.get(profile) + if expected is not None and int(expected) < 0: + raise AphisEvidenceError(f"negative expected row count for {profile}") + if expected is not None and captured_rows > int(expected): + raise AphisEvidenceError( + f"expected row count underreported for {profile}: captured={captured_rows}, expected={expected}" + ) duplicate_keys = {key for key, count in occurrences.items() if count > 1} profile_failures = [item for item in failures if _text(item.get("profile")) == profile] observed = quarantined = 0 @@ -307,14 +324,11 @@ def build_packet( timeline.append(evidence) private_rows.append({"evidence": evidence, "record": _safe_record(record)}) timeline.extend(_document_events(key, document_refs.get(key, ()))) - adapter_quarantine = [dict(item) for item in quarantine_rows_by_profile.get(profile, ())] for item in adapter_quarantine: record = item.get("record") if isinstance(item.get("record"), Mapping) else {} timeline.append({"state": "quarantined", "profile": profile, "source_record_key": record.get("source_record_key"), "reason": "adapter_quarantine", "reasons": list(item.get("reasons", ()))}) - expected = expected_rows.get(profile) - captured_rows = int(profile_input_rows.get(profile, len(rows) + len(adapter_quarantine))) if expected is not None and captured_rows < int(expected): timeline.append({"state": "not_observed", "profile": profile, "period": None, "missing_count": int(expected) - captured_rows, @@ -558,6 +572,8 @@ def run_from_handoffs( provenance: dict[tuple[str, str], dict[str, Any]] = {} failures: list[dict[str, Any]] = [] observed_input_rows: dict[str, int] = {} + evidence_origins: list[str] = [] + capture_classifications: list[str] = [] for profile in PROFILES: handoff = handoff_dirs.get(profile) if handoff is None: @@ -571,6 +587,12 @@ def run_from_handoffs( source_url = _text(manifest.get("source_url")) retrieved = _text(manifest.get("retrieved_at_utc")) digest = _text(manifest.get("source_artifact_sha256") or manifest.get("checksum_sha256")) + origin = _text(manifest.get("evidence_origin")) + classification = _text(manifest.get("capture_classification")) + if origin: + evidence_origins.append(origin) + if classification: + capture_classifications.append(classification) observed_input_rows[profile] = len(rows) provenance[("us.aphis", profile)] = { "artifact_sha256": digest, @@ -607,13 +629,17 @@ def run_from_handoffs( document_refs=document_refs, quarantine_rows_by_profile=quarantine_rows_by_profile, ) + def _consensus(values: list[str], fallback: str) -> str: + distinct = set(values) + return next(iter(distinct)) if len(distinct) == 1 else fallback + # ``test_only`` is the existing private/no-publication gate on this - # handoff contract. It does not mean the source is synthetic. Keep source - # origin explicit so real government evidence is not mislabeled as a test - # fixture merely because publication is blocked. + # handoff contract. It does not establish source origin. Only explicit, + # consistent handoff metadata may classify the evidence; otherwise retain + # an unknown state rather than using a URL or source name as a heuristic. for value in (packet["row_free_summary"], packet["private_packet"]): - value["evidence_origin"] = "government-sourced" - value["capture_classification"] = "real-retained-source-handoff" + value["evidence_origin"] = _consensus(evidence_origins, "unknown") + value["capture_classification"] = _consensus(capture_classifications, "unknown-retained-handoff") manifest = write_packet(packet_dir, packet) return {"packet": packet, "manifest": manifest, "graph": graph, "input_failures": failures} diff --git a/pipeline/sources/us/accountability/current_identity.py b/pipeline/sources/us/accountability/current_identity.py index 89871ac..79f3869 100644 --- a/pipeline/sources/us/accountability/current_identity.py +++ b/pipeline/sources/us/accountability/current_identity.py @@ -147,10 +147,20 @@ def _provenance( profile_keys.append("annual_reports") values = None if source_record_key: - values = ( - provenance.get((source_id, profile, source_record_key)) - or provenance.get(f"{source_id}:{profile}:{source_record_key}") - ) + exact_profiles = [profile] + if profile == "amendments": + # Amendments are often carried in the annual-report artifact, but + # their source-record key remains version-specific. Prefer that + # exact row key under either explicit profile spelling before any + # profile-level fallback. + exact_profiles.append("annual_reports") + for exact_profile in exact_profiles: + values = ( + provenance.get((source_id, exact_profile, source_record_key)) + or provenance.get(f"{source_id}:{exact_profile}:{source_record_key}") + ) + if values: + break for profile_key in profile_keys: if not values: values = ( diff --git a/pipeline/sources/us/accountability/test_aphis_evidence.py b/pipeline/sources/us/accountability/test_aphis_evidence.py index 18b84db..8a9fea1 100644 --- a/pipeline/sources/us/accountability/test_aphis_evidence.py +++ b/pipeline/sources/us/accountability/test_aphis_evidence.py @@ -134,6 +134,8 @@ def test_source_local_handoffs_build_exact_private_paths(self): (handoff / "manifest.json").write_text(json.dumps({ "contract_version": "us-aphis-observation-handoff-v1", "source_id": "us.aphis", "profile": profile, + "evidence_origin": "synthetic", + "capture_classification": "synthetic-test-fixture", "source_url": f"https://example.invalid/{profile}", "retrieved_at_utc": "2026-09-19T00:00:00Z", "source_artifact_sha256": "a" * 64, @@ -151,8 +153,8 @@ def test_source_local_handoffs_build_exact_private_paths(self): self.assertGreater(summary["links"]["quarantined_count"], 0) self.assertNotIn("link_missing_or_invalid_provenance", summary["links"]["excluded_reasons"]) self.assertEqual(summary["publication_status"], "not_eligible") - self.assertEqual(summary["evidence_origin"], "government-sourced") - self.assertEqual(summary["capture_classification"], "real-retained-source-handoff") + self.assertEqual(summary["evidence_origin"], "synthetic") + self.assertEqual(summary["capture_classification"], "synthetic-test-fixture") def test_handoff_accounting_keeps_adapter_quarantine_distinct_from_missing_rows(self): with tempfile.TemporaryDirectory() as directory: @@ -186,6 +188,25 @@ def test_handoff_accounting_keeps_adapter_quarantine_distinct_from_missing_rows( self.assertEqual(inspections["quarantined_rows"], 1) self.assertNotIn("not_observed", summary["timeline_state_counts"]) + def test_explicit_input_total_must_reconcile_with_accepted_and_quarantine(self): + records = self._records() + with self.assertRaisesRegex(ValueError, "input row reconciliation failed"): + build_packet( + records_by_profile=records, + provenance={}, + profile_input_rows={"inspections": len(records["inspections"]) + 1}, + quarantine_rows_by_profile={"inspections": []}, + ) + + def test_expected_total_cannot_be_lower_than_captured_input(self): + records = self._records() + with self.assertRaisesRegex(ValueError, "expected row count underreported"): + build_packet( + records_by_profile=records, + provenance={}, + expected_rows={"inspections": len(records["inspections"]) - 1}, + ) + def test_aggregate_metadata_hash_cannot_be_link_provenance(self): records = self._records() registration_key = records["registrations"][0]["source_record_key"] diff --git a/pipeline/sources/us/accountability/test_current_identity.py b/pipeline/sources/us/accountability/test_current_identity.py index bce66fe..72c2d44 100644 --- a/pipeline/sources/us/accountability/test_current_identity.py +++ b/pipeline/sources/us/accountability/test_current_identity.py @@ -69,6 +69,21 @@ def test_amended_annual_reports_remain_source_versioned_and_linkable(self): self.assertEqual(amendment_links[0]["assertion_status"], "candidate") self.assertTrue(any(entity["entity_type"] == "amendments" for entity in graph["entities"])) + def test_amendment_row_uses_exact_annual_artifact_provenance_before_profile_fallback(self): + payload = load_fixture() + amendment = copy.deepcopy(payload["aphis"]["annual_reports"][0]) + amendment["source_record_key"] = "amendments:00-B-TEST-001:2025:2" + amendment["normalized"]["evidence_type"] = "amendments" + payload["aphis"]["annual_reports"].append(amendment) + payload["provenance"][("us.aphis", "annual_reports", amendment["source_record_key"])] = { + "artifact_sha256": "e" * 64, + "source_url": "https://example.invalid/annual-page.csv", + "retrieved_at_utc": "2026-09-19T00:02:00Z", + } + graph = self.build(payload) + link = next(item for item in graph["candidates"] if item["right"]["profile"] == "amendments") + self.assertEqual(link["evidence"]["provenance"]["us.aphis:amendments"]["artifact_sha256"], "e" * 64) + def test_same_name_different_address_is_not_a_join(self): payload = load_fixture() payload["aphis"]["annual_reports"][0]["normalized"]["certificate_number"] = "" From 7a08758a0672a4c541d2553573bdc85414800807 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sat, 19 Sep 2026 11:25:49 -0700 Subject: [PATCH 297/311] Require complete APHIS origin metadata consensus --- .../sources/us/accountability/aphis_evidence.py | 16 ++++++++-------- .../us/accountability/test_aphis_evidence.py | 8 ++++++++ 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/pipeline/sources/us/accountability/aphis_evidence.py b/pipeline/sources/us/accountability/aphis_evidence.py index 1d1aed2..793ffd3 100644 --- a/pipeline/sources/us/accountability/aphis_evidence.py +++ b/pipeline/sources/us/accountability/aphis_evidence.py @@ -572,8 +572,8 @@ def run_from_handoffs( provenance: dict[tuple[str, str], dict[str, Any]] = {} failures: list[dict[str, Any]] = [] observed_input_rows: dict[str, int] = {} - evidence_origins: list[str] = [] - capture_classifications: list[str] = [] + evidence_origins: dict[str, str | None] = {profile: None for profile in PROFILES} + capture_classifications: dict[str, str | None] = {profile: None for profile in PROFILES} for profile in PROFILES: handoff = handoff_dirs.get(profile) if handoff is None: @@ -589,10 +589,8 @@ def run_from_handoffs( digest = _text(manifest.get("source_artifact_sha256") or manifest.get("checksum_sha256")) origin = _text(manifest.get("evidence_origin")) classification = _text(manifest.get("capture_classification")) - if origin: - evidence_origins.append(origin) - if classification: - capture_classifications.append(classification) + evidence_origins[profile] = origin + capture_classifications[profile] = classification observed_input_rows[profile] = len(rows) provenance[("us.aphis", profile)] = { "artifact_sha256": digest, @@ -629,8 +627,10 @@ def run_from_handoffs( document_refs=document_refs, quarantine_rows_by_profile=quarantine_rows_by_profile, ) - def _consensus(values: list[str], fallback: str) -> str: - distinct = set(values) + def _consensus(values: Mapping[str, str | None], fallback: str) -> str: + if not values or any(not value for value in values.values()): + return fallback + distinct = set(values.values()) return next(iter(distinct)) if len(distinct) == 1 else fallback # ``test_only`` is the existing private/no-publication gate on this diff --git a/pipeline/sources/us/accountability/test_aphis_evidence.py b/pipeline/sources/us/accountability/test_aphis_evidence.py index 8a9fea1..4255abc 100644 --- a/pipeline/sources/us/accountability/test_aphis_evidence.py +++ b/pipeline/sources/us/accountability/test_aphis_evidence.py @@ -155,6 +155,14 @@ def test_source_local_handoffs_build_exact_private_paths(self): self.assertEqual(summary["publication_status"], "not_eligible") self.assertEqual(summary["evidence_origin"], "synthetic") self.assertEqual(summary["capture_classification"], "synthetic-test-fixture") + annual_manifest = root / "annual_reports" / "manifest.json" + annual_payload = json.loads(annual_manifest.read_text(encoding="utf-8")) + annual_payload.pop("evidence_origin") + annual_payload.pop("capture_classification") + annual_manifest.write_text(json.dumps(annual_payload), encoding="utf-8") + mixed = run_from_handoffs(handoffs, root / "mixed-packet")["packet"]["row_free_summary"] + self.assertEqual(mixed["evidence_origin"], "unknown") + self.assertEqual(mixed["capture_classification"], "unknown-retained-handoff") def test_handoff_accounting_keeps_adapter_quarantine_distinct_from_missing_rows(self): with tempfile.TemporaryDirectory() as directory: From 8dc35b988bc14760f8fe86caffce619f7e274417 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sat, 19 Sep 2026 11:47:21 -0700 Subject: [PATCH 298/311] Verify APHIS original page lineage --- .../us/accountability/aphis_evidence.py | 216 +++++++++++++++++- .../us/accountability/current_identity.py | 11 +- .../us/accountability/test_aphis_evidence.py | 98 ++++++++ 3 files changed, 315 insertions(+), 10 deletions(-) diff --git a/pipeline/sources/us/accountability/aphis_evidence.py b/pipeline/sources/us/accountability/aphis_evidence.py index 793ffd3..79aae57 100644 --- a/pipeline/sources/us/accountability/aphis_evidence.py +++ b/pipeline/sources/us/accountability/aphis_evidence.py @@ -10,7 +10,9 @@ from __future__ import annotations import argparse +import csv import hashlib +import io import json import re from collections import Counter, defaultdict @@ -58,6 +60,148 @@ def _json(path: Path) -> Mapping[str, Any]: return value +def _verified_lineage_pages( + manifest_path: str | Path, + page_root: str | Path, +) -> dict[int, dict[str, Any]]: + """Verify and parse the explicitly named original capture pages. + + A derived handoff may carry lineage metadata, but that metadata is not + evidence by itself. The capture manifest and page bytes are checked here + so a correct-looking embedded hash cannot point at a missing, tampered, or + wrong page. + """ + manifest_file = Path(manifest_path).resolve() + manifest = _json(manifest_file) + pages = manifest.get("pages") + if not isinstance(pages, list) or not pages: + raise AphisEvidenceError(f"lineage manifest {manifest_file} lacks a non-empty pages list") + root = Path(page_root).resolve() + if not root.is_dir(): + raise AphisEvidenceError(f"lineage page root is not a directory: {root}") + verified: dict[int, dict[str, Any]] = {} + for item in pages: + if not isinstance(item, Mapping): + raise AphisEvidenceError(f"lineage manifest {manifest_file} has a malformed page entry") + ordinal = item.get("ordinal") + filename = _text(item.get("file")) + declared_hash = _text(item.get("sha256")) + declared_size = item.get("byte_size") + source_url = _text(item.get("source_url")) + if not isinstance(ordinal, int) or ordinal < 1 or not filename: + raise AphisEvidenceError(f"lineage manifest {manifest_file} has invalid page identity") + if ordinal in verified or not declared_hash or not _SHA256.fullmatch(declared_hash): + raise AphisEvidenceError(f"lineage manifest {manifest_file} has invalid page hash") + try: + declared_size = int(declared_size) + except (TypeError, ValueError) as exc: + raise AphisEvidenceError(f"lineage manifest {manifest_file} has invalid page size") from exc + if declared_size < 0 or not source_url: + raise AphisEvidenceError(f"lineage manifest {manifest_file} has incomplete page metadata") + candidate = (root / filename).resolve() + try: + candidate.relative_to(root) + except ValueError as exc: + raise AphisEvidenceError(f"lineage page escapes declared root: {filename}") from exc + if not candidate.is_file(): + raise AphisEvidenceError(f"lineage page is missing: {filename}") + actual_size = candidate.stat().st_size + actual_hash = _sha256(candidate) + if actual_size != declared_size or actual_hash.lower() != declared_hash.lower(): + raise AphisEvidenceError(f"lineage page bytes do not match capture manifest: {filename}") + try: + with candidate.open("r", encoding="utf-8-sig", newline="") as handle: + text = handle.read() + reader = csv.DictReader(io.StringIO(text), strict=True) + headers = reader.fieldnames + rows = list(reader) + except (OSError, UnicodeDecodeError, csv.Error) as exc: + raise AphisEvidenceError(f"lineage page is not a strict UTF-8 CSV: {filename}") from exc + if not headers or any(header is None or not str(header).strip() for header in headers): + raise AphisEvidenceError(f"lineage page has invalid CSV headers: {filename}") + if any(None in row for row in rows): + raise AphisEvidenceError(f"lineage page has malformed CSV rows: {filename}") + verified[ordinal] = { + "sha256": declared_hash.lower(), + "byte_size": declared_size, + "source_url": source_url, + "page_retrieved_at_utc": _text(item.get("page_retrieved_at_utc")) or "unknown", + "rows": rows, + } + return verified + + +def _string_row(value: Mapping[Any, Any]) -> dict[str, str]: + return {str(key): "" if item is None else str(item) for key, item in value.items()} + + +def _verify_row_lineage( + record: Mapping[str, Any], + pages: Mapping[int, Mapping[str, Any]], + *, + handoff: Path, +) -> Mapping[str, Any]: + normalized = record.get("normalized") if isinstance(record.get("normalized"), Mapping) else {} + lineage = normalized.get("source_capture_lineage") if isinstance(normalized, Mapping) else None + if not isinstance(lineage, Mapping): + raise AphisEvidenceError(f"lineage-aware APHIS handoff row lacks source_capture_lineage: {handoff}") + page_hash = _text(lineage.get("page_sha256")) + page_url = _text(lineage.get("source_url")) + page_ordinal = lineage.get("page_ordinal") + page_row = lineage.get("page_row") + page_size = lineage.get("page_byte_size") + if not page_hash or not _SHA256.fullmatch(page_hash) or not page_url: + raise AphisEvidenceError(f"lineage-aware APHIS handoff row has invalid page provenance: {handoff}") + if not isinstance(page_ordinal, int) or page_ordinal < 1 or not isinstance(page_row, int) or page_row < 1: + raise AphisEvidenceError(f"lineage-aware APHIS handoff row has invalid page position: {handoff}") + try: + page_size = int(page_size) + except (TypeError, ValueError) as exc: + raise AphisEvidenceError(f"lineage-aware APHIS handoff row has invalid page size: {handoff}") from exc + if page_size < 0: + raise AphisEvidenceError(f"lineage-aware APHIS handoff row has invalid page size: {handoff}") + page = pages.get(page_ordinal) + if page is None: + raise AphisEvidenceError(f"lineage-aware APHIS handoff row references unknown page: {handoff}") + if ( + page_hash.lower() != _text(page.get("sha256")) + or page_size != page.get("byte_size") + or page_url != page.get("source_url") + ): + raise AphisEvidenceError(f"lineage-aware APHIS handoff row disagrees with capture manifest: {handoff}") + source_rows = page.get("rows") + if not isinstance(source_rows, list) or page_row > len(source_rows): + raise AphisEvidenceError(f"lineage-aware APHIS handoff row is outside original page: {handoff}") + source_values = record.get("source_values") + if not isinstance(source_values, Mapping): + raise AphisEvidenceError(f"lineage-aware APHIS handoff row does not match original page row: {handoff}") + source_values = _string_row(source_values) + expected_capture_values = { + "__capture_page_byte_size": str(page_size), + "__capture_page_ordinal": str(page_ordinal), + "__capture_page_retrieved_at_utc": str(_text(lineage.get("page_retrieved_at_utc")) or page.get("page_retrieved_at_utc") or "unknown"), + "__capture_page_row": str(page_row), + "__capture_page_sha256": page_hash.lower(), + "__capture_source_url": page_url, + } + for key in (key for key in source_values if key.startswith("__capture_")): + if key not in expected_capture_values or source_values[key] != expected_capture_values[key]: + raise AphisEvidenceError(f"lineage-aware APHIS handoff row has invalid embedded capture metadata: {handoff}") + source_values = {key: value for key, value in source_values.items() if not key.startswith("__capture_")} + if source_values != _string_row(source_rows[page_row - 1]): + raise AphisEvidenceError(f"lineage-aware APHIS handoff row does not match original page row: {handoff}") + return { + "artifact_sha256": page_hash.lower(), + "source_url": page_url, + "artifact_classification": "original_page", + "page_sha256": page_hash.lower(), + "page_ordinal": page_ordinal, + "page_row": page_row, + "page_byte_size": page_size, + "page_retrieved_at_utc": _text(lineage.get("page_retrieved_at_utc")) or page.get("page_retrieved_at_utc") or "unknown", + } + + def _artifact_entries(manifest: Mapping[str, Any]) -> list[dict[str, Any]]: """Normalize Wave 1/Wave 2 artifact metadata without reading rows.""" profiles = manifest.get("profiles") @@ -212,7 +356,14 @@ def _provenance_for( return None if verified_artifact_hashes is not None and digest.lower() not in verified_artifact_hashes: return None - return {"artifact_sha256": digest.lower(), "source_url": url, "retrieved_at_utc": retrieved} + result = {"artifact_sha256": digest.lower(), "source_url": url, "retrieved_at_utc": retrieved} + for key in ( + "artifact_classification", "page_sha256", "page_ordinal", "page_row", + "page_byte_size", "page_retrieved_at_utc", "derived_artifact_sha256", + ): + if value.get(key) is not None: + result[key] = value[key] + return result def _safe_record(record: Mapping[str, Any]) -> dict[str, Any]: @@ -555,6 +706,8 @@ def run_from_handoffs( document_refs: Mapping[str, Iterable[Mapping[str, Any]]] | None = None, profile_input_rows: Mapping[str, int] | None = None, quarantine_rows_by_profile: Mapping[str, Iterable[Mapping[str, Any]]] | None = None, + lineage_manifest_paths: Mapping[str, str | Path] | None = None, + lineage_page_roots: Mapping[str, str | Path] | None = None, ) -> dict[str, Any]: """Consume lane-specific APHIS observation handoffs into one packet. @@ -569,11 +722,22 @@ def run_from_handoffs( that excluded rows were absent. """ records: dict[str, list[dict[str, Any]]] = {profile: [] for profile in PROFILES} + quarantine_rows = { + profile: list((quarantine_rows_by_profile or {}).get(profile, ())) + for profile in PROFILES + } + lineage_manifest_paths = dict(lineage_manifest_paths or {}) + lineage_page_roots = dict(lineage_page_roots or {}) provenance: dict[tuple[str, str], dict[str, Any]] = {} failures: list[dict[str, Any]] = [] observed_input_rows: dict[str, int] = {} evidence_origins: dict[str, str | None] = {profile: None for profile in PROFILES} capture_classifications: dict[str, str | None] = {profile: None for profile in PROFILES} + lineage_rows: dict[str, int] = {profile: 0 for profile in PROFILES} + lineage_unknown_retrieval_rows: dict[str, int] = {profile: 0 for profile in PROFILES} + lineage_quarantine_rows: dict[str, int] = {profile: 0 for profile in PROFILES} + artifact_classifications: dict[str, str | None] = {profile: None for profile in PROFILES} + verified_pages: dict[str, dict[int, dict[str, Any]]] = {} for profile in PROFILES: handoff = handoff_dirs.get(profile) if handoff is None: @@ -589,25 +753,55 @@ def run_from_handoffs( digest = _text(manifest.get("source_artifact_sha256") or manifest.get("checksum_sha256")) origin = _text(manifest.get("evidence_origin")) classification = _text(manifest.get("capture_classification")) + artifact_classification = _text(manifest.get("source_artifact_classification")) evidence_origins[profile] = origin capture_classifications[profile] = classification + artifact_classifications[profile] = artifact_classification observed_input_rows[profile] = len(rows) provenance[("us.aphis", profile)] = { "artifact_sha256": digest, "source_url": source_url, "retrieved_at_utc": retrieved, } + if artifact_classification == "derived_staging_with_original_page_lineage": + lineage_manifest = lineage_manifest_paths.get(profile) + lineage_root = lineage_page_roots.get(profile) + if lineage_manifest is None or lineage_root is None: + raise AphisEvidenceError( + f"lineage-aware APHIS handoff requires explicit capture manifest and page root: {handoff}" + ) + verified_pages[profile] = _verified_lineage_pages(lineage_manifest, lineage_root) + for row in rows: row_copy = dict(row) - row_copy["_retained_artifact"] = { - "artifact_sha256": digest, - "source_url": source_url, - "retrieved_at_utc": retrieved, - } + if artifact_classification == "derived_staging_with_original_page_lineage": + row_copy["_retained_artifact"] = { + **_verify_row_lineage(row, verified_pages[profile], handoff=Path(handoff).resolve()), + "retrieved_at_utc": retrieved, + "derived_artifact_sha256": digest, + } + lineage_rows[profile] += 1 + if row_copy["_retained_artifact"]["page_retrieved_at_utc"] == "unknown": + lineage_unknown_retrieval_rows[profile] += 1 + else: + row_copy["_retained_artifact"] = { + "artifact_sha256": digest, + "source_url": source_url, + "retrieved_at_utc": retrieved, + "artifact_classification": artifact_classification or "source_artifact", + } records[profile].append(row_copy) - if digest: + if row_copy["_retained_artifact"].get("artifact_sha256"): provenance[("us.aphis", profile, str(row["source_record_key"]))] = dict(row_copy["_retained_artifact"]) + if artifact_classification == "derived_staging_with_original_page_lineage": + for item in quarantine_rows[profile]: + quarantined_record = item.get("record") if isinstance(item, Mapping) else None + if not isinstance(quarantined_record, Mapping): + continue + _verify_row_lineage(quarantined_record, verified_pages[profile], handoff=Path(handoff).resolve()) + lineage_quarantine_rows[profile] += 1 + from pipeline.sources.us.accountability.current_identity import build_current_identity_graph graph = build_current_identity_graph( @@ -625,7 +819,7 @@ def run_from_handoffs( input_failures=failures, profile_input_rows=profile_input_rows or observed_input_rows, document_refs=document_refs, - quarantine_rows_by_profile=quarantine_rows_by_profile, + quarantine_rows_by_profile=quarantine_rows, ) def _consensus(values: Mapping[str, str | None], fallback: str) -> str: if not values or any(not value for value in values.values()): @@ -640,6 +834,12 @@ def _consensus(values: Mapping[str, str | None], fallback: str) -> str: for value in (packet["row_free_summary"], packet["private_packet"]): value["evidence_origin"] = _consensus(evidence_origins, "unknown") value["capture_classification"] = _consensus(capture_classifications, "unknown-retained-handoff") + value["lineage"] = { + "rows_with_original_page_lineage": dict(lineage_rows), + "rows_with_unknown_page_retrieval": dict(lineage_unknown_retrieval_rows), + "quarantined_rows_with_original_page_lineage": dict(lineage_quarantine_rows), + "source_artifact_classification": dict(artifact_classifications), + } manifest = write_packet(packet_dir, packet) return {"packet": packet, "manifest": manifest, "graph": graph, "input_failures": failures} diff --git a/pipeline/sources/us/accountability/current_identity.py b/pipeline/sources/us/accountability/current_identity.py index 79f3869..2842238 100644 --- a/pipeline/sources/us/accountability/current_identity.py +++ b/pipeline/sources/us/accountability/current_identity.py @@ -132,7 +132,7 @@ def _provenance( source_id: str, profile: str, source_record_key: str | None = None, -) -> dict[str, str] | None: +) -> dict[str, Any] | None: """Resolve row provenance, then profile/source fallback metadata. A real capture can span multiple retained source artifacts. Prefer the @@ -177,11 +177,18 @@ def _provenance( retrieved = _text(values.get("retrieved_at_utc")) if not digest or not _HEX64.fullmatch(digest.lower()) or not url or not _valid_datetime(retrieved): return None - return { + result: dict[str, Any] = { "artifact_sha256": digest.lower(), "source_url": url, "retrieved_at_utc": retrieved, } + for key in ( + "artifact_classification", "page_sha256", "page_ordinal", "page_row", + "page_byte_size", "page_retrieved_at_utc", "derived_artifact_sha256", + ): + if values.get(key) is not None: + result[key] = values[key] + return result def _suppressed(record: Mapping[str, Any]) -> bool: diff --git a/pipeline/sources/us/accountability/test_aphis_evidence.py b/pipeline/sources/us/accountability/test_aphis_evidence.py index 4255abc..b8003ea 100644 --- a/pipeline/sources/us/accountability/test_aphis_evidence.py +++ b/pipeline/sources/us/accountability/test_aphis_evidence.py @@ -1,3 +1,4 @@ +import csv import hashlib import json import shutil @@ -317,6 +318,103 @@ def test_wave2_replay_verifies_input_manifest_and_uses_row_hashes(self): self.assertGreater(summary["links"]["quarantined_count"], 0) self.assertNotIn("link_missing_or_invalid_provenance", summary["links"]["excluded_reasons"]) + def _lineage_handoffs(self, root, *, wrong_row_profile=None, missing_page_profile=None, tampered_page_profile=None): + source = json.loads((Path(__file__).parent / "fixtures" / "current_identity.json").read_text(encoding="utf-8"))["aphis"] + handoffs = {} + lineage_manifests = {} + page_roots = {} + page_hashes = {} + for profile in ("registrations", "annual_reports", "inspections"): + profile_root = root / profile + page_root = profile_root / "raw-pages" + handoff = profile_root / "candidate-handoff" + page_root.mkdir(parents=True) + handoff.mkdir() + row = json.loads(json.dumps(source[profile][0])) + fieldnames = list(row["source_values"]) + page = page_root / "page-01.csv" + with page.open("w", encoding="utf-8", newline="") as handle: + writer = csv.DictWriter(handle, fieldnames=fieldnames) + writer.writeheader() + writer.writerow(row["source_values"]) + page_hash = hashlib.sha256(page.read_bytes()).hexdigest() + page_hashes[profile] = page_hash + source_url = f"https://example.invalid/original/{profile}" + capture_manifest = profile_root / "capture-manifest.json" + capture_manifest.write_text(json.dumps({"pages": [{ + "ordinal": 1, "file": "page-01.csv", "sha256": page_hash, + "byte_size": page.stat().st_size, "source_url": source_url, + "page_retrieved_at_utc": "unknown", + }]}), encoding="utf-8") + lineage_manifests[profile] = capture_manifest + page_roots[profile] = page_root + normalized = dict(row["normalized"]) + normalized["source_capture_lineage"] = { + "page_sha256": page_hash, "page_ordinal": 1, "page_row": 1, + "page_byte_size": page.stat().st_size, "source_url": source_url, + "page_retrieved_at_utc": "unknown", + } + row["normalized"] = normalized + if profile == wrong_row_profile: + row["source_values"] = dict(row["source_values"], City="Wrongville") + payload = (json.dumps(row, ensure_ascii=False, sort_keys=True) + "\n").encode("utf-8") + (handoff / "records.jsonl").write_bytes(payload) + handoff_manifest = { + "contract_version": "us-aphis-observation-handoff-v1", "source_id": "us.aphis", + "profile": profile, "evidence_origin": "synthetic", + "capture_classification": "synthetic-test-fixture", + "source_artifact_classification": "derived_staging_with_original_page_lineage", + "source_url": f"https://example.invalid/derived/{profile}", + "retrieved_at_utc": "2026-09-19T00:00:00Z", "source_artifact_sha256": "d" * 64, + "normalized_rows": 1, "normalized_sha256": hashlib.sha256(payload).hexdigest(), + "graph_candidate_emission": False, "auto_merge": False, "release_state": "not-created", + "publication_state": "private-candidate", "review_state": "review_required", + "privacy_gate": "pending", "coordinate_gate": "review_required", "test_only": True, + "row_payloads_included": True, + } + (handoff / "manifest.json").write_text(json.dumps(handoff_manifest), encoding="utf-8") + handoffs[profile] = handoff + if profile == missing_page_profile: + page.unlink() + elif profile == tampered_page_profile: + page.write_bytes(page.read_bytes() + b"tampered") + return handoffs, lineage_manifests, page_roots, page_hashes + + def test_lineage_verifies_original_bytes_and_keeps_derived_hash_separate(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + handoffs, manifests, page_roots, page_hashes = self._lineage_handoffs(root) + result = run_from_handoffs( + handoffs, root / "packet", lineage_manifest_paths=manifests, lineage_page_roots=page_roots, + ) + summary = result["packet"]["row_free_summary"] + self.assertEqual(summary["lineage"]["rows_with_original_page_lineage"], { + "registrations": 1, "annual_reports": 1, "inspections": 1, + }) + registration = next( + row for row in result["packet"]["private_packet"]["rows"] + if row["evidence"].get("profile") == "registrations" + ) + provenance = registration["evidence"]["provenance"] + self.assertEqual(provenance["artifact_sha256"], page_hashes["registrations"]) + self.assertEqual(provenance["derived_artifact_sha256"], "d" * 64) + self.assertEqual(provenance["artifact_classification"], "original_page") + + def test_lineage_rejects_wrong_payload_missing_page_and_tampered_bytes(self): + cases = ( + ("wrong row", {"wrong_row_profile": "annual_reports"}, "does not match original page row"), + ("missing page", {"missing_page_profile": "inspections"}, "lineage page is missing"), + ("tampered bytes", {"tampered_page_profile": "registrations"}, "bytes do not match capture manifest"), + ) + for label, options, message in cases: + with self.subTest(label=label), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + handoffs, manifests, page_roots, _ = self._lineage_handoffs(root, **options) + with self.assertRaisesRegex(ValueError, message): + run_from_handoffs( + handoffs, root / "packet", lineage_manifest_paths=manifests, lineage_page_roots=page_roots, + ) + if __name__ == "__main__": unittest.main() From e3ce775df7105c8cb82ff5d8e8330169cea1ee1b Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sat, 19 Sep 2026 11:51:44 -0700 Subject: [PATCH 299/311] Record APHIS evidence consumer integration --- docs/sprint02-integration-ledger.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/sprint02-integration-ledger.md b/docs/sprint02-integration-ledger.md index 13d74d1..696fec6 100644 --- a/docs/sprint02-integration-ledger.md +++ b/docs/sprint02-integration-ledger.md @@ -36,6 +36,7 @@ Kickoff validation: `npm ci` completed with no vulnerabilities; `python -m unitt - APHIS registration/report and inspection lanes are still in progress; their real handoffs require independent replay before integration. - APHIS inspection code chain `addcef0c` -> `4518bce6` -> `8b4d3c68` is QA-approved and integrated as `e946d15e`. The authoritative replay accepted 1,075 input rows, 1,071 candidates, and 4 exact-duplicate quarantines, with every source row mapped to verified original-page lineage. Public release/import remains blocked. - APHIS registration/annual-report per-row lineage is not yet accepted; the core/evidence owners are correcting that gap before integration. +- The authoritative APHIS evidence-consumer stack `8430572b` -> `4ee87737` -> `3124696e` -> `285ef119` -> `da17cc34` -> `9b026c13` is code-QA approved and integrated as lane commits through `8dc35b98`. The integrated checks fail closed on quarantine accounting, origin metadata consensus, and original-page lineage; this does not accept the still-held annual/registration rows. ## Integration ledger @@ -46,9 +47,9 @@ Kickoff validation: `npm ci` completed with no vulnerabilities; `python -m unitt | FSIS current parity | Lane 3 handoff under private storage | Pending source handoff and review | | France candidate | Lane 4 handoff under private storage | QA-approved and integrated for private candidate/replay | | Italy candidate | Lane 5 handoff under private storage | QA-approved and integrated for private candidate/replay | -| Evidence integration | Lane 6 existing APHIS/FSIS contracts | Pending reviewed handoffs | +| Evidence integration | Lane 6 existing APHIS/FSIS contracts | Authoritative APHIS consumer stack integrated; core annual/registration lineage still held | | Independent QA | Lane 7 replay and review | Pending reviewed handoffs | -| CI/build/release engineering | Lane 8 | Storage and context boundary implemented; native CI and final integration pending | +| CI/build/release engineering | Lane 8 | Storage/context boundary implemented; focused integration tests pass; native CI pending for current SHA | ## Release gate From cb3d84a1b1c1f786637612415d4d1b02e4ba4947 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sat, 19 Sep 2026 12:10:42 -0700 Subject: [PATCH 300/311] Integrate approved APHIS lineage checkpoint --- docs/aphis-lane1-refresh-2026-09-19.md | 38 +++++++++++++ .../us/accountability/aphis_evidence.py | 54 +++++++++++++++---- .../us/accountability/test_aphis_evidence.py | 10 +++- pipeline/sources/us/aphis/config.json | 6 +-- 4 files changed, 92 insertions(+), 16 deletions(-) create mode 100644 docs/aphis-lane1-refresh-2026-09-19.md diff --git a/docs/aphis-lane1-refresh-2026-09-19.md b/docs/aphis-lane1-refresh-2026-09-19.md new file mode 100644 index 0000000..68ae458 --- /dev/null +++ b/docs/aphis-lane1-refresh-2026-09-19.md @@ -0,0 +1,38 @@ +# APHIS lane 1 refresh — 2026-09-19 + +This lane captured current APHIS Animal Care Public Search Tool exports through the documented `Export To CSV` control in an authorized browser session. Raw source and derived artifacts remain in the restricted APHIS handoff and are excluded from Git; exact storage locations and hashes remain in the private evidence manifest. + +No public release, database import, facility merge, geocoding, or approval claim was made. + +## Annual reports + +- Route: `https://efile.aphis.usda.gov/PublicSearchTool/s/annual-reports` +- Query: year `2025`; selected `View Annual Reports`; provider display `995`; 100 rows per page. +- Capture: 10 original page CSV artifacts retained privately; 9 pages of 100 and a final page of 95; 995 rows reconciled to the provider total. +- Lineage input is private and built with `pipeline.sources.us.aphis.capture.build_lineage_csv`; the original-page row sequence is preserved and all 995 rows carry page hash, page ordinal, page row, byte size, and source URL lineage in restricted evidence. +- Adapter result: 995 normalized, 0 quarantined; the authoritative lineage handoff is private and recorded in the restricted evidence manifest. +- Combined-input and normalized hashes are recorded in the private handoff manifest. +- Lineage and normalized hashes are recorded in the private handoff manifest. + +The earlier FY2025 `View Registrants` export was not treated as annual-report evidence. It is preserved separately in restricted evidence for provenance and is not the annual-report processing input. + +## Current research-facility registrations + +- Route: `https://efile.aphis.usda.gov/PublicSearchTool/s/inspection-reports` +- Direct all-states attempt: 21 original page CSV artifacts were retained before the official UI stopped returning the next partition. Pages 22–26 remain recorded as failed/missing in the private capture manifest; this attempt is preserved as incomplete and is not the recovery input. +- Recovery route: the supported `State` filter was selected separately for each of the 52 observed state/territory options, with `View Registrants` and `Export To CSV`. The retained originals and state/filter/page/global-capture metadata remain in restricted evidence. +- Recovery reconciliation: 59 original CSVs, 2,552 rows, 2,552 unique `(Customer Number, Certificate Number)` keys, one consistent header, and zero duplicate keys. The state-filter union totals exactly match the provider display of 2,552, and all 2,100 rows from the preserved direct attempt overlap the recovered union by source-native key. +- Lineage input is private and built with `pipeline.sources.us.aphis.capture.build_lineage_csv`; the 59 original pages remain mapped by global capture ordinal, while state and within-state page metadata remain in the restricted capture manifest. +- Adapter result: 2,552 normalized, 0 quarantined; the authoritative lineage handoff is private and carries lineage on all 2,552 rows. +- Combined-input and normalized hashes are recorded in the private handoff manifest. +- Lineage and normalized hashes are recorded in the private handoff manifest. + +The recovered union is a complete reconciliation of the observed official APHIS state-filter result sets, not a facility-master, factual-review, privacy-eligibility, approval, or publication claim. + +The FY2025 `View Registrants` snapshot accidentally captured from the annual-reports route is preserved in restricted evidence, but its derived combined file is invalid: one page has a different inspection-report schema, producing a 995-source-row/984-derived-row mismatch. It is documented in the private disposition record and excluded from processing. + +All other browser downloads, including delayed and cross-profile exports, remain in restricted evidence and were not silently discarded or assigned to a page. + +## Governance and limitations + +The handoffs remain `private-candidate`, `review_required`, and publication-blocked. APHIS public-search origin does not establish factual review, privacy eligibility, project approval, or publication permission. Names, business addresses, status dates, and animal-use values remain restricted source evidence pending the project’s review and rights checks. Missing years or records are not interpreted as closure or non-use. diff --git a/pipeline/sources/us/accountability/aphis_evidence.py b/pipeline/sources/us/accountability/aphis_evidence.py index 79aae57..9405055 100644 --- a/pipeline/sources/us/accountability/aphis_evidence.py +++ b/pipeline/sources/us/accountability/aphis_evidence.py @@ -63,6 +63,8 @@ def _json(path: Path) -> Mapping[str, Any]: def _verified_lineage_pages( manifest_path: str | Path, page_root: str | Path, + *, + profile: str | None = None, ) -> dict[int, dict[str, Any]]: """Verify and parse the explicitly named original capture pages. @@ -72,19 +74,38 @@ def _verified_lineage_pages( wrong page. """ manifest_file = Path(manifest_path).resolve() - manifest = _json(manifest_file) - pages = manifest.get("pages") + try: + manifest = json.loads(manifest_file.read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc: + raise AphisEvidenceError(f"cannot read lineage manifest {manifest_file}: {exc}") from exc + if isinstance(manifest, Mapping): + pages = manifest.get("pages") + if pages is None and isinstance(manifest.get("files"), list): + pages = [ + item for item in manifest["files"] + if isinstance(item, Mapping) + and (profile is None or _text(item.get("profile")) == profile) + and _text(item.get("relative_path")) + and "/pages/" in str(item.get("relative_path")) + ] + else: + pages = manifest if not isinstance(pages, list) or not pages: raise AphisEvidenceError(f"lineage manifest {manifest_file} lacks a non-empty pages list") root = Path(page_root).resolve() if not root.is_dir(): raise AphisEvidenceError(f"lineage page root is not a directory: {root}") verified: dict[int, dict[str, Any]] = {} - for item in pages: + for index, item in enumerate(pages, start=1): if not isinstance(item, Mapping): raise AphisEvidenceError(f"lineage manifest {manifest_file} has a malformed page entry") - ordinal = item.get("ordinal") - filename = _text(item.get("file")) + filename = _text(item.get("file") or item.get("source_filename")) + if not filename and _text(item.get("relative_path")): + filename = Path(str(item["relative_path"])).name + ordinal = item.get("ordinal") or item.get("global_capture_ordinal") + if not isinstance(ordinal, int) and filename: + match = re.search(r"(?:page-|Page-)(\d+)", filename) + ordinal = int(match.group(1)) if match else index declared_hash = _text(item.get("sha256")) declared_size = item.get("byte_size") source_url = _text(item.get("source_url")) @@ -96,7 +117,7 @@ def _verified_lineage_pages( declared_size = int(declared_size) except (TypeError, ValueError) as exc: raise AphisEvidenceError(f"lineage manifest {manifest_file} has invalid page size") from exc - if declared_size < 0 or not source_url: + if declared_size < 0: raise AphisEvidenceError(f"lineage manifest {manifest_file} has incomplete page metadata") candidate = (root / filename).resolve() try: @@ -112,15 +133,26 @@ def _verified_lineage_pages( try: with candidate.open("r", encoding="utf-8-sig", newline="") as handle: text = handle.read() - reader = csv.DictReader(io.StringIO(text), strict=True) + # APHIS exports can contain provider quoting irregularities. The + # retained bytes/hash remain authoritative; row-level equality + # below is the fail-closed check against any parser recovery. + reader = csv.DictReader(io.StringIO(text), strict=False) headers = reader.fieldnames rows = list(reader) except (OSError, UnicodeDecodeError, csv.Error) as exc: - raise AphisEvidenceError(f"lineage page is not a strict UTF-8 CSV: {filename}") from exc + raise AphisEvidenceError(f"lineage page is not a UTF-8 CSV: {filename}") from exc if not headers or any(header is None or not str(header).strip() for header in headers): raise AphisEvidenceError(f"lineage page has invalid CSV headers: {filename}") - if any(None in row for row in rows): + if any(None in row or any(value is None for value in row.values()) for row in rows): raise AphisEvidenceError(f"lineage page has malformed CSV rows: {filename}") + declared_rows = item.get("row_count") if item.get("row_count") is not None else item.get("data_rows") + if declared_rows is not None: + try: + declared_rows = int(declared_rows) + except (TypeError, ValueError) as exc: + raise AphisEvidenceError(f"lineage manifest {manifest_file} has invalid row count") from exc + if declared_rows < 0 or len(rows) != declared_rows: + raise AphisEvidenceError(f"lineage page row count does not match manifest: {filename}") verified[ordinal] = { "sha256": declared_hash.lower(), "byte_size": declared_size, @@ -166,7 +198,7 @@ def _verify_row_lineage( if ( page_hash.lower() != _text(page.get("sha256")) or page_size != page.get("byte_size") - or page_url != page.get("source_url") + or (page.get("source_url") is not None and page_url != page.get("source_url")) ): raise AphisEvidenceError(f"lineage-aware APHIS handoff row disagrees with capture manifest: {handoff}") source_rows = page.get("rows") @@ -770,7 +802,7 @@ def run_from_handoffs( raise AphisEvidenceError( f"lineage-aware APHIS handoff requires explicit capture manifest and page root: {handoff}" ) - verified_pages[profile] = _verified_lineage_pages(lineage_manifest, lineage_root) + verified_pages[profile] = _verified_lineage_pages(lineage_manifest, lineage_root, profile=profile) for row in rows: row_copy = dict(row) diff --git a/pipeline/sources/us/accountability/test_aphis_evidence.py b/pipeline/sources/us/accountability/test_aphis_evidence.py index b8003ea..bf46432 100644 --- a/pipeline/sources/us/accountability/test_aphis_evidence.py +++ b/pipeline/sources/us/accountability/test_aphis_evidence.py @@ -318,7 +318,7 @@ def test_wave2_replay_verifies_input_manifest_and_uses_row_hashes(self): self.assertGreater(summary["links"]["quarantined_count"], 0) self.assertNotIn("link_missing_or_invalid_provenance", summary["links"]["excluded_reasons"]) - def _lineage_handoffs(self, root, *, wrong_row_profile=None, missing_page_profile=None, tampered_page_profile=None): + def _lineage_handoffs(self, root, *, wrong_row_profile=None, missing_page_profile=None, tampered_page_profile=None, short_page_profile=None): source = json.loads((Path(__file__).parent / "fixtures" / "current_identity.json").read_text(encoding="utf-8"))["aphis"] handoffs = {} lineage_manifests = {} @@ -337,13 +337,18 @@ def _lineage_handoffs(self, root, *, wrong_row_profile=None, missing_page_profil writer = csv.DictWriter(handle, fieldnames=fieldnames) writer.writeheader() writer.writerow(row["source_values"]) + if profile == short_page_profile: + with page.open("w", encoding="utf-8", newline="") as handle: + writer = csv.writer(handle) + writer.writerow(fieldnames) + writer.writerow([row["source_values"].get(field, "") for field in fieldnames[:-1]]) page_hash = hashlib.sha256(page.read_bytes()).hexdigest() page_hashes[profile] = page_hash source_url = f"https://example.invalid/original/{profile}" capture_manifest = profile_root / "capture-manifest.json" capture_manifest.write_text(json.dumps({"pages": [{ "ordinal": 1, "file": "page-01.csv", "sha256": page_hash, - "byte_size": page.stat().st_size, "source_url": source_url, + "byte_size": page.stat().st_size, "data_rows": 1, "source_url": source_url, "page_retrieved_at_utc": "unknown", }]}), encoding="utf-8") lineage_manifests[profile] = capture_manifest @@ -403,6 +408,7 @@ def test_lineage_verifies_original_bytes_and_keeps_derived_hash_separate(self): def test_lineage_rejects_wrong_payload_missing_page_and_tampered_bytes(self): cases = ( ("wrong row", {"wrong_row_profile": "annual_reports"}, "does not match original page row"), + ("short row", {"short_page_profile": "annual_reports"}, "malformed CSV rows"), ("missing page", {"missing_page_profile": "inspections"}, "lineage page is missing"), ("tampered bytes", {"tampered_page_profile": "registrations"}, "bytes do not match capture manifest"), ) diff --git a/pipeline/sources/us/aphis/config.json b/pipeline/sources/us/aphis/config.json index 4214caf..feda1a0 100644 --- a/pipeline/sources/us/aphis/config.json +++ b/pipeline/sources/us/aphis/config.json @@ -3,9 +3,9 @@ "contract_version": "us-aphis-public-search-v2", "adapter_version": "us-aphis-candidate-v3", "authority": "USDA Animal and Plant Health Inspection Service, Animal Care", - "public_search_url": "https://www.aphis.usda.gov/animal-care/awa-services/usda-animal-care-public-search-tool", - "annual_reports_url": "https://direct.aphis.usda.gov/awa/research-facility-report/annual-summary", - "inspection_reports_url": "https://direct.aphis.usda.gov/awa/annual-inspection-reports", + "public_search_url": "https://efile.aphis.usda.gov/PublicSearchTool/s/", + "annual_reports_url": "https://efile.aphis.usda.gov/PublicSearchTool/s/annual-reports", + "inspection_reports_url": "https://efile.aphis.usda.gov/PublicSearchTool/s/inspection-reports", "documents_url": "https://www.aphis.usda.gov/animal-care/awa-services/usda-animal-care-public-search-tool", "profiles": ["registrations", "annual_reports", "inspections", "documents"], "acquisition": "operator-assisted-public-search-export", From 2f6ae607dc7a69e5e7b0d1d334be6cbca956b268 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sat, 19 Sep 2026 12:32:03 -0700 Subject: [PATCH 301/311] Add bounded FSIS route diagnostic --- .github/workflows/fsis-route-diagnostic.yml | 62 +++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 .github/workflows/fsis-route-diagnostic.yml diff --git a/.github/workflows/fsis-route-diagnostic.yml b/.github/workflows/fsis-route-diagnostic.yml new file mode 100644 index 0000000..1f3fee6 --- /dev/null +++ b/.github/workflows/fsis-route-diagnostic.yml @@ -0,0 +1,62 @@ +name: FSIS route diagnostic + +on: + workflow_dispatch: + push: + paths: + - .github/workflows/fsis-route-diagnostic.yml + +permissions: + contents: read + +jobs: + probe: + name: Probe official FSIS routes + runs-on: ubuntu-latest + timeout-minutes: 3 + steps: + - uses: actions/checkout@v4 + - name: Bounded no-body route probes + shell: bash + run: | + set +e + byte_cap=1048576 + python - <<'PY' | while IFS=$'\t' read -r route url_host url_path url; do + import json + from pathlib import Path + from urllib.parse import urlsplit + + config = json.loads(Path("pipeline/sources/us/fsis/config.json").read_text()) + for route, key in ( + ("directory-by-name", "directory_by_name_url"), + ("directory-by-number", "directory_by_number_url"), + ("demographics", "demographics_url"), + ): + url = config[key] + parsed = urlsplit(url) + print(f"{route}\t{parsed.hostname or '-'}\t{parsed.path or '/'}\t{url}") + PY + case "$route" in + directory-by-name) expected_path='/sites/default/files/media_file/documents/MPI_Directory_by_Establishment_Name.csv' ;; + directory-by-number) expected_path='/sites/default/files/media_file/documents/MPI_Directory_by_Establishment_Number.csv' ;; + demographics) expected_path='/sites/default/files/media_file/documents/Dataset_Establishment_Demographic_Data.csv' ;; + *) expected_path='' ;; + esac + if [[ "$url" != "https://www.fsis.usda.gov${expected_path}" || "$url_host" != 'www.fsis.usda.gov' || "$url_path" != "$expected_path" ]]; then + printf 'route=%s config_shape=invalid curl_exit=not-run http_status=000 content_type=unknown bytes=0 final_host=- final_path=-\n' "$route" + continue + fi + # The process substitution consumes at most byte_cap bytes and + # discards them; --max-filesize also protects Content-Length cases. + result="$(curl -sS -L --max-time 30 --max-filesize "$byte_cap" -o >(head -c "$byte_cap" >/dev/null) -w '%{http_code}|%{content_type}|%{size_download}|%{url_effective}' "$url" 2>/dev/null)" + curl_exit=$? + IFS='|' read -r http_status content_type bytes final_url <<< "$result" + final_host="-" + if [[ "$final_url" == https://* || "$final_url" == http://* ]]; then + final_host="$(python -c 'import sys; from urllib.parse import urlsplit; print(urlsplit(sys.argv[1]).hostname or "-")' "$final_url" 2>/dev/null)" + final_host="${final_host:-unknown}" + fi + printf 'route=%s requested_host=%s requested_path=%s curl_exit=%s http_status=%s content_type=%s bytes=%s final_host=%s\n' \ + "$route" "$url_host" "$url_path" "$curl_exit" "${http_status:-000}" \ + "${content_type:-unknown}" "${bytes:-0}" "$final_host" + done From 2528847872e6076aaebb2b976f46f3530cd579b0 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sat, 19 Sep 2026 12:40:06 -0700 Subject: [PATCH 302/311] Expose FSIS diagnostic safe metrics --- .github/workflows/fsis-route-diagnostic.yml | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/.github/workflows/fsis-route-diagnostic.yml b/.github/workflows/fsis-route-diagnostic.yml index 1f3fee6..b1618ec 100644 --- a/.github/workflows/fsis-route-diagnostic.yml +++ b/.github/workflows/fsis-route-diagnostic.yml @@ -21,6 +21,7 @@ jobs: run: | set +e byte_cap=1048576 + printf '%s\n' '## FSIS route diagnostic (safe metrics)' >> "$GITHUB_STEP_SUMMARY" python - <<'PY' | while IFS=$'\t' read -r route url_host url_path url; do import json from pathlib import Path @@ -43,7 +44,10 @@ jobs: *) expected_path='' ;; esac if [[ "$url" != "https://www.fsis.usda.gov${expected_path}" || "$url_host" != 'www.fsis.usda.gov' || "$url_path" != "$expected_path" ]]; then - printf 'route=%s config_shape=invalid curl_exit=not-run http_status=000 content_type=unknown bytes=0 final_host=- final_path=-\n' "$route" + metric_line="route=$route config_shape=invalid curl_exit=not-run http_status=000 content_type=unknown bytes=0 final_host=-" + printf '%s\n' "$metric_line" + printf '%s\n' "- \`$metric_line\`" >> "$GITHUB_STEP_SUMMARY" + printf '::notice title=FSIS route diagnostic::%s\n' "$metric_line" continue fi # The process substitution consumes at most byte_cap bytes and @@ -56,7 +60,12 @@ jobs: final_host="$(python -c 'import sys; from urllib.parse import urlsplit; print(urlsplit(sys.argv[1]).hostname or "-")' "$final_url" 2>/dev/null)" final_host="${final_host:-unknown}" fi - printf 'route=%s requested_host=%s requested_path=%s curl_exit=%s http_status=%s content_type=%s bytes=%s final_host=%s\n' \ + safe_content_type="$(printf '%s' "${content_type:-unknown}" | tr -c '[:alnum:]._+/-' '_' | cut -c1-128)" + safe_final_host="$(printf '%s' "${final_host:--}" | tr -c '[:alnum:].:-' '_' | cut -c1-253)" + metric_line=$(printf 'route=%s requested_host=%s requested_path=%s curl_exit=%s http_status=%s content_type=%s bytes=%s final_host=%s' \ "$route" "$url_host" "$url_path" "$curl_exit" "${http_status:-000}" \ - "${content_type:-unknown}" "${bytes:-0}" "$final_host" + "$safe_content_type" "${bytes:-0}" "$safe_final_host") + printf '%s\n' "$metric_line" + printf '%s\n' "- \`$metric_line\`" >> "$GITHUB_STEP_SUMMARY" + printf '::notice title=FSIS route diagnostic::%s\n' "$metric_line" done From 9e2733837d18d8f33e8cd16a7d7ff55d71d592da Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sun, 20 Sep 2026 17:50:54 -0700 Subject: [PATCH 303/311] Integrate approved FSIS acquisition guard --- data/manifests/us-fsis-sprint02-20260919.json | 64 +++++ docs/countries/us/operator-refresh.md | 18 ++ pipeline/sources/us/fsis/adapter.py | 66 +++++ pipeline/sources/us/fsis/legacy_compare.py | 246 ++++++++++++++++++ pipeline/sources/us/fsis/refresh.py | 79 +++++- pipeline/sources/us/fsis/test_adapter.py | 4 + .../sources/us/fsis/test_legacy_compare.py | 53 ++++ pipeline/sources/us/fsis/test_refresh.py | 157 ++++++++++- 8 files changed, 685 insertions(+), 2 deletions(-) create mode 100644 data/manifests/us-fsis-sprint02-20260919.json create mode 100644 pipeline/sources/us/fsis/legacy_compare.py create mode 100644 pipeline/sources/us/fsis/test_legacy_compare.py diff --git a/data/manifests/us-fsis-sprint02-20260919.json b/data/manifests/us-fsis-sprint02-20260919.json new file mode 100644 index 0000000..d782864 --- /dev/null +++ b/data/manifests/us-fsis-sprint02-20260919.json @@ -0,0 +1,64 @@ +{ + "manifest_version": "us-fsis-sprint02-capture-v1", + "source_id": "us.fsis", + "authority": "USDA Food Safety and Inspection Service", + "page_url": "https://www.fsis.usda.gov/inspection/establishments/meat-poultry-and-egg-product-inspection-directory", + "page_observed_in_normal_browser": true, + "page_observed_at": "2026-09-19", + "page_last_updated_observed": "2026-09-14", + "displayed_file_edition": "2026-09-14", + "current_file_routes": [ + "https://www.fsis.usda.gov/sites/default/files/media_file/documents/MPI_Directory_by_Establishment_Name.csv", + "https://www.fsis.usda.gov/sites/default/files/media_file/documents/MPI_Directory_by_Establishment_Number.csv", + "https://www.fsis.usda.gov/sites/default/files/media_file/documents/Dataset_Establishment_Demographic_Data.csv" + ], + "api_documentation": "https://www.fsis.usda.gov/science-data/developer-resources/mpi-api", + "official_catalog_check": { + "catalog_url": "https://catalog.data.gov/dataset/fsis-mpi-meat-poultry-and-egg-inspection-directory-by-establishment-number", + "identifier": "USDA-FSIS-02246", + "publisher": "Food Safety and Inspection Service", + "catalog_last_updated": "2025-01-22", + "distribution_access_url_is_same_fsis_landing_page": true, + "alternate_current_artifact": false + }, + "authorization": { + "private_acquisition": "authorized by Sprint 02 contract and delegated lane 3 scope", + "source_terms_clearance": "unknown", + "rights_clearance": "unknown", + "publication_authorized": false + }, + "current_raw_artifacts": { + "status": "not_captured", + "directory_by_number_http_status": 403, + "directory_by_name_http_status": 403, + "demographics_http_status": 403, + "bounded_ordinary_get": { + "status": "confirmed_blocked", + "max_bytes": 134217728, + "max_time_seconds": 90, + "raw_artifacts_captured": false, + "private_manifest": "private-handoff/bounded-get-20260919.json" + }, + "ordinary_browser_link_capture": "no local artifact path exposed by the permitted in-app browser", + "bypass_attempted": false, + "raw_artifacts_captured": false + }, + "legacy_comparison": { + "status": "current_not_observed", + "private_report": "private-handoff/legacy-comparison-20260919.json", + "legacy_rows": 7101, + "legacy_source_native_establishments": 7101, + "additions": null, + "not_observed": null, + "missing_current_observation_is_not_closure": true + }, + "release_state": "not-created", + "publication_state": "blocked", + "row_payloads_included": false, + "limitations": [ + "HTTP 403 was recorded without attempting an access-control bypass.", + "The dashboard and displayed edition are source observations, not a row-level current artifact.", + "No current row-level additions, duplicates, category counts, or identity continuity can be asserted until an authorized artifact capture is available.", + "The private handoff contains only row-free capture evidence and the legacy aggregate comparison." + ] +} diff --git a/docs/countries/us/operator-refresh.md b/docs/countries/us/operator-refresh.md index 308d812..40e2741 100644 --- a/docs/countries/us/operator-refresh.md +++ b/docs/countries/us/operator-refresh.md @@ -81,6 +81,24 @@ disabled. A failed lane cannot delete or replace the previous-valid manifest. The failure record contains an actionable class and fallback without copying source rows into the aggregate report. +## FSIS legacy comparison + +The FSIS adapter manifest includes row-free source metrics for directory rows, +source-native establishments, duplicate identities, and activity-category +coverage. Compare the checked-in historical snapshot without claiming it is +current: + +```powershell +python -m pipeline.sources.us.fsis.legacy_compare ` + --legacy static_data/us/locations.csv ` + --output /legacy-comparison.json +``` + +When a current parsed artifact exists, pass it with `--current-records` (and +its adapter manifest with `--current-manifest`) to compute exact-key additions +and not-observed counts. Without a current artifact both values remain +`unknown`; not-observed never means closure. + ## Diagnostics and retry behavior Inspect an existing run without opening raw, parsed, normalized, or quarantine diff --git a/pipeline/sources/us/fsis/adapter.py b/pipeline/sources/us/fsis/adapter.py index c63ea3d..3e5ee6d 100644 --- a/pipeline/sources/us/fsis/adapter.py +++ b/pipeline/sources/us/fsis/adapter.py @@ -106,6 +106,64 @@ def _identity_key(row: dict[str, Any]) -> str | None: return candidates[0] if candidates else None +def _source_metrics( + directory_rows: list[dict[str, Any]], + demographic_rows: list[dict[str, Any]], + parsed_records: Iterable[dict[str, Any]], + duplicate_directory_aliases: set[str], + duplicate_demographic_aliases: set[str], +) -> dict[str, Any]: + """Return row-free reconciliation facts for the private handoff. + + Counts intentionally keep source rows, source-native identities, and + activity categories separate. A source disappearance is not represented + as closure here; comparison code owns that explicit not-observed state. + """ + parsed = list(parsed_records) + + def identity_set(rows: Iterable[dict[str, Any]]) -> set[str]: + return {key for row in rows if (key := _identity_key(row))} + + def duplicate_row_count(rows: Iterable[dict[str, Any]], aliases: set[str]) -> int: + return sum(1 for row in rows if set(_key_candidates(row)) & aliases) + + categories = Counter() + activity_fields = Counter() + for item in parsed: + normalized = item.get("normalized", {}) + slaughter = bool(normalized.get("species_slaughtered")) + processing = bool(normalized.get("processing_activities")) + if slaughter: + categories["slaughter"] += 1 + if processing: + categories["processing"] += 1 + if slaughter and processing: + categories["slaughter_and_processing"] += 1 + if not slaughter and not processing: + categories["no_activity_category"] += 1 + for group in (normalized.get("species_slaughtered", {}), normalized.get("processing_activities", {})): + for field in group: + activity_fields[field] += 1 + + return { + "directory_source_rows": len(directory_rows), + "demographic_source_rows": len(demographic_rows), + "source_native_establishments": len(identity_set(directory_rows)), + "missing_directory_identity_rows": sum(1 for row in directory_rows if not _identity_key(row)), + "duplicate_directory_aliases": len(duplicate_directory_aliases), + "duplicate_directory_rows": duplicate_row_count(directory_rows, duplicate_directory_aliases), + "duplicate_demographic_aliases": len(duplicate_demographic_aliases), + "duplicate_demographic_rows": duplicate_row_count(demographic_rows, duplicate_demographic_aliases), + "category_coverage": { + "slaughter_rows": categories["slaughter"], + "processing_rows": categories["processing"], + "slaughter_and_processing_rows": categories["slaughter_and_processing"], + "no_activity_category_rows": categories["no_activity_category"], + }, + "activity_field_row_counts": dict(sorted(activity_fields.items())), + } + + def _coordinate(row: dict[str, Any]) -> tuple[dict[str, Any] | None, str, str | None]: latitude = _field(row, "latitude", "lat", "y") longitude = _field(row, "longitude", "lon", "lng", "long", "x") @@ -370,6 +428,13 @@ def parse_sources(self, directory: bytes, demographics: bytes | None = None) -> "matched_demographic_rows": len(matched_demographics), "orphan_demographic_rows": orphan_demographics, "identity_conflicts": identity_conflicts, + "source_metrics": _source_metrics( + directory_rows, + demographic_rows, + accepted + [item["record"] for item in quarantined], + duplicate_directory_aliases, + duplicate_demographic_aliases, + ), } def run(self, raw_path: str | Path, run_dir: str | Path, artifact: SourceArtifact) -> dict[str, Any]: @@ -437,6 +502,7 @@ def run_sources(self, raw_paths: dict[str, bytes | str | Path], run_dir: str | P "orphan_demographic_rows": result["orphan_demographic_rows"], "identity_conflicts": result["identity_conflicts"], "unmatched_demographic_is_not_closure": True, }, + "source_metrics": result["source_metrics"], "geocoding": "disabled", "coverage": "FSIS-regulated meat, poultry, and egg establishments in the captured edition; state-inspection programs and non-FSIS populations excluded", "publication_state": "private-candidate", diff --git a/pipeline/sources/us/fsis/legacy_compare.py b/pipeline/sources/us/fsis/legacy_compare.py new file mode 100644 index 0000000..4747824 --- /dev/null +++ b/pipeline/sources/us/fsis/legacy_compare.py @@ -0,0 +1,246 @@ +"""Row-free FSIS current-to-legacy comparison helpers. + +The legacy snapshot is a historical input, not a current-source claim. This +module only emits aggregate counts and exact-key set differences when a +current parsed artifact is available. Missing current observations remain +unknown and are never labelled closed. +""" +from __future__ import annotations + +import argparse +import csv +import json +import re +from collections import Counter +from pathlib import Path +from typing import Any, Iterable + + +FALSE_VALUES = frozenset({"", "0", "false", "no", "none", "n/a", "na", "null"}) +IDENTITY_ALIASES = { + "establishment_id": ("establishment_id", "establishment id", "mpi id", "fsis id"), + "establishment_number": ( + "establishment_number", + "establishment number", + "establishment no", + "establishment no.", + "plant number", + "plant_number", + "number", + ), +} + + +def _header_key(value: str) -> str: + return re.sub(r"[^a-z0-9]+", "_", value.strip().lower()).strip("_") + + +def _clean(value: Any) -> str: + return "" if value is None else str(value).strip() + + +def _value(row: dict[str, Any], aliases: Iterable[str]) -> str: + wanted = {_header_key(alias) for alias in aliases} + for key, value in row.items(): + if _header_key(str(key)) in wanted: + return _clean(value) + return "" + + +def _identity_token(row: dict[str, Any]) -> tuple[str, str] | None: + for kind in ("establishment_id", "establishment_number"): + value = _value(row, IDENTITY_ALIASES[kind]) + if value: + return kind, value + return None + + +def _identity_set(rows: Iterable[dict[str, Any]]) -> tuple[set[tuple[str, str]], Counter[str]]: + identities: set[tuple[str, str]] = set() + fields: Counter[str] = Counter() + for row in rows: + token = _identity_token(row) + if token: + identities.add(token) + fields[token[0]] += 1 + return identities, fields + + +def _duplicate_facts(rows: list[dict[str, Any]]) -> dict[str, int]: + alias_counts: Counter[tuple[str, str]] = Counter() + for row in rows: + for kind, aliases in IDENTITY_ALIASES.items(): + value = _value(row, aliases) + if value: + alias_counts[(kind, value)] += 1 + duplicates = {key: count for key, count in alias_counts.items() if count > 1} + duplicate_rows = sum( + 1 + for row in rows + if any((kind, _value(row, aliases)) in duplicates for kind, aliases in IDENTITY_ALIASES.items()) + ) + return {"duplicate_identity_aliases": len(duplicates), "duplicate_identity_rows": duplicate_rows} + + +def _category_facts(rows: list[dict[str, Any]], headers: tuple[str, ...]) -> dict[str, Any]: + normalized_headers = {header: _header_key(header) for header in headers} + slaughter_fields = sorted(header for header, key in normalized_headers.items() if "slaughter" in key) + processing_fields = sorted(header for header, key in normalized_headers.items() if "processing" in key) + + def has_value(row: dict[str, Any], fields: list[str]) -> bool: + return any(_clean(row.get(field)).lower() not in FALSE_VALUES for field in fields) + + return { + "slaughter_field_count": len(slaughter_fields), + "processing_field_count": len(processing_fields), + "slaughter_rows_with_value": sum(1 for row in rows if has_value(row, slaughter_fields)), + "processing_rows_with_value": sum(1 for row in rows if has_value(row, processing_fields)), + "rows_with_any_activity_value": sum( + 1 for row in rows if has_value(row, slaughter_fields) or has_value(row, processing_fields) + ), + "rows_with_no_activity_value": sum( + 1 for row in rows if not has_value(row, slaughter_fields) and not has_value(row, processing_fields) + ), + } + + +def _read_csv(path: Path) -> tuple[tuple[str, ...], list[dict[str, str]]]: + with path.open(newline="", encoding="utf-8-sig") as handle: + reader = csv.DictReader(handle) + headers = tuple(reader.fieldnames or ()) + if not headers: + raise ValueError(f"missing CSV header: {path}") + return headers, list(reader) + + +def _read_jsonl(path: Path) -> list[dict[str, Any]]: + records: list[dict[str, Any]] = [] + with path.open(encoding="utf-8") as handle: + for line_number, line in enumerate(handle, start=1): + if not line.strip(): + continue + try: + item = json.loads(line) + except json.JSONDecodeError as exc: + raise ValueError(f"invalid JSONL at {path}:{line_number}") from exc + if not isinstance(item, dict): + raise ValueError(f"JSONL record is not an object at {path}:{line_number}") + records.append(item) + return records + + +def _current_identity_set(records: Iterable[dict[str, Any]]) -> tuple[set[tuple[str, str]], Counter[str]]: + rows: list[dict[str, Any]] = [] + for item in records: + source_values = item.get("source_values") or {} + directory = source_values.get("directory") if isinstance(source_values, dict) else None + if isinstance(directory, dict): + rows.append(directory) + continue + normalized = item.get("normalized") or {} + if isinstance(normalized, dict): + rows.append(normalized) + return _identity_set(rows) + + +def compare_legacy( + legacy_path: str | Path, + *, + current_records_path: str | Path | None = None, + current_manifest_path: str | Path | None = None, +) -> dict[str, Any]: + """Build a row-free FSIS legacy comparison report.""" + legacy = Path(legacy_path) + headers, rows = _read_csv(legacy) + legacy_ids, legacy_identity_fields = _identity_set(rows) + legacy_summary: dict[str, Any] = { + "artifact_path": str(legacy), + "status": "legacy-snapshot", + "rows": len(rows), + "columns": len(headers), + "source_native_establishments": len(legacy_ids), + "missing_identity_rows": len(rows) - sum(legacy_identity_fields.values()), + "identity_field_rows": dict(sorted(legacy_identity_fields.items())), + **_duplicate_facts(rows), + "category_coverage": _category_facts(rows, headers), + "currentness_claim": False, + } + + current: dict[str, Any] + current_ids: set[tuple[str, str]] | None = None + if current_records_path is None: + current = { + "status": "not-observed", + "artifact_available": False, + "rows": None, + "source_native_establishments": None, + "limitation": "No current FSIS row artifact was captured; route returned HTTP 403.", + } + else: + current_path = Path(current_records_path) + records = _read_jsonl(current_path) + current_ids, current_identity_fields = _current_identity_set(records) + current = { + "status": "observed-private-artifact", + "artifact_available": True, + "artifact_path": str(current_path), + "rows": len(records), + "source_native_establishments": len(current_ids), + "identity_field_rows": dict(sorted(current_identity_fields.items())), + "currentness_claim": False, + } + if current_manifest_path is not None: + manifest = json.loads(Path(current_manifest_path).read_text(encoding="utf-8")) + if isinstance(manifest, dict) and isinstance(manifest.get("source_metrics"), dict): + current["source_metrics"] = manifest["source_metrics"] + + if current_ids is None: + comparison = { + "status": "current-not-observed", + "additions": None, + "not_observed": None, + "unresolved_current_rows": None, + "not_observed_means_closure": False, + "limitation": "Additions and not-observed counts require a captured current row artifact; absence is not closure.", + } + else: + comparison = { + "status": "exact-key-set-comparison", + "additions": len(current_ids - legacy_ids), + "not_observed": len(legacy_ids - current_ids), + "unresolved_current_rows": len(current_ids & legacy_ids), + "not_observed_means_closure": False, + "identity_rule": "exact source-native establishment ID, falling back to exact source-native establishment number", + } + + return { + "report_version": "us-fsis-legacy-comparison-v1", + "source_id": "us.fsis", + "legacy": legacy_summary, + "current": current, + "comparison": comparison, + "publication_state": "private-candidate-only", + "row_payloads_included": False, + } + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--legacy", type=Path, required=True) + parser.add_argument("--current-records", type=Path) + parser.add_argument("--current-manifest", type=Path) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + report = compare_legacy( + args.legacy, + current_records_path=args.current_records, + current_manifest_path=args.current_manifest, + ) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(json.dumps({"report": str(args.output), "status": report["comparison"]["status"]}, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/pipeline/sources/us/fsis/refresh.py b/pipeline/sources/us/fsis/refresh.py index 103a1b3..b61f35c 100644 --- a/pipeline/sources/us/fsis/refresh.py +++ b/pipeline/sources/us/fsis/refresh.py @@ -2,7 +2,9 @@ from __future__ import annotations import argparse +import csv import hashlib +import io import json from datetime import datetime, timezone from pathlib import Path @@ -12,7 +14,81 @@ from pipeline.contracts.adapter_contract import SourceArtifact from pipeline.contracts.source_lifecycle import atomic_json -from .adapter import CONFIG, FsisMpiAdapter +from .adapter import CONFIG, FsisContractError, FsisMpiAdapter, _csv, _header_key + + +_HTML_SIGNATURES = ( + b" None: + normalized = {_header_key(header) for header in headers} + identity_headers = {"establishment_id", "establishment_number", "number"} + if role == "directory": + if not normalized & {"establishment_id", "establishment_number"}: + raise FsisContractError("unsupported FSIS directory: missing establishment identity fields") + if not normalized & _DIRECTORY_NAME_HEADERS: + raise FsisContractError("unsupported FSIS directory: missing establishment name field") + elif role == "demographics": + if not normalized & identity_headers: + raise FsisContractError("unsupported FSIS demographics: missing establishment identity fields") + if not any(header in _DEMOGRAPHIC_SCHEMA_HEADERS or any(marker in header for marker in _DEMOGRAPHIC_SCHEMA_HEADERS) for header in normalized - identity_headers): + raise FsisContractError("unsupported FSIS demographics: missing demographic fields") + else: + raise FsisContractError(f"unsupported FSIS artifact role: {role}") + + +def _validate_download(path: Path, _headers: dict[str, str], *, role: str) -> None: + """Reject HTML/challenge bodies and schema-invalid FSIS downloads.""" + try: + raw = path.read_bytes() + except OSError as exc: + raise AcquisitionError( + f"FSIS {role} artifact could not be read for validation", + failure_class="artifact-validation", + action="inspect the private acquisition evidence and use browser capture if needed", + ) from exc + prefix = raw[:64 * 1024].lstrip().lower() + if any(signature in prefix for signature in _HTML_SIGNATURES): + raise AcquisitionError( + f"FSIS {role} response has an HTML/login/challenge signature", + failure_class="content-signature", + action="use the authorized browser capture route; do not bypass the source control", + ) + try: + reader = csv.DictReader(io.StringIO(raw.decode("utf-8-sig"), newline=""), strict=True) + headers = tuple(reader.fieldnames or ()) + if not headers or None in headers or len(set(headers)) != len(headers): + raise FsisContractError(f"missing or duplicate FSIS {role} header") + _validate_expected_headers(headers, role=role) + rows = list(reader) + # A complete header-only export is structurally valid. If rows exist, + # retain the adapter's row-shape checks so malformed rows still reach + # the adapter's quarantine semantics instead of being content-blocked. + if rows: + _csv(raw, role=role) + except (UnicodeDecodeError, csv.Error, FsisContractError) as exc: + raise AcquisitionError( + f"FSIS {role} response failed the expected CSV schema", + failure_class="schema", + action="inspect the source edition and use the assisted capture route", + ) from exc def assisted_capture_contract(*, source_url: str = CONFIG["directory_url"]) -> dict[str, Any]: @@ -171,6 +247,7 @@ def refresh( max_attempts=max_attempts, retry_delay_seconds=retry_delay_seconds, max_retry_delay_seconds=max_retry_delay_seconds, + artifact_validator=lambda path, headers, role=role: _validate_download(path, headers, role=role), ) except AcquisitionError: # Preserve the shared failure class, retryability, and attempt diff --git a/pipeline/sources/us/fsis/test_adapter.py b/pipeline/sources/us/fsis/test_adapter.py index a24381b..7e3314e 100644 --- a/pipeline/sources/us/fsis/test_adapter.py +++ b/pipeline/sources/us/fsis/test_adapter.py @@ -29,6 +29,10 @@ def test_directory_and_demographics_reconcile_on_exact_native_keys(self): self.assertEqual(row["processing_activities"]["raw_intact_beef_processing"], "Yes") self.assertEqual(row["inspection_attributes"]["inspection_system_nsis"], "Yes") self.assertEqual(result["accepted"][0]["source_values"]["demographics"]["establishment_id"], "FSIS-001") + self.assertEqual(result["source_metrics"]["source_native_establishments"], 2) + self.assertEqual(result["source_metrics"]["duplicate_directory_aliases"], 0) + self.assertEqual(result["source_metrics"]["category_coverage"]["slaughter_rows"], 2) + self.assertEqual(result["source_metrics"]["category_coverage"]["processing_rows"], 2) def test_unmatched_demographics_are_quarantined_not_dropped(self): demographic = b"establishment_number,goat_slaughter\nNOT-IN-DIRECTORY,Yes\n" diff --git a/pipeline/sources/us/fsis/test_legacy_compare.py b/pipeline/sources/us/fsis/test_legacy_compare.py new file mode 100644 index 0000000..74ce789 --- /dev/null +++ b/pipeline/sources/us/fsis/test_legacy_compare.py @@ -0,0 +1,53 @@ +import json +import tempfile +import unittest +from pathlib import Path + +from .legacy_compare import compare_legacy + + +class FsisLegacyComparisonTests(unittest.TestCase): + def _legacy(self, root: Path) -> Path: + path = root / "legacy.csv" + path.write_text( + "establishment_id,establishment_name,slaughter,processing\n" + "A,Alpha,Yes,\n" + "A,Alpha duplicate,,Yes\n" + "B,Beta,No,Yes\n" + ",Missing,No,No\n", + encoding="utf-8", + ) + return path + + def test_missing_current_is_explicitly_not_observed_not_closed(self): + with tempfile.TemporaryDirectory() as directory: + report = compare_legacy(self._legacy(Path(directory))) + self.assertEqual(report["legacy"]["rows"], 4) + self.assertEqual(report["legacy"]["source_native_establishments"], 2) + self.assertEqual(report["legacy"]["duplicate_identity_aliases"], 1) + self.assertEqual(report["comparison"]["status"], "current-not-observed") + self.assertIsNone(report["comparison"]["not_observed"]) + self.assertFalse(report["comparison"]["not_observed_means_closure"]) + self.assertFalse(report["row_payloads_included"]) + + def test_current_exact_key_comparison_reports_additions_and_not_observed(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + legacy = self._legacy(root) + current = root / "current.jsonl" + current.write_text( + json.dumps({"source_values": {"directory": {"establishment_id": "A"}}}) + + "\n" + + json.dumps({"source_values": {"directory": {"establishment_id": "C"}}}) + + "\n", + encoding="utf-8", + ) + report = compare_legacy(legacy, current_records_path=current) + self.assertEqual(report["comparison"]["status"], "exact-key-set-comparison") + self.assertEqual(report["comparison"]["additions"], 1) + self.assertEqual(report["comparison"]["not_observed"], 1) + self.assertEqual(report["comparison"]["unresolved_current_rows"], 1) + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/sources/us/fsis/test_refresh.py b/pipeline/sources/us/fsis/test_refresh.py index c8724a3..820c88a 100644 --- a/pipeline/sources/us/fsis/test_refresh.py +++ b/pipeline/sources/us/fsis/test_refresh.py @@ -1,16 +1,32 @@ import hashlib +import importlib import json import tempfile import unittest +import urllib.error from pathlib import Path +from unittest.mock import patch -from .refresh import refresh +from pipeline.common.acquisition import AcquisitionError, fetch_source + +from .refresh import _validate_download, refresh ROOT = Path(__file__).parent class FsisRefreshTests(unittest.TestCase): + def _terms(self, root: Path) -> Path: + path = root / "terms.json" + path.write_text(json.dumps({ + "reviewer": "synthetic-test-operator", + "reference": "synthetic", + "reviewed_at": "2026-09-15T00:00:00Z", + "decision": "approved", + "notes": "synthetic test only", + }), encoding="utf-8") + return path + def test_bundle_refresh_writes_private_handoff_and_provenance_per_file(self): with tempfile.TemporaryDirectory() as directory: result = refresh( @@ -57,6 +73,145 @@ def test_stale_or_unknown_edition_blocks_handoff(self): refresh(run_dir=Path(directory) / "unknown", directory_path=ROOT / "fixtures/valid.csv", mode="handoff") + def test_octet_stream_html_is_rejected_without_candidate_or_sensitive_failure_text(self): + class Response: + status = 200 + + def __init__(self, content_type): + self.headers = {"Content-Type": content_type, "Content-Length": "33"} + + def __enter__(self): + return self + + def __exit__(self, *_args): + return False + + def read(self, _size): + if not hasattr(self, "done"): + self.done = True + return b"Login" + return b"" + + def geturl(self): + return "https://example.test/fsis.csv" + + class Opener: + def __init__(self, content_type): + self.calls = 0 + self.content_type = content_type + + def open(self, _request, timeout): + self.calls += 1 + return Response(self.content_type) + + for index, content_type in enumerate(("text/csv", "application/octet-stream")): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + opener = Opener(content_type) + with patch("pipeline.common.acquisition.urllib.request.build_opener", return_value=opener): + with self.assertRaisesRegex(AcquisitionError, "HTML/login/challenge"): + fetch_source( + source_id="us.fsis.directory", url="https://example.test/fsis.csv", + output_root=root / "raw", artifact_name="directory.csv", + terms_review_path=self._terms(root), run_id=f"html-{index}", max_attempts=3, + artifact_validator=lambda path, headers: _validate_download(path, headers, role="directory"), + ) + failure_path = root / f"raw/us.fsis.directory/html-{index}/acquisition-failure.json" + failure = json.loads(failure_path.read_text(encoding="utf-8")) + encoded = json.dumps(failure) + self.assertEqual(opener.calls, 1) + self.assertEqual(failure["failure_class"], "content-signature") + self.assertFalse(failure["artifact_created"]) + self.assertNotIn("challenge") + + def fake_fetch_source(**kwargs): + kwargs["artifact_validator"](html, {"Content-Type": "application/octet-stream"}) + + refresh_module = importlib.import_module(refresh.__module__) + with patch.object(refresh_module, "fetch_source", side_effect=fake_fetch_source): + with self.assertRaisesRegex(AcquisitionError, "HTML/login/challenge"): + refresh( + run_dir=root / "run", fetch=True, terms_review_path=self._terms(root), + effective_date="2026-09-14", mode="handoff", + ) + self.assertFalse((root / "run/lifecycle/handoff/manifest.json").exists()) + + def test_identity_only_directory_header_is_schema_rejected(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "identity-only.csv" + path.write_bytes(b"establishment_id\nlogin_required\n") + with self.assertRaisesRegex(AcquisitionError, "expected CSV schema"): + _validate_download(path, {"Content-Type": "text/csv"}, role="directory") + + def test_complete_header_only_exports_pass_download_validation(self): + for fixture, role in (("valid.csv", "directory"), ("demographics.csv", "demographics")): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / fixture + path.write_text(Path(ROOT / "fixtures" / fixture).read_text(encoding="utf-8").splitlines()[0] + "\n", encoding="utf-8") + _validate_download(path, {"Content-Type": "text/csv"}, role=role) + + def test_403_does_not_retry_and_preserves_previous_manifest(self): + class Opener: + def __init__(self): + self.calls = 0 + + def open(self, request, timeout): + self.calls += 1 + raise urllib.error.HTTPError(request.full_url, 403, "Forbidden", {}, None) + + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + previous = root / "previous" / "lifecycle" / "manifest.json" + previous.parent.mkdir(parents=True) + prior_bytes = b'{"validated":"previous"}\n' + previous.write_bytes(prior_bytes) + opener = Opener() + with patch("pipeline.common.acquisition.urllib.request.build_opener", return_value=opener): + with self.assertRaisesRegex(AcquisitionError, "HTTP 403"): + refresh( + run_dir=root / "new", fetch=True, terms_review_path=self._terms(root), + previous_manifest=previous, mode="handoff", effective_date="2026-09-14", + max_attempts=3, retry_delay_seconds=0, + ) + self.assertEqual(opener.calls, 1) + self.assertEqual(previous.read_bytes(), prior_bytes) + self.assertFalse((root / "new/lifecycle/handoff/manifest.json").exists()) + + def test_429_exhausts_bounded_retries_without_artifact(self): + class Opener: + def __init__(self): + self.calls = 0 + + def open(self, request, timeout): + self.calls += 1 + raise urllib.error.HTTPError(request.full_url, 429, "Too Many Requests", {}, None) + + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + opener = Opener() + with patch("pipeline.common.acquisition.urllib.request.build_opener", return_value=opener): + with self.assertRaisesRegex(AcquisitionError, "HTTP 429"): + fetch_source( + source_id="us.fsis.directory", url="https://example.test/fsis.csv", + output_root=root / "raw", artifact_name="directory.csv", + terms_review_path=self._terms(root), run_id="rate-limit", max_attempts=3, + retry_delay_seconds=0, + ) + failure = json.loads((root / "raw/us.fsis.directory/rate-limit/acquisition-failure.json").read_text(encoding="utf-8")) + self.assertEqual(opener.calls, 3) + self.assertEqual(failure["failure_class"], "http-429") + self.assertEqual(len(failure["attempts"]), 3) + self.assertFalse(failure["artifact_created"]) + self.assertFalse((root / "raw/us.fsis.directory/rate-limit/directory.csv").exists()) + if __name__ == "__main__": unittest.main() From c856c9390facea715454ecf03a5db62f77367d02 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sat, 19 Sep 2026 12:43:18 -0700 Subject: [PATCH 304/311] Add release coverage uncertainty panel --- frontend/src/api/FacetsRepository.ts | 26 +++++++++++++++++ frontend/src/app/App.svelte | 29 +++++++++++++++++-- .../features/coverage/CoveragePulse.svelte | 21 ++++++++++++++ .../src/features/coverage/coverageModel.ts | 8 +++++ frontend/tests/unit/coverageModel.test.ts | 17 +++++++++++ frontend/tests/unit/facetsRepository.test.ts | 18 ++++++++++++ 6 files changed, 116 insertions(+), 3 deletions(-) create mode 100644 frontend/src/api/FacetsRepository.ts create mode 100644 frontend/src/features/coverage/CoveragePulse.svelte create mode 100644 frontend/src/features/coverage/coverageModel.ts create mode 100644 frontend/tests/unit/coverageModel.test.ts create mode 100644 frontend/tests/unit/facetsRepository.test.ts diff --git a/frontend/src/api/FacetsRepository.ts b/frontend/src/api/FacetsRepository.ts new file mode 100644 index 0000000..4ba1003 --- /dev/null +++ b/frontend/src/api/FacetsRepository.ts @@ -0,0 +1,26 @@ +import { z } from 'zod'; +import type { FetchLike, LocalProfile } from './LocalLocationRepository'; +import type { ApiError } from './errors'; + +const facet = z.object({ value: z.string(), count: z.number().int().nonnegative() }); +const responseSchema = z.object({ api_version: z.literal('v2'), meta: z.object({ profile: z.enum(['official', 'secondary', 'community']), release_id: z.string(), ruleset_version: z.string(), release_created_at: z.string(), coverage_scope: z.string().min(1), count_semantics: z.string().min(1), filters: z.object({ country_code: z.string().nullable().optional(), region: z.string().nullable().optional(), category: z.string().nullable().optional(), source_type: z.string().nullable().optional(), display_precision: z.string().nullable().optional(), lifecycle_status: z.string().nullable().optional() }) }), dimensions: z.record(z.string(), z.array(facet)) }); +export type Facet = Readonly<{ value: string; count: number }>; +export type FacetResponse = Readonly<{ profile: LocalProfile; releaseId: string; ruleset: string; coverageScope: string; countSemantics: string; dimensions: Readonly> }>; +const fail = (kind: ApiError['kind'], message: string): ApiError => Object.assign(new Error(message), { kind }); +export type FacetFilters = Readonly<{ country_code?: string | undefined; region?: string | undefined; category?: string | undefined; source_type?: string | undefined; display_precision?: string | undefined; lifecycle_status?: string | undefined }>; + +export class FacetsRepository { + constructor(private readonly fetcher: FetchLike = globalThis.fetch, private readonly baseUrl = '') {} + async get(profile: LocalProfile, filters: FacetFilters, expected: { releaseId: string; ruleset: string }, signal?: AbortSignal): Promise { + const params = new URLSearchParams({ profile }); + for (const [key, value] of Object.entries(filters)) if (value) params.set(key, value); + let response: Response; + try { const init: RequestInit = { cache: 'no-store' }; if (signal) init.signal = signal; response = await this.fetcher.call(globalThis, `${this.baseUrl}/api/v2/discovery/facets?${params}`, init); } + catch (error) { if (error instanceof DOMException && error.name === 'AbortError') throw fail('aborted', 'Facet request was aborted.'); throw fail('network', 'Coverage summary could not connect to the V2 service.'); } + if (!response.ok) throw fail(response.status >= 500 ? 'unavailable' : response.status === 429 ? 'rate-limited' : 'http', `Coverage summary request failed with status ${response.status}.`); + const parsed = responseSchema.safeParse(await response.json()); + if (!parsed.success) throw fail('invalid-contract', 'Coverage summary response was rejected safely.'); + if (parsed.data.meta.profile !== profile || parsed.data.meta.release_id !== expected.releaseId || parsed.data.meta.ruleset_version !== expected.ruleset) throw fail('invalid-contract', 'Coverage summary belongs to a different profile or promoted release.'); + return { profile, releaseId: parsed.data.meta.release_id, ruleset: parsed.data.meta.ruleset_version, coverageScope: parsed.data.meta.coverage_scope, countSemantics: parsed.data.meta.count_semantics, dimensions: parsed.data.dimensions }; + } +} diff --git a/frontend/src/app/App.svelte b/frontend/src/app/App.svelte index 7ec7d32..114b9bd 100644 --- a/frontend/src/app/App.svelte +++ b/frontend/src/app/App.svelte @@ -21,6 +21,8 @@ import { TestReleaseCsvExportRepository } from '../api/TestReleaseCsvExportRepository'; import DevReviewPanel from '../features/devPreview/DevReviewPanel.svelte'; import type { ApiError } from '../api/errors'; + import { FacetsRepository, type FacetResponse } from '../api/FacetsRepository'; + import CoveragePulse from '../features/coverage/CoveragePulse.svelte'; let profile: Profile = 'curated'; let selected: Location | undefined = locations[0]; @@ -57,6 +59,11 @@ let manifestSha256: string | undefined; let repo = new LocalLocationRepository(); let csvRepo = new LocalCsvExportRepository(); + let facetsRepo = new FacetsRepository(); + let facets: FacetResponse | undefined; + let facetsStatus: 'idle' | 'loading' | 'ready' | 'error' | 'unavailable' = 'idle'; + let facetsError = ''; + let facetsAbort: AbortController | undefined; let metadata: FilterMetadata | undefined; let metadataStatus: 'idle' | 'loading' | 'ready' | 'error' = 'idle'; let nextCursor: string | null = null; @@ -145,7 +152,7 @@ if (!append) activeListKey = queryKey; listGeneration += 1; const generation = listGeneration; listAbort?.abort(); const controller = new AbortController(); listAbort = controller; lastRemoteQuery = queryKey; - if (!append) { invalidateDetail(); localStatus = 'loading'; localFailure = 'unknown'; localError = ''; pagingError = ''; manifestSha256 = undefined; loaded = []; selected = undefined; nextCursor = null; coverageNote = ''; coverageScope = ''; countSemantics = ''; } else { paging = true; pagingError = ''; } + if (!append) { invalidateDetail(); localStatus = 'loading'; localFailure = 'unknown'; localError = ''; pagingError = ''; manifestSha256 = undefined; loaded = []; selected = undefined; nextCursor = null; coverageNote = ''; coverageScope = ''; countSemantics = ''; facetsAbort?.abort(); facets = undefined; facetsStatus = 'idle'; facetsError = ''; } else { paging = true; pagingError = ''; } try { const result = await repo.list(requestProfile, { q: search.trim() || undefined, country_code: region === 'all' ? undefined : region, region: subregion.trim() || undefined, category: category === 'all' ? undefined : category, source_type: sourceType === 'all' ? undefined : sourceType, display_precision: displayPrecision === 'all' ? undefined : displayPrecision, lifecycle_status: lifecycleStatus === 'all' ? undefined : lifecycleStatus, cursor, limit: 100 }, controller.signal); if (generation !== listGeneration) return; @@ -153,6 +160,7 @@ loaded = append ? [...loaded, ...result.locations] : result.locations; if (!selected) selected = result.locations[0]; release = result.releaseId; ruleset = result.ruleset; nextCursor = result.nextCursor; coverageNote = result.coverageNote; coverageScope = result.coverageScope ?? ''; countSemantics = result.countSemantics ?? ''; localStatus = 'ready'; localFailure = 'unknown'; paging = false; pagingError = ''; + if (!append) void loadFacets(result.releaseId, result.ruleset ?? ''); if (!append) await syncRoute(); } catch (error) { paging = false; if (!append) activeListKey = ''; if (generation !== listGeneration) return; @@ -164,6 +172,20 @@ } }; + const loadFacets = async (releaseId: string, releaseRuleset: string) => { + facetsAbort?.abort(); + const controller = new AbortController(); facetsAbort = controller; facetsStatus = 'loading'; facetsError = ''; + if (search.trim()) { facetsStatus = 'ready'; facets = undefined; return; } + try { + const result = await facetsRepo.get(apiProfile, { country_code: region === 'all' ? undefined : region, region: subregion.trim() || undefined, category: category === 'all' ? undefined : category, source_type: sourceType === 'all' ? undefined : sourceType, display_precision: displayPrecision === 'all' ? undefined : displayPrecision, lifecycle_status: lifecycleStatus === 'all' ? undefined : lifecycleStatus }, { releaseId, ruleset: releaseRuleset }, controller.signal); + if (controller.signal.aborted || release !== result.releaseId || ruleset !== result.ruleset || apiProfile !== result.profile) return; + facets = result; facetsStatus = 'ready'; + } catch (error) { + if (controller.signal.aborted) return; + facets = undefined; facetsStatus = error && typeof error === 'object' && 'kind' in error && (error as { kind: string }).kind === 'unavailable' ? 'unavailable' : 'error'; facetsError = error instanceof Error ? error.message : 'Coverage summary was rejected safely.'; + } + }; + const downloadCsv = async () => { if (devPreviewMode || !eligibleExport || exportBusy) return; exportBusy = true; exportError = ''; @@ -206,14 +228,14 @@ const route = parseRoute(window.location.hash); if (route.kind !== 'not-found') profile = localMode ? toApiProfile(route.profile) : route.profile; if (localMode) { search = params.get('q') ?? ''; region = params.get('country_code') ?? 'all'; subregion = params.get('region') ?? ''; category = params.get('category') ?? 'all'; sourceType = params.get('source_type') ?? 'all'; displayPrecision = params.get('display_precision') ?? 'all'; lifecycleStatus = params.get('lifecycle_status') ?? 'all'; - try { const api = params.get('api') ?? undefined; repo = new LocalLocationRepository(globalThis.fetch, api); csvRepo = new LocalCsvExportRepository(globalThis.fetch, api ?? ''); metadataStatus = 'loading'; void new FilterMetadataRepository(globalThis.fetch, api ?? '').get().then((value) => { metadata = value; metadataStatus = 'ready'; }).catch(() => { metadataStatus = 'error'; }); } + try { const api = params.get('api') ?? undefined; repo = new LocalLocationRepository(globalThis.fetch, api); csvRepo = new LocalCsvExportRepository(globalThis.fetch, api ?? ''); facetsRepo = new FacetsRepository(globalThis.fetch, api ?? ''); metadataStatus = 'loading'; void new FilterMetadataRepository(globalThis.fetch, api ?? '').get().then((value) => { metadata = value; metadataStatus = 'ready'; }).catch(() => { metadataStatus = 'error'; }); } catch (error) { localStatus = 'error'; localFailure = 'network'; localError = error instanceof Error ? error.message : 'Local API origin was rejected safely.'; } } if (devPreviewMode) { previewStatus = import.meta.env.DEV ? 'idle' : 'blocked'; if (!import.meta.env.DEV) previewError = 'Private candidate preview is unavailable in production builds.'; } else if (localMode && localStatus !== 'error') void loadLocal(); else if (!localMode) void syncRoute(); const onHashChange = () => { const next = parseRoute(window.location.hash); if (next.kind !== 'not-found' && toApiProfile(next.profile) !== toApiProfile(profile)) { profile = localMode ? toApiProfile(next.profile) : next.profile; if (!localMode) void syncRoute(); } else void syncRoute(); }; const onPopState = () => { const current = new URLSearchParams(window.location.search); if (localMode) { search = current.get('q') ?? ''; region = current.get('country_code') ?? 'all'; subregion = current.get('region') ?? ''; category = current.get('category') ?? 'all'; sourceType = current.get('source_type') ?? 'all'; displayPrecision = current.get('display_precision') ?? 'all'; lifecycleStatus = current.get('lifecycle_status') ?? 'all'; } onHashChange(); }; - window.addEventListener('hashchange', onHashChange); window.addEventListener('popstate', onPopState); return () => { listAbort?.abort(); detailAbort?.abort(); window.removeEventListener('hashchange', onHashChange); window.removeEventListener('popstate', onPopState); }; + window.addEventListener('hashchange', onHashChange); window.addEventListener('popstate', onPopState); return () => { listAbort?.abort(); detailAbort?.abort(); facetsAbort?.abort(); window.removeEventListener('hashchange', onHashChange); window.removeEventListener('popstate', onPopState); }; }); @@ -234,6 +256,7 @@ {#if localMode && localStatus === 'loading'}
    Loading the {profileLabelText.toLowerCase()}…
    {:else if localMode && (localStatus === 'error' || localStatus === 'no-release')}{:else if localMode && detailStatus === 'loading'}
    Loading the selected local record…
    {:else if localMode && detailStatus === 'error'}{:else}

    02 / FILTER & COMPARE

    Results {visibleLocations.length}{localMode && nextCursor ? ' on first page' : ''}

    {localMode ? 'Current eligible response' : 'Fictional demonstration data'} · {profileLabelText}

    {#if localMode}
    VISIBLE FACILITY RECORDS{visibleLocations.length}{nextCursor ? '+' : ''}
    DENOMINATORNot available

    {countSemantics || 'Counts refer to eligible public facility projection rows, not animals or a story-wide total.'} Scope: {coverageScope || 'selected promoted release public facilities'}. {nextCursor ? 'This is a partial page.' : 'This response has no further page.'} Legacy status is not inferred: the current V2 record contract has no legacy field.

    {coverageNote} {nextCursor ? 'Only the first page is loaded. Search and filters below may miss later records; counts and map points are partial.' : 'All records in this response are loaded; search applies to those records.'}

    {/if} + {#if !devPreviewMode}{/if}
    {#if selected}

    03 / RECORD DETAIL

    {selected.name}

    {selected.region} · {selected.category}

    {#if isUnreviewedCommunity(selected)}

    {selected.evidence?.publicationWarning ?? 'Unreviewed community claim — not verified by Until Every Cage'}

    {/if}
    OBSERVED{selected.observed}
    MAP STATUS{precisionLabel(selected)}
    {#if selected.evidence}
    Source origin
    {sourceLabel(selected.evidence.sourceType)}
    Factual review
    {selected.evidence.factualReviewStatus}{selected.evidence.reviewerRole ? ` · ${selected.evidence.reviewerRole}` : ' · reviewer role unavailable'}
    Privacy screening
    {selected.evidence.privacyScreeningStatus}
    Project approval
    {selected.evidence.projectApproval}
    Published profile
    {selected.evidence.publicationProfile ?? 'unavailable'} · release {release}
    Source
    {selected.evidence.provenanceSource ?? selected.source} · {selected.evidence.sourceId}
    Source rights
    {selected.evidence.sourceRightsStatus}
    Retrieved
    {selected.evidence.retrievedAt}
    Lifecycle
    {lifecycleLabel(selected)}
    Legacy status
    Not supplied by the current V2 contract
    Observation count
    {selected.evidence.observationCount ?? 'unavailable'}
    {/if}

    READ WITH CARE

    This is an observation in a particular release, not a guarantee of current operation. Source origin, factual review, privacy screening, and project approval are separate signals.

    {:else}

    Select a record to inspect its evidence context.

    {/if}
    {#if selected}

    RECORD / {selected.id}

    {/if} {/if} diff --git a/frontend/src/features/coverage/CoveragePulse.svelte b/frontend/src/features/coverage/CoveragePulse.svelte new file mode 100644 index 0000000..42987a1 --- /dev/null +++ b/frontend/src/features/coverage/CoveragePulse.svelte @@ -0,0 +1,21 @@ + +
    +

    {localMode ? 'EVIDENCE COVERAGE' : 'SYNTHETIC COVERAGE'}

    What this release makes visible

    {context}
    + {#if localMode && status === 'loading'}

    Reading release-wide coverage…

    + {:else if localMode && (status === 'error' || status === 'unavailable')}

    Coverage summary is unavailable for this response. The visible records remain available. {error}

    + {:else if localMode && searchActive}

    Coverage dimensions describe the structured filters and selected profile. Free-text search is excluded because this endpoint does not apply it.

    + {:else if dimensions.length === 0}

    No coverage dimensions are available for this view.

    + {:else}
    {#each dimensions as dimension}

    {dimension.label}

    {dimension.note}

    {#each dimension.values as item}
    {displayValue(item.value)}
    {item.count}
    {/each}
    {/each}
    {/if} +

    Counts are eligible public facility projection rows after current suppression. They are not animal counts, story-wide totals, or a measure of suffering. Each dimension is a separate view; values across dimensions must not be added.

    + {#if localMode && ruleset}

    Pinned to release {releaseId} · ruleset {ruleset}

    {/if} +
    + diff --git a/frontend/src/features/coverage/coverageModel.ts b/frontend/src/features/coverage/coverageModel.ts new file mode 100644 index 0000000..3027977 --- /dev/null +++ b/frontend/src/features/coverage/coverageModel.ts @@ -0,0 +1,8 @@ +import type { Location } from '../../domain/location'; +import type { Facet } from '../../api/FacetsRepository'; +export type CoverageDimension = Readonly<{ key: string; label: string; note: string; values: readonly Facet[] }>; +const labels: Record = { display_precision: 'Map precision', source_type: 'Source origin', lifecycle_status: 'Lifecycle' }; +const notes: Record = { display_precision: 'How precisely a public record can be placed.', source_type: 'Where the published evidence originated.', lifecycle_status: 'What the source says about observation status.' }; +export const dimensionsFromFacets = (facets: Readonly>): readonly CoverageDimension[] => ['display_precision', 'source_type', 'lifecycle_status'].map(key => ({ key, label: labels[key] ?? key, note: notes[key] ?? '', values: facets[key] ?? [] })).filter(dimension => dimension.values.length > 0); +export const dimensionsFromLocations = (locations: readonly Location[]): readonly CoverageDimension[] => { const counts = new Map>(); for (const location of locations) { const values = { display_precision: location.evidence?.displayPrecision ?? 'unmapped', source_type: location.evidence?.sourceType ?? 'unknown', lifecycle_status: location.evidence?.lifecycleStatus ?? 'status_unknown' }; for (const [key, value] of Object.entries(values)) { const dimension = counts.get(key) ?? new Map(); dimension.set(value, (dimension.get(value) ?? 0) + 1); counts.set(key, dimension); } } return dimensionsFromFacets(Object.fromEntries([...counts].map(([key, values]) => [key, [...values].map(([value, count]) => ({ value, count }))]))); }; +export const displayValue = (value: string): string => ({ exact: 'Exact public point', city: 'City-level point', unmapped: 'Unmapped', official: 'Government-sourced', secondary: 'Secondary-sourced', user_submitted: 'Community-submitted', active_observed: 'Active observed', explicitly_closed: 'Explicitly closed', not_seen_recently: 'Not observed recently', status_unknown: 'Status unknown' }[value] ?? value); diff --git a/frontend/tests/unit/coverageModel.test.ts b/frontend/tests/unit/coverageModel.test.ts new file mode 100644 index 0000000..a5d8264 --- /dev/null +++ b/frontend/tests/unit/coverageModel.test.ts @@ -0,0 +1,17 @@ +import { describe, expect, it } from 'vitest'; +import { dimensionsFromFacets, dimensionsFromLocations, displayValue } from '../../src/features/coverage/coverageModel'; + +describe('coverage model', () => { + it('keeps dimensions separate and preserves facet counts', () => { + const result = dimensionsFromFacets({ display_precision: [{ value: 'exact', count: 4 }], source_type: [{ value: 'official', count: 4 }], lifecycle_status: [{ value: 'active_observed', count: 3 }] }); + expect(result.map(dimension => dimension.key)).toEqual(['display_precision', 'source_type', 'lifecycle_status']); + expect(result[0]?.values[0]?.count).toBe(4); + }); + + it('derives explicitly synthetic fixture dimensions from evidence fields', () => { + const result = dimensionsFromLocations([{ id: 'one', name: 'One', region: 'DK', category: 'dairy', lat: null, lon: null, observed: '2026', source: 'Test', evidence: { sourceType: 'official', factualReviewStatus: 'recorded', reviewerRole: null, privacyScreeningStatus: 'passed', projectApproval: 'approved', publicationProfile: 'official', publicationWarning: null, sourceId: 'one', sourceUrl: 'https://example.test/one', provenanceSource: 'Test', sourceRightsStatus: 'cleared', retrievedAt: '2026', displayPrecision: 'unmapped', lifecycleStatus: 'status_unknown', observationCount: null } }]); + expect(result.find(dimension => dimension.key === 'display_precision')?.values).toEqual([{ value: 'unmapped', count: 1 }]); + }); + + it('uses human labels for public vocabulary values', () => expect(displayValue('user_submitted')).toBe('Community-submitted')); +}); diff --git a/frontend/tests/unit/facetsRepository.test.ts b/frontend/tests/unit/facetsRepository.test.ts new file mode 100644 index 0000000..7e2a715 --- /dev/null +++ b/frontend/tests/unit/facetsRepository.test.ts @@ -0,0 +1,18 @@ +import { describe, expect, it, vi } from 'vitest'; +import { FacetsRepository } from '../../src/api/FacetsRepository'; + +const body = (overrides: Record = {}) => ({ api_version: 'v2', meta: { profile: 'official', release_id: 'release-1', ruleset_version: 'rules-1', release_created_at: '2026-01-01T00:00:00Z', coverage_scope: 'selected_promoted_release_public_facilities', count_semantics: 'Counts are eligible public facility projection rows after current suppression; they are not story-wide or animal counts.', filters: { country_code: null, region: null, category: null, source_type: null, display_precision: null, lifecycle_status: null } }, dimensions: { display_precision: [{ value: 'exact', count: 2 }], source_type: [{ value: 'official', count: 2 }], lifecycle_status: [{ value: 'active_observed', count: 2 }] }, ...overrides }); + +describe('FacetsRepository', () => { + it('validates and pins profile, release, and ruleset', async () => { + const fetcher = vi.fn().mockResolvedValue(new Response(JSON.stringify(body()))); + const result = await new FacetsRepository(fetcher).get('official', {}, { releaseId: 'release-1', ruleset: 'rules-1' }); + expect(result.dimensions.display_precision?.[0]?.count).toBe(2); + expect(fetcher.mock.calls[0]?.[0]).toContain('profile=official'); + }); + + it('rejects a facet snapshot from a different release', async () => { + const fetcher = vi.fn().mockResolvedValue(new Response(JSON.stringify(body({ meta: { ...body().meta, release_id: 'old-release' } })))); + await expect(new FacetsRepository(fetcher).get('official', {}, { releaseId: 'release-1', ruleset: 'rules-1' })).rejects.toThrow(/different profile or promoted release/); + }); +}); From d875369d20e2839c9f0e3f3cd115165c61ca7e63 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sun, 20 Sep 2026 17:54:02 -0700 Subject: [PATCH 305/311] fix coverage facet contract validation --- frontend/src/api/FacetsRepository.ts | 13 ++++++++++++- frontend/tests/unit/facetsRepository.test.ts | 15 +++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/frontend/src/api/FacetsRepository.ts b/frontend/src/api/FacetsRepository.ts index 4ba1003..b3e2d50 100644 --- a/frontend/src/api/FacetsRepository.ts +++ b/frontend/src/api/FacetsRepository.ts @@ -9,6 +9,11 @@ export type FacetResponse = Readonly<{ profile: LocalProfile; releaseId: string; const fail = (kind: ApiError['kind'], message: string): ApiError => Object.assign(new Error(message), { kind }); export type FacetFilters = Readonly<{ country_code?: string | undefined; region?: string | undefined; category?: string | undefined; source_type?: string | undefined; display_precision?: string | undefined; lifecycle_status?: string | undefined }>; +const normalizedFilter = (value: string | null | undefined): string | null => { + const normalized = value?.trim(); + return normalized ? normalized : null; +}; + export class FacetsRepository { constructor(private readonly fetcher: FetchLike = globalThis.fetch, private readonly baseUrl = '') {} async get(profile: LocalProfile, filters: FacetFilters, expected: { releaseId: string; ruleset: string }, signal?: AbortSignal): Promise { @@ -18,9 +23,15 @@ export class FacetsRepository { try { const init: RequestInit = { cache: 'no-store' }; if (signal) init.signal = signal; response = await this.fetcher.call(globalThis, `${this.baseUrl}/api/v2/discovery/facets?${params}`, init); } catch (error) { if (error instanceof DOMException && error.name === 'AbortError') throw fail('aborted', 'Facet request was aborted.'); throw fail('network', 'Coverage summary could not connect to the V2 service.'); } if (!response.ok) throw fail(response.status >= 500 ? 'unavailable' : response.status === 429 ? 'rate-limited' : 'http', `Coverage summary request failed with status ${response.status}.`); - const parsed = responseSchema.safeParse(await response.json()); + let payload: unknown; + try { payload = await response.json(); } catch { throw fail('invalid-contract', 'Coverage summary response was not valid JSON.'); } + const parsed = responseSchema.safeParse(payload); if (!parsed.success) throw fail('invalid-contract', 'Coverage summary response was rejected safely.'); if (parsed.data.meta.profile !== profile || parsed.data.meta.release_id !== expected.releaseId || parsed.data.meta.ruleset_version !== expected.ruleset) throw fail('invalid-contract', 'Coverage summary belongs to a different profile or promoted release.'); + const responseFilters = parsed.data.meta.filters; + for (const key of ['country_code', 'region', 'category', 'source_type', 'display_precision', 'lifecycle_status'] as const) { + if (normalizedFilter(filters[key]) !== normalizedFilter(responseFilters[key])) throw fail('invalid-contract', 'Coverage summary belongs to a different filter scope.'); + } return { profile, releaseId: parsed.data.meta.release_id, ruleset: parsed.data.meta.ruleset_version, coverageScope: parsed.data.meta.coverage_scope, countSemantics: parsed.data.meta.count_semantics, dimensions: parsed.data.dimensions }; } } diff --git a/frontend/tests/unit/facetsRepository.test.ts b/frontend/tests/unit/facetsRepository.test.ts index 7e2a715..10e2e3b 100644 --- a/frontend/tests/unit/facetsRepository.test.ts +++ b/frontend/tests/unit/facetsRepository.test.ts @@ -15,4 +15,19 @@ describe('FacetsRepository', () => { const fetcher = vi.fn().mockResolvedValue(new Response(JSON.stringify(body({ meta: { ...body().meta, release_id: 'old-release' } })))); await expect(new FacetsRepository(fetcher).get('official', {}, { releaseId: 'release-1', ruleset: 'rules-1' })).rejects.toThrow(/different profile or promoted release/); }); + + it('rejects a facet snapshot from a different requested filter scope', async () => { + const fetcher = vi.fn().mockResolvedValue(new Response(JSON.stringify(body({ meta: { ...body().meta, filters: { ...body().meta.filters, category: 'slaughter' } } })))); + await expect(new FacetsRepository(fetcher).get('official', { category: 'dairy' }, { releaseId: 'release-1', ruleset: 'rules-1' })).rejects.toMatchObject({ kind: 'invalid-contract' }); + }); + + it('accepts normalized filter metadata from the backend', async () => { + const fetcher = vi.fn().mockResolvedValue(new Response(JSON.stringify(body({ meta: { ...body().meta, filters: { ...body().meta.filters, region: 'North Coast' } } })))); + await expect(new FacetsRepository(fetcher).get('official', { region: ' North Coast ' }, { releaseId: 'release-1', ruleset: 'rules-1' })).resolves.toMatchObject({ releaseId: 'release-1' }); + }); + + it('classifies malformed successful JSON as an invalid contract', async () => { + const fetcher = vi.fn().mockResolvedValue(new Response('{not-json')); + await expect(new FacetsRepository(fetcher).get('official', {}, { releaseId: 'release-1', ruleset: 'rules-1' })).rejects.toMatchObject({ kind: 'invalid-contract' }); + }); }); From 4105701131fb990499c75b39c66560e527a04a89 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sun, 20 Sep 2026 18:26:52 -0700 Subject: [PATCH 306/311] Integrate approved FSIS provenance fixes --- pipeline/sources/us/fsis/refresh.py | 2 +- pipeline/sources/us/fsis/test_refresh.py | 28 ++++++++++++++++++++++++ pipeline/sources/us/refresh.py | 4 ++-- pipeline/sources/us/test_refresh.py | 21 ++++++++++++++++++ 4 files changed, 52 insertions(+), 3 deletions(-) diff --git a/pipeline/sources/us/fsis/refresh.py b/pipeline/sources/us/fsis/refresh.py index b61f35c..05ba270 100644 --- a/pipeline/sources/us/fsis/refresh.py +++ b/pipeline/sources/us/fsis/refresh.py @@ -263,7 +263,7 @@ def refresh( observed_at = retrieved_at_utc or utc_now() metadata["directory"] = _local_facts( paths["directory"], role="directory", - source_url=CONFIG.get("directory_by_number_url") or source_url, + source_url=source_url, retrieved_at_utc=observed_at, effective_date=effective_date, ) if demographics_path is not None: diff --git a/pipeline/sources/us/fsis/test_refresh.py b/pipeline/sources/us/fsis/test_refresh.py index 820c88a..d111b89 100644 --- a/pipeline/sources/us/fsis/test_refresh.py +++ b/pipeline/sources/us/fsis/test_refresh.py @@ -39,6 +39,10 @@ def test_bundle_refresh_writes_private_handoff_and_provenance_per_file(self): ) manifest = result["manifest"] self.assertTrue(result["candidate_created"]) + self.assertEqual( + manifest["source_artifacts"]["directory"]["source_url"], + "https://www.fsis.usda.gov/inspection/establishments/meat-poultry-and-egg-product-inspection-directory", + ) self.assertEqual(manifest["release_state"], "not-created") self.assertEqual(manifest["publication_state"], "private-candidate") self.assertEqual(manifest["source_artifacts"]["demographics"]["byte_size"], len((ROOT / "fixtures/demographics.csv").read_bytes())) @@ -49,6 +53,30 @@ def test_bundle_refresh_writes_private_handoff_and_provenance_per_file(self): self.assertEqual(handoff["handoff_artifact_role"], "directory") self.assertEqual(handoff["source_artifacts"]["demographics"]["sha256"], manifest["source_artifacts"]["demographics"]["sha256"]) self.assertEqual(handoff["bundle_artifact"]["sha256"], manifest["sha256"]) + self.assertEqual( + handoff["source_artifacts"]["directory"]["source_url"], + manifest["source_artifacts"]["directory"]["source_url"], + ) + + def test_operator_directory_url_is_preserved_in_local_artifact_provenance(self): + with tempfile.TemporaryDirectory() as directory: + source_url = "https://www.fsis.usda.gov/sites/default/files/media_file/documents/MPI_Directory_by_Establishment_Name.csv" + result = refresh( + run_dir=Path(directory) / "run", + directory_path=ROOT / "fixtures/valid.csv", + source_url=source_url, + retrieved_at_utc="2026-09-20T00:00:00Z", + effective_date="2026-09-14", + mode="handoff", + ) + lifecycle = result["manifest"] + self.assertEqual(lifecycle["source_url"], source_url) + self.assertEqual(lifecycle["source_artifacts"]["directory"]["source_url"], source_url) + handoff = json.loads((Path(directory) / "run/lifecycle/handoff/manifest.json").read_text(encoding="utf-8")) + self.assertEqual(handoff["bundle_artifact"]["source_url"], source_url) + self.assertEqual(handoff["source_artifacts"]["directory"]["source_url"], source_url) + normalized = (Path(directory) / "run/lifecycle/handoff/normalized/records.jsonl").read_text(encoding="utf-8") + self.assertIn('"source_id": "us.fsis"', normalized) def test_schema_drift_blocks_handoff_after_previous_manifest(self): with tempfile.TemporaryDirectory() as directory: diff --git a/pipeline/sources/us/refresh.py b/pipeline/sources/us/refresh.py index ab5b1b6..b4eb230 100644 --- a/pipeline/sources/us/refresh.py +++ b/pipeline/sources/us/refresh.py @@ -17,7 +17,7 @@ from pipeline.contracts.source_lifecycle import atomic_json from .aphis.refresh import refresh as refresh_aphis -from .fsis.refresh import refresh as refresh_fsis +from .fsis.refresh import CONFIG as FSIS_CONFIG, refresh as refresh_fsis REPORT_VERSION = "us-operator-report-v1" @@ -208,7 +208,7 @@ def _run_one(spec: dict[str, Any], base: Path, root: Path, mode: str, retry: dic directory_path=directory, demographics_path=demographics, fetch=fetch, - source_url=spec.get("source_url") or None, + source_url=spec.get("source_url") or FSIS_CONFIG["directory_url"], retrieved_at_utc=spec.get("retrieved_at_utc"), effective_date=spec.get("effective_date"), mode=mode, diff --git a/pipeline/sources/us/test_refresh.py b/pipeline/sources/us/test_refresh.py index 4c6c1ee..4dea9e3 100644 --- a/pipeline/sources/us/test_refresh.py +++ b/pipeline/sources/us/test_refresh.py @@ -94,6 +94,27 @@ def test_failed_lane_reports_action_and_preserves_previous_valid_manifest(self): self.assertEqual(fsis["failure"]["failure_class"], "validation-or-runtime") self.assertTrue((root / "second/fsis/failure-report.json").exists()) + def test_fsis_source_url_defaults_in_orchestration_and_preserves_explicit_override(self): + default_url = "https://www.fsis.usda.gov/inspection/establishments/meat-poultry-and-egg-product-inspection-directory" + explicit_url = "https://www.fsis.usda.gov/sites/default/files/media_file/documents/MPI_Directory_by_Establishment_Name.csv" + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + base_plan = self._plan( + directory=ROOT / "fsis/fixtures/valid.csv", + demographics=ROOT / "fsis/fixtures/demographics.csv", + aphis=ROOT / "aphis/fixtures/annual_reports.csv", + ) + for index, value in enumerate(("omitted", None)): + plan = {**base_plan, "sources": [dict(base_plan["sources"][0])]} + if value != "omitted": + plan["sources"][0]["source_url"] = value + report = run_us_refresh(plan, plan_base=root, run_root=root / f"default-{index}") + self.assertEqual(report["sources"][0]["acquisition"]["roles"]["directory"]["source_url"], default_url) + plan = {**base_plan, "sources": [dict(base_plan["sources"][0])]} + plan["sources"][0]["source_url"] = explicit_url + report = run_us_refresh(plan, plan_base=root, run_root=root / "explicit") + self.assertEqual(report["sources"][0]["acquisition"]["roles"]["directory"]["source_url"], explicit_url) + if __name__ == "__main__": unittest.main() From c9af8eaec0d78a43fc12fddeb59e999324810e09 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sun, 20 Sep 2026 18:38:54 -0700 Subject: [PATCH 307/311] Refresh Sprint 02 acceptance ledger --- docs/sprint02-integration-ledger.md | 34 +++++++++++++---------------- 1 file changed, 15 insertions(+), 19 deletions(-) diff --git a/docs/sprint02-integration-ledger.md b/docs/sprint02-integration-ledger.md index 696fec6..d6638b1 100644 --- a/docs/sprint02-integration-ledger.md +++ b/docs/sprint02-integration-ledger.md @@ -1,15 +1,15 @@ # Sprint 02 integration and storage ledger -Status: partial reviewed country checkpoint, 2026-09-19. France and Italy are accepted for private candidate/replay integration; this is not a release approval or a claim that Sprint 02 source acquisition is complete. +Status: partial reviewed candidate/replay checkpoint, 2026-09-20. France, Italy, APHIS core/inspection, evidence integration, and the accepted FSIS backend scope are integrated for private candidate/replay use; this is not a release approval or a claim that Sprint 02 source acquisition is complete. ## Ownership and baseline - Contract: [`SPRINT-02-CONTRACT.md`](SPRINT-02-CONTRACT.md) - Required baseline: `5570b6ab42e74dc5227e08c2cd7c1ac7f08aab96` -- Lane worktree: `C:\Users\pnael\.codex\worktrees\8a23\UntilEveryCage` +- Lane worktree: managed integration worktree (local path intentionally omitted) - Lane branch: `codex/sprint02-integration-lane8` - Integration target: `origin/eli/front-end-overhaul` -- Raw evidence boundary: `C:\New Projects\UntilEveryCage\.private\sprint02-20260919\` +- Raw evidence boundary: private Sprint 02 evidence root outside Git (local path intentionally omitted) ## Storage checkpoint @@ -18,7 +18,6 @@ The shared private root exists with separate directories for `aphis-core`, `aphi Verification commands: ```powershell -git -C 'C:\New Projects\UntilEveryCage' check-ignore -v --no-index '.private/sprint02-20260919/aphis-core/probe.bin' git check-ignore -v --no-index '.private/sprint02-20260919/probe.bin' powershell -ExecutionPolicy Bypass -File pipeline/tests/verify-docker-context.ps1 ``` @@ -29,28 +28,25 @@ Kickoff validation: `npm ci` completed with no vulnerabilities; `python -m unitt ## Reviewed country checkpoint -- France commit `707ac93e` was independently replayed from retained Section I/II raw artifacts and approved by QA; it is integrated as `fbf4e682` on `eli/front-end-overhaul` with unresolved identity signals preserved, no automatic merges, and publication/DB import blocked. -- Italy commits `e04adecf` and `291ad21` were independently replayed byte-for-byte and approved by QA; they are integrated in the same checkpoint with quarantine, location/privacy, rights, and publication gates preserved. -- The combined local validation passed 256 pipeline tests, 25 Jest tests, 7 developer tests, doctor, and diff checks. Native GitHub Actions run [98](https://github.com/eliperez-dev/UntilEveryCage/actions/runs/35461131905) succeeded for exact SHA `fbf4e6824792078a4c3aa5ac1f730e0629039224`. -- FSIS current files remain blocked after bounded ordinary GETs to the three displayed official routes returned HTTP 403; no response body was retained. The row-free evidence is private at `C:\New Projects\UntilEveryCage\.private\sprint02-20260919\fsis\handoff\bounded-get-20260919.json`. -- APHIS registration/report and inspection lanes are still in progress; their real handoffs require independent replay before integration. -- APHIS inspection code chain `addcef0c` -> `4518bce6` -> `8b4d3c68` is QA-approved and integrated as `e946d15e`. The authoritative replay accepted 1,075 input rows, 1,071 candidates, and 4 exact-duplicate quarantines, with every source row mapped to verified original-page lineage. Public release/import remains blocked. -- APHIS registration/annual-report per-row lineage is not yet accepted; the core/evidence owners are correcting that gap before integration. -- The authoritative APHIS evidence-consumer stack `8430572b` -> `4ee87737` -> `3124696e` -> `285ef119` -> `da17cc34` -> `9b026c13` is code-QA approved and integrated as lane commits through `8dc35b98`. The integrated checks fail closed on quarantine accounting, origin metadata consensus, and original-page lineage; this does not accept the still-held annual/registration rows. +- France and Italy are QA-approved and integrated for private candidate/replay use with unresolved identity signals, quarantine, location/privacy, rights, and publication gates preserved. +- APHIS core/annual-report and inspection v4 handoffs have final QA acceptance and are integrated for private candidate/replay use. Row-free handoffs, source lineage, quarantine accounting, and publication/DB-import gates remain enforced; public release/import remains blocked. +- Lane 6 evidence-consumer contracts have final QA acceptance in the current integrated checkpoint. The consumer fails closed on quarantine accounting, origin metadata consensus, and original-page lineage; this is not publication approval. +- FSIS backend guard, operator-directory URL provenance, and missing-orchestration-URL fallback are accepted in checkpoint `41057011`. QA also recorded successful browser-backed acquisition evidence for the establishment-name directory and demographics; the reusable browser-acquisition implementation remains a separate author lane. +- Earlier bounded ordinary GETs to official FSIS routes returned HTTP 403. That transport diagnostic is retained as context and does not override the separately reviewed browser-backed evidence; no response body, raw row, private path, or artifact hash is recorded here. ## Integration ledger | Area | Owner/interface | Acceptance state | | --- | --- | --- | -| APHIS registrations/reports | Lane 1 handoff under private storage | Pending per-row lineage correction and QA | -| APHIS inspections | Lane 2 handoff under private storage | QA-approved and integrated for private candidate/replay | -| FSIS current parity | Lane 3 handoff under private storage | Pending source handoff and review | +| APHIS registrations/reports | Lane 1 handoff | QA-approved and integrated for private candidate/replay; publication/import blocked | +| APHIS inspections | Lane 2 handoff | Final QA-approved and integrated for private candidate/replay | +| FSIS current parity | Lane 3 handoff | Backend guard/provenance/fallback accepted; browser-acquisition implementation remains separate | | France candidate | Lane 4 handoff under private storage | QA-approved and integrated for private candidate/replay | | Italy candidate | Lane 5 handoff under private storage | QA-approved and integrated for private candidate/replay | -| Evidence integration | Lane 6 existing APHIS/FSIS contracts | Authoritative APHIS consumer stack integrated; core annual/registration lineage still held | -| Independent QA | Lane 7 replay and review | Pending reviewed handoffs | -| CI/build/release engineering | Lane 8 | Storage/context boundary implemented; focused integration tests pass; native CI pending for current SHA | +| Evidence integration | Lane 6 existing APHIS/FSIS contracts | Final QA-approved and integrated; fail-closed lineage and quarantine gates retained | +| Independent QA | Lane 7 replay and review | Current handoffs reviewed; durable row-free final signoff in preparation | +| CI/build/release engineering | Lane 8 | Checkpoint `41057011` accepted; 282 pipeline tests passed and hosted CI [run 116](https://github.com/eliperez-dev/UntilEveryCage/actions/runs/35551000683) passed for exact SHA `4105701131fb990499c75b39c66560e527a04a89` | ## Release gate -No public release, deployment, or publication approval is implied. Before final integration, lane 8 must verify reviewed commits, migration reservations, private-artifact availability, reproducible commands, native Linux CI for the exact integrated SHA, and the remaining coverage/privacy/rights limitations. Failed acquisition leaves the previous validated release available subject to current restrictions. +No public release, deployment, or publication approval is implied. Before release, maintainers must still verify migration reservations, private-artifact availability, reproducible commands, exact-SHA CI, and the remaining coverage/privacy/rights limitations. Failed acquisition leaves the previous validated release available subject to current restrictions. From 56b22705087788c6195002f96e0c132d1abf92e8 Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sun, 20 Sep 2026 19:41:05 -0700 Subject: [PATCH 308/311] Integrate bounded Firefox FSIS acquisition --- docs/countries/us/operator-refresh.md | 14 +- docs/sprint02-integration-ledger.md | 8 +- pipeline/sources/us/fsis/README.md | 44 ++- .../sources/us/fsis/firefox_acquisition.py | 337 ++++++++++++++++++ pipeline/sources/us/fsis/refresh.py | 64 +++- .../us/fsis/test_firefox_acquisition.py | 176 +++++++++ pipeline/sources/us/refresh.py | 3 + 7 files changed, 621 insertions(+), 25 deletions(-) create mode 100644 pipeline/sources/us/fsis/firefox_acquisition.py create mode 100644 pipeline/sources/us/fsis/test_firefox_acquisition.py diff --git a/docs/countries/us/operator-refresh.md b/docs/countries/us/operator-refresh.md index 40e2741..f3d7570 100644 --- a/docs/countries/us/operator-refresh.md +++ b/docs/countries/us/operator-refresh.md @@ -37,6 +37,8 @@ plan file: "sources": [ { "source": "fsis", + "acquisition_method": "http", + "acquisition_authorization": "private-captures/fsis-acquisition-authorization.json", "directory": "private-captures/fsis-directory.csv", "demographics": "private-captures/fsis-demographics.csv", "retrieved_at_utc": "2026-09-18T00:00:00Z", @@ -111,8 +113,16 @@ python -m pipeline.sources.us.refresh ` --as-of-utc 2026-09-18T12:00:00Z ``` -Network acquisition is still opt-in and requires the source-specific approved -terms record. The shared acquisition primitive retries only bounded network, +Network acquisition is still opt-in. The `http` method requires the +source-specific approved terms record; the `firefox` method requires a typed +owner acquisition-authorization record for the exact FSIS routes and may +proceed for private staging while terms remain unknown. FSIS plans may set +`acquisition_method` to `firefox` for a fresh +temporary Firefox/Selenium browser capture; the browser route preserves exact +URL, timestamp, hash, byte size, and browser/runtime provenance and accepts a +navigation timeout only after a complete schema-valid file is present. It does +not import profiles or credentials and is not a stealth or endpoint-discovery +framework. The shared acquisition primitive retries only bounded network, rate-limit, timeout, and interrupted-download failures. It uses temporary partial files, removes them after an interrupted read, and records every attempt. HTTP 403, HTML/login/challenge responses, invalid content types, diff --git a/docs/sprint02-integration-ledger.md b/docs/sprint02-integration-ledger.md index d6638b1..da80406 100644 --- a/docs/sprint02-integration-ledger.md +++ b/docs/sprint02-integration-ledger.md @@ -1,6 +1,6 @@ # Sprint 02 integration and storage ledger -Status: partial reviewed candidate/replay checkpoint, 2026-09-20. France, Italy, APHIS core/inspection, evidence integration, and the accepted FSIS backend scope are integrated for private candidate/replay use; this is not a release approval or a claim that Sprint 02 source acquisition is complete. +Status: partial reviewed candidate/replay checkpoint, 2026-09-20. France, Italy, APHIS core/inspection, evidence integration, and the accepted FSIS backend plus bounded Firefox method are integrated for private candidate/replay use; this is not a release approval or a claim that Sprint 02 source acquisition is complete. ## Ownership and baseline @@ -31,8 +31,8 @@ Kickoff validation: `npm ci` completed with no vulnerabilities; `python -m unitt - France and Italy are QA-approved and integrated for private candidate/replay use with unresolved identity signals, quarantine, location/privacy, rights, and publication gates preserved. - APHIS core/annual-report and inspection v4 handoffs have final QA acceptance and are integrated for private candidate/replay use. Row-free handoffs, source lineage, quarantine accounting, and publication/DB-import gates remain enforced; public release/import remains blocked. - Lane 6 evidence-consumer contracts have final QA acceptance in the current integrated checkpoint. The consumer fails closed on quarantine accounting, origin metadata consensus, and original-page lineage; this is not publication approval. -- FSIS backend guard, operator-directory URL provenance, and missing-orchestration-URL fallback are accepted in checkpoint `41057011`. QA also recorded successful browser-backed acquisition evidence for the establishment-name directory and demographics; the reusable browser-acquisition implementation remains a separate author lane. -- Earlier bounded ordinary GETs to official FSIS routes returned HTTP 403. That transport diagnostic is retained as context and does not override the separately reviewed browser-backed evidence; no response body, raw row, private path, or artifact hash is recorded here. +- FSIS backend guard, operator-directory URL provenance, missing-orchestration-URL fallback, and the bounded Firefox acquisition method are accepted in the final candidate checkpoint. The source-author factual handoff still must provide two complete establishment-name directory runs plus one demographics run; until that row-free proof is received, FSIS source-acquisition acceptance is incomplete. +- Earlier bounded ordinary GETs to official FSIS routes returned HTTP 403. That transport diagnostic is retained as context; it neither establishes nor negates the pending factual browser handoff, and no response body, raw row, private path, or artifact hash is recorded here. ## Integration ledger @@ -40,7 +40,7 @@ Kickoff validation: `npm ci` completed with no vulnerabilities; `python -m unitt | --- | --- | --- | | APHIS registrations/reports | Lane 1 handoff | QA-approved and integrated for private candidate/replay; publication/import blocked | | APHIS inspections | Lane 2 handoff | Final QA-approved and integrated for private candidate/replay | -| FSIS current parity | Lane 3 handoff | Backend guard/provenance/fallback accepted; browser-acquisition implementation remains separate | +| FSIS current parity | Lane 3 handoff | Backend guard/provenance/fallback and bounded Firefox method integrated; factual two-directory-plus-demographics handoff proof pending | | France candidate | Lane 4 handoff under private storage | QA-approved and integrated for private candidate/replay | | Italy candidate | Lane 5 handoff under private storage | QA-approved and integrated for private candidate/replay | | Evidence integration | Lane 6 existing APHIS/FSIS contracts | Final QA-approved and integrated; fail-closed lineage and quarantine gates retained | diff --git a/pipeline/sources/us/fsis/README.md b/pipeline/sources/us/fsis/README.md index e3f5781..145422e 100644 --- a/pipeline/sources/us/fsis/README.md +++ b/pipeline/sources/us/fsis/README.md @@ -9,10 +9,16 @@ APHIS observations, and non-FSIS populations remain outside this source. The official page currently exposes a directory export by establishment name, a directory export by establishment number, and a supplemental establishment- -demographic CSV. Direct links may return HTTP 403. The refresh command never -bypasses that control: use an authorized operator-assisted capture or a -terms-reviewed bounded fetch. HTML, login pages, 403 responses, unsupported -content types, malformed CSV, and schema drift fail closed. +demographic CSV. Direct HTTP links may return HTTP 403 while a normal Firefox +download succeeds on the same source route. The refresh command has an +explicit `--acquisition-method` choice: `http` uses the bounded HTTP primitive; +`firefox` uses a fresh temporary Firefox profile and Selenium download settings. +The browser method records the exact URL, retrieval time, hash, byte size, +browser/runtime version, navigation method, and whether a navigation timeout +occurred after a complete validated file appeared. It never imports a user +profile, credentials, cookies, or hidden endpoints. HTML, login pages, 403 +responses, unsupported content types, malformed CSV, incomplete downloads, and +schema drift fail closed. For an operator-assisted capture: @@ -23,6 +29,36 @@ python -m pipeline.sources.us.fsis.refresh \ --run-dir --mode dry-run ``` +For an owner-authorized normal-browser capture, install the optional Selenium +package in the operator runtime and install system Firefox; these are not +project runtime dependencies: + +```text +python -m pip install selenium +``` + +Then choose the browser method explicitly: + +```text +python -m pipeline.sources.us.fsis.refresh \ + --fetch --acquisition-method firefox \ + --source-url https://www.fsis.usda.gov/sites/default/files/media_file/documents/MPI_Directory_by_Establishment_Name.csv \ + --acquisition-authorization \ + --run-dir --mode dry-run +``` + +The authorization record must state `status=authorized`, an owner basis, +source scope, private/no-public restrictions, and `terms_status=unknown` or +`pending_review`. It records acquisition permission separately; it is not a +redistribution or licensing approval. Supply `--terms-review` only when a +separate approved terms record exists; otherwise the browser metadata records +terms as unknown and publication remains blocked. Each browser attempt is bounded to two fresh sessions by default, retains only +validated CSV bytes and row-free metadata, and removes incomplete/invalid +download bodies after preserving the failure record. A browser navigation +timeout is accepted only after the completed file passes the byte bound and +the source-role CSV validator. The demographic route is acquired in the same +run; missing demographics remain an explicit directory-only profile. + Use `--mode handoff` only after reviewing the private manifest and quarantine. `--raw` remains a directory-only compatibility alias. `--fetch` requires a terms-review JSON and fetches the configured directory-by-number and diff --git a/pipeline/sources/us/fsis/firefox_acquisition.py b/pipeline/sources/us/fsis/firefox_acquisition.py new file mode 100644 index 0000000..c2c7ca6 --- /dev/null +++ b/pipeline/sources/us/fsis/firefox_acquisition.py @@ -0,0 +1,337 @@ +"""Bounded Firefox download acquisition for the private FSIS refresh. + +This is an explicit source method, not a stealth or anti-bot framework. Each +attempt uses a fresh temporary Firefox profile and a caller-selected private +download directory. Browser navigation timeouts are tolerated only when a +complete downloaded file passes size and source-schema validation. +""" +from __future__ import annotations + +import hashlib +import shutil +import tempfile +import time +import uuid +from pathlib import Path +from typing import Any, Callable + +from pipeline.common.acquisition import AcquisitionError, require_terms_review, utc_now +from pipeline.contracts.source_lifecycle import atomic_json + + +def _load_selenium() -> Any: + try: + import selenium + from selenium import webdriver + from selenium.common.exceptions import TimeoutException + from selenium.webdriver.common.by import By + except ImportError as error: + raise AcquisitionError( + "Firefox acquisition requires the optional Selenium package and an installed Firefox browser", + failure_class="configuration", + action="install the documented Firefox/Selenium runtime, then rerun with --acquisition-method firefox", + ) from error + return {"selenium": selenium, "webdriver": webdriver, "TimeoutException": TimeoutException, "By": By} + + +def _open_driver(download_dir: Path) -> tuple[Any, dict[str, Any]]: + runtime = _load_selenium() + options = runtime["webdriver"].FirefoxOptions() + options.add_argument("-headless") + options.set_preference("browser.download.folderList", 2) + options.set_preference("browser.download.dir", str(download_dir)) + options.set_preference("browser.download.useDownloadDir", True) + options.set_preference("browser.download.alwaysOpenPanel", False) + options.set_preference("browser.helperApps.neverAsk.saveToDisk", "text/csv,application/csv,application/octet-stream") + options.set_preference("pdfjs.disabled", True) + # Do not set options.profile: Selenium creates a fresh disposable profile. + driver = runtime["webdriver"].Firefox(options=options) + capabilities = getattr(driver, "capabilities", {}) or {} + return driver, { + "browser": "Firefox", + "browser_version": capabilities.get("browserVersion", "unknown"), + "selenium_version": getattr(runtime["selenium"], "__version__", "unknown"), + "profile": "fresh-temporary", + "headless": True, + "timeout_exception": runtime["TimeoutException"], + "by": runtime["By"], + } + + +def _complete_download(download_dir: Path, *, max_bytes: int) -> Path | None: + partials = [path for path in download_dir.iterdir() if path.is_file() and path.name.endswith(".part")] + complete = [path for path in download_dir.iterdir() if path.is_file() and not path.name.endswith(".part")] + if partials or len(complete) != 1: + return None + path = complete[0] + size = path.stat().st_size + if size <= 0 or size > max_bytes: + raise AcquisitionError( + f"Firefox download size {size} is outside the allowed bound", + failure_class="browser-download-size", + action="inspect the private browser attempt and source edition", + ) + return path + + +def _remove_downloads(download_dir: Path) -> None: + try: + paths = tuple(download_dir.iterdir()) + except OSError: + return + for path in paths: + if path.is_file(): + try: + path.unlink(missing_ok=True) + except OSError: + # Cleanup must never mask the authoritative acquisition or + # validation failure. The private failure ledger remains the + # source of truth for the attempt outcome. + continue + + +def _validate_acquisition_authorization(value: dict[str, Any] | None, *, source_id: str, url: str) -> dict[str, Any]: + if not isinstance(value, dict) or value.get("status") != "authorized": + raise AcquisitionError( + "Firefox acquisition requires an explicit owner acquisition-authorization record", + failure_class="authorization", + action="provide a scoped authorization record; keep redistribution terms and publication blocked until separately reviewed", + ) + for field in ("basis", "scope"): + if not isinstance(value.get(field), str) or not value[field].strip(): + raise AcquisitionError( + f"acquisition authorization requires a non-empty {field}", + failure_class="authorization", + action="provide a scoped authorization record with owner basis and source scope", + ) + restrictions = value.get("restrictions") + if not isinstance(restrictions, list) or not restrictions or not all(isinstance(item, str) and item.strip() for item in restrictions): + raise AcquisitionError( + "acquisition authorization restrictions must be a non-empty list of strings", + failure_class="authorization", + action="provide typed private/no-public restrictions in the authorization record", + ) + allowed_routes = value.get("allowed_routes") + if not isinstance(allowed_routes, list) or not all( + isinstance(item, dict) + and all(isinstance(item.get(field), str) and item[field].strip() for field in ("source_id", "role", "url")) + for item in allowed_routes + ): + raise AcquisitionError( + "acquisition authorization requires typed allowed_routes entries", + failure_class="authorization", + action="bind authorization to exact FSIS source IDs and official URLs", + ) + expected_role = "demographics" if source_id.endswith(".demographics") else "directory" + if not any(item["source_id"] == source_id and item["role"] == expected_role and item["url"] == url for item in allowed_routes): + raise AcquisitionError( + f"acquisition authorization does not cover {source_id} at the configured official URL", + failure_class="authorization", + action="add the exact approved FSIS route to the private authorization record", + ) + terms_status = value.get("terms_status", "unknown") + if terms_status not in {"unknown", "pending_review"}: + raise AcquisitionError( + "Firefox acquisition authorization cannot claim redistribution terms approval", + failure_class="authorization", + action="record terms as unknown or pending_review; publication remains blocked", + ) + return { + "status": "authorized", + "basis": value["basis"], + "scope": value["scope"], + "restrictions": restrictions, + "allowed_routes": allowed_routes, + "recorded_at_utc": value.get("recorded_at_utc"), + "terms_status": terms_status, + "publication_status": "not_eligible", + } + + +def acquire_firefox( + *, + source_id: str, + page_url: str, + url: str, + output_root: str | Path, + artifact_name: str, + terms_review_path: str | Path | None, + acquisition_authorization: dict[str, Any] | None, + run_id: str | None = None, + max_attempts: int = 2, + max_bytes: int = 128 * 1024 * 1024, + navigation_timeout_seconds: float = 60.0, + download_timeout_seconds: float = 90.0, + effective_date: str | None = None, + publication_date: str | None = None, + code_version: str = "unknown", + config_version: str = "unknown", + coverage: str | None = None, + rights_caveat: str | None = None, + privacy_caveat: str | None = None, + artifact_validator: Callable[[Path, dict[str, str]], None] | None = None, + driver_opener: Callable[[Path], tuple[Any, dict[str, Any]]] = _open_driver, +) -> dict[str, Any]: + if not source_id or not page_url or not url or not artifact_name: + raise AcquisitionError("source_id, page_url, url, and artifact_name are required", failure_class="configuration") + if max_bytes <= 0 or navigation_timeout_seconds <= 0 or download_timeout_seconds <= 0: + raise AcquisitionError("Firefox acquisition bounds must be positive", failure_class="configuration") + if not 1 <= max_attempts <= 3: + raise AcquisitionError("Firefox max_attempts must be between 1 and 3", failure_class="configuration") + authorization = _validate_acquisition_authorization(acquisition_authorization, source_id=source_id, url=url) + terms_review = require_terms_review(Path(terms_review_path)) if terms_review_path is not None else { + "status": "unknown", + "required_before_publication": True, + } + run_id = run_id or (utc_now().replace(":", "").replace("-", "") + "-" + uuid.uuid4().hex[:8]) + run_dir = Path(output_root) / source_id / run_id + artifact_path = run_dir / artifact_name + attempts: list[dict[str, Any]] = [] + requested_at = utc_now() + last_runtime: dict[str, Any] = {} + + for attempt_number in range(1, max_attempts + 1): + attempt_dir = run_dir / "browser-attempts" / f"attempt-{attempt_number}" + attempt_dir.mkdir(parents=True, exist_ok=True) + # Keep the transient browser download path short. Firefox on Windows + # can silently fail to complete a download when the private run path + # is deeply nested near the legacy MAX_PATH boundary. + download_dir = Path(tempfile.mkdtemp(prefix="fsis-firefox-")) + driver = None + navigation_mode = "direct-official-url" + timeout_after_download_start = False + try: + driver, runtime = driver_opener(download_dir) + last_runtime = runtime + driver.set_page_load_timeout(navigation_timeout_seconds) + driver.get(page_url) + by = runtime.get("by") + if by is not None: + link_deadline = time.monotonic() + min(30.0, navigation_timeout_seconds) + while time.monotonic() < link_deadline: + if any(link.get_attribute("href") == url for link in driver.find_elements(by.CSS_SELECTOR, "a[href]")): + break + time.sleep(0.5) + for control in driver.find_elements(by.CSS_SELECTOR, "button, summary, [role='button']"): + label = " ".join(control.text.split()) + if "Active Establishment MPI Data Files and Other References" in label and control.is_displayed() and control.is_enabled(): + driver.execute_script("arguments[0].scrollIntoView({block: 'center'});", control) + control.send_keys("\ue007") + time.sleep(2) + break + # The configured official URL is the observed public download + # route. Navigate to it directly after the landing-page check so + # duplicate/collapsed links cannot change the selected artifact. + try: + driver.get(url) + except runtime["timeout_exception"]: + timeout_after_download_start = True + deadline = time.monotonic() + download_timeout_seconds + downloaded = None + while time.monotonic() < deadline: + downloaded = _complete_download(download_dir, max_bytes=max_bytes) + if downloaded is not None: + break + time.sleep(0.5) + if downloaded is None: + observed_files = [] + for path in sorted(download_dir.iterdir()): + if path.is_file(): + observed_files.append({"name": path.name, "byte_size": path.stat().st_size}) + raise AcquisitionError( + "Firefox did not produce one complete CSV download before the bounded timeout", + failure_class="browser-download-timeout", + retryable=True, + action="inspect the private browser attempt or use the authorized operator capture route", + ) + raw_size = downloaded.stat().st_size + digest = hashlib.sha256(downloaded.read_bytes()).hexdigest() + if artifact_validator is not None: + artifact_validator(downloaded, {}) + if artifact_path.exists(): + existing_digest = hashlib.sha256(artifact_path.read_bytes()).hexdigest() + if existing_digest != digest: + raise AcquisitionError("existing browser artifact differs from newly acquired bytes", failure_class="artifact-collision", action="use a new run id and preserve both observations") + else: + artifact_path.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(downloaded, artifact_path) + attempts.append({ + "attempt": attempt_number, + "outcome": "success", + "navigation_mode": navigation_mode, + "navigation_timeout_after_download_start": timeout_after_download_start, + "artifact_verified": True, + }) + metadata = { + "acquisition_method": "firefox_browser_download", + "method_choice": "firefox", + "source_id": source_id, + "artifact": artifact_name, + "artifact_path": str(artifact_path), + "run_id": run_id, + "requested_url": url, + "final_url": url, + "page_url": page_url, + "requested_at_utc": requested_at, + "retrieved_at_utc": utc_now(), + "effective_date": effective_date or "unknown", + "publication_date": publication_date, + "sha256": digest, + "byte_size": raw_size, + "code_version": code_version, + "config_version": config_version, + "coverage": coverage, + "rights_caveat": rights_caveat, + "privacy_caveat": privacy_caveat, + "terms_review": terms_review, + "acquisition_authorization": authorization, + "attempts": attempts, + "browser": {key: value for key, value in last_runtime.items() if key not in {"timeout_exception", "by"}}, + "navigation_mode": navigation_mode, + "navigation_timeout_after_download_start": timeout_after_download_start, + "retention": {"class": "restricted-research-evidence", "public_exposure": False, "review_required": True}, + } + atomic_json(run_dir / "acquisition-metadata.json", metadata) + return metadata + except AcquisitionError as error: + attempt_record = {"attempt": attempt_number, "outcome": "failed", "failure_class": error.failure_class, "retryable": error.retryable, "navigation_mode": navigation_mode} + if error.failure_class == "browser-download-timeout": + attempt_record["observed_files"] = locals().get("observed_files", []) + attempts.append(attempt_record) + if attempt_number == max_attempts or not error.retryable: + atomic_json(run_dir / "acquisition-failure.json", { + "schema_version": "acquisition-failure-v1", + "source_id": source_id, + "run_id": run_id, + "failure_class": error.failure_class, + "retryable": error.retryable, + "error": str(error), + "action": error.action, + "attempts": attempts, + "artifact_created": False, + "public_exposure": False, + }) + raise + except Exception as error: + safe_error = AcquisitionError( + f"Firefox acquisition failed: {type(error).__name__}", + failure_class="browser-runtime", + retryable=False, + action="inspect the private browser attempt and runtime setup", + ) + atomic_json(run_dir / "acquisition-failure.json", { + "schema_version": "acquisition-failure-v1", "source_id": source_id, "run_id": run_id, + "failure_class": safe_error.failure_class, "retryable": False, "error": str(safe_error), + "action": safe_error.action, "attempts": attempts, "artifact_created": False, "public_exposure": False, + }) + raise safe_error from error + finally: + if driver is not None: + try: + driver.quit() + except Exception: + pass + if not any(item.get("outcome") == "success" and item.get("attempt") == attempt_number for item in attempts): + _remove_downloads(download_dir) + shutil.rmtree(download_dir, ignore_errors=True) + raise AcquisitionError("Firefox acquisition retry loop did not complete", failure_class="runtime") diff --git a/pipeline/sources/us/fsis/refresh.py b/pipeline/sources/us/fsis/refresh.py index 05ba270..f7ec301 100644 --- a/pipeline/sources/us/fsis/refresh.py +++ b/pipeline/sources/us/fsis/refresh.py @@ -15,6 +15,7 @@ from pipeline.contracts.source_lifecycle import atomic_json from .adapter import CONFIG, FsisContractError, FsisMpiAdapter, _csv, _header_key +from .firefox_acquisition import acquire_firefox _HTML_SIGNATURES = ( @@ -160,6 +161,18 @@ def _artifact(metadata: dict[str, Any]) -> SourceArtifact: ) +def _load_acquisition_authorization(path: str | Path | None) -> dict[str, Any] | None: + if path is None: + return None + try: + value = json.loads(Path(path).read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as error: + raise ValueError(f"acquisition authorization cannot be read: {error}") from error + if not isinstance(value, dict): + raise ValueError("acquisition authorization must be a JSON object") + return value + + def _drift(manifest: dict[str, Any], previous_manifest: str | Path | None) -> dict[str, Any]: if previous_manifest is None: return {"checked": False, "blocked": False, "alarms": []} @@ -199,6 +212,8 @@ def refresh( directory_path: str | Path | None = None, demographics_path: str | Path | None = None, fetch: bool = False, + acquisition_method: str = "http", + acquisition_authorization: dict[str, Any] | None = None, source_url: str = CONFIG["directory_url"], retrieved_at_utc: str | None = None, effective_date: str | None = None, @@ -221,34 +236,48 @@ def refresh( raise ValueError("specify a directory artifact or fetch") if mode not in {"dry-run", "handoff"}: raise ValueError("mode must be dry-run or handoff") + if acquisition_method not in {"http", "firefox"}: + raise ValueError("acquisition_method must be http or firefox") root = Path(run_dir) metadata: dict[str, Any] = {} paths: dict[str, Path] = {} if fetch: - if terms_review_path is None: + if acquisition_method == "http" and terms_review_path is None: raise ValueError("terms_review_path is required for network acquisition") + if acquisition_method == "firefox" and acquisition_authorization is None: + raise ValueError("acquisition_authorization is required for Firefox acquisition") routes = { - "directory": CONFIG.get("directory_by_number_url") or CONFIG["data_url"], + "directory": source_url if source_url != CONFIG["directory_url"] else CONFIG.get("directory_by_number_url") or CONFIG["data_url"], "demographics": CONFIG.get("demographics_url"), } for role, url in routes.items(): if not url: raise ValueError(f"missing configured FSIS {role} URL") try: - acquired = fetch_source( - source_id=f"{CONFIG['source_id']}.{role}", url=url, output_root=root / "acquisition", - artifact_name=f"{role}.csv", terms_review_path=terms_review_path, max_bytes=max_bytes, - allowed_content_types=("text/csv", "application/csv", "application/octet-stream"), - code_version=CONFIG["adapter_version"], config_version=CONFIG["contract_version"], - coverage="FSIS MPI edition only; state-inspection and APHIS populations excluded", - rights_caveat="terms review retained with run", privacy_caveat="private staging; privacy review pending", - effective_date=effective_date, - max_attempts=max_attempts, - retry_delay_seconds=retry_delay_seconds, - max_retry_delay_seconds=max_retry_delay_seconds, - artifact_validator=lambda path, headers, role=role: _validate_download(path, headers, role=role), - ) + common = { + "source_id": f"{CONFIG['source_id']}.{role}", "url": url, "artifact_name": f"{role}.csv", + "terms_review_path": terms_review_path, "max_bytes": max_bytes, + "code_version": CONFIG["adapter_version"], "config_version": CONFIG["contract_version"], + "coverage": "FSIS MPI edition only; state-inspection and APHIS populations excluded", + "rights_caveat": "terms review retained with run", "privacy_caveat": "private staging; privacy review pending", + "effective_date": effective_date, + } + if acquisition_method == "firefox": + acquired = acquire_firefox( + **common, acquisition_authorization=acquisition_authorization, + page_url=CONFIG["directory_url"], output_root=root / "acquisition", + max_attempts=max_attempts, + artifact_validator=lambda path, headers, role=role: _validate_download(path, headers, role=role), + ) + else: + acquired = fetch_source( + **common, output_root=root / "acquisition", + allowed_content_types=("text/csv", "application/csv", "application/octet-stream"), + max_attempts=max_attempts, retry_delay_seconds=retry_delay_seconds, + max_retry_delay_seconds=max_retry_delay_seconds, + artifact_validator=lambda path, headers, role=role: _validate_download(path, headers, role=role), + ) except AcquisitionError: # Preserve the shared failure class, retryability, and attempt # ledger for the aggregate operator report. The role remains @@ -331,6 +360,8 @@ def main() -> int: parser.add_argument("--demographics", type=Path) parser.add_argument("--run-dir", type=Path, required=True) parser.add_argument("--source-url", default=CONFIG["directory_url"]) + parser.add_argument("--acquisition-method", choices=("http", "firefox"), default="http") + parser.add_argument("--acquisition-authorization", type=Path, help="private owner authorization JSON required by the Firefox method") parser.add_argument("--retrieved-at-utc") parser.add_argument("--effective-date") parser.add_argument("--previous-manifest", type=Path) @@ -343,9 +374,12 @@ def main() -> int: parser.add_argument("--max-age-days", type=int, default=14) args = parser.parse_args() try: + acquisition_authorization = _load_acquisition_authorization(args.acquisition_authorization) result = refresh( run_dir=args.run_dir, raw_path=args.raw, directory_path=args.directory, demographics_path=args.demographics, fetch=args.fetch, source_url=args.source_url, retrieved_at_utc=args.retrieved_at_utc, + acquisition_method=args.acquisition_method, + acquisition_authorization=acquisition_authorization, effective_date=args.effective_date, mode=args.mode, terms_review_path=args.terms_review, previous_manifest=args.previous_manifest, max_bytes=args.max_bytes, max_attempts=args.max_attempts, retry_delay_seconds=args.retry_delay_seconds, diff --git a/pipeline/sources/us/fsis/test_firefox_acquisition.py b/pipeline/sources/us/fsis/test_firefox_acquisition.py new file mode 100644 index 0000000..eaf53bd --- /dev/null +++ b/pipeline/sources/us/fsis/test_firefox_acquisition.py @@ -0,0 +1,176 @@ +import json +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +from pipeline.common.acquisition import AcquisitionError + +from .firefox_acquisition import _remove_downloads, acquire_firefox + + +ROOT = Path(__file__).parent + + +class FakeTimeout(Exception): + pass + + +class FakeBy: + CSS_SELECTOR = "css selector" + + +class FakeDriver: + capabilities = {"browserVersion": "synthetic-firefox"} + + def __init__(self, download_dir: Path, payload: bytes, *, timeout_after_download: bool = False): + self.download_dir = download_dir + self.payload = payload + self.timeout_after_download = timeout_after_download + self.urls: list[str] = [] + self.quit_called = False + + def set_page_load_timeout(self, _seconds): + return None + + def get(self, url): + self.urls.append(url) + if url.endswith(".csv"): + (self.download_dir / "source.csv").write_bytes(self.payload) + if self.timeout_after_download: + raise FakeTimeout("download completed while navigation remained pending") + + def find_elements(self, _selector, _value): + return [] + + def quit(self): + self.quit_called = True + + +def _terms(root: Path) -> Path: + path = root / "terms.json" + path.write_text(json.dumps({ + "reviewer": "synthetic-test-operator", + "reference": "synthetic", + "reviewed_at": "2026-09-15T00:00:00Z", + "decision": "approved", + "notes": "synthetic test only", + }), encoding="utf-8") + return path + + +def _authorization() -> dict: + return { + "status": "authorized", + "basis": "synthetic test owner authorization", + "scope": "official FSIS test route; private staging only", + "allowed_routes": [{"source_id": "us.fsis.directory", "role": "directory", "url": "https://example.test/directory.csv"}], + "restrictions": ["no public release", "no credentials", "no bypass"], + "recorded_at_utc": "2026-09-20T00:00:00Z", + "terms_status": "unknown", + } + + +class FirefoxAcquisitionTests(unittest.TestCase): + def test_cleanup_failure_does_not_escape(self): + with tempfile.TemporaryDirectory() as directory: + download_dir = Path(directory) + (download_dir / "partial.part").write_bytes(b"partial") + with patch.object(Path, "unlink", side_effect=OSError("synthetic cleanup failure")): + _remove_downloads(download_dir) + + def test_complete_file_survives_navigation_timeout_and_records_browser_provenance(self): + payload = (ROOT / "fixtures/valid.csv").read_bytes() + drivers: list[FakeDriver] = [] + + def opener(download_dir): + driver = FakeDriver(download_dir, payload, timeout_after_download=True) + drivers.append(driver) + return driver, {"browser": "Firefox", "browser_version": "synthetic-firefox", "selenium_version": "synthetic-selenium", "profile": "fresh-temporary", "timeout_exception": FakeTimeout, "by": FakeBy} + + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + metadata = acquire_firefox( + source_id="us.fsis.directory", page_url="https://example.test/page", + url="https://example.test/directory.csv", output_root=root / "raw", + artifact_name="directory.csv", terms_review_path=None, run_id="browser-timeout", + acquisition_authorization=_authorization(), + navigation_timeout_seconds=0.1, + artifact_validator=lambda path, _headers: self.assertEqual(path.read_bytes(), payload), driver_opener=opener, + ) + self.assertEqual(metadata["acquisition_method"], "firefox_browser_download") + self.assertTrue(metadata["navigation_timeout_after_download_start"]) + self.assertEqual(metadata["browser"]["browser_version"], "synthetic-firefox") + self.assertEqual(metadata["terms_review"]["status"], "unknown") + self.assertEqual(metadata["acquisition_authorization"]["status"], "authorized") + self.assertTrue(drivers[0].quit_called) + self.assertTrue((root / "raw/us.fsis.directory/browser-timeout/directory.csv").exists()) + + def test_firefox_requires_owner_authorization_separately_from_terms(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + with self.assertRaisesRegex(AcquisitionError, "authorization"): + acquire_firefox( + source_id="us.fsis.directory", page_url="https://example.test/page", + url="https://example.test/directory.csv", output_root=root / "raw", + artifact_name="directory.csv", terms_review_path=None, acquisition_authorization=None, + run_id="missing-authorization", driver_opener=lambda _path: (_ for _ in ()).throw(AssertionError("driver must not open")), + ) + + def test_firefox_rejects_unbound_or_malformed_authorization(self): + payload = (ROOT / "fixtures/valid.csv").read_bytes() + + def opener(download_dir): + return FakeDriver(download_dir, payload), {"browser": "Firefox", "browser_version": "synthetic", "selenium_version": "synthetic", "profile": "fresh-temporary", "timeout_exception": FakeTimeout, "by": FakeBy} + + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + unbound = _authorization() + unbound["allowed_routes"] = [{"source_id": "us.fsis.demographics", "role": "demographics", "url": "https://example.test/demographics.csv"}] + with self.assertRaisesRegex(AcquisitionError, "does not cover"): + acquire_firefox( + source_id="us.fsis.directory", page_url="https://example.test/page", url="https://example.test/directory.csv", + output_root=root / "raw", artifact_name="directory.csv", terms_review_path=None, + acquisition_authorization=unbound, run_id="unbound", navigation_timeout_seconds=0.1, driver_opener=opener, + ) + malformed = _authorization() + malformed["restrictions"] = "no-public-release" + with self.assertRaisesRegex(AcquisitionError, "restrictions"): + acquire_firefox( + source_id="us.fsis.directory", page_url="https://example.test/page", url="https://example.test/directory.csv", + output_root=root / "raw", artifact_name="directory.csv", terms_review_path=None, + acquisition_authorization=malformed, run_id="malformed", navigation_timeout_seconds=0.1, driver_opener=opener, + ) + def test_invalid_completed_file_fails_closed_without_retaining_body(self): + drivers: list[FakeDriver] = [] + + def opener(download_dir): + driver = FakeDriver(download_dir, b"challenge", timeout_after_download=True) + drivers.append(driver) + return driver, {"browser": "Firefox", "browser_version": "synthetic-firefox", "selenium_version": "synthetic-selenium", "profile": "fresh-temporary", "timeout_exception": FakeTimeout, "by": FakeBy} + + def reject(_path, _headers): + raise AcquisitionError("schema rejected", failure_class="content-signature", action="use assisted capture") + + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + with self.assertRaisesRegex(AcquisitionError, "schema rejected"): + acquire_firefox( + source_id="us.fsis.directory", page_url="https://example.test/page", + url="https://example.test/directory.csv", output_root=root / "raw", + artifact_name="directory.csv", terms_review_path=_terms(root), run_id="browser-invalid", + acquisition_authorization=_authorization(), + navigation_timeout_seconds=0.1, + artifact_validator=reject, driver_opener=opener, + ) + failure_path = root / "raw/us.fsis.directory/browser-invalid/acquisition-failure.json" + failure = json.loads(failure_path.read_text(encoding="utf-8")) + self.assertEqual(failure["failure_class"], "content-signature") + self.assertFalse(failure["artifact_created"]) + self.assertNotIn("challenge", failure_path.read_text(encoding="utf-8")) + self.assertFalse((root / "raw/us.fsis.directory/browser-invalid/directory.csv").exists()) + self.assertTrue(drivers[0].quit_called) + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/sources/us/refresh.py b/pipeline/sources/us/refresh.py index b4eb230..97e13cd 100644 --- a/pipeline/sources/us/refresh.py +++ b/pipeline/sources/us/refresh.py @@ -203,12 +203,15 @@ def _run_one(spec: dict[str, Any], base: Path, root: Path, mode: str, retry: dic directory = _path(spec.get("directory"), base) demographics = _path(spec.get("demographics"), base) fetch = bool(spec.get("fetch")) + authorization = _read_json(_path(spec.get("acquisition_authorization"), base)) if spec.get("acquisition_authorization") else None result = refresh_fsis( run_dir=source_root, directory_path=directory, demographics_path=demographics, fetch=fetch, source_url=spec.get("source_url") or FSIS_CONFIG["directory_url"], + acquisition_method=str(spec.get("acquisition_method") or "http"), + acquisition_authorization=authorization, retrieved_at_utc=spec.get("retrieved_at_utc"), effective_date=spec.get("effective_date"), mode=mode, From 8e52216edc3dc373066adaf302136b0e79cb547d Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sun, 20 Sep 2026 21:17:49 -0700 Subject: [PATCH 309/311] docs: establish canonical product readiness roadmap --- docs/PRODUCT-READINESS.md | 206 ++++++++++++++++++ docs/README.md | 8 + docs/V2-REVIEW-CLEANUP-2026-09-13.md | 35 --- docs/archive/README.md | 30 +++ .../audits}/DEAD-CODE-AUDIT-2026-09-16.md | 16 +- .../audits/V2-REVIEW-CLEANUP-2026-09-13.md | 35 +++ .../country-rehearsal-2026-09-15.json | 0 .../country-rehearsal-2026-09-15.md | 0 .../archive/research/v2-ideas.md | 2 +- .../sprints}/V2-BACKEND-PHASE-0-CLOSEOUT.md | 2 +- .../sprints}/V2-IMPLEMENTATION-TODO.md | 18 +- .../sprints}/V2-INTEGRATION-BASELINE.md | 16 +- .../sprints}/V2-SPRINT-2026-09-13.md | 0 .../sprint01-integration-execution.md | 0 docs/governance/v2-mvp-claim-evidence.json | 6 +- docs/source-status.json | 2 +- pipeline/tests/test_product_readiness.py | 68 ++++++ 17 files changed, 379 insertions(+), 65 deletions(-) create mode 100644 docs/PRODUCT-READINESS.md delete mode 100644 docs/V2-REVIEW-CLEANUP-2026-09-13.md create mode 100644 docs/archive/README.md rename docs/{ => archive/audits}/DEAD-CODE-AUDIT-2026-09-16.md (81%) create mode 100644 docs/archive/audits/V2-REVIEW-CLEANUP-2026-09-13.md rename docs/{ => archive/rehearsals}/country-rehearsal-2026-09-15.json (100%) rename docs/{ => archive/rehearsals}/country-rehearsal-2026-09-15.md (100%) rename v2-ideas.md => docs/archive/research/v2-ideas.md (98%) rename docs/{ => archive/sprints}/V2-BACKEND-PHASE-0-CLOSEOUT.md (96%) rename docs/{ => archive/sprints}/V2-IMPLEMENTATION-TODO.md (90%) rename docs/{ => archive/sprints}/V2-INTEGRATION-BASELINE.md (56%) rename docs/{ => archive/sprints}/V2-SPRINT-2026-09-13.md (100%) rename docs/{ => archive/sprints}/sprint01-integration-execution.md (100%) create mode 100644 pipeline/tests/test_product_readiness.py diff --git a/docs/PRODUCT-READINESS.md b/docs/PRODUCT-READINESS.md new file mode 100644 index 0000000..13ebeb9 --- /dev/null +++ b/docs/PRODUCT-READINESS.md @@ -0,0 +1,206 @@ +# Product readiness and V2 roadmap + +**Canonical authority:** this document is the sole product-level readiness and +overall V2 roadmap authority. It answers what is complete, what is verified, +what blocks release, and what comes next. Supporting documents provide +evidence for particular systems, sources, policies, or review packets; they do +not replace this page or make an overall completeness claim. + +## Authority and scope + +This page covers product completeness for the V2 platform and the controlled +replacement of the current V1 public application. It does not grant publication +approval, source permission, privacy clearance, or maintainer authority. The +governing policy is [ETHICS.md](ETHICS.md); its implementation checklist is +[governance/policy-implementation-todo.md](governance/policy-implementation-todo.md). + +Use [source-status.json](source-status.json) for source-level status and +[governance/v2-mvp-claim-evidence.json](governance/v2-mvp-claim-evidence.json) +for implementation claims. A count in this document is an aggregate candidate +or evidence count, never a count of approved, accurate, operating, or +publishable facilities unless explicitly labelled that way. + +## Verified checkpoint + +- **Baseline:** repository checkpoint `56b22705087788c6195002f96e0c132d1abf92e8`. +- **Current public product:** V1 remains production and the public default. +- **V2 frontend:** Svelte/TypeScript fixture and local synthetic preview; it is + not the production replacement and has no configured external tile service. +- **Public release:** no V2 public release has been created or promoted. +- **Evidence:** the current [Sprint 02 integration ledger](sprint02-integration-ledger.md), + [source status baseline](source-status.md), [current reacquisition notes](current-reacquisition.md), + and [current geospatial readiness](current-geospatial-readiness.md) are the + latest supporting records at this checkpoint. + +### Current private candidate evidence + +These figures are retained only as safe aggregates; raw and restricted payloads +remain private. They describe acquisition/normalization handoffs, not release +readiness. + +| Evidence family | Aggregate observed | State and limitation | +| --- | ---: | --- | +| APHIS registrations | 2,552 | normalized private candidate; review, privacy, and approval remain open | +| APHIS FY2025 annual reports | 995 | separate evidence family; not a facility master | +| APHIS inspections | 1,075 input / 1,071 accepted / 4 exact duplicates | inspection observations, not a complete inspection history | +| APHIS combined packet | 4,618 accepted; 2,121 identity links held | identity links remain held for review; no silent merge | +| FSIS current directory | 7,241 | private current candidate; source terms, identity, location, and review remain open | +| FSIS legacy comparison | 7,101 | legacy V1-derived comparison; not a currentness claim | +| France | 2,517 accepted | private candidate; source and publication gates remain open | +| Italy 853/2004 | 47,375 input / 41,849 accepted / 5,526 quarantined | repeated recognition/activity identities require review | + +Supporting source evidence includes the [APHIS refresh](aphis-lane1-refresh-2026-09-19.md), +[US source boundary](countries/us/README.md), [France handoff](countries/france/sprint02-handoff-20260919.md), +and [Italy review packet](review-packet-italy.md). No private rows or payloads +belong in this roadmap. + +## Status vocabulary + +Use exactly one status for each roadmap item or gate: + +- **Complete and verified** — implemented and supported by linked, reproducible evidence. +- **Implemented, not exercised** — code or documentation exists but the intended path has not been verified. +- **In progress** — active work has a defined owner and next action. +- **Blocked: engineering** — a technical dependency prevents progress. +- **Blocked: human review** — terms, privacy, factual review, approval, or another authorized decision is required. +- **Deferred** — intentionally sequenced after a named roadmap milestone. +- **Not started** — no implementation or review work has begun. + +Do not infer a green product or publication state from passing tests. Acquisition, +import, review, geocoding, approval, promotion, and publication are separate +states and must be reported separately. + +## Current product state + +| Workstream | Status | Current truth | Next action | +| --- | --- | --- | --- | +| Backend and database | Complete and verified | Rust/Axum API, PostGIS schema, migrations, release/profile concepts, provenance, suppression-aware projections, and candidate/import boundaries exist and are tested in synthetic/local environments. | Close the remaining live-wire contract and durable release/revocation gaps. | +| API contract | Implemented, not exercised | V2 contracts and generated/hand-authored schemas exist, but the complete frontend-facing contract is not yet frozen against a reviewed real release. | Product-convergence sprint: freeze DTOs, release identity, evidence identifiers, source dates, review state, and revocation semantics. | +| Frontend product | In progress | V2 is a developer preview using fixtures/local synthetic data; V1 vanilla JavaScript remains public. | Approve information architecture and visual direction, then build a production Svelte frontend. | +| Data acquisition | In progress | Several current private captures and source adapters exist; many countries remain reconnaissance-only or adapter/fixture-only. | Select one bounded first release and complete its source-specific terms and provenance review. | +| Identity and reconciliation | Blocked: human review | Source-local identities are preserved; held links and quarantines remain substantial. Cross-source merges are not automatic. | Review candidate identity links and publish only scoped, evidenced relationships. | +| Geospatial readiness | In progress | Geocoding is disabled or tightly bounded in current private handoffs; provider, precision, privacy, and review state must remain explicit. | Complete source-specific coordinate/privacy review and a production geocoder/provider decision. | +| Privacy and suppression | In progress | Policy and synthetic suppression paths exist, but independent durable restriction, replay, cache, and cross-V1/V2 propagation controls remain release gates. | Implement and exercise the durable ledger, pre-service restore gate, and suppression crosswalk. | +| Operations and deployment | Blocked: engineering | Local/private environments and runbooks exist; production proxy trust, visitor/provider audit, artifact inventory, rollback, and operational ownership are not fully verified. | Complete deployment/provider audit and an operator-run private release drill. | +| Publication and release authority | Blocked: human review | No current candidate has completed all source, privacy, factual, project-approval, and release-authority gates. | Obtain authorized review for the bounded first release; do not infer approval from acquisition or tests. | + +### Data lifecycle states + +The product readiness state is not a single data count: + +```text +acquisition → normalization/quarantine → candidate import → factual/privacy review +→ geocoding review → project approval for a named release/profile +→ promotion → publication +``` + +An acquired or imported record is not reviewed. A reviewed record is not +approved. An approved record is not promoted. A promoted record is not public +until publication is explicitly verified. Suppression and removal can interrupt +the chain at any stage and also apply to older releases, caches, exports, +reimports, and restores. + +## Launch gates + +All gates below must be green for a controlled V2 public cutover. “Green” means +the linked evidence exists and the relevant human decision is recorded where +required. + +| Gate | Status | Evidence | Next action | +| --- | --- | --- | --- | +| Governing ethics and privacy controls | In progress | [ETHICS.md](ETHICS.md), [policy checklist](governance/policy-implementation-todo.md) | Close outstanding implementation controls and verify behavior, not just prose. | +| Source terms and redistribution | Blocked: human review | [source rights decisions](architecture/source-rights-decisions.md), source-specific assessments | Record terms decision for each source in the first release. | +| Candidate acquisition and provenance | In progress | [source status](source-status.json), [Sprint 02 ledger](sprint02-integration-ledger.md) | Re-run selected sources with retained provenance and safe aggregate validation. | +| Identity and factual review | Blocked: human review | [US source boundary](countries/us/README.md), [Italy packet](review-packet-italy.md) | Adjudicate held links, quarantines, and contradictions without name/address guessing. | +| Coordinate and address privacy | In progress | [geospatial readiness](current-geospatial-readiness.md), [geocoding operator](geocoding-operator.md) | Complete precision, residential/private-location, provider, and review-state checks. | +| API and release contract | Implemented, not exercised | [V2 API contract](api/v2-contract.md), [MVP claim evidence](governance/v2-mvp-claim-evidence.md) | Freeze frontend contract against a named reviewed candidate release. | +| Suppression and revocation | Blocked: engineering | [suppression runbook](governance/suppression-runbook.md), [release manifest guidance](architecture/release-manifest-verification.md) | Add durable restriction ledger, replay gate, cache invalidation, and V1↔V2 crosswalk. | +| Frontend accessibility and performance | In progress | [frontend README](../frontend/README.md), [reviewed demonstration plan](reviewed-demonstration-release.md) | Redesign and test responsive, accessible, performant real-data-shaped views. | +| Deployment and visitor privacy | Blocked: engineering | [visitor privacy inventory](governance/visitor-privacy-inventory.md), [production operations](deployment/production-operations.md) | Verify proxy trust, logging/provider disclosures, rollback, monitoring, and ownership. | +| Authorized release approval | Blocked: human review | [reviewed demonstration release](reviewed-demonstration-release.md), [ETHICS.md](ETHICS.md) | Name the release/profile, record approval, and verify every public projection. | +| Public cutover and rollback | Not started | [V1/V2 reconciliation](architecture/v1-v2-reconciliation.md) | Run private E2E, parallel comparison, cutover rehearsal, then obtain explicit launch approval. | + +## Roadmap + +### C1 — Repository clarity and canonical readiness + +**Status: In progress.** Establish this page as the sole product-level roadmap, +archive historical overall-roadmap inputs, clarify the Denmark launcher, and +add deterministic documentation consistency checks. This sprint must not alter +data, migration history, publication behavior, or policy meaning. + +### Product convergence + +**Status: Not started.** Freeze the minimum V2 API contract, choose the first +bounded real-data release family, and approve a product information architecture +and visual direction that make evidence, uncertainty, accountability, and action +legible. Keep fixture/local synthetic data available while the design converges. + +### Production V2 frontend + +**Status: Deferred.** Build the redesigned Svelte frontend against the frozen +contract, including responsive/accessibility/performance work and complete +empty, restricted, error, provenance, and uncertainty states. Keep V1 routes and +rollback available during transition. + +### First reviewed private release + +**Status: Not started.** Select one bounded candidate family (FSIS is the current +leading candidate, subject to terms/privacy/review decisions), complete the +acquisition → import → review → geocoding → approval chain, and create a private +named release. APHIS research evidence remains a separate family unless an +explicit scoped relationship is approved. + +### Private end-to-end trial + +**Status: Not started.** Run the production-shaped frontend and API against the +reviewed private release. Test discovery, maps, profiles, evidence, exports, +suppression, revocation, mobile, accessibility, performance, and operator +recovery without public promotion. + +### V1/V2 parallel comparison + +**Status: Not started.** Compare route behavior, identity/suppression outcomes, +coverage labels, and user-critical journeys. Resolve differences explicitly; +do not silently replace V1 records or call absence closure. + +### Controlled cutover + +**Status: Not started.** Obtain explicit maintainer approval, deploy the named +release with public access paused during migration, verify current restrictions, +provider settings, rollback, monitoring, and visitor-facing disclosures, then +switch traffic while retaining a defined V1 rollback window. + +### Expansion + +**Status: Deferred.** Add further countries, source families, accountability +evidence, and richer story experiences only after the first complete release path +is repeatable and safe. Each addition gets its own source, privacy, identity, +geospatial, approval, and publication decision. + +## Update discipline + +- Update this page in the same integration change as any material readiness change. +- Every **Complete and verified** claim must link to repository evidence or a named, + reproducible test result. +- Keep aggregate counts separate from row-level payloads; never copy private data here. +- Keep acquisition, import, review, geocoding, approval, promotion, and publication + states explicit; do not collapse them into “complete.” +- Record the checkpoint commit/date and the evidence scope whenever the baseline changes. +- If evidence conflicts, mark the item blocked or in progress and record the conflict + rather than choosing the more favorable claim. +- Source-specific status belongs in `source-status.json` and its supporting packet; + this document links to it and summarizes only the product consequence. + +## Decision-log rules + +Record a decision here when it changes product scope, release sequencing, a launch +gate, a canonical entrypoint, or the meaning of a readiness status. Each entry +must include date, decision, scope, evidence, owner/authority, and the next review +point. Policy amendments belong in [governance/ethics-changelog.md](governance/ethics-changelog.md), +and source rights decisions belong in [architecture/source-rights-decisions.md](architecture/source-rights-decisions.md). + +| Date | Decision | Scope | Evidence / authority | Next review | +| --- | --- | --- | --- | --- | +| 2026-09-20 | Establish this document as the sole product-level readiness and overall V2 roadmap authority. | V2 product completeness and V1 replacement sequencing. | C1 approved scope; governing policy remains [ETHICS.md](ETHICS.md). | At the next integration sprint or any material gate change. | +| 2026-09-20 | Keep V1 public and V2 private/local until a reviewed named release completes all launch gates. | All public application surfaces. | [V2 API contract](api/v2-contract.md), [source status](source-status.json), [reviewed release guidance](reviewed-demonstration-release.md). | Before private E2E trial. | diff --git a/docs/README.md b/docs/README.md index ac2b3c5..0548b8e 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,5 +1,13 @@ # Documentation guide +## Product readiness authority + +[PRODUCT-READINESS.md](PRODUCT-READINESS.md) is the sole product-level source of +truth for V2 completeness, launch gates, and the overall roadmap. Supporting +documents may provide source, architecture, policy, or review evidence, but no +other active document is the overall V2 roadmap. Dated historical planning and +integration inputs are preserved in [archive/README.md](archive/README.md). + Documentation is grouped by the kind of decision it records: - [ETHICS.md](ETHICS.md) — governing policy for credibility, provenance, uncertainty, privacy, and publication; takes precedence over conflicting supporting guidance. diff --git a/docs/V2-REVIEW-CLEANUP-2026-09-13.md b/docs/V2-REVIEW-CLEANUP-2026-09-13.md deleted file mode 100644 index 0f3def7..0000000 --- a/docs/V2-REVIEW-CLEANUP-2026-09-13.md +++ /dev/null @@ -1,35 +0,0 @@ -# V2 review cleanup — 2026-09-13 - -Status: post-`integration-v2-2026-09-13` working-tree review, not part of the tag at `d853721`, a production release, or publication approval. The tagged baseline and its maintainer-reported CI pass remain documented in [V2-INTEGRATION-BASELINE.md](V2-INTEGRATION-BASELINE.md). This note records scoped fixes and remaining evidence gaps under [ETHICS.md](ETHICS.md). - -## Confirmed fixes in this review round - -| Area | Review outcome and in-repo evidence | -| --- | --- | -| Release-scoped publication | [Migration 022](../pipeline/migrations/022_publication_safety_scopes.sql) scopes publication decisions to a release, withholds ambiguous older decisions, accounts for restricted observations, and computes public-only history counts. [Validation](../pipeline/scripts/stages/validate-release.py) and [promotion](../pipeline/scripts/stages/promote-release.py) recheck approval and suppression against that release; neither creates a human approval event. | -| Community CSV | The [V2 export handler](../src/lib.rs) admits eligible community-profile rows, adds per-row factual-review, approval, profile notice and unreviewed-claim warning context, and identifies the selected export profile. [Synthetic API checks](../pipeline/tests/e2e/test_public_surface_safety.py) cover CSV/API parity and current suppression. | -| Country pipelines | The [shared orchestrator](../pipeline/common/orchestrator.py) and [delta comparison](../pipeline/common/delta.py) retain restricted, human-gated states and apply source-qualified suppression/identity handling. [UK composition](../pipeline/sources/uk/approved/compose.py) keeps FSA/FSS identities separate and blocks release creation; the [Germany BLtU adapter](../pipeline/germany/bltu_adapter.py) remains restricted and quarantines unresolved input. | -| Visitor-facing context | The [Svelte preview](../frontend/src/app/App.svelte) clears stale local responses, keeps direct-link community warnings and evidence context visible, and labels first-page-only results; [local browser safety tests](../frontend/tests/e2e/local-safety.spec.ts) use mocked V2 responses. The [static V2 adapter](../static/modules/v2Adapter.js), [contract](../static/modules/v2Contract.js), and [export](../static/modules/ExportManager.js) preserve publication context and avoid presenting a paginated loaded page as a complete export. | -| Rate limiting and CI | The [Axum middleware](../src/main.rs) keys limits by socket peer by default and accepts a parsed forwarded address only with explicit proxy trust. The [CI workflow](../.github/workflows/tests.yml) adds static Jest, public-surface E2E, gate self-test, and Svelte local-safety fixture coverage; the [combined gate](../pipeline/tests/run-v2-gate.ps1) now propagates standard/Jest failures. | -| Restore and release integrity | The [two-stage synthetic backup drill](../pipeline/tests/e2e/backup-restore.ps1) now uses a [portless Compose override](../pipeline/tests/e2e/docker-compose.backup-restore.yml), rejects an old restore before replay, then checks the current restriction after replay. The portless drill passed twice in local scoped verification. [Promotion](../pipeline/scripts/stages/promote-release.py) stores a canonical hashed manifest with an explicitly declared artifact list, and [verification guidance](architecture/release-manifest-verification.md) states its limits. | -| Local synthetic launcher | [local-v2.ps1](../pipeline/scripts/maintenance/local-v2.ps1) now explicitly passes `--no-distributed-artifacts` when promoting its synthetic release. A [narrow launcher contract test](../pipeline/tests/test_local_v2_start_contract.py) checks the parsed command and a mocked start branch without starting services; its scoped run passed. This declaration applies only to that local fixture. | - -### A/B regression chain and resolution - -The post-fix regression uses one synthetic source record in independent releases A and B. An ambiguous older source-only approval must not qualify either candidate; a release-A review permits A but does not let B inherit approval. A later B denial must block B's validation/promotion without relabeling or withdrawing the still-promoted A; a later B approval must not change A's review context. Current suppression still blocks publication regardless of either approval. [Migration 022](../pipeline/migrations/022_publication_safety_scopes.sql) provides a release-keyed current-review view (while its source-keyed compatibility view prefers a promoted decision), [validation/promotion](../pipeline/tests/test_publication_scoped_stages.py) exercise those gates, and the [Rust list/detail/CSV queries](../src/lib.rs) join review by both source record and release. A [two-profile HTTP E2E](../pipeline/tests/e2e/test_public_surface_safety.py) checks that a B denial removes B while A remains correctly labeled in list, detail, and CSV. These are synthetic regression checks, not publication decisions. - -### Final local verification - -The independent verifier reports the **final assembled local cleanup gate green, with no retries**: the standard runner passed 72 Rust tests and 102 Python tests (5 skipped) after migration 022; four sequential API E2E modules passed 36/36, including the cross-profile regression; the portless two-stage backup/restore drill passed; root Jest passed 19/19; frontend unit tests passed 41/41; and Playwright passed 42/42 across Chromium, Firefox, and WebKit. Frontend check, lint, boundary, and build; `cargo fmt`; `git diff --check`; and the PowerShell gate self-test (3/3) also passed. The verifier reports no disposable Docker project remains and persistent databases were untouched. These results supersede the earlier worker-scoped-only test status. **Remote CI for this cleanup round remains pending**; the older tag's reported CI pass is separate evidence. - -Migration 022 and the updated V2 API must be deployed together. Keep V2 public access paused throughout that transition: the old API does not use the new release-keyed review contract, and the updated API requires its database view. Resume only after migration/API compatibility and current restriction checks are verified, subject to the open release gates below. - -## Open launch blockers - -- A reviewed V1↔V2 identity/suppression crosswalk is still needed. [Public-surface tests](../pipeline/tests/e2e/test_public_surface_safety.py) check V1 compatibility but explicitly do not claim cross-system suppression propagation. -- The backup drill replays a synthetic restriction from its test fixture. Production needs an **independent durable restriction ledger**, replay and an enforced pre-service gate before any old-backup restore can serve public data; see the [drill](../pipeline/tests/e2e/backup-restore.ps1) and [policy checklist](governance/policy-implementation-todo.md). -- `UEC_TRUST_PROXY` needs deployment-specific reverse-proxy boundary configuration and verification. The code path alone does not establish which forwarding headers are trustworthy; see [API contract](architecture/api-location-contract.md) and [middleware](../src/main.rs). -- Manifest creation requires an operator-complete inventory of every distributed artifact; the tool cannot discover omissions. Database promotion and writing the manifest file are separate, non-atomic steps requiring recovery/verification before distribution; see [manifest guidance](architecture/release-manifest-verification.md). -- Svelte and static V2 views still operate on loaded pages, so client-side search/map/count/export scope can be partial. The [Svelte preview](../frontend/src/app/App.svelte) and [static export](../static/modules/ExportManager.js) label this limit, but full traversal and release-consistent product behavior remain unfinished. -- The new Svelte safety browser tests mock V2. The [real-backend browser test](../frontend/tests/e2e/local-backend.spec.ts) remains opt-in and is not part of the reviewed remote fixture-browser job; a full real-backend frontend E2E gate remains open. -- Source-specific acquisition/redistribution terms, privacy screening, suppression operations, authorized human release review, and deployment/visitor-provider audits remain release gates. See [Germany assessment](germany-source-assessment.md), [FSS assessment](countries/uk/fss-approved-establishments-source-assessment.md), [policy checklist](governance/policy-implementation-todo.md), and [ETHICS.md](ETHICS.md). No code or passing test supplies legal clearance or publication authority. diff --git a/docs/archive/README.md b/docs/archive/README.md new file mode 100644 index 0000000..9837545 --- /dev/null +++ b/docs/archive/README.md @@ -0,0 +1,30 @@ +# Archived planning and evidence + +This directory preserves dated planning, integration, audit, rehearsal, and +research material that is no longer the active overall roadmap. Nothing here was +deleted; the files remain historical evidence and may describe an earlier +checkpoint, test scope, or proposal. + +The sole current product-level readiness and roadmap authority is +[docs/PRODUCT-READINESS.md](../PRODUCT-READINESS.md). Current policy, +architecture, country workflows, source status, and review packets remain in +their active locations unless listed below. + +## Archive index + +| Archived document | Category | Status / use | Current authority | +| --- | --- | --- | --- | +| [DEAD-CODE-AUDIT-2026-09-16.md](audits/DEAD-CODE-AUDIT-2026-09-16.md) | Audit | Historical repository cleanup findings; no deletion authorization. | [PRODUCT-READINESS.md](../PRODUCT-READINESS.md) and active docs guide. | +| [V2-REVIEW-CLEANUP-2026-09-13.md](audits/V2-REVIEW-CLEANUP-2026-09-13.md) | Cleanup evidence | Historical post-tag review and local verification; not a release approval. | [PRODUCT-READINESS.md](../PRODUCT-READINESS.md) and [ETHICS.md](../ETHICS.md). | +| [V2-SPRINT-2026-09-13.md](sprints/V2-SPRINT-2026-09-13.md) | Sprint evidence | Historical integration record; its local test results are checkpoint-specific. | [PRODUCT-READINESS.md](../PRODUCT-READINESS.md). | +| [sprint01-integration-execution.md](sprints/sprint01-integration-execution.md) | Sprint evidence | Historical Sprint 01 execution handoff. | [PRODUCT-READINESS.md](../PRODUCT-READINESS.md). | +| [V2-INTEGRATION-BASELINE.md](sprints/V2-INTEGRATION-BASELINE.md) | Baseline | Historical non-production integration checkpoint. | [PRODUCT-READINESS.md](../PRODUCT-READINESS.md). | +| [V2-BACKEND-PHASE-0-CLOSEOUT.md](sprints/V2-BACKEND-PHASE-0-CLOSEOUT.md) | Baseline | Historical backend closeout evidence; not production authorization. | [PRODUCT-READINESS.md](../PRODUCT-READINESS.md). | +| [V2-IMPLEMENTATION-TODO.md](sprints/V2-IMPLEMENTATION-TODO.md) | Roadmap input | Superseded execution checklist; retained as historical context. | [PRODUCT-READINESS.md](../PRODUCT-READINESS.md). | +| [country-rehearsal-2026-09-15.md](rehearsals/country-rehearsal-2026-09-15.md) | Rehearsal | Private/test-only country rehearsal; no release was created. | [source-status.json](../source-status.json) and [PRODUCT-READINESS.md](../PRODUCT-READINESS.md). | +| [country-rehearsal-2026-09-15.json](rehearsals/country-rehearsal-2026-09-15.json) | Rehearsal evidence | Row-free aggregate rehearsal manifest; historical and non-public. | [source-status.json](../source-status.json). | +| [v2-ideas.md](research/v2-ideas.md) | Research/proposal | Historical product-direction input; proposals are not readiness claims. | [PRODUCT-READINESS.md](../PRODUCT-READINESS.md). | + +Archive files may contain links written for their original location. Links to +active authority are intentionally redirected where practical; historical links +are not evidence of current status. diff --git a/docs/DEAD-CODE-AUDIT-2026-09-16.md b/docs/archive/audits/DEAD-CODE-AUDIT-2026-09-16.md similarity index 81% rename from docs/DEAD-CODE-AUDIT-2026-09-16.md rename to docs/archive/audits/DEAD-CODE-AUDIT-2026-09-16.md index 6508258..823ddc1 100644 --- a/docs/DEAD-CODE-AUDIT-2026-09-16.md +++ b/docs/archive/audits/DEAD-CODE-AUDIT-2026-09-16.md @@ -26,13 +26,15 @@ but were not removed or redesigned. legacy or research inputs referenced by the source inventory and country crosswalks. They are data evidence, not dead code, and the governing policy requires preserving provenance and recovery boundaries. No files were deleted. -* `v2-ideas.md` remains a proposed roadmap and is explicitly linked by - `docs/V2-IMPLEMENTATION-TODO.md`. It is not presented as completed status, so - it is retained rather than silently removed. -* `docs/V2-SPRINT-2026-09-13.md`, `docs/V2-INTEGRATION-BASELINE.md`, and - `docs/V2-REVIEW-CLEANUP-2026-09-13.md` are dated integration evidence with - explicit non-production and evidence-scope language. Their overlap is - historical reporting, not redundant current instructions. +* The former `docs/archive/research/v2-ideas.md` proposal and + `docs/archive/sprints/V2-IMPLEMENTATION-TODO.md` are now preserved under the + archive; current completeness is tracked only in + `docs/PRODUCT-READINESS.md`. +* `docs/archive/sprints/V2-SPRINT-2026-09-13.md`, + `docs/archive/sprints/V2-INTEGRATION-BASELINE.md`, and + `docs/archive/audits/V2-REVIEW-CLEANUP-2026-09-13.md` are dated integration + evidence with explicit non-production and evidence-scope language. Their + overlap is historical reporting, not redundant current instructions. * `docs/PIPELINE-MIGRATION.md` was last updated on 2026-09-16 and documents the shared artifact-boundary migration. Its “next consolidation target” language is source-specific status, not an unused entrypoint; it is retained. diff --git a/docs/archive/audits/V2-REVIEW-CLEANUP-2026-09-13.md b/docs/archive/audits/V2-REVIEW-CLEANUP-2026-09-13.md new file mode 100644 index 0000000..11421da --- /dev/null +++ b/docs/archive/audits/V2-REVIEW-CLEANUP-2026-09-13.md @@ -0,0 +1,35 @@ +# V2 review cleanup — 2026-09-13 + +Status: post-`integration-v2-2026-09-13` working-tree review, not part of the tag at `d853721`, a production release, or publication approval. The tagged baseline and its maintainer-reported CI pass remain documented in [V2-INTEGRATION-BASELINE.md](../sprints/V2-INTEGRATION-BASELINE.md). This note records scoped fixes and remaining evidence gaps under [ETHICS.md](../../ETHICS.md). + +## Confirmed fixes in this review round + +| Area | Review outcome and in-repo evidence | +| --- | --- | +| Release-scoped publication | [Migration 022](../../../pipeline/migrations/022_publication_safety_scopes.sql) scopes publication decisions to a release, withholds ambiguous older decisions, accounts for restricted observations, and computes public-only history counts. [Validation](../../../pipeline/scripts/stages/validate-release.py) and [promotion](../../../pipeline/scripts/stages/promote-release.py) recheck approval and suppression against that release; neither creates a human approval event. | +| Community CSV | The [V2 export handler](../../../src/lib.rs) admits eligible community-profile rows, adds per-row factual-review, approval, profile notice and unreviewed-claim warning context, and identifies the selected export profile. [Synthetic API checks](../../../pipeline/tests/e2e/test_public_surface_safety.py) cover CSV/API parity and current suppression. | +| Country pipelines | The [shared orchestrator](../../../pipeline/common/orchestrator.py) and [delta comparison](../../../pipeline/common/delta.py) retain restricted, human-gated states and apply source-qualified suppression/identity handling. [UK composition](../../../pipeline/sources/uk/approved/compose.py) keeps FSA/FSS identities separate and blocks release creation; the [Germany BLtU adapter](../../../pipeline/germany/bltu_adapter.py) remains restricted and quarantines unresolved input. | +| Visitor-facing context | The [Svelte preview](../../../frontend/src/app/App.svelte) clears stale local responses, keeps direct-link community warnings and evidence context visible, and labels first-page-only results; [local browser safety tests](../../../frontend/tests/e2e/local-safety.spec.ts) use mocked V2 responses. The [static V2 adapter](../../../static/modules/v2Adapter.js), [contract](../../../static/modules/v2Contract.js), and [export](../../../static/modules/ExportManager.js) preserve publication context and avoid presenting a paginated loaded page as a complete export. | +| Rate limiting and CI | The [Axum middleware](../../../src/main.rs) keys limits by socket peer by default and accepts a parsed forwarded address only with explicit proxy trust. The [CI workflow](../../../.github/workflows/tests.yml) adds static Jest, public-surface E2E, gate self-test, and Svelte local-safety fixture coverage; the [combined gate](../../../pipeline/tests/run-v2-gate.ps1) now propagates standard/Jest failures. | +| Restore and release integrity | The [two-stage synthetic backup drill](../../../pipeline/tests/e2e/backup-restore.ps1) now uses a [portless Compose override](../../../pipeline/tests/e2e/docker-compose.backup-restore.yml), rejects an old restore before replay, then checks the current restriction after replay. The portless drill passed twice in local scoped verification. [Promotion](../../../pipeline/scripts/stages/promote-release.py) stores a canonical hashed manifest with an explicitly declared artifact list, and [verification guidance](../../architecture/release-manifest-verification.md) states its limits. | +| Local synthetic launcher | [local-v2.ps1](../../../pipeline/scripts/maintenance/local-v2.ps1) now explicitly passes `--no-distributed-artifacts` when promoting its synthetic release. A [narrow launcher contract test](../../../pipeline/tests/test_local_v2_start_contract.py) checks the parsed command and a mocked start branch without starting services; its scoped run passed. This declaration applies only to that local fixture. | + +### A/B regression chain and resolution + +The post-fix regression uses one synthetic source record in independent releases A and B. An ambiguous older source-only approval must not qualify either candidate; a release-A review permits A but does not let B inherit approval. A later B denial must block B's validation/promotion without relabeling or withdrawing the still-promoted A; a later B approval must not change A's review context. Current suppression still blocks publication regardless of either approval. [Migration 022](../../../pipeline/migrations/022_publication_safety_scopes.sql) provides a release-keyed current-review view (while its source-keyed compatibility view prefers a promoted decision), [validation/promotion](../../../pipeline/tests/test_publication_scoped_stages.py) exercise those gates, and the [Rust list/detail/CSV queries](../../../src/lib.rs) join review by both source record and release. A [two-profile HTTP E2E](../../../pipeline/tests/e2e/test_public_surface_safety.py) checks that a B denial removes B while A remains correctly labeled in list, detail, and CSV. These are synthetic regression checks, not publication decisions. + +### Final local verification + +The independent verifier reports the **final assembled local cleanup gate green, with no retries**: the standard runner passed 72 Rust tests and 102 Python tests (5 skipped) after migration 022; four sequential API E2E modules passed 36/36, including the cross-profile regression; the portless two-stage backup/restore drill passed; root Jest passed 19/19; frontend unit tests passed 41/41; and Playwright passed 42/42 across Chromium, Firefox, and WebKit. Frontend check, lint, boundary, and build; `cargo fmt`; `git diff --check`; and the PowerShell gate self-test (3/3) also passed. The verifier reports no disposable Docker project remains and persistent databases were untouched. These results supersede the earlier worker-scoped-only test status. **Remote CI for this cleanup round remains pending**; the older tag's reported CI pass is separate evidence. + +Migration 022 and the updated V2 API must be deployed together. Keep V2 public access paused throughout that transition: the old API does not use the new release-keyed review contract, and the updated API requires its database view. Resume only after migration/API compatibility and current restriction checks are verified, subject to the open release gates below. + +## Open launch blockers + +- A reviewed V1↔V2 identity/suppression crosswalk is still needed. [Public-surface tests](../../../pipeline/tests/e2e/test_public_surface_safety.py) check V1 compatibility but explicitly do not claim cross-system suppression propagation. +- The backup drill replays a synthetic restriction from its test fixture. Production needs an **independent durable restriction ledger**, replay and an enforced pre-service gate before any old-backup restore can serve public data; see the [drill](../../../pipeline/tests/e2e/backup-restore.ps1) and [policy checklist](../../governance/policy-implementation-todo.md). +- `UEC_TRUST_PROXY` needs deployment-specific reverse-proxy boundary configuration and verification. The code path alone does not establish which forwarding headers are trustworthy; see [API contract](../../architecture/api-location-contract.md) and [middleware](../../../src/main.rs). +- Manifest creation requires an operator-complete inventory of every distributed artifact; the tool cannot discover omissions. Database promotion and writing the manifest file are separate, non-atomic steps requiring recovery/verification before distribution; see [manifest guidance](../../architecture/release-manifest-verification.md). +- Svelte and static V2 views still operate on loaded pages, so client-side search/map/count/export scope can be partial. The [Svelte preview](../../../frontend/src/app/App.svelte) and [static export](../../../static/modules/ExportManager.js) label this limit, but full traversal and release-consistent product behavior remain unfinished. +- The new Svelte safety browser tests mock V2. The [real-backend browser test](../../../frontend/tests/e2e/local-backend.spec.ts) remains opt-in and is not part of the reviewed remote fixture-browser job; a full real-backend frontend E2E gate remains open. +- Source-specific acquisition/redistribution terms, privacy screening, suppression operations, authorized human release review, and deployment/visitor-provider audits remain release gates. See [Germany assessment](../../germany-source-assessment.md), [FSS assessment](../../countries/uk/fss-approved-establishments-source-assessment.md), [policy checklist](../../governance/policy-implementation-todo.md), and [ETHICS.md](../../ETHICS.md). No code or passing test supplies legal clearance or publication authority. diff --git a/docs/country-rehearsal-2026-09-15.json b/docs/archive/rehearsals/country-rehearsal-2026-09-15.json similarity index 100% rename from docs/country-rehearsal-2026-09-15.json rename to docs/archive/rehearsals/country-rehearsal-2026-09-15.json diff --git a/docs/country-rehearsal-2026-09-15.md b/docs/archive/rehearsals/country-rehearsal-2026-09-15.md similarity index 100% rename from docs/country-rehearsal-2026-09-15.md rename to docs/archive/rehearsals/country-rehearsal-2026-09-15.md diff --git a/v2-ideas.md b/docs/archive/research/v2-ideas.md similarity index 98% rename from v2-ideas.md rename to docs/archive/research/v2-ideas.md index 42b6e6c..d2882f6 100644 --- a/v2-ideas.md +++ b/docs/archive/research/v2-ideas.md @@ -4,7 +4,7 @@ Status: proposed scope, ready to refine. Product names and technology choices re ## First priority: establish an auditable, continuously maintained data system -This roadmap is governed by [docs/ETHICS.md](docs/ETHICS.md). Every preservation, immutable-history, and reconstruction requirement below is subject to its controlled retention/removal exceptions. Removed sensitive material must not be reconstructed or republished. Government source origin is not a factual accuracy guarantee. [Policy implementation tasks](docs/governance/policy-implementation-todo.md) track the removal runbook, release checks, public policy page, and end-to-end verification; these are publication requirements, not implemented guarantees. +This roadmap is governed by [docs/ETHICS.md](../../ETHICS.md). Every preservation, immutable-history, and reconstruction requirement below is subject to its controlled retention/removal exceptions. Removed sensitive material must not be reconstructed or republished. Government source origin is not a factual accuracy guarantee. [Policy implementation tasks](../../governance/policy-implementation-todo.md) track the removal runbook, release checks, public policy page, and end-to-end verification; these are publication requirements, not implemented guarantees. The current datasets are approximately a year old according to the developer and must all be preserved and migrated with an explicit `legacy` tag. Their exact source dates may be unknown. The migration date must never be presented as the date the underlying information was verified. diff --git a/docs/V2-BACKEND-PHASE-0-CLOSEOUT.md b/docs/archive/sprints/V2-BACKEND-PHASE-0-CLOSEOUT.md similarity index 96% rename from docs/V2-BACKEND-PHASE-0-CLOSEOUT.md rename to docs/archive/sprints/V2-BACKEND-PHASE-0-CLOSEOUT.md index 7151654..faa0ea1 100644 --- a/docs/V2-BACKEND-PHASE-0-CLOSEOUT.md +++ b/docs/archive/sprints/V2-BACKEND-PHASE-0-CLOSEOUT.md @@ -45,4 +45,4 @@ These do not invalidate the verified local/CI backend foundation, but they preve ## Handoff decision -The backend foundation is complete and certified for frontend contract work and integration. It is not yet authorized as the public production replacement for V1. The release blockers above remain linked to [V2-IMPLEMENTATION-TODO.md](V2-IMPLEMENTATION-TODO.md) and the governing [ETHICS.md](ETHICS.md). +The backend foundation is complete and certified for frontend contract work and integration. It is not yet authorized as the public production replacement for V1. The release blockers above remain linked to the [canonical product readiness roadmap](../../PRODUCT-READINESS.md) and the governing [ETHICS.md](../../ETHICS.md). diff --git a/docs/V2-IMPLEMENTATION-TODO.md b/docs/archive/sprints/V2-IMPLEMENTATION-TODO.md similarity index 90% rename from docs/V2-IMPLEMENTATION-TODO.md rename to docs/archive/sprints/V2-IMPLEMENTATION-TODO.md index 2aafc2d..558cb0a 100644 --- a/docs/V2-IMPLEMENTATION-TODO.md +++ b/docs/archive/sprints/V2-IMPLEMENTATION-TODO.md @@ -2,7 +2,7 @@ This is the execution plan for the V2 overhaul: an auditable backend/data platform and a scalable frontend discovery experience. -Current evidence is summarized in [V2-BACKEND-PHASE-0-CLOSEOUT.md](V2-BACKEND-PHASE-0-CLOSEOUT.md). That report distinguishes verified backend behavior from unresolved public-production blockers. +Current evidence is summarized in [V2-BACKEND-PHASE-0-CLOSEOUT.md](V2-BACKEND-PHASE-0-CLOSEOUT.md). That report distinguishes verified backend behavior from unresolved public-production blockers. The current overall roadmap is [PRODUCT-READINESS.md](../../PRODUCT-READINESS.md). The current production application remains V1 while this plan is executed. V2 must not be promoted merely because the API or frontend exists; promotion requires the relevant data, privacy, release, accessibility, performance, and rollback gates below. @@ -21,13 +21,13 @@ At the [2026-09-13 integration baseline](V2-INTEGRATION-BASELINE.md) (`integrati ### 2026-09-13 integration evidence -The tag annotation records a maintainer-reported CI pass, not a hosted run independently checked here. [Existing backend closeout evidence](V2-BACKEND-PHASE-0-CLOSEOUT.md) and the dated Phase 0/1 results below document earlier local and isolated test passes; [the CI workflow](../.github/workflows/tests.yml) now covers the Svelte fixture browser gate and [country adapter tests](../pipeline/tests/run-standard.ps1). See the [integration baseline](V2-INTEGRATION-BASELINE.md) for the exact scope and remaining release/ethics blockers. No tests were rerun for this documentation update. +The tag annotation records a maintainer-reported CI pass, not a hosted run independently checked here. [Existing backend closeout evidence](V2-BACKEND-PHASE-0-CLOSEOUT.md) and the dated Phase 0/1 results below document earlier local and isolated test passes; [the CI workflow](../../../.github/workflows/tests.yml) now covers the Svelte fixture browser gate and [country adapter tests](../../../pipeline/tests/run-standard.ps1). See the [integration baseline](V2-INTEGRATION-BASELINE.md) for the exact scope and remaining release/ethics blockers. No tests were rerun for this documentation update. ### 2026-09-13 post-tag review cleanup -The [review cleanup record](V2-REVIEW-CLEANUP-2026-09-13.md) distinguishes fixes now in the working tree and a verified green final local gate from remote CI, which remains pending. Migration 022, community CSV, restricted country staging, static/Svelte context, rate limiting, CI coverage, backup-drill staging, and release-manifest checks address specific findings. They do not close the Phase 0 suppression-propagation item or the Phase 5–7 launch gates. The V1↔V2 suppression crosswalk, independent durable replay before an old-backup restore can serve, deployment proxy trust, complete artifact inventory/atomic manifest delivery, partial loaded-page behavior, real-backend frontend E2E, source terms, and authorized human release review remain open; leave the broader phase checkboxes unchanged. +The [review cleanup record](../audits/V2-REVIEW-CLEANUP-2026-09-13.md) distinguishes fixes now in the working tree and a verified green final local gate from remote CI, which remains pending. Migration 022, community CSV, restricted country staging, static/Svelte context, rate limiting, CI coverage, backup-drill staging, and release-manifest checks address specific findings. They do not close the Phase 0 suppression-propagation item or the Phase 5–7 launch gates. The V1↔V2 suppression crosswalk, independent durable replay before an old-backup restore can serve, deployment proxy trust, complete artifact inventory/atomic manifest delivery, partial loaded-page behavior, real-backend frontend E2E, source terms, and authorized human release review remain open; leave the broader phase checkboxes unchanged. -Post-fix A/B regression checks now cover release-scoped review in migration 022 and validation/promotion, plus Rust list/detail/CSV behavior through a two-profile synthetic HTTP E2E: B's later denial cannot revoke or relabel promoted A, and B cannot inherit A's approval. The local synthetic launcher explicitly declares `--no-distributed-artifacts`, with a narrow contract test; the portless two-stage backup drill passed twice in scoped local verification and in the final gate. The independent verifier reports a no-retry final local pass: standard 72 Rust and 102 Python (5 skipped), four sequential API E2E modules 36/36, root Jest 19/19, frontend unit 41/41, Playwright 42/42 across three browsers, plus frontend check/lint/boundary/build, `cargo fmt`, `git diff --check`, and PowerShell gate self-test 3/3. No disposable Docker project remained; persistent databases were untouched. **Remote CI remains pending.** Deploy migration 022 and the updated V2 API together with V2 public access paused during the transition, then verify compatibility and current restrictions before any eligible access resumes. See the [review cleanup record](V2-REVIEW-CLEANUP-2026-09-13.md); production crosswalk, independent suppression replay, and the other release blockers remain open. +Post-fix A/B regression checks now cover release-scoped review in migration 022 and validation/promotion, plus Rust list/detail/CSV behavior through a two-profile synthetic HTTP E2E: B's later denial cannot revoke or relabel promoted A, and B cannot inherit A's approval. The local synthetic launcher explicitly declares `--no-distributed-artifacts`, with a narrow contract test; the portless two-stage backup drill passed twice in scoped local verification and in the final gate. The independent verifier reports a no-retry final local pass: standard 72 Rust and 102 Python (5 skipped), four sequential API E2E modules 36/36, root Jest 19/19, frontend unit 41/41, Playwright 42/42 across three browsers, plus frontend check/lint/boundary/build, `cargo fmt`, `git diff --check`, and PowerShell gate self-test 3/3. No disposable Docker project remained; persistent databases were untouched. **Remote CI remains pending.** Deploy migration 022 and the updated V2 API together with V2 public access paused during the transition, then verify compatibility and current restrictions before any eligible access resumes. See the [review cleanup record](../audits/V2-REVIEW-CLEANUP-2026-09-13.md); production crosswalk, independent suppression replay, and the other release blockers remain open. ### 2026-09-15 final integration evidence @@ -256,8 +256,8 @@ backend foundation ## Governing references -- [V2 design and implementation roadmap](../v2-ideas.md) -- [V2 location API contract](architecture/api-location-contract.md) -- [V2 data pipeline plan](architecture/data-pipeline-plan.md) -- [Ethics policy](ETHICS.md) -- [Policy implementation checklist](governance/policy-implementation-todo.md) +- [Archived V2 design and implementation roadmap](../research/v2-ideas.md) +- [V2 location API contract](../../architecture/api-location-contract.md) +- [V2 data pipeline plan](../../architecture/data-pipeline-plan.md) +- [Ethics policy](../../ETHICS.md) +- [Policy implementation checklist](../../governance/policy-implementation-todo.md) diff --git a/docs/V2-INTEGRATION-BASELINE.md b/docs/archive/sprints/V2-INTEGRATION-BASELINE.md similarity index 56% rename from docs/V2-INTEGRATION-BASELINE.md rename to docs/archive/sprints/V2-INTEGRATION-BASELINE.md index 5f06a5f..ed06df8 100644 --- a/docs/V2-INTEGRATION-BASELINE.md +++ b/docs/archive/sprints/V2-INTEGRATION-BASELINE.md @@ -4,18 +4,18 @@ Date: 2026-09-13. Checkpoint: annotated tag `integration-v2-2026-09-13` at commi ## What is integrated -- The current public application remains V1. A separate Svelte 5/TypeScript preview uses synthetic fixtures by default, with a local map, detail, profile warning, and limited export preview. It is staged only through an explicit preview command; no external map tiles are configured. See [frontend/README.md](../frontend/README.md), [App.svelte](../frontend/src/app/App.svelte), and the [fixture browser tests](../frontend/tests/e2e/fixture-platform.spec.ts). -- `?mode=local-v2` opts that preview into a loopback V2 API and a seeded synthetic release. The local helper starts PostGIS and Axum, while the development proxy connects the frontend; the local list/detail test is opt-in. This is a development integration path, not production frontend wiring. See [frontend/README.md](../frontend/README.md), [local-v2.ps1](../pipeline/scripts/maintenance/local-v2.ps1), [LocalLocationRepository.ts](../frontend/src/api/LocalLocationRepository.ts), and [local-backend.spec.ts](../frontend/tests/e2e/local-backend.spec.ts). -- Denmark has controlled acquisition and private staging: network retrieval needs an operator terms review, bounded development geocoding needs a separate per-run review, and database import and release promotion remain separate. See [pipeline/README.md](../pipeline/README.md) and [run-denmark-pipeline.py](../pipeline/run-denmark-pipeline.py). -- Germany BLtU has restricted, non-release staging with synthetic adapter tests. A previously acquired export remains private research evidence; automated reacquisition and public redistribution await a source-specific human terms decision. See [Germany assessment](germany-source-assessment.md) and [Germany adapter README](../pipeline/germany/README.md). -- UK FSA, FSS, and composition adapters are synthetic-fixture-only, with no live acquisition or release. The FSS assessment conditionally recommends restricted staging of the published CSV after provenance and privacy checks, while recurring retrieval and public release remain gated. See [UK composition](../pipeline/sources/uk/approved/README.md), [FSA adapter](../pipeline/sources/uk/fsa_approved/README.md), [FSS adapter](../pipeline/sources/uk/fss_approved/README.md), and [FSS assessment](countries/uk/fss-approved-establishments-source-assessment.md). +- The current public application remains V1. A separate Svelte 5/TypeScript preview uses synthetic fixtures by default, with a local map, detail, profile warning, and limited export preview. It is staged only through an explicit preview command; no external map tiles are configured. See [frontend/README.md](../../../frontend/README.md), [App.svelte](../../../frontend/src/app/App.svelte), and the [fixture browser tests](../../../frontend/tests/e2e/fixture-platform.spec.ts). +- `?mode=local-v2` opts that preview into a loopback V2 API and a seeded synthetic release. The local helper starts PostGIS and Axum, while the development proxy connects the frontend; the local list/detail test is opt-in. This is a development integration path, not production frontend wiring. See [frontend/README.md](../../../frontend/README.md), [local-v2.ps1](../../../pipeline/scripts/maintenance/local-v2.ps1), [LocalLocationRepository.ts](../../../frontend/src/api/LocalLocationRepository.ts), and [local-backend.spec.ts](../../../frontend/tests/e2e/local-backend.spec.ts). +- Denmark has controlled acquisition and private staging: network retrieval needs an operator terms review, bounded development geocoding needs a separate per-run review, and database import and release promotion remain separate. See [pipeline/README.md](../../../pipeline/README.md) and [run-denmark-pipeline.py](../../../pipeline/run-denmark-pipeline.py). +- Germany BLtU has restricted, non-release staging with synthetic adapter tests. A previously acquired export remains private research evidence; automated reacquisition and public redistribution await a source-specific human terms decision. See [Germany assessment](../../germany-source-assessment.md) and [Germany adapter README](../../../pipeline/germany/README.md). +- UK FSA, FSS, and composition adapters are synthetic-fixture-only, with no live acquisition or release. The FSS assessment conditionally recommends restricted staging of the published CSV after provenance and privacy checks, while recurring retrieval and public release remain gated. See [UK composition](../../../pipeline/sources/uk/approved/README.md), [FSA adapter](../../../pipeline/sources/uk/fsa_approved/README.md), [FSS adapter](../../../pipeline/sources/uk/fss_approved/README.md), and [FSS assessment](../../countries/uk/fss-approved-establishments-source-assessment.md). ## Evidence and remaining gates -The earlier [backend closeout](V2-BACKEND-PHASE-0-CLOSEOUT.md) records 65 Rust and 57 Python passes, isolated public/community/seeded API E2E passes (5/5/12), migration checks, and synthetic backup/restore; the [implementation TODO](V2-IMPLEMENTATION-TODO.md) also records 13 Jest passes and later contract-gate results. These are dated repository evidence, not tests rerun for this documentation change. The [CI workflow](../.github/workflows/tests.yml) defines standard, API E2E, backup/restore, and frontend check/test/lint/build/three-browser fixture jobs, including the reconciled country adapter tests in the [standard runner](../pipeline/tests/run-standard.ps1). +The earlier [backend closeout](V2-BACKEND-PHASE-0-CLOSEOUT.md) records 65 Rust and 57 Python passes, isolated public/community/seeded API E2E passes (5/5/12), migration checks, and synthetic backup/restore; the [implementation TODO](V2-IMPLEMENTATION-TODO.md) also records 13 Jest passes and later contract-gate results. These are dated repository evidence, not tests rerun for this documentation change. The [CI workflow](../../../.github/workflows/tests.yml) defines standard, API E2E, backup/restore, and frontend check/test/lint/build/three-browser fixture jobs, including the reconciled country adapter tests in the [standard runner](../../../pipeline/tests/run-standard.ps1). -Public V2 release remains blocked by incomplete live-wire contract and revocation semantics, source coverage and terms decisions, privacy screening and suppression propagation across API, map, export, caches, history, reimports, and restores, plus human review and release authority. Deployment TLS/provider configuration, visitor-data and provider logging/retention audits, correction/removal and legal-demand operations, and browser accessibility, mobile, and performance review also remain open. See [frontend gaps](../frontend/README.md), [backend blockers](V2-BACKEND-PHASE-0-CLOSEOUT.md), [policy implementation checklist](governance/policy-implementation-todo.md), and governing [ETHICS.md](ETHICS.md). No provider logging practice or jurisdiction-specific legal position is verified by this checkpoint. +Public V2 release remains blocked by incomplete live-wire contract and revocation semantics, source coverage and terms decisions, privacy screening and suppression propagation across API, map, export, caches, history, reimports, and restores, plus human review and release authority. Deployment TLS/provider configuration, visitor-data and provider logging/retention audits, correction/removal and legal-demand operations, and browser accessibility, mobile, and performance review also remain open. See [frontend gaps](../../../frontend/README.md), [backend blockers](V2-BACKEND-PHASE-0-CLOSEOUT.md), [policy implementation checklist](../../governance/policy-implementation-todo.md), and governing [ETHICS.md](../../ETHICS.md). No provider logging practice or jurisdiction-specific legal position is verified by this checkpoint. ## Post-tag review cleanup -The [2026-09-13 review cleanup](V2-REVIEW-CLEANUP-2026-09-13.md) records uncommitted fixes made after this tag and their remaining gates. It does not revise the tagged checkpoint or convert worker-scoped test passes into a combined-gate or remote-CI pass. V1 remains the public default; V2 publication still needs the cross-system suppression, restore, deployment, source, and human release decisions listed there. +The [2026-09-13 review cleanup](../audits/V2-REVIEW-CLEANUP-2026-09-13.md) records uncommitted fixes made after this tag and their remaining gates. It does not revise the tagged checkpoint or convert worker-scoped test passes into a combined-gate or remote-CI pass. V1 remains the public default; V2 publication still needs the cross-system suppression, restore, deployment, source, and human release decisions listed there. diff --git a/docs/V2-SPRINT-2026-09-13.md b/docs/archive/sprints/V2-SPRINT-2026-09-13.md similarity index 100% rename from docs/V2-SPRINT-2026-09-13.md rename to docs/archive/sprints/V2-SPRINT-2026-09-13.md diff --git a/docs/sprint01-integration-execution.md b/docs/archive/sprints/sprint01-integration-execution.md similarity index 100% rename from docs/sprint01-integration-execution.md rename to docs/archive/sprints/sprint01-integration-execution.md diff --git a/docs/governance/v2-mvp-claim-evidence.json b/docs/governance/v2-mvp-claim-evidence.json index 1933daa..dc81cd3 100644 --- a/docs/governance/v2-mvp-claim-evidence.json +++ b/docs/governance/v2-mvp-claim-evidence.json @@ -52,7 +52,7 @@ "claim": "Legacy data can be retained as explicitly private legacy evidence rather than relabeled as fresh verification.", "status": "implemented_tested", "evidence": [ - {"path": "v2-ideas.md", "line": 13}, + {"path": "docs/archive/research/v2-ideas.md", "line": 13}, {"path": "docs/real-v2-rehearsal.md", "line": 1}, {"path": "pipeline/tests/test_real_v2_rehearsal.py", "line": 1} ], @@ -356,7 +356,7 @@ "claim": "The flagship scale story is a finished, citable, accessible public experience that connects the horror of scale to local evidence without overclaiming.", "status": "prototype_only", "evidence": [ - {"path": "v2-ideas.md", "line": 242}, + {"path": "docs/archive/research/v2-ideas.md", "line": 242}, {"path": "docs/story/flagship-scale-evidence-spec.md", "line": 1}, {"path": "docs/story/storyboard.md", "line": 1} ], @@ -368,7 +368,7 @@ "claim": "An invited activist, journalist, or researcher can independently start the MVP, inspect provenance, use filters/exports, understand limitations, and provide feedback.", "status": "planned", "evidence": [ - {"path": "v2-ideas.md", "line": 29}, + {"path": "docs/archive/research/v2-ideas.md", "line": 29}, {"path": "docs/api/v2-contract.md", "line": 1}, {"path": "docs/governance/policy-implementation-todo.md", "line": 39} ], diff --git a/docs/source-status.json b/docs/source-status.json index e2b8971..4394ef7 100644 --- a/docs/source-status.json +++ b/docs/source-status.json @@ -1,7 +1,7 @@ { "schema_version": "1.0", "purpose": "Evidence-backed source readiness baseline; not a runtime health monitor or publication approval register.", - "latest_private_rehearsal": "docs/country-rehearsal-2026-09-15.json", + "latest_private_rehearsal": "docs/archive/rehearsals/country-rehearsal-2026-09-15.json", "latest_country_capture": "data/manifests/de-be-private-candidates-2026-09-17.json", "status_vocabulary": { "metadata": ["verified", "partial", "unknown"], diff --git a/pipeline/tests/test_product_readiness.py b/pipeline/tests/test_product_readiness.py new file mode 100644 index 0000000..36f9d08 --- /dev/null +++ b/pipeline/tests/test_product_readiness.py @@ -0,0 +1,68 @@ +"""Deterministic checks for the canonical product-readiness documentation.""" + +import re +import unittest +from pathlib import Path +from urllib.parse import unquote, urlsplit + + +ROOT = Path(__file__).resolve().parents[2] +LOCAL_LINK = re.compile(r"\[[^\]]+\]\(([^)]+)\)") +CANONICAL_DECLARATION = re.compile( + r"\*\*Canonical authority:\*\*\s*this document is the sole product-level\s+" + r"readiness\s+and\s+overall V2 roadmap authority\." +) + + +def local_markdown_targets(path: Path): + text = path.read_text(encoding="utf-8") + for match in LOCAL_LINK.finditer(text): + target = match.group(1).strip() + parsed = urlsplit(target) + if parsed.scheme or parsed.netloc or target.startswith("#"): + continue + relative = Path(unquote(parsed.path)) + if not relative: + continue + yield path.parent / relative + + +class ProductReadinessDocumentationTests(unittest.TestCase): + def test_exactly_one_canonical_overall_roadmap_declaration(self): + matches = [] + for path in (ROOT / "docs").rglob("*.md"): + for match in CANONICAL_DECLARATION.finditer(path.read_text(encoding="utf-8")): + matches.append((path, match.start())) + self.assertEqual(len(matches), 1) + self.assertEqual(matches[0][0], ROOT / "docs" / "PRODUCT-READINESS.md") + + def test_product_readiness_and_archive_index_links_resolve(self): + checked = [ROOT / "docs" / "PRODUCT-READINESS.md", ROOT / "docs" / "archive" / "README.md"] + missing = [] + for path in checked: + for target in local_markdown_targets(path): + if not target.resolve().exists(): + missing.append(f"{path.relative_to(ROOT)} -> {target}") + self.assertEqual(missing, []) + + def test_historical_overall_roadmap_inputs_are_archived(self): + expected = { + "docs/archive/audits/DEAD-CODE-AUDIT-2026-09-16.md", + "docs/archive/audits/V2-REVIEW-CLEANUP-2026-09-13.md", + "docs/archive/research/v2-ideas.md", + "docs/archive/sprints/V2-BACKEND-PHASE-0-CLOSEOUT.md", + "docs/archive/sprints/V2-IMPLEMENTATION-TODO.md", + "docs/archive/sprints/V2-INTEGRATION-BASELINE.md", + "docs/archive/sprints/V2-SPRINT-2026-09-13.md", + "docs/archive/sprints/sprint01-integration-execution.md", + "docs/archive/rehearsals/country-rehearsal-2026-09-15.md", + "docs/archive/rehearsals/country-rehearsal-2026-09-15.json", + } + for relative in expected: + self.assertTrue((ROOT / relative).is_file(), relative) + self.assertFalse((ROOT / "docs" / "V2-IMPLEMENTATION-TODO.md").exists()) + self.assertFalse((ROOT / "v2-ideas.md").exists()) + + +if __name__ == "__main__": + unittest.main() From 79c38897f552b69ddff8d72f9637af0b8b137e5f Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sun, 20 Sep 2026 21:11:06 -0700 Subject: [PATCH 310/311] docs: deprecate historical Denmark launcher --- docs/archive/sprints/V2-INTEGRATION-BASELINE.md | 2 +- pipeline/ONBOARDING.md | 10 ++++++---- pipeline/README.md | 17 ++++++++++++++--- pipeline/run-denmark-pipeline.py | 7 ++++++- pipeline/sources/denmark/README.md | 7 ++++--- pipeline/tests/test_denmark_entrypoints.py | 10 ++++++++++ 6 files changed, 41 insertions(+), 12 deletions(-) diff --git a/docs/archive/sprints/V2-INTEGRATION-BASELINE.md b/docs/archive/sprints/V2-INTEGRATION-BASELINE.md index ed06df8..ed90c16 100644 --- a/docs/archive/sprints/V2-INTEGRATION-BASELINE.md +++ b/docs/archive/sprints/V2-INTEGRATION-BASELINE.md @@ -6,7 +6,7 @@ Date: 2026-09-13. Checkpoint: annotated tag `integration-v2-2026-09-13` at commi - The current public application remains V1. A separate Svelte 5/TypeScript preview uses synthetic fixtures by default, with a local map, detail, profile warning, and limited export preview. It is staged only through an explicit preview command; no external map tiles are configured. See [frontend/README.md](../../../frontend/README.md), [App.svelte](../../../frontend/src/app/App.svelte), and the [fixture browser tests](../../../frontend/tests/e2e/fixture-platform.spec.ts). - `?mode=local-v2` opts that preview into a loopback V2 API and a seeded synthetic release. The local helper starts PostGIS and Axum, while the development proxy connects the frontend; the local list/detail test is opt-in. This is a development integration path, not production frontend wiring. See [frontend/README.md](../../../frontend/README.md), [local-v2.ps1](../../../pipeline/scripts/maintenance/local-v2.ps1), [LocalLocationRepository.ts](../../../frontend/src/api/LocalLocationRepository.ts), and [local-backend.spec.ts](../../../frontend/tests/e2e/local-backend.spec.ts). -- Denmark has controlled acquisition and private staging: network retrieval needs an operator terms review, bounded development geocoding needs a separate per-run review, and database import and release promotion remain separate. See [pipeline/README.md](../../../pipeline/README.md) and [run-denmark-pipeline.py](../../../pipeline/run-denmark-pipeline.py). +- Denmark has controlled acquisition and private staging: network retrieval needs an operator terms review, bounded development geocoding needs a separate per-run review, and database import and release promotion remain separate. Use the [canonical source-owned runner](../../../pipeline/sources/denmark/run-denmark-pipeline.py); the root-level [runner](../../../pipeline/run-denmark-pipeline.py) is a deprecated one-release compatibility wrapper. See [pipeline README](../../../pipeline/README.md). - Germany BLtU has restricted, non-release staging with synthetic adapter tests. A previously acquired export remains private research evidence; automated reacquisition and public redistribution await a source-specific human terms decision. See [Germany assessment](../../germany-source-assessment.md) and [Germany adapter README](../../../pipeline/germany/README.md). - UK FSA, FSS, and composition adapters are synthetic-fixture-only, with no live acquisition or release. The FSS assessment conditionally recommends restricted staging of the published CSV after provenance and privacy checks, while recurring retrieval and public release remain gated. See [UK composition](../../../pipeline/sources/uk/approved/README.md), [FSA adapter](../../../pipeline/sources/uk/fsa_approved/README.md), [FSS adapter](../../../pipeline/sources/uk/fss_approved/README.md), and [FSS assessment](../../countries/uk/fss-approved-establishments-source-assessment.md). diff --git a/pipeline/ONBOARDING.md b/pipeline/ONBOARDING.md index 04832b8..f91ddac 100644 --- a/pipeline/ONBOARDING.md +++ b/pipeline/ONBOARDING.md @@ -72,9 +72,10 @@ python pipeline/sources/denmark/run-denmark-pipeline.py path/to/private/Smileyda Acquisition requires an operator-approved terms review and is intentionally opt-in. Candidate import, geocoding, release validation, and promotion are -separate commands and separate gates. The older `pipeline/run-denmark-pipeline.py` -path remains a compatibility launcher; new source-specific documentation -should link to the source-owned path. +separate commands and separate gates. The older +`pipeline/run-denmark-pipeline.py` path remains a deprecated compatibility +launcher for one release; new source-specific documentation should link to the +source-owned path. ## Known onboarding friction @@ -82,7 +83,8 @@ should link to the source-owned path. describes the private V2 pipeline; contributors must choose the pipeline path before running `cargo run`. - Both historical and source-owned Denmark launchers exist. The source-owned - launcher is canonical; the historical path is retained for compatibility. + launcher is canonical; the historical path is deprecated and retained for + one release for compatibility. - Full local V2 API startup requires Docker/Postgres and a Rust build. It is not required for adapter contract tests and must use only the disposable local configuration in `pipeline/scripts/maintenance/local-v2.ps1`. diff --git a/pipeline/README.md b/pipeline/README.md index 15136ce..c575a59 100644 --- a/pipeline/README.md +++ b/pipeline/README.md @@ -20,13 +20,24 @@ It writes `data/manifests/legacy-files.csv` with one row per legacy input, inclu ## Controlled Denmark acquisition -The Denmark wrapper archives an artifact only; it never imports, validates for publication, promotes a release, or alters application data. Network retrieval is intentionally opt-in and requires an operator-authored terms review JSON with `reviewer`, `reference`, `reviewed_at`, `decision: "approved"`, and `notes`. +The canonical Denmark launcher is +`pipeline/sources/denmark/run-denmark-pipeline.py`. It archives an artifact +only; it never imports, validates for publication, promotes a release, or +alters application data. Network retrieval is intentionally opt-in and +requires an operator-authored terms review JSON with `reviewer`, `reference`, +`reviewed_at`, `decision: "approved"`, and `notes`. ```powershell python pipeline/scripts/stages/acquire-denmark-smiley.py --fetch --terms-review ``` -Raw XML and its deterministic `acquisition-metadata.json` are written under ignored `data/raw/dk.smiley//`. For offline development, use `--local-file path/to/synthetic.xml`; it needs no terms review and records that distinction. Existing staging runs can continue to take an already archived local XML path. `run-denmark-pipeline.py --fetch --terms-review ...` uses the wrapper first, then performs staging only; database import and release promotion remain separate commands. +Raw XML and its deterministic `acquisition-metadata.json` are written under +ignored `data/raw/dk.smiley//`. For offline development, use +`--local-file path/to/synthetic.xml`; it needs no terms review and records that +distinction. Existing staging runs can continue to take an already archived +local XML path. Database import and release promotion remain separate +commands. The historical `pipeline/run-denmark-pipeline.py` wrapper remains +available for one release as a deprecated compatibility path. ## Status vocabulary @@ -72,7 +83,7 @@ Script organization and execution conventions are documented in `scripts/README. The orchestrator runs the auditable stages in order and leaves database import as an explicit separate action: ```powershell -python pipeline/run-denmark-pipeline.py data/raw/denmark-smiley//Smileydata.xml +python pipeline/sources/denmark/run-denmark-pipeline.py data/raw/denmark-smiley//Smileydata.xml ``` Add `--geocode-limit 100` to run a bounded DAWA development sample. Every run gets numbered stage directories and a `pipeline-manifest.json` containing output sizes and SHA-256 checksums. diff --git a/pipeline/run-denmark-pipeline.py b/pipeline/run-denmark-pipeline.py index 3daf3f3..273d564 100644 --- a/pipeline/run-denmark-pipeline.py +++ b/pipeline/run-denmark-pipeline.py @@ -1,5 +1,10 @@ #!/usr/bin/env python3 -"""Compatibility shim for Denmark's source-owned staging runner.""" +"""Deprecated compatibility shim for Denmark's source-owned staging runner. + +Use ``pipeline/sources/denmark/run-denmark-pipeline.py`` for new commands. +This wrapper remains available for one release so existing operator scripts can +migrate without a behavior change. +""" from __future__ import annotations from pathlib import Path diff --git a/pipeline/sources/denmark/README.md b/pipeline/sources/denmark/README.md index 06595c7..1f8d370 100644 --- a/pipeline/sources/denmark/README.md +++ b/pipeline/sources/denmark/README.md @@ -1,9 +1,10 @@ # Denmark source entry points Denmark-specific commands are exposed from this directory so country logic has -a stable home as more sources are added. The historical commands under -`pipeline/scripts/stages/` and `pipeline/run-denmark-pipeline.py` remain valid -compatibility paths and retain their argument behavior. +a stable home as more sources are added. The source-owned runner below is the +canonical entrypoint. The historical `pipeline/run-denmark-pipeline.py` +launcher remains valid for one release as a deprecated compatibility path and +retains its argument behavior. The current launchers intentionally delegate to the established implementations to avoid duplicating acquisition, parsing, normalization, classification, and diff --git a/pipeline/tests/test_denmark_entrypoints.py b/pipeline/tests/test_denmark_entrypoints.py index c13bff6..7c9dbb9 100644 --- a/pipeline/tests/test_denmark_entrypoints.py +++ b/pipeline/tests/test_denmark_entrypoints.py @@ -8,6 +8,16 @@ class DenmarkEntrypointTests(unittest.TestCase): + def test_source_owned_entrypoint_is_canonical(self): + canonical = ROOT / "sources/denmark/run-denmark-pipeline.py" + self.assertIn("Source-owned entry point", canonical.read_text(encoding="utf-8")) + + def test_root_entrypoint_is_deprecated_compatibility_wrapper(self): + legacy = ROOT / "run-denmark-pipeline.py" + text = legacy.read_text(encoding="utf-8") + self.assertIn("Deprecated compatibility shim", text) + self.assertIn("one release", text) + def test_new_entrypoint_preserves_help_contract(self): result = subprocess.run( [sys.executable, str(ROOT / "sources/denmark/run-denmark-pipeline.py"), "--help"], From 73c848578d23e07d17b0cb32e8f13ce074d6a84e Mon Sep 17 00:00:00 2001 From: Eli Perez Date: Sun, 20 Sep 2026 21:12:56 -0700 Subject: [PATCH 311/311] test: guard migration identity by full filename stem --- docs/PIPELINE-MIGRATION.md | 22 ++++++++++++++++++++++ pipeline/tests/test_apply_migrations.py | 14 ++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/docs/PIPELINE-MIGRATION.md b/docs/PIPELINE-MIGRATION.md index bf0a829..cded03c 100644 --- a/docs/PIPELINE-MIGRATION.md +++ b/docs/PIPELINE-MIGRATION.md @@ -18,3 +18,25 @@ surfaces disabled, and treats disappearance as `not-observed`, never closure. Failures preserve the previous validated release and emit a restricted failure report. Acquisition terms approval and publication approval remain separate human gates. + +## Migration runner contract + +`pipeline/scripts/maintenance/apply-migrations.py` discovers every `*.sql` file +in the migration directory and sorts the paths lexically. The migration identity +stored in `uec.schema_migrations.version` is the complete filename stem, not the +numeric prefix. Consequently, `020_release_manifests.sql` and +`020_suppression_aware_v2_history.sql` have distinct identities and are both +applied in that lexical order. Migration files must not be renamed, edited, +deleted, squashed, or resequenced after they have been applied; a new change +gets a new filename. + +The runner records the SHA-256 digest of each migration and refuses to execute +an already-recorded identity when its file content has changed. Database setup +and each migration run in separate transactions. A failed migration rolls back +its SQL and ledger insert, while earlier successful migrations remain recorded +so a subsequent run can retry and resume at the failed identity. + +The database connection is retried up to ten times after an operational +connection error, waiting one second between attempts. Connection retries do +not alter migration ordering or checksum behavior. These rules are tested by +`pipeline/tests/test_apply_migrations.py` without connecting to a database. diff --git a/pipeline/tests/test_apply_migrations.py b/pipeline/tests/test_apply_migrations.py index 3a8ab6f..d431b23 100644 --- a/pipeline/tests/test_apply_migrations.py +++ b/pipeline/tests/test_apply_migrations.py @@ -63,6 +63,20 @@ def execute(self, query, params=None): class MigrationRunnerTests(unittest.TestCase): + def test_existing_020_migrations_use_distinct_full_stem_identities(self): + migration_dir = Path(__file__).parents[1] / "migrations" + + files = MODULE.migration_files(migration_dir) + names = [path.name for path in files] + stems = [path.stem for path in files] + + first = "020_release_manifests.sql" + second = "020_suppression_aware_v2_history.sql" + self.assertIn(first, names) + self.assertIn(second, names) + self.assertNotEqual(Path(first).stem, Path(second).stem) + self.assertLess(stems.index(Path(first).stem), stems.index(Path(second).stem)) + def test_failed_migration_is_not_recorded_and_retry_resumes(self): with tempfile.TemporaryDirectory() as directory: root = Path(directory)