Draft: front end overhaul

.dockerignore

@@ -1,4 +1,19 @@
 target
+data/
+private/
+.private/
+**/.private/
+staging/
+.tmp/
+tmp/
+**/.env
+**/.env.*
+*.env
+*.secret
+*.secrets
+**/*secret*
+**/*credential*
+**/*token*
 Old scripts
 Old CSVs
 .git

.github/workflows/fsis-route-diagnostic.yml

@@ -0,0 +1,71 @@
+name: FSIS route diagnostic
+
+on:
+  workflow_dispatch:
+  push:
+    paths:
+      - .github/workflows/fsis-route-diagnostic.yml
+
+permissions:
+  contents: read
+
+jobs:
+  probe:
+    name: Probe official FSIS routes
+    runs-on: ubuntu-latest
+    timeout-minutes: 3
+    steps:
+      - uses: actions/checkout@v4
+      - name: Bounded no-body route probes
+        shell: bash
+        run: |
+          set +e
+          byte_cap=1048576
+          printf '%s\n' '## FSIS route diagnostic (safe metrics)' >> "$GITHUB_STEP_SUMMARY"
+          python - <<'PY' | while IFS=$'\t' read -r route url_host url_path url; do
+          import json
+          from pathlib import Path
+          from urllib.parse import urlsplit
+
+          config = json.loads(Path("pipeline/sources/us/fsis/config.json").read_text())
+          for route, key in (
+              ("directory-by-name", "directory_by_name_url"),
+              ("directory-by-number", "directory_by_number_url"),
+              ("demographics", "demographics_url"),
+          ):
+              url = config[key]
+              parsed = urlsplit(url)
+              print(f"{route}\t{parsed.hostname or '-'}\t{parsed.path or '/'}\t{url}")
+          PY
+            case "$route" in
+              directory-by-name) expected_path='/sites/default/files/media_file/documents/MPI_Directory_by_Establishment_Name.csv' ;;
+              directory-by-number) expected_path='/sites/default/files/media_file/documents/MPI_Directory_by_Establishment_Number.csv' ;;
+              demographics) expected_path='/sites/default/files/media_file/documents/Dataset_Establishment_Demographic_Data.csv' ;;
+              *) expected_path='' ;;
+            esac
+            if [[ "$url" != "https://www.fsis.usda.gov${expected_path}" || "$url_host" != 'www.fsis.usda.gov' || "$url_path" != "$expected_path" ]]; then
+              metric_line="route=$route config_shape=invalid curl_exit=not-run http_status=000 content_type=unknown bytes=0 final_host=-"
+              printf '%s\n' "$metric_line"
+              printf '%s\n' "- \`$metric_line\`" >> "$GITHUB_STEP_SUMMARY"
+              printf '::notice title=FSIS route diagnostic::%s\n' "$metric_line"
+              continue
+            fi
+            # The process substitution consumes at most byte_cap bytes and
+            # discards them; --max-filesize also protects Content-Length cases.
+            result="$(curl -sS -L --max-time 30 --max-filesize "$byte_cap" -o >(head -c "$byte_cap" >/dev/null) -w '%{http_code}|%{content_type}|%{size_download}|%{url_effective}' "$url" 2>/dev/null)"
+            curl_exit=$?
+            IFS='|' read -r http_status content_type bytes final_url <<< "$result"
+            final_host="-"
+            if [[ "$final_url" == https://* || "$final_url" == http://* ]]; then
+              final_host="$(python -c 'import sys; from urllib.parse import urlsplit; print(urlsplit(sys.argv[1]).hostname or "-")' "$final_url" 2>/dev/null)"
+              final_host="${final_host:-unknown}"
+            fi
+            safe_content_type="$(printf '%s' "${content_type:-unknown}" | tr -c '[:alnum:]._+/-' '_' | cut -c1-128)"
+            safe_final_host="$(printf '%s' "${final_host:--}" | tr -c '[:alnum:].:-' '_' | cut -c1-253)"
+            metric_line=$(printf 'route=%s requested_host=%s requested_path=%s curl_exit=%s http_status=%s content_type=%s bytes=%s final_host=%s' \
+              "$route" "$url_host" "$url_path" "$curl_exit" "${http_status:-000}" \
+              "$safe_content_type" "${bytes:-0}" "$safe_final_host")
+            printf '%s\n' "$metric_line"
+            printf '%s\n' "- \`$metric_line\`" >> "$GITHUB_STEP_SUMMARY"
+            printf '::notice title=FSIS route diagnostic::%s\n' "$metric_line"
+          done

.github/workflows/tests.yml

@@ -13,7 +13,7 @@ jobs:
       - uses: actions/setup-python@v5
         with:
           python-version: '3.11'
-      - run: pip install psycopg[binary]
+      - run: pip install -r pipeline/requirements.txt
       - name: Run standard tests in the canonical clean environment
         shell: pwsh
         run: ./pipeline/tests/run-standard.ps1
@@ -26,22 +26,66 @@ jobs:
       - uses: actions/setup-python@v5
         with:
           python-version: '3.11'
-      - run: pip install psycopg[binary]
-      - name: Run Docker-backed API E2E tests
-        env:
-          UEC_RUN_E2E: '1'
-        run: |
-          python -m unittest pipeline.tests.e2e.test_public_api -v
-          python -m unittest pipeline.tests.e2e.test_community_api -v
-          python -m unittest pipeline.tests.e2e.test_seeded_api -v
+      - run: pip install -r pipeline/requirements.txt
+      - name: Verify private Docker build context
+        shell: pwsh
+        run: ./pipeline/tests/verify-docker-context.ps1
+      - name: Run Docker-backed API E2E tests (core and extended)
+        shell: pwsh
+        run: ./pipeline/tests/e2e/run-suite.ps1 -Suite full
 
   backup-restore:
     # The PostGIS image is Linux-only; Ubuntu includes PowerShell Core for the drill.
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+      - run: pip install -r pipeline/requirements.txt
       - name: Run synthetic backup/restore gate
         shell: pwsh
         env:
           UEC_RUN_E2E: '1'
         run: ./pipeline/tests/e2e/backup-restore.ps1
+
+  static-jest:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: '22'
+          cache: npm
+          cache-dependency-path: package-lock.json
+      - run: npm ci
+      - run: npm test
+
+  v2-gate-self-test:
+    runs-on: windows-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Check V2 gate failure and success reporting
+        shell: pwsh
+        run: ./pipeline/tests/test-v2-gate.ps1
+
+  frontend:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: frontend
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: '22'
+          cache: npm
+          cache-dependency-path: frontend/package-lock.json
+      - run: npm ci
+      - run: npm run check
+      - run: npm test
+      - run: npm run lint
+      - run: npm run boundary
+      - run: npm run build
+      - run: npx playwright install --with-deps chromium firefox webkit
+      - run: npx playwright test tests/e2e/fixture-platform.spec.ts tests/e2e/local-safety.spec.ts tests/e2e/private-preview.spec.ts --project=chromium --project=firefox --project=webkit --workers=2

.gitignore

@@ -1,10 +1,14 @@
 /target/
+/.tmp/
 /node_modules/
 /snapshots/
+/static/v2-preview/
+/.private/
 /\.shuttle*
 /Secrets*.toml
 __pycache__/
 /data/raw/
+/data/restricted/
 /data/raw/**/*.xml
 /data/raw/**/*.csv
 /data/raw/**/*.xlsx
@@ -17,3 +21,4 @@ __pycache__/
 !/data/raw/**/
 !/data/raw/**/metadata.json
 !/data/reports/.gitkeep
+/data/graph-rehearsal/

AGENTS.md

@@ -6,12 +6,26 @@ The current public application is a Rust/Axum backend in `src/` with a vanilla J
 
 ## Build, Test, and Development Commands
 
-- `cargo run` starts the current application on port 8000.
-- `npm test` runs the Jest suite.
+- `python scripts/dev.py --help` is the preferred V2 developer entrypoint; use
+  `python scripts/dev.py --json doctor` before starting local services.
+- Run `npm ci` once at the repository root before `npm test`; `npm test` runs
+  the legacy/static Jest suite. The Svelte preview has separate dependencies:
+  run `npm --prefix frontend ci` before using its commands.
+- `cargo run` starts the current application on port 8000 when its configured
+  database is available; `python scripts/dev.py up` is the reproducible local
+  V2 setup path.
 - `npm run test:coverage` runs Jest with coverage.
 - `powershell -ExecutionPolicy Bypass -File pipeline/scripts/maintenance/build-legacy-manifest.ps1` regenerates the legacy file inventory and SHA-256 manifest in `data/manifests/`.
 - `python pipeline/scripts/diagnostics/inspect-denmark-smiley.py static_data/dk/Smiley_xml.xml` inspects the Danish XML without transforming it.
 
+For a clean database-backed validation, use
+`pwsh -NoProfile -ExecutionPolicy Bypass -File pipeline/tests/run-standard.ps1`.
+The persistent `uec-local-v2` database rejects changed migration checksums on
+purpose. If local setup reports `migration checksum changed after application`,
+the named volume belongs to an older checkout; preserve it unless it is known
+to be disposable, and follow the reset/troubleshooting instructions in
+`docs/development.md` rather than editing migration history.
+
 ## Data Credibility and Provenance
 
 Start with the [ethics summary](docs/ETHICS-SUMMARY.md) for orientation; it does not replace the full policy. Visitor-data handling, legal demands, and publication authority also follow ETHICS.md sections 11–14. Do not claim no logging, legal immunity, a backup reviewer, or enforced approval controls without evidence. Pause new publication needing human approval if no authorized reviewer is available; compliant acquisition and eligible existing releases may continue, and urgent suppression follows policy. Assess preservation obligations before exceptional deletion when a legal demand is involved; escalate to the responsible maintainer rather than making novel legal judgments.