fix(overture): harden integration lifecycle

Author: Medformatik

apps/api/.env.example

@@ -200,9 +200,10 @@ PASSKEY_ORIGIN=http://127.0.0.1:3000,openmapx://
 # default: https://raw.githubusercontent.com/openmapx/community-extensions/main/catalog.json
 # EXTENSION_CATALOG_URL=
 
-# Overture Maps POI integration (monthly data pull by data-manager)
+# Overture Maps POI integration (weekly release check by data-manager)
 # OVERTURE_ENABLED=false
-# OVERTURE_SYNC_CRON=0 5 1 * *
+# OVERTURE_SYNC_CRON=0 5 * * 2
+# OVERTURE_CONFLATION_RETRY_CRON=*/15 * * * *
 
 # Instance display name shown in the admin UI header. Optional; falls back to
 # the value set in /admin/settings.

apps/web/src/components/admin/services/OvertureMaintenance.test.tsx

@@ -33,4 +33,39 @@ describe("OvertureMaintenance", () => {
     expect(markup).toContain("completed");
     expect(markup).not.toContain("completed · complete");
   });
+
+  it("uses phase-specific progress denominators", async () => {
+    const { overtureProgress } = await import("./OvertureMaintenance");
+    expect(
+      overtureProgress({
+        status: "running",
+        phase: "score",
+        placeCount: 4_000_000,
+        extractedCount: 1_400_000,
+        processedCount: 700_000,
+      }),
+    ).toEqual({ value: 50, label: "700,000 of 1,400,000 OSM POIs scored" });
+    expect(
+      overtureProgress({
+        status: "running",
+        phase: "assign",
+        componentCount: 200,
+        assignmentCursor: 50,
+      }),
+    ).toEqual({ value: 25, label: "50 of 200 components assigned" });
+    expect(overtureProgress({ status: "running", phase: "publish" })).toEqual({
+      value: null,
+      label: "Validating and publishing links",
+    });
+  });
+
+  it("allows only an expired running lease to be resumed", async () => {
+    const { canResumeOvertureLinks } = await import("./OvertureMaintenance");
+    expect(
+      canResumeOvertureLinks({ status: "running", stalled: false }, "europe/germany", false),
+    ).toBe(false);
+    expect(
+      canResumeOvertureLinks({ status: "running", stalled: true }, "europe/germany", false),
+    ).toBe(true);
+  });
 });

apps/web/src/components/admin/services/OvertureMaintenance.tsx

@@ -26,7 +26,11 @@ interface OvertureStatus {
   placeCount?: number;
   status?: string;
   phase?: string;
+  emittedCount?: number | null;
+  extractedCount?: number | null;
   processedCount?: number | null;
+  componentCount?: number | null;
+  assignmentCursor?: number | null;
   linkedCount?: number | null;
   attemptCount?: number;
   lastError?: string | null;
@@ -36,6 +40,54 @@ interface OvertureStatus {
 
 type Operation = "overture-sync" | "overture-conflate";
 
+export function overtureProgress(status: OvertureStatus | undefined): {
+  value: number | null;
+  label: string;
+} | null {
+  if (status?.status !== "running") return null;
+  if (
+    status.phase === "score" &&
+    status.extractedCount &&
+    status.processedCount !== null &&
+    status.processedCount !== undefined
+  ) {
+    return {
+      value: Math.min(100, (status.processedCount / status.extractedCount) * 100),
+      label: `${status.processedCount.toLocaleString()} of ${status.extractedCount.toLocaleString()} OSM POIs scored`,
+    };
+  }
+  if (
+    status.phase === "assign" &&
+    status.componentCount &&
+    status.assignmentCursor !== null &&
+    status.assignmentCursor !== undefined
+  ) {
+    return {
+      value: Math.min(100, (status.assignmentCursor / status.componentCount) * 100),
+      label: `${status.assignmentCursor.toLocaleString()} of ${status.componentCount.toLocaleString()} components assigned`,
+    };
+  }
+  const label =
+    status.phase === "extract"
+      ? `${status.emittedCount?.toLocaleString() ?? "0"} OSM geometries streamed`
+      : status.phase === "publish"
+        ? "Validating and publishing links"
+        : "Preparing Overture conflation";
+  return { value: null, label };
+}
+
+export function canResumeOvertureLinks(
+  status: OvertureStatus | undefined,
+  region: string,
+  operationPending: boolean,
+): boolean {
+  return (
+    region.trim().length > 0 &&
+    !operationPending &&
+    (status?.status !== "running" || status.stalled === true)
+  );
+}
+
 export function OvertureMaintenance({ apiUrl }: { apiUrl: string }) {
   const showToast = useAdminToast();
   const queryClient = useQueryClient();
@@ -83,10 +135,7 @@ export function OvertureMaintenance({ apiUrl }: { apiUrl: string }) {
   });
 
   const status = statusQuery.data;
-  const progress =
-    status?.placeCount && status.processedCount !== null && status.processedCount !== undefined
-      ? Math.min(100, (status.processedCount / status.placeCount) * 100)
-      : null;
+  const progress = overtureProgress(status);
   const visiblePhase =
     status?.status === "completed" && status.phase === "complete" ? undefined : status?.phase;
 
@@ -136,6 +185,13 @@ export function OvertureMaintenance({ apiUrl }: { apiUrl: string }) {
           {status.lastError}
         </Alert>
       )}
+      {status?.stalled && (
+        <Alert severity="warning" sx={{ mt: 1.5 }}>
+          The worker heartbeat is stale
+          {status.heartbeatAgeMs ? ` (${Math.floor(status.heartbeatAgeMs / 60_000)} minutes)` : ""}.
+          The expired lease can be reclaimed safely with Resume links.
+        </Alert>
+      )}
 
       {status?.ok && (
         <Stack direction="row" sx={{ gap: 3, flexWrap: "wrap", mt: 1.5 }}>
@@ -157,12 +213,14 @@ export function OvertureMaintenance({ apiUrl }: { apiUrl: string }) {
           ))}
         </Stack>
       )}
-      {progress !== null && status?.status === "running" && (
+      {progress && (
         <Box sx={{ mt: 1.5 }}>
-          <LinearProgress variant="determinate" value={progress} />
+          <LinearProgress
+            variant={progress.value === null ? "indeterminate" : "determinate"}
+            value={progress.value ?? undefined}
+          />
           <Typography variant="caption" sx={{ color: "text.secondary" }}>
-            {status.processedCount?.toLocaleString()} of {status.placeCount?.toLocaleString()}{" "}
-            places
+            {progress.label}
           </Typography>
         </Box>
       )}
@@ -189,7 +247,7 @@ export function OvertureMaintenance({ apiUrl }: { apiUrl: string }) {
         <Button
           variant="outlined"
           startIcon={<MergeIcon />}
-          disabled={!region.trim() || operation.isPending || status?.status === "running"}
+          disabled={!canResumeOvertureLinks(status, region, operation.isPending)}
           onClick={() => setConfirmOperation("overture-conflate")}
         >
           Resume links

docs/docs/features/overture-places.md

@@ -12,8 +12,12 @@ than enrich OSM records: a place present only in Overture is returned as its own
 search result and opens as a complete place card with a stable GERS identifier.
 
 The integration is optional. Without it, category search continues to use OSM
-through Overpass. With it enabled, OpenMapX queries both sources, matches records
-that describe the same real-world place, and keeps unmatched Overture records as
+through Overpass. Activating it has two layers: enable the data lifecycle with
+`OVERTURE_ENABLED=true`, then enable `poi-overture` for search and
+`knowledge-overture` for place-card enrichment in **Admin → Integrations**.
+These runtime integrations are independent, so an operator may enable either or
+both. With both enabled, OpenMapX queries both sources, matches records that
+describe the same real-world place, and keeps unmatched Overture records as
 coverage gap-fill.
 
 ## Imported fields
@@ -69,11 +73,14 @@ release starts with a durable `pending` link state. The rebuild records
 `extract`, `score`, `assign`, `publish`, and `complete` phases, keyset and
 component cursors, source fingerprint, heartbeats, attempt counts, separate
 emitted-geometry and unique-POI counts, and per-phase durations in PostGIS. A
-failed or interrupted attempt leaves the Places release active and is retried
-every six hours by default. A retry resumes its saved phase: completed OSM
+failed or interrupted attempt leaves the Places release active. Recovery is
+attempted once at data-manager startup and every 15 minutes thereafter. A retry resumes its saved phase: completed OSM
 extraction is not repeated, scoring continues after its last committed keyset
 cursor, and assignment continues after its last completed graph component. If
-the local PBF fingerprint changes, the state safely restarts from extraction.
+the local PBF fingerprint changes—even after a previously completed run—the
+state safely restarts from extraction. Across Overture releases, an unchanged
+OSM table and its fingerprint are moved into the new release in constant time,
+so only Overture-dependent scoring and assignment repeat.
 
 The rebuild is designed for country-scale inputs. OSM GeoJSON sequences are
 parsed directly into fixed-size Postgres batches and published with an atomic
@@ -87,7 +94,11 @@ country-wide scored graph as one object. The completely assigned next-link
 snapshot is durable and replaces the live link table in one transaction, so a
 failed attempt can never expose a partial link set. A PostgreSQL advisory lock
 serializes schema swaps, OSM snapshot publication, and link rebuilding across
-data-manager processes.
+data-manager processes. Temporary filtered PBFs are removed after extraction.
+Capacity checks measure the host `/data` filesystem before a pull and the
+PostgreSQL container's actual data filesystem before staging or conflation.
+Each reserves estimated working space plus a safety margin and fails with an
+actionable byte estimate instead of filling either filesystem.
 
 The staged Places snapshot is checked against release-pinned, human-reviewed
 category cases before activation. After link assignment, the final fused
@@ -98,7 +109,10 @@ including known upstream category mistakes and duplicates.
 
 After a complete ingest, fused quality check, and link publication, OpenMapX
 retains the active local release snapshot and one predecessor by default. Older
-release directories are pruned; incomplete refreshes never trigger pruning.
+release directories are pruned whether publication completed in the original
+sync or an independent retry; incomplete refreshes never trigger pruning.
+Durable finalization markers prevent completed retries from repeatedly
+truncating work tables or rescanning release directories.
 
 Run the complete workflow manually with:
 

docs/docs/install/configuration.md

@@ -286,14 +286,16 @@ for what each class covers.
 
 ## Overture Maps POIs
 
-Schedules and switches for the monthly regional Overture Maps Places refresh.
+Schedules and switches for the regional Overture Maps Places refresh.
 
 | Variable           | Description                                                                                  | Required / Default        |
 | ------------------ | ------------------------------------------------------------------------------------------- | ------------------------- |
 | `OVERTURE_ENABLED`  | Set to `true` to discover and atomically import newer Overture Places releases for `OPENMAPX_REGION`. | Optional. Default unset |
-| `OVERTURE_SYNC_CRON`| Cron schedule for checking and refreshing the regional snapshot. | Optional. Default `0 5 1 * *` (monthly) |
-| `OVERTURE_CONFLATION_RETRY_CRON` | Retry an incomplete OSM↔Overture link rebuild without downloading or importing Places again. | Optional. Default `15 */6 * * *` (every six hours) |
+| `OVERTURE_SYNC_CRON`| Cron schedule for checking and refreshing the regional snapshot. | Optional. Default `0 5 * * 2` (weekly, Tuesday 05:00 UTC) |
+| `OVERTURE_CONFLATION_RETRY_CRON` | Retry an incomplete OSM↔Overture link rebuild without downloading or importing Places again. Recovery also runs once at startup. | Optional. Default `*/15 * * * *` |
 | `OVERTURE_RELEASE_RETENTION` | Number of completed local release snapshots to retain, including the active release. Applied only after fused quality validation and link publication. | Optional. Default `2`; range 1–12 |
+| `OVERTURE_DISK_RESERVE_BYTES` | Free-space safety reserve kept beyond estimated pull/ingest working space. | Optional. Default `5368709120` (5 GiB) |
+| `OVERTURE_FIRST_PULL_ESTIMATE_BYTES` | Working-space estimate for the first regional pull when no earlier snapshot exists. Later pulls use prior local snapshot sizes. | Optional. Default `2147483648` (2 GiB) |
 
 The job uses Overture's official STAC catalog, skips an installed release,
 resolves exact spatially relevant Places assets, validates a local release
@@ -303,6 +305,11 @@ retry schedule with resumable extraction, scoring, assignment, and publication
 phases: a missing PBF or failed rebuild does not roll back a valid Places
 release. See [Overture Places](../features/overture-places.md).
 
+After the first successful ingest, enable `poi-overture` in **Admin →
+Integrations** to return Overture search results and `knowledge-overture` to
+enrich matched OSM place cards. `OVERTURE_ENABLED` maintains the data but does
+not silently enable either runtime integration.
+
 ## Natural-language search
 
 [Natural-language search](../features/natural-language-search.md) (the

infra/docker/.env.example

@@ -395,11 +395,13 @@ THUNDERFOREST_API_KEY=
 # default: https://raw.githubusercontent.com/openmapx/community-extensions/main/catalog.json
 # EXTENSION_CATALOG_URL=
 
-# Overture Maps POI integration (monthly Places pull + independent link retries)
+# Overture Maps POI integration (weekly release check + independent link retries)
 # OVERTURE_ENABLED=false
-# OVERTURE_SYNC_CRON=0 5 1 * *
-# OVERTURE_CONFLATION_RETRY_CRON=15 */6 * * *
+# OVERTURE_SYNC_CRON=0 5 * * 2
+# OVERTURE_CONFLATION_RETRY_CRON=*/15 * * * *
 # OVERTURE_RELEASE_RETENTION=2
+# OVERTURE_DISK_RESERVE_BYTES=5368709120
+# OVERTURE_FIRST_PULL_ESTIMATE_BYTES=2147483648
 
 # Data-use policy. By default every data source is served (the reference
 # deployment is non-commercial). A commercial operator can exclude

integrations/knowledge-overture/provider.ts

@@ -74,8 +74,7 @@ function parseOsmRef(ref: string): { type: string; id: bigint } | null {
  * Resolve a GERS id for the given place.
  *
  * Phase 1 — link-first: query poi_conflation_link using the OSM ref from
- * context.ids.osm. The table is empty until plan 03, so this always falls
- * through for now (expected).
+ * context.ids.osm.
  *
  * Phase 2 — spatial+name: find candidates within 150 m of the place
  * coordinates and filter with Overture's own taxonomy hierarchy; pick the one whose name
@@ -84,7 +83,7 @@ function parseOsmRef(ref: string): { type: string; id: bigint } | null {
  * context.ids may be undefined (the neighborhoods call site passes a partial
  * Place with no ids) — falls through to the spatial path without throwing.
  */
-async function resolveGers(
+export async function resolveGers(
   database: DatabaseClient,
   osmTags: Record<string, string>,
   context: KnowledgeContext | undefined,
@@ -131,25 +130,45 @@ async function resolveGers(
     sql = `
       SELECT gers_id, name
       FROM overture_places.places
-      WHERE ST_DWithin(geom::geography, ST_MakePoint($1, $2)::geography, 150)
+      WHERE geom && ST_Envelope(
+        ST_Buffer(
+          ST_SetSRID(ST_MakePoint($1, $2), 4326)::geography,
+          150
+        )::geometry
+      )
+        AND ST_DWithin(
+          geom::geography,
+          ST_SetSRID(ST_MakePoint($1, $2), 4326)::geography,
+          150
+        )
         AND (
           basic_category = ANY($3::TEXT[])
           OR taxonomy_primary = ANY($3::TEXT[])
           OR taxonomy_hierarchy && $3::TEXT[]
           OR taxonomy_alternates && $3::TEXT[]
         )
         AND (operating_status IS NULL OR operating_status <> 'permanently_closed')
-      ORDER BY geom <-> ST_MakePoint($1, $2)::geometry
+      ORDER BY geom <-> ST_SetSRID(ST_MakePoint($1, $2), 4326)
       LIMIT 5
     `;
     params = [lng, lat, concepts];
   } else {
     sql = `
       SELECT gers_id, name
       FROM overture_places.places
-      WHERE ST_DWithin(geom::geography, ST_MakePoint($1, $2)::geography, 150)
+      WHERE geom && ST_Envelope(
+        ST_Buffer(
+          ST_SetSRID(ST_MakePoint($1, $2), 4326)::geography,
+          150
+        )::geometry
+      )
+        AND ST_DWithin(
+          geom::geography,
+          ST_SetSRID(ST_MakePoint($1, $2), 4326)::geography,
+          150
+        )
         AND (operating_status IS NULL OR operating_status <> 'permanently_closed')
-      ORDER BY geom <-> ST_MakePoint($1, $2)::geometry
+      ORDER BY geom <-> ST_SetSRID(ST_MakePoint($1, $2), 4326)
       LIMIT 5
     `;
     params = [lng, lat];

packages/cli/src/commands/data.ts

@@ -679,13 +679,26 @@ export function registerDataCommands(program: Command): void {
           );
           process.exit(1);
         }
+        if (result.status === "already_running") {
+          log.warn(
+            result.message ??
+              "Overture conflation is still running under another worker; no work was claimed",
+          );
+          return;
+        }
         const linked = result.linked ?? 0;
-        log.ok(
-          `Overture conflation complete: ${linked} link${linked === 1 ? "" : "s"} written` +
-            (result.extracted !== undefined && result.candidates !== undefined
-              ? ` from ${result.extracted} OSM POIs and ${result.candidates} accepted edges`
-              : ""),
-        );
+        if (result.status === "already_completed") {
+          log.ok(
+            `Overture conflation was already complete: ${linked} link${linked === 1 ? "" : "s"} active`,
+          );
+        } else {
+          log.ok(
+            `Overture conflation complete: ${linked} link${linked === 1 ? "" : "s"} written` +
+              (result.extracted !== undefined && result.candidates !== undefined
+                ? ` from ${result.extracted} OSM POIs and ${result.candidates} accepted edges`
+                : ""),
+          );
+        }
       } catch (err) {
         log.err(`overture-conflate failed: ${(err as Error).message}`);
         dataManagerHint();