|
| 1 | +"""Fetch vector features from Overture Maps. |
| 2 | +
|
| 3 | +Overture is an alternative to the Overpass API that ``osmnx`` queries. Overpass |
| 4 | +is a live query service that routinely rate-limits or times out on dense urban |
| 5 | +bounding boxes; Overture serves static, monthly-released GeoParquet from cloud |
| 6 | +storage, so a fetch is a bounded range-read rather than a server-side query. |
| 7 | +
|
| 8 | +The underlying data is largely the same: Overture's ``base`` and |
| 9 | +``transportation`` themes are derived from OpenStreetMap. The ``buildings`` |
| 10 | +theme adds machine-learning-derived footprints on top of OSM. |
| 11 | +""" |
| 12 | + |
| 13 | +import logging |
| 14 | +import time |
| 15 | +from typing import Any, Optional |
| 16 | + |
| 17 | +import geopandas as gpd |
| 18 | +from overturemaps import core |
| 19 | + |
| 20 | +# Overture type names for each kind of feature OWM targets, chosen to match the |
| 21 | +# OSM tag sets in target_builders: OSM_water_tags -> water, OSM_roads_tags |
| 22 | +# (highway=*) -> road segments, OSM_buildings_tags -> buildings. |
| 23 | +OVERTURE_TYPES: dict[str, str] = { |
| 24 | + "water": "water", |
| 25 | + "roads": "segment", |
| 26 | + "buildings": "building", |
| 27 | +} |
| 28 | + |
| 29 | +# The transportation theme also carries "rail" and "water" segments. Only |
| 30 | +# "road" is kept so negative targets match what highway=* returned from OSM. |
| 31 | +OVERTURE_SUBTYPES: dict[str, frozenset[str]] = { |
| 32 | + "roads": frozenset({"road"}), |
| 33 | +} |
| 34 | + |
| 35 | +# Everything seaward of the OSM coastline is a single "ocean" water feature. |
| 36 | +# OSM_water_tags has no coastline equivalent, so this is signal OWM did not |
| 37 | +# previously receive - but it is pinned to the coastline vector regardless of |
| 38 | +# tide, hence the opt-out in build_targets. |
| 39 | +OCEAN_SUBTYPE = "ocean" |
| 40 | + |
| 41 | +# Overture files a few landforms under the water theme's "physical" subtype: |
| 42 | +# a cape is a headland, a blowhole is a coastal rock formation, and a shoal is |
| 43 | +# a sandbank that is routinely exposed. Rasterizing them as positive water |
| 44 | +# targets marks land as water. Capes and blowholes are usually points (which |
| 45 | +# combine_vector_targets drops anyway), but shoals do appear as polygons. |
| 46 | +# OSM_water_tags never selected these - it queried natural=water/strait, not |
| 47 | +# natural=cape/shoal - so dropping them also keeps the two sources aligned. |
| 48 | +OVERTURE_LAND_CLASSES = frozenset({"cape", "blowhole", "shoal"}) |
| 49 | + |
| 50 | +# Only these are needed downstream: combine_vector_targets uses geometry alone, |
| 51 | +# and the subtype/class pair is kept for filtering, cache inspection and |
| 52 | +# debugging. |
| 53 | +# Dropping the rest keeps the concat in combine_vector_targets and the parquet |
| 54 | +# cache free of Overture's deeply nested attribute columns. |
| 55 | +KEEP_COLUMNS = ["subtype", "class", "geometry"] |
| 56 | + |
| 57 | +# Overture is served from S3, so failures are transient far more often than not: |
| 58 | +# a throttled range read, a dropped connection, a timeout on a dense bbox. The |
| 59 | +# client itself does not retry, so a single blip would otherwise cost the scene |
| 60 | +# all of its vector targets. Delays are 2s, 4s - short enough not to stall a |
| 61 | +# batch run, long enough to clear a throttle. |
| 62 | +MAX_ATTEMPTS = 3 |
| 63 | +BACKOFF_SECONDS = 2.0 |
| 64 | + |
| 65 | + |
| 66 | +def get_overture_features( |
| 67 | + gdf_bounds_4326: gpd.GeoDataFrame, |
| 68 | + kind: str, |
| 69 | + include_ocean: bool = True, |
| 70 | +) -> gpd.GeoDataFrame: |
| 71 | + """Download Overture features of ``kind`` within a bounding box. |
| 72 | +
|
| 73 | + ``kind`` is one of ``water``, ``roads`` or ``buildings``. ``include_ocean`` |
| 74 | + only applies to ``water``. |
| 75 | + """ |
| 76 | + if kind not in OVERTURE_TYPES: |
| 77 | + raise ValueError( |
| 78 | + f"Unknown Overture feature kind: {kind!r}. " |
| 79 | + f"Expected one of {sorted(OVERTURE_TYPES)}." |
| 80 | + ) |
| 81 | + overture_type = OVERTURE_TYPES[kind] |
| 82 | + bounds = gdf_bounds_4326.total_bounds |
| 83 | + bbox = ( |
| 84 | + float(bounds[0]), |
| 85 | + float(bounds[1]), |
| 86 | + float(bounds[2]), |
| 87 | + float(bounds[3]), |
| 88 | + ) |
| 89 | + |
| 90 | + features = _fetch_with_retry(overture_type=overture_type, bbox=bbox, kind=kind) |
| 91 | + |
| 92 | + if features.empty: |
| 93 | + logging.info(f"No {kind} features found within bbox: {bbox}") |
| 94 | + return gpd.GeoDataFrame() |
| 95 | + |
| 96 | + features = _filter_features(features, kind=kind, include_ocean=include_ocean) |
| 97 | + if features.empty: |
| 98 | + logging.info(f"No {kind} features left after filtering") |
| 99 | + return gpd.GeoDataFrame() |
| 100 | + |
| 101 | + features = features[[c for c in KEEP_COLUMNS if c in features.columns]] |
| 102 | + features = features.set_crs("EPSG:4326", allow_override=True) |
| 103 | + features = gpd.clip(features, gdf_bounds_4326) |
| 104 | + return gpd.GeoDataFrame(features) |
| 105 | + |
| 106 | + |
| 107 | +def _fetch_once( |
| 108 | + overture_type: str, |
| 109 | + bbox: tuple[float, float, float, float], |
| 110 | +) -> Optional[gpd.GeoDataFrame]: |
| 111 | + """Run a single fetch attempt, or return None if Overture gave no reader. |
| 112 | +
|
| 113 | + ``record_batch_reader`` streams lazily, so a failed range read surfaces out |
| 114 | + of ``from_arrow`` rather than out of the reader call. Both live here so the |
| 115 | + retry above covers the whole request. |
| 116 | + """ |
| 117 | + # stac=True lets the client resolve the release and target only the |
| 118 | + # relevant files via the STAC catalogue. Without it the whole dataset |
| 119 | + # listing is opened, which measured ~5x slower per request. |
| 120 | + reader = core.record_batch_reader(overture_type, bbox=bbox, stac=True) |
| 121 | + if reader is None: |
| 122 | + return None |
| 123 | + return gpd.GeoDataFrame.from_arrow(reader) |
| 124 | + |
| 125 | + |
| 126 | +def _bbox_has_no_files( |
| 127 | + overture_type: str, |
| 128 | + bbox: tuple[float, float, float, float], |
| 129 | +) -> bool: |
| 130 | + """Report whether STAC finds no files at all intersecting ``bbox``. |
| 131 | +
|
| 132 | + ``record_batch_reader`` returns None both when the bbox is genuinely empty |
| 133 | + (STAC matched zero files, e.g. open ocean or Antarctica) and when opening |
| 134 | + the dataset failed. Only the second is an error, and the two are |
| 135 | + indistinguishable from the None alone. ``_prepare_query`` is what returns |
| 136 | + None for the empty case, so ask it directly. It is private, so an |
| 137 | + overturemaps release that drops or renames it makes this report False and |
| 138 | + the caller treats the None as an error - the conservative reading. |
| 139 | + """ |
| 140 | + prepare_query = getattr(core, "_prepare_query", None) |
| 141 | + if prepare_query is None: |
| 142 | + return False |
| 143 | + try: |
| 144 | + return prepare_query(overture_type, bbox=bbox, stac=True) is None |
| 145 | + except Exception as e: |
| 146 | + logging.debug(f"Could not check STAC file coverage for {overture_type}: {e}") |
| 147 | + return False |
| 148 | + |
| 149 | + |
| 150 | +def _fetch_with_retry( |
| 151 | + overture_type: str, |
| 152 | + bbox: tuple[float, float, float, float], |
| 153 | + kind: str, |
| 154 | +) -> gpd.GeoDataFrame: |
| 155 | + """Fetch ``overture_type`` over ``bbox``, retrying transient failures. |
| 156 | +
|
| 157 | + Returns an empty frame when the bbox genuinely holds no features. Raises |
| 158 | + once the attempts are exhausted, so a persistent failure fails the build |
| 159 | + rather than passing an empty frame off as "no water here". |
| 160 | + """ |
| 161 | + last_error: Optional[Exception] = None |
| 162 | + |
| 163 | + for attempt in range(1, MAX_ATTEMPTS + 1): |
| 164 | + try: |
| 165 | + features = _fetch_once(overture_type, bbox) |
| 166 | + except Exception as e: |
| 167 | + last_error = e |
| 168 | + else: |
| 169 | + if features is not None: |
| 170 | + return features |
| 171 | + if _bbox_has_no_files(overture_type, bbox): |
| 172 | + logging.info(f"Overture has no {kind} files intersecting bbox: {bbox}") |
| 173 | + return gpd.GeoDataFrame() |
| 174 | + last_error = RuntimeError( |
| 175 | + f"Overture returned no reader for type {overture_type!r} " |
| 176 | + f"within bbox: {bbox}" |
| 177 | + ) |
| 178 | + |
| 179 | + if attempt < MAX_ATTEMPTS: |
| 180 | + delay = BACKOFF_SECONDS * 2 ** (attempt - 1) |
| 181 | + logging.warning( |
| 182 | + f"Overture {kind} fetch failed (attempt {attempt}/{MAX_ATTEMPTS}): " |
| 183 | + f"{last_error}. Retrying in {delay:.0f}s" |
| 184 | + ) |
| 185 | + time.sleep(delay) |
| 186 | + |
| 187 | + raise RuntimeError( |
| 188 | + f"Overture {kind} fetch failed after {MAX_ATTEMPTS} attempts for bbox: " |
| 189 | + f"{bbox}. This usually indicates a network or cloud storage error." |
| 190 | + ) from last_error |
| 191 | + |
| 192 | + |
| 193 | +def _filter_features( |
| 194 | + features: gpd.GeoDataFrame, |
| 195 | + kind: str, |
| 196 | + include_ocean: bool, |
| 197 | +) -> gpd.GeoDataFrame: |
| 198 | + """Restrict features to those matching the equivalent OSM tag sets.""" |
| 199 | + if kind == "water" and "class" in features.columns: |
| 200 | + features = features[~features["class"].isin(OVERTURE_LAND_CLASSES)] |
| 201 | + |
| 202 | + if "subtype" not in features.columns: |
| 203 | + return gpd.GeoDataFrame(features) |
| 204 | + |
| 205 | + keep: Any = OVERTURE_SUBTYPES.get(kind) |
| 206 | + if keep is not None: |
| 207 | + features = features[features["subtype"].isin(keep)] |
| 208 | + |
| 209 | + if kind == "water" and not include_ocean: |
| 210 | + features = features[features["subtype"] != OCEAN_SUBTYPE] |
| 211 | + |
| 212 | + return gpd.GeoDataFrame(features) |
0 commit comments