Skip to content

Commit 7ca07ad

Browse files
watmildontsmock
watmildon
authored and
tsmock
committed
See #24875: Read overture sources from the STAC catalog
The pmtiles catalog used for the overture sources no longer exists (403). Overture now publishes releases at https://stac.overturemaps.org/catalog.json and tiles at https://overturemaps-extras-us-west-2.s3.us-west-2.amazonaws.com/tiles/<release>/<theme>.pmtiles When the source URL points at the STAC catalog, create the sources for the latest release. Only the last few releases are kept, so these sources have an id without the release in it; user entries then follow new releases instead of being dropped. The old catalog format is still supported. sources.json (pages branch) needs to point at the STAC catalog for this to take effect.
1 parent 65c92ed commit 7ca07ad

2 files changed

Lines changed: 141 additions & 17 deletions

File tree

‎src/main/java/org/openstreetmap/josm/plugins/mapwithai/io/mapwithai/OvertureSourceReader.java‎

Lines changed: 69 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
import java.util.EnumSet;
1111
import java.util.List;
1212
import java.util.Objects;
13+
import java.util.regex.Pattern;
1314
import java.util.stream.Stream;
1415

1516
import org.openstreetmap.josm.data.Bounds;
@@ -31,6 +32,17 @@
3132
* Read data from overture sources
3233
*/
3334
public class OvertureSourceReader extends CommonSourceReader<List<MapWithAIInfo>> implements Closeable {
35+
/**
36+
* The location of the tiles for an overture release. The STAC catalog only lists
37+
* the geoparquet files, so this comes from
38+
* <a href="https://docs.overturemaps.org/examples/overture-tiles/">the overture
39+
* tiles documentation</a>.
40+
*/
41+
static final String TILES_URL = "https://overturemaps-extras-us-west-2.s3.us-west-2.amazonaws.com/tiles/{release}/{theme}.pmtiles";
42+
/** The themes with tiles. Transportation is not listed since it currently causes crashes. */
43+
private static final List<String> TILE_THEMES = List.of("addresses", "base", "buildings", "divisions", "places");
44+
/** The expected format for release ids (they are used to build URLs) */
45+
private static final Pattern RELEASE_PATTERN = Pattern.compile("[0-9A-Za-z][0-9A-Za-z._-]*");
3446
private final MapWithAIInfo source;
3547

3648
public OvertureSourceReader(MapWithAIInfo source) {
@@ -44,9 +56,39 @@ public List<MapWithAIInfo> parseJson(JsonParser jsonParser) {
4456
if (jsonObject.containsKey("releases")) {
4557
return parseRoot(jsonObject);
4658
}
59+
// The STAC catalog (https://stac.overturemaps.org/catalog.json), see #24875
60+
if (jsonObject.get("latest")instanceof JsonString latest) {
61+
return parseStacRoot(latest.getString());
62+
}
4763
return Collections.emptyList();
4864
}
4965

66+
/**
67+
* Create the sources for the latest release from the STAC catalog. Overture
68+
* only keeps tiles for the most recent releases, so the sources keep the same
69+
* id between releases. This means that user entries are updated to the new
70+
* release instead of being dropped.
71+
*
72+
* @param releaseId The latest release
73+
* @return The sources for that release
74+
*/
75+
private List<MapWithAIInfo> parseStacRoot(String releaseId) {
76+
if (!RELEASE_PATTERN.matcher(releaseId).matches()) {
77+
Logging.warn("MapWithAI: Unexpected overture release id: {0}", releaseId);
78+
return Collections.emptyList();
79+
}
80+
final var info = new ArrayList<MapWithAIInfo>(TILE_THEMES.size());
81+
for (var theme : TILE_THEMES) {
82+
final var uri = URI.create(TILES_URL.replace("{release}", releaseId).replace("{theme}", theme));
83+
final var themeInfo = buildSource(uri, releaseId, theme);
84+
if (themeInfo != null) {
85+
themeInfo.setId(this.source.getName() + ": " + theme);
86+
info.add(themeInfo);
87+
}
88+
}
89+
return info;
90+
}
91+
5092
private List<MapWithAIInfo> parseRoot(JsonObject jsonObject) {
5193
final var info = new ArrayList<MapWithAIInfo>(6 * 4);
5294
final var releases = jsonObject.get("releases");
@@ -123,24 +165,34 @@ private MapWithAIInfo buildSource(URI uri, String releaseId, String theme) {
123165
info.setId(info.getName());
124166
if (uri.getPath().endsWith(".pmtiles")) {
125167
info.setSourceType(MapWithAIType.PMTILES);
126-
// Set additional information
127-
try {
128-
final var header = PMTiles.readHeader(uri);
129-
final var metadata = PMTiles.readMetadata(header);
130-
final var bounds = new Bounds(header.minLatitude(), header.minLongitude(), header.maxLatitude(),
131-
header.maxLongitude());
132-
info.setBounds(new ImageryInfo.ImageryBounds(bounds.encodeAsString(","), ","));
133-
if (metadata.containsKey("name") && metadata.get("name")instanceof JsonString name) {
134-
info.setName(name.getString() + " - " + releaseId);
135-
}
136-
if (metadata.containsKey("description")
137-
&& metadata.get("description")instanceof JsonString description) {
138-
info.setDescription(description.getString());
139-
}
140-
} catch (IOException ioException) {
141-
Logging.error(ioException);
142-
}
168+
readTileInformation(info, uri, releaseId);
143169
}
144170
return info;
145171
}
172+
173+
/**
174+
* Read additional information (bounds, name, description) from the tiles
175+
*
176+
* @param info The info to update
177+
* @param uri The location of the tiles
178+
* @param releaseId The release id
179+
*/
180+
void readTileInformation(MapWithAIInfo info, URI uri, String releaseId) {
181+
try {
182+
final var header = PMTiles.readHeader(uri);
183+
final var metadata = PMTiles.readMetadata(header);
184+
final var bounds = new Bounds(header.minLatitude(), header.minLongitude(), header.maxLatitude(),
185+
header.maxLongitude());
186+
info.setBounds(new ImageryInfo.ImageryBounds(bounds.encodeAsString(","), ","));
187+
if (metadata.containsKey("name") && metadata.get("name")instanceof JsonString name) {
188+
info.setName(name.getString() + " - " + releaseId);
189+
}
190+
if (metadata.containsKey("description")
191+
&& metadata.get("description")instanceof JsonString description) {
192+
info.setDescription(description.getString());
193+
}
194+
} catch (IOException ioException) {
195+
Logging.error(ioException);
196+
}
197+
}
146198
}

‎src/test/unit/org/openstreetmap/josm/plugins/mapwithai/io/mapwithai/OvertureSourceReaderTest.java‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
// License: GPL. For details, see LICENSE file.
2+
package org.openstreetmap.josm.plugins.mapwithai.io.mapwithai;
3+
4+
import static org.junit.jupiter.api.Assertions.assertEquals;
5+
import static org.junit.jupiter.api.Assertions.assertTrue;
6+
7+
import java.io.IOException;
8+
import java.io.StringReader;
9+
import java.net.URI;
10+
import java.util.List;
11+
12+
import org.junit.jupiter.api.Test;
13+
import org.openstreetmap.josm.plugins.mapwithai.data.mapwithai.MapWithAICategory;
14+
import org.openstreetmap.josm.plugins.mapwithai.data.mapwithai.MapWithAIInfo;
15+
import org.openstreetmap.josm.plugins.mapwithai.data.mapwithai.MapWithAIType;
16+
import org.openstreetmap.josm.testutils.annotations.BasicPreferences;
17+
18+
import jakarta.json.Json;
19+
20+
/**
21+
* Test class for {@link OvertureSourceReader}
22+
*/
23+
@BasicPreferences
24+
class OvertureSourceReaderTest {
25+
private static List<MapWithAIInfo> parse(String json) throws IOException {
26+
final var source = new MapWithAIInfo("Overture", "https://stac.overturemaps.org/catalog.json");
27+
source.setSourceType(MapWithAIType.OVERTURE);
28+
try (var reader = new OvertureSourceReader(source) {
29+
@Override
30+
void readTileInformation(MapWithAIInfo info, URI uri, String releaseId) {
31+
// Don't read the (remote) pmtiles headers in tests
32+
}
33+
};
34+
var sr = new StringReader(json);
35+
var parser = Json.createParser(sr)) {
36+
parser.next();
37+
return reader.parseJson(parser);
38+
}
39+
}
40+
41+
/**
42+
* Non-regression test for #24875: the old pmtiles catalog no longer exists. Use
43+
* the STAC catalog instead.
44+
*/
45+
@Test
46+
void testStacCatalog() throws IOException {
47+
final var infos = parse("""
48+
{"type":"Catalog","id":"Overture Releases","stac_version":"1.1.0",
49+
"links":[{"rel":"child","href":"https://stac.overturemaps.org/2026-09-23.1/catalog.json","latest":true}],
50+
"latest":"2026-09-23.1"}""");
51+
assertEquals(5, infos.size());
52+
for (var info : infos) {
53+
assertEquals(MapWithAIType.PMTILES, info.getSourceType());
54+
assertTrue(info.getUrl().startsWith(
55+
"https://overturemaps-extras-us-west-2.s3.us-west-2.amazonaws.com/tiles/2026-09-23.1/"),
56+
info.getUrl());
57+
assertTrue(info.getUrl().endsWith(".pmtiles"), info.getUrl());
58+
assertTrue(info.hasValidUrl());
59+
}
60+
final var buildings = infos.stream().filter(i -> i.getUrl().endsWith("/buildings.pmtiles")).findFirst()
61+
.orElseThrow();
62+
assertEquals(MapWithAICategory.BUILDING, buildings.getCategory());
63+
// The id must not change between releases
64+
assertEquals("Overture: buildings", buildings.getId());
65+
}
66+
67+
@Test
68+
void testBadCatalog() throws IOException {
69+
assertTrue(parse("{\"type\":\"Catalog\"}").isEmpty());
70+
assertTrue(parse("{\"latest\":\"../../something else\"}").isEmpty());
71+
}
72+
}

0 commit comments

Comments
 (0)