See #24875: Take the overture tiles from the STAC catalog instead of a hardcoded bucket

The theme catalogs link to the tiles ("rel": "pmtiles", currently on
tiles.overturemaps.org), so follow root -> latest release -> theme and use
that link. Themes without a tiles link are skipped with a warning. This costs
one small (cached) request per theme, but means the plugin follows Overture
if the tiles move again.

Author: watmildon

src/main/java/org/openstreetmap/josm/plugins/mapwithai/io/mapwithai/CommonSourceReader.java

@@ -10,6 +10,7 @@
 import org.openstreetmap.josm.tools.Utils;
 
 import jakarta.json.Json;
+import jakarta.json.JsonObject;
 import jakarta.json.stream.JsonParser;
 import jakarta.json.stream.JsonParsingException;
 
@@ -54,6 +55,31 @@ public Optional<T> parse() throws IOException {
         return Optional.empty();
     }
 
+    /**
+     * Read a JSON object from another URL, using the same cache settings as the
+     * main source.
+     *
+     * @param url The url to read
+     * @return The object, or {@code null} if the document is not a JSON object
+     * @throws IOException if any I/O error occurs
+     */
+    protected JsonObject readObject(String url) throws IOException {
+        final var file = new CachedFile(url).setMaxAge(CachedFile.DAYS)
+                .setCachingStrategy(CachedFile.CachingStrategy.IfModifiedSince);
+        if (this.clearCache) {
+            file.clear();
+        }
+        file.setFastFail(this.fastFail);
+        try (file; JsonParser reader = Json.createParser(file.getContentReader())) {
+            if (reader.hasNext() && reader.next() == JsonParser.Event.START_OBJECT) {
+                return reader.getObject();
+            }
+        } catch (JsonParsingException jsonParsingException) {
+            Logging.error(jsonParsingException);
+        }
+        return null;
+    }
+
     /**
      * Parses MapWithAI entry sources
      *

src/main/java/org/openstreetmap/josm/plugins/mapwithai/io/mapwithai/OvertureSourceReader.java

@@ -10,7 +10,6 @@
 import java.util.EnumSet;
 import java.util.List;
 import java.util.Objects;
-import java.util.regex.Pattern;
 import java.util.stream.Stream;
 
 import org.openstreetmap.josm.data.Bounds;
@@ -32,17 +31,6 @@
  * Read data from overture sources
  */
 public class OvertureSourceReader extends CommonSourceReader<List<MapWithAIInfo>> implements Closeable {
-    /**
-     * The location of the tiles for an overture release. The STAC catalog only lists
-     * the geoparquet files, so this comes from
-     * <a href="https://docs.overturemaps.org/examples/overture-tiles/">the overture
-     * tiles documentation</a>.
-     */
-    static final String TILES_URL = "https://overturemaps-extras-us-west-2.s3.us-west-2.amazonaws.com/tiles/{release}/{theme}.pmtiles";
-    /** The themes with tiles. Transportation is not listed since it currently causes crashes. */
-    private static final List<String> TILE_THEMES = List.of("addresses", "base", "buildings", "divisions", "places");
-    /** The expected format for release ids (they are used to build URLs) */
-    private static final Pattern RELEASE_PATTERN = Pattern.compile("[0-9A-Za-z][0-9A-Za-z._-]*");
     private final MapWithAIInfo source;
 
     public OvertureSourceReader(MapWithAIInfo source) {
@@ -57,38 +45,87 @@ public List<MapWithAIInfo> parseJson(JsonParser jsonParser) {
             return parseRoot(jsonObject);
         }
         // The STAC catalog (https://stac.overturemaps.org/catalog.json), see #24875
-        if (jsonObject.get("latest")instanceof JsonString latest) {
-            return parseStacRoot(latest.getString());
+        if ("Catalog".equals(jsonObject.getString("type", null)) && jsonObject.containsKey("links")) {
+            return parseStacRoot(jsonObject);
         }
         return Collections.emptyList();
     }
 
     /**
-     * Create the sources for the latest release from the STAC catalog. Overture
-     * only keeps tiles for the most recent releases, so the sources keep the same
-     * id between releases. This means that user entries are updated to the new
+     * Create the sources for the latest release from the STAC catalog. The root
+     * catalog links to the releases, a release links to its themes, and a theme
+     * links to its tiles ({@code "rel": "pmtiles"}).
+     * <p>
+     * Overture only keeps the last two releases, so the sources keep the same id
+     * between releases. This means that user entries are updated to the new
      * release instead of being dropped.
      *
-     * @param releaseId The latest release
-     * @return The sources for that release
+     * @param root The root catalog
+     * @return The sources for the latest release
      */
-    private List<MapWithAIInfo> parseStacRoot(String releaseId) {
-        if (!RELEASE_PATTERN.matcher(releaseId).matches()) {
-            Logging.warn("MapWithAI: Unexpected overture release id: {0}", releaseId);
+    private List<MapWithAIInfo> parseStacRoot(JsonObject root) {
+        final var latest = root.getString("latest", null);
+        final var releases = getLinks(root, "child").toList();
+        var release = latest == null ? null
+                : releases.stream().filter(link -> link.getString("href").contains('/' + latest + '/')).findFirst()
+                        .orElse(null);
+        if (release == null) {
+            release = releases.stream().filter(link -> link.getBoolean("latest", false)).findFirst().orElse(null);
+        }
+        if (release == null) {
+            Logging.warn("MapWithAI: No overture release found in {0}", this.source.getUrl());
             return Collections.emptyList();
         }
-        final var info = new ArrayList<MapWithAIInfo>(TILE_THEMES.size());
-        for (var theme : TILE_THEMES) {
-            final var uri = URI.create(TILES_URL.replace("{release}", releaseId).replace("{theme}", theme));
-            final var themeInfo = buildSource(uri, releaseId, theme);
-            if (themeInfo != null) {
-                themeInfo.setId(this.source.getName() + ": " + theme);
-                info.add(themeInfo);
+        // The title is "<release> Overture Release"
+        final var releaseId = latest != null ? latest : release.getString("title", "").split(" ", 2)[0];
+        final var info = new ArrayList<MapWithAIInfo>(6);
+        try {
+            final var releaseCatalog = readObject(release.getString("href"));
+            if (releaseCatalog == null) {
+                return info;
+            }
+            for (var theme : getLinks(releaseCatalog, "child").toList()) {
+                final var themeCatalog = readObject(theme.getString("href"));
+                if (themeCatalog == null) {
+                    continue;
+                }
+                final var themeId = themeCatalog.getString("id", theme.getString("title", ""));
+                final var tiles = getLinks(themeCatalog, "pmtiles").findFirst();
+                if (tiles.isEmpty()) {
+                    Logging.warn("MapWithAI: Overture theme {0} has no tiles in release {1}", themeId, releaseId);
+                    continue;
+                }
+                final var themeInfo = buildSource(URI.create(tiles.get().getString("href")), releaseId, themeId);
+                if (themeInfo != null) {
+                    themeInfo.setId(this.source.getName() + ": " + themeId);
+                    info.add(themeInfo);
+                }
             }
+        } catch (IOException | IllegalArgumentException e) {
+            Logging.warn("MapWithAI: Could not read the overture catalog for release {0}: {1}", releaseId,
+                    e.getMessage());
+            Logging.trace(e);
         }
         return info;
     }
 
+    /**
+     * Get the links of a STAC object with a specific relation
+     *
+     * @param stac The STAC object (catalog or collection)
+     * @param rel  The relation to look for
+     * @return The links with an {@code href}
+     */
+    private static Stream<JsonObject> getLinks(JsonObject stac, String rel) {
+        final var links = stac.get("links");
+        if (links instanceof JsonArray array) {
+            return array.stream().filter(JsonObject.class::isInstance).map(JsonObject.class::cast)
+                    .filter(link -> rel.equals(link.getString("rel", null)) && link.containsKey("href")
+                            && link.get("href").getValueType() == JsonValue.ValueType.STRING);
+        }
+        return Stream.empty();
+    }
+
     private List<MapWithAIInfo> parseRoot(JsonObject jsonObject) {
         final var info = new ArrayList<MapWithAIInfo>(6 * 4);
         final var releases = jsonObject.get("releases");

src/test/unit/org/openstreetmap/josm/plugins/mapwithai/io/mapwithai/OvertureSourceReaderTest.java

@@ -8,6 +8,7 @@
 import java.io.StringReader;
 import java.net.URI;
 import java.util.List;
+import java.util.Map;
 
 import org.junit.jupiter.api.Test;
 import org.openstreetmap.josm.plugins.mapwithai.data.mapwithai.MapWithAICategory;
@@ -16,23 +17,61 @@
 import org.openstreetmap.josm.testutils.annotations.BasicPreferences;
 
 import jakarta.json.Json;
+import jakarta.json.JsonObject;
 
 /**
  * Test class for {@link OvertureSourceReader}
  */
 @BasicPreferences
 class OvertureSourceReaderTest {
+    private static final String STAC = "https://stac.overturemaps.org/";
+    private static final String ROOT = """
+            {"type":"Catalog","id":"Overture Releases","stac_version":"1.1.0","latest":"2026-09-23.1",
+            "links":[{"rel":"child","href":"https://stac.overturemaps.org/2026-08-19.0/catalog.json","title":"2026-08-19.0 Overture Release"},
+            {"rel":"child","href":"https://stac.overturemaps.org/2026-09-23.1/catalog.json","title":"2026-09-23.1 Overture Release","latest":true},
+            {"rel":"self","href":"https://stac.overturemaps.org/catalog.json"}]}""";
+    private static final String RELEASE = """
+            {"type":"Catalog","id":"2026-09-23.1","stac_version":"1.1.0",
+            "links":[{"rel":"root","href":"https://stac.overturemaps.org/catalog.json"},
+            {"rel":"child","href":"https://stac.overturemaps.org/2026-09-23.1/addresses/catalog.json","title":"addresses"},
+            {"rel":"child","href":"https://stac.overturemaps.org/2026-09-23.1/buildings/catalog.json","title":"buildings"},
+            {"rel":"child","href":"https://stac.overturemaps.org/2026-09-23.1/divisions/catalog.json","title":"divisions"},
+            {"rel":"child","href":"https://stac.overturemaps.org/2026-09-23.1/transportation/catalog.json","title":"transportation"}]}""";
+
+    private static String theme(String theme, boolean tiles) {
+        return "{\"type\":\"Catalog\",\"id\":\"" + theme + "\",\"links\":[{\"rel\":\"root\",\"href\":\"" + STAC
+                + "catalog.json\"}" + (tiles ? ",{\"rel\":\"pmtiles\",\"href\":\"https://tiles.overturemaps.org/2026-09-23.1/"
+                        + theme + ".pmtiles\",\"type\":\"application/vnd.pmtiles\"}" : "")
+                + "]}";
+    }
+
+    private static final Map<String, String> CATALOGS = Map.of(STAC + "2026-09-23.1/catalog.json", RELEASE,
+            STAC + "2026-09-23.1/addresses/catalog.json", theme("addresses", true),
+            STAC + "2026-09-23.1/buildings/catalog.json", theme("buildings", true),
+            STAC + "2026-09-23.1/divisions/catalog.json", theme("divisions", false),
+            STAC + "2026-09-23.1/transportation/catalog.json", theme("transportation", true));
+
     private static List<MapWithAIInfo> parse(String json) throws IOException {
-        final var source = new MapWithAIInfo("Overture", "https://stac.overturemaps.org/catalog.json");
+        final var source = new MapWithAIInfo("Overture", STAC + "catalog.json");
         source.setSourceType(MapWithAIType.OVERTURE);
         try (var reader = new OvertureSourceReader(source) {
+            @Override
+            protected JsonObject readObject(String url) throws IOException {
+                // Serve the canned catalogs instead of going to the network
+                if (!CATALOGS.containsKey(url)) {
+                    throw new IOException("Unexpected url: " + url);
+                }
+                try (var parser = Json.createParser(new StringReader(CATALOGS.get(url)))) {
+                    parser.next();
+                    return parser.getObject();
+                }
+            }
+
             @Override
             void readTileInformation(MapWithAIInfo info, URI uri, String releaseId) {
                 // Don't read the (remote) pmtiles headers in tests
             }
-        };
-                var sr = new StringReader(json);
-                var parser = Json.createParser(sr)) {
+        }; var sr = new StringReader(json); var parser = Json.createParser(sr)) {
             parser.next();
             return reader.parseJson(parser);
         }
@@ -44,29 +83,39 @@ void readTileInformation(MapWithAIInfo info, URI uri, String releaseId) {
      */
     @Test
     void testStacCatalog() throws IOException {
-        final var infos = parse("""
-                {"type":"Catalog","id":"Overture Releases","stac_version":"1.1.0",
-                "links":[{"rel":"child","href":"https://stac.overturemaps.org/2026-09-23.1/catalog.json","latest":true}],
-                "latest":"2026-09-23.1"}""");
-        assertEquals(5, infos.size());
+        final var infos = parse(ROOT);
+        // divisions has no tiles, transportation is skipped
+        assertEquals(2, infos.size(), infos.toString());
         for (var info : infos) {
             assertEquals(MapWithAIType.PMTILES, info.getSourceType());
-            assertTrue(info.getUrl().startsWith(
-                    "https://overturemaps-extras-us-west-2.s3.us-west-2.amazonaws.com/tiles/2026-09-23.1/"),
-                    info.getUrl());
-            assertTrue(info.getUrl().endsWith(".pmtiles"), info.getUrl());
             assertTrue(info.hasValidUrl());
+            assertTrue(info.getName().contains("2026-09-23.1"), info.getName());
         }
         final var buildings = infos.stream().filter(i -> i.getUrl().endsWith("/buildings.pmtiles")).findFirst()
                 .orElseThrow();
+        assertEquals("https://tiles.overturemaps.org/2026-09-23.1/buildings.pmtiles", buildings.getUrl());
         assertEquals(MapWithAICategory.BUILDING, buildings.getCategory());
         // The id must not change between releases
         assertEquals("Overture: buildings", buildings.getId());
+        final var addresses = infos.stream().filter(i -> i.getUrl().endsWith("/addresses.pmtiles")).findFirst()
+                .orElseThrow();
+        assertEquals(MapWithAICategory.ADDRESS, addresses.getCategory());
+    }
+
+    /**
+     * The {@code latest} field is optional in STAC; fall back to the link marked as
+     * latest.
+     */
+    @Test
+    void testStacCatalogWithoutLatest() throws IOException {
+        assertEquals(2, parse(ROOT.replace("\"latest\":\"2026-09-23.1\",", "")).size());
     }
 
     @Test
     void testBadCatalog() throws IOException {
         assertTrue(parse("{\"type\":\"Catalog\"}").isEmpty());
-        assertTrue(parse("{\"latest\":\"../../something else\"}").isEmpty());
+        assertTrue(parse("{\"type\":\"Catalog\",\"links\":[]}").isEmpty());
+        assertTrue(parse("{\"type\":\"Catalog\",\"latest\":\"2000-01-01.0\",\"links\":[{\"rel\":\"child\","
+                + "\"href\":\"https://stac.overturemaps.org/2000-01-01.0/catalog.json\"}]}").isEmpty());
     }
 }