Skip to content

Commit dc502eb

Browse files
MedformatikMedformatik
committed
fix(transit-registry): pin catalog and validate endpoints
1 parent 1300f4c commit dc502eb

15 files changed

Lines changed: 973 additions & 28 deletions

‎infra/docker/transport-apis.lock.json‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
{
2+
"$schema": "./transport-apis.lock.schema.json",
3+
"schemaVersion": 1,
4+
"source": "public-transport-transport-apis",
5+
"ref": "v1",
6+
"commit": "58aec5b1b7c876f133c9d1336739d0f61211b74e",
7+
"entryCount": 85,
8+
"lockedAt": "2026-08-02T22:14:09.534Z",
9+
"lockedBy": "dev@fwcr.de",
10+
"comment": "Pinned commit of public-transport/transport-apis consumed by integrations/transit-dynamic-registry. Bump via `pnpm openmapx transit-registry bump`."
11+
}

‎infra/docker/transport-apis.lock.schema.json‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
{
2+
"$schema": "https://json-schema.org/draft/2020-12/schema",
3+
"type": "object",
4+
"required": ["schemaVersion", "source", "ref", "commit", "entryCount", "lockedAt", "lockedBy"],
5+
"properties": {
6+
"$schema": { "type": "string" },
7+
"schemaVersion": { "const": 1 },
8+
"source": { "const": "public-transport-transport-apis" },
9+
"ref": { "type": "string", "minLength": 1 },
10+
"commit": { "type": "string", "pattern": "^[0-9a-f]{40}$" },
11+
"entryCount": { "type": "integer", "minimum": 1 },
12+
"lockedAt": { "type": "string", "format": "date-time" },
13+
"lockedBy": { "type": "string", "minLength": 1 },
14+
"comment": { "type": "string" }
15+
},
16+
"additionalProperties": false
17+
}

‎integrations/transit-dynamic-registry/__tests__/pinned-fetch.test.ts‎

Lines changed: 138 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,138 @@
1+
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
2+
import { TRANSPORT_APIS_COMMIT } from "../pin";
3+
4+
const mockFetch = vi.fn();
5+
vi.stubGlobal("fetch", mockFetch);
6+
7+
const { fetchRegistryEntries } = await import("../fetcher");
8+
9+
const VALID_ENTRY = {
10+
name: "ÖBB",
11+
type: { hafasMgate: {} },
12+
coverage: { realtimeCoverage: { region: ["AT"] } },
13+
options: { endpoint: "https://fahrplan.oebb.at/bin/mgate.exe" },
14+
};
15+
16+
function makeJsonResponse(data: unknown, ok = true): Response {
17+
return {
18+
ok,
19+
status: ok ? 200 : 500,
20+
json: () => Promise.resolve(data),
21+
} as unknown as Response;
22+
}
23+
24+
function listing(...paths: string[]) {
25+
return { files: paths.map((path) => ({ name: `/${path}` })) };
26+
}
27+
28+
beforeEach(() => {
29+
vi.clearAllMocks();
30+
});
31+
32+
afterEach(() => {
33+
vi.clearAllMocks();
34+
});
35+
36+
describe("pinned transit registry fetches", () => {
37+
it("fetches both the listing and files at the immutable revision", async () => {
38+
mockFetch
39+
.mockResolvedValueOnce(makeJsonResponse(listing("data/at/oebb-hafas-mgate.json")))
40+
.mockResolvedValueOnce(makeJsonResponse(VALID_ENTRY));
41+
42+
const entries = await fetchRegistryEntries();
43+
expect(entries).toHaveLength(1);
44+
45+
for (const [input] of mockFetch.mock.calls) {
46+
const url = String(input);
47+
expect(url).toContain(TRANSPORT_APIS_COMMIT);
48+
expect(url).not.toContain("transport-apis@HEAD");
49+
expect(url).not.toContain("transport-apis@v1");
50+
expect(url).not.toContain("/transport-apis/v1/");
51+
}
52+
});
53+
54+
it("pins the GitHub fallback tree and raw file URLs", async () => {
55+
mockFetch
56+
.mockRejectedValueOnce(new Error("JSDelivr unavailable"))
57+
.mockResolvedValueOnce(
58+
makeJsonResponse({
59+
tree: [{ path: "data/at/oebb-hafas-mgate.json", type: "blob" }],
60+
}),
61+
)
62+
.mockResolvedValueOnce(makeJsonResponse(VALID_ENTRY));
63+
64+
const entries = await fetchRegistryEntries();
65+
expect(entries).toHaveLength(1);
66+
expect(String(mockFetch.mock.calls[1]?.[0])).toBe(
67+
`https://api.github.com/repos/public-transport/transport-apis/git/trees/${TRANSPORT_APIS_COMMIT}?recursive=1`,
68+
);
69+
expect(String(mockFetch.mock.calls[2]?.[0])).toContain(
70+
`/${TRANSPORT_APIS_COMMIT}/data/at/oebb-hafas-mgate.json`,
71+
);
72+
});
73+
74+
it("drops a private-host endpoint while preserving a valid sibling", async () => {
75+
mockFetch
76+
.mockResolvedValueOnce(
77+
makeJsonResponse(listing("data/at/oebb-hafas-mgate.json", "data/us/private-otp.json")),
78+
)
79+
.mockResolvedValueOnce(makeJsonResponse(VALID_ENTRY))
80+
.mockResolvedValueOnce(
81+
makeJsonResponse({
82+
...VALID_ENTRY,
83+
name: "Private",
84+
type: { otpGraphQl: {} },
85+
options: { endpoint: "http://127.0.0.1:8080/otp" },
86+
}),
87+
);
88+
89+
const entries = await fetchRegistryEntries();
90+
expect(entries).toHaveLength(1);
91+
expect(entries[0]?.name).toBe("ÖBB");
92+
});
93+
94+
it("drops a credentialed plain-HTTP endpoint", async () => {
95+
mockFetch
96+
.mockResolvedValueOnce(makeJsonResponse(listing("data/us/insecure-otp.json")))
97+
.mockResolvedValueOnce(
98+
makeJsonResponse({
99+
...VALID_ENTRY,
100+
name: "Insecure",
101+
type: { otpGraphQl: {} },
102+
options: { endpoint: "http://api.example.org/graphql", apiKey: "x" },
103+
}),
104+
);
105+
106+
await expect(fetchRegistryEntries()).resolves.toEqual([]);
107+
});
108+
109+
it.each([
110+
["array", []],
111+
["missing type", { name: "Missing type", coverage: VALID_ENTRY.coverage, options: {} }],
112+
["string options", { ...VALID_ENTRY, options: "unexpected" }],
113+
])("drops an unexpected %s registry shape without losing a good sibling", async (_name, bad) => {
114+
mockFetch
115+
.mockResolvedValueOnce(makeJsonResponse(listing("data/at/good.json", "data/at/bad.json")))
116+
.mockResolvedValueOnce(makeJsonResponse(VALID_ENTRY))
117+
.mockResolvedValueOnce(makeJsonResponse(bad));
118+
119+
const entries = await fetchRegistryEntries();
120+
expect(entries).toHaveLength(1);
121+
expect(entries[0]?.name).toBe("ÖBB");
122+
});
123+
124+
it("continues dropping entries without coverage", async () => {
125+
mockFetch
126+
.mockResolvedValueOnce(makeJsonResponse(listing("data/xx/no-coverage.json")))
127+
.mockResolvedValueOnce(
128+
makeJsonResponse({
129+
...VALID_ENTRY,
130+
name: "No coverage",
131+
coverage: {},
132+
options: {},
133+
}),
134+
);
135+
136+
await expect(fetchRegistryEntries()).resolves.toEqual([]);
137+
});
138+
});

‎integrations/transit-dynamic-registry/__tests__/validate-endpoint.test.ts‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
import { describe, expect, it } from "vitest";
2+
import { registryEndpointRejection } from "../validate-endpoint";
3+
4+
describe("registryEndpointRejection", () => {
5+
it("allows entries without an endpoint", () => {
6+
expect(registryEndpointRejection({})).toBeNull();
7+
});
8+
9+
it("allows a public HTTPS endpoint", () => {
10+
expect(
11+
registryEndpointRejection({ endpoint: "https://fahrplan.oebb.at/bin/mgate.exe" }),
12+
).toBeNull();
13+
});
14+
15+
it("allows public HTTP when no credential is present", () => {
16+
expect(registryEndpointRejection({ endpoint: "http://api.example.org/graphql" })).toBeNull();
17+
});
18+
19+
it.each([
20+
"http://127.0.0.1:8080/graphql",
21+
"http://localhost:8080/graphql",
22+
"http://10.0.0.5/otp",
23+
"http://[::1]/otp",
24+
"ftp://example.org/otp",
25+
"file:///etc/passwd",
26+
"not-a-url",
27+
])("rejects unsafe URL %s", (endpoint) => {
28+
expect(registryEndpointRejection({ endpoint })).toBe("not-public-http");
29+
});
30+
31+
it("rejects a non-string endpoint", () => {
32+
expect(registryEndpointRejection({ endpoint: 42 })).toBe("not-a-string");
33+
});
34+
35+
it.each([
36+
{ endpoint: "http://api.example.org/graphql", apiKey: "x" },
37+
{ endpoint: "http://api.example.org/graphql", auth: { token: "x" } },
38+
])("rejects credentials sent over plain HTTP", (options) => {
39+
expect(registryEndpointRejection(options)).toBe("insecure-with-credential");
40+
});
41+
42+
it("allows credentials over HTTPS", () => {
43+
expect(
44+
registryEndpointRejection({ endpoint: "https://api.example.org/graphql", apiKey: "x" }),
45+
).toBeNull();
46+
});
47+
});

‎integrations/transit-dynamic-registry/fetcher.ts‎

Lines changed: 46 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,16 @@
11
import { type BBox, fetchJson } from "@openmapx/core";
22
import type { CacheClient } from "@openmapx/integration-framework";
33
import { COUNTRY_BBOXES } from "./country-bboxes";
4+
import {
5+
TRANSPORT_APIS_COMMIT,
6+
TRANSPORT_APIS_GITHUB_TREE_URL,
7+
TRANSPORT_APIS_JSDELIVR_CDN_BASE,
8+
TRANSPORT_APIS_JSDELIVR_PKG_URL,
9+
TRANSPORT_APIS_RAW_BASE,
10+
} from "./pin";
411
import type { CoverageTier, ProtocolType, RegistryEntry } from "./registry-types";
12+
import { registryEndpointRejection } from "./validate-endpoint";
513

6-
// JSDelivr CDN — mirrors GitHub without API rate limits
7-
// @HEAD resolves to the repo's default branch (no releases/tags needed)
8-
const JSDELIVR_PKG_URL =
9-
"https://data.jsdelivr.com/v1/packages/gh/public-transport/transport-apis@HEAD";
10-
const JSDELIVR_CDN_BASE = "https://cdn.jsdelivr.net/gh/public-transport/transport-apis@HEAD";
11-
// GitHub API fallback
12-
const GITHUB_TREE_URL =
13-
"https://api.github.com/repos/public-transport/transport-apis/git/trees/v1?recursive=1";
14-
const RAW_BASE = "https://raw.githubusercontent.com/public-transport/transport-apis/v1";
1514
const REGISTRY_CACHE_KEY = "transit:registry";
1615
const REGISTRY_CACHE_TTL = 172800; // 48 hours
1716
const MAX_CONCURRENT = 10;
@@ -134,10 +133,31 @@ function parseCoverageTier(
134133
}
135134

136135
// biome-ignore lint/suspicious/noExplicitAny: external JSON
137-
function parseEntry(path: string, json: any): RegistryEntry | null {
138-
const protocol = parseProtocol(json.type ?? {});
136+
export function parseEntry(path: string, json: any): RegistryEntry | null {
137+
if (!json || typeof json !== "object" || Array.isArray(json)) return null;
138+
139+
const type = json.type;
140+
const protocol = parseProtocol(
141+
type && typeof type === "object" && !Array.isArray(type) ? type : {},
142+
);
139143
if (!protocol) return null;
140144

145+
const rawOptions = json.options;
146+
if (
147+
rawOptions !== undefined &&
148+
rawOptions !== null &&
149+
(typeof rawOptions !== "object" || Array.isArray(rawOptions))
150+
) {
151+
console.warn(`[transit-registry] Dropping ${path}: options rejected (not-an-object)`);
152+
return null;
153+
}
154+
const options = (rawOptions ?? {}) as Record<string, unknown>;
155+
const rejection = registryEndpointRejection(options);
156+
if (rejection) {
157+
console.warn(`[transit-registry] Dropping ${path}: endpoint rejected (${rejection})`);
158+
return null;
159+
}
160+
141161
const id = idFromPath(path);
142162
const slug = slugFromPath(path);
143163
const prefix = `${slug}:`;
@@ -171,7 +191,7 @@ function parseEntry(path: string, json: any): RegistryEntry | null {
171191
protocol,
172192
supportedLanguages: json.supportedLanguages ?? [],
173193
timezone: json.timezone,
174-
options: json.options ?? {},
194+
options,
175195
coverage: { bbox, tiers },
176196
attribution: json.attribution
177197
? {
@@ -232,21 +252,21 @@ function collectDataPaths(
232252
return [...new Set(paths)];
233253
}
234254

235-
/** Fetch file listing via JSDelivr @HEAD (no auth, no rate limits). */
255+
/** Fetch file listing via jsDelivr (no auth, no rate limits). */
236256
async function fetchPathsFromJsdelivr(): Promise<string[]> {
237-
const json = await fetchJson<JsDelivrFile>(JSDELIVR_PKG_URL, {
257+
const json = await fetchJson<JsDelivrFile>(TRANSPORT_APIS_JSDELIVR_PKG_URL, {
238258
timeoutMs: 15_000,
239-
headers: githubAuthHeaders(JSDELIVR_PKG_URL),
259+
headers: githubAuthHeaders(TRANSPORT_APIS_JSDELIVR_PKG_URL),
240260
errorMessage: ({ status }) => `JSDelivr listing: ${status}`,
241261
});
242262
return collectDataPaths(json);
243263
}
244264

245265
/** Fetch file listing via GitHub Tree API (requires GITHUB_TOKEN for reliable access). */
246266
async function fetchPathsFromGithub(): Promise<string[]> {
247-
const tree = await fetchJson<{ tree: GitTreeEntry[] }>(GITHUB_TREE_URL, {
267+
const tree = await fetchJson<{ tree: GitTreeEntry[] }>(TRANSPORT_APIS_GITHUB_TREE_URL, {
248268
timeoutMs: 15_000,
249-
headers: githubAuthHeaders(GITHUB_TREE_URL),
269+
headers: githubAuthHeaders(TRANSPORT_APIS_GITHUB_TREE_URL),
250270
errorMessage: ({ status }) => `GitHub tree: ${status}`,
251271
});
252272
return tree.tree
@@ -281,11 +301,13 @@ async function fetchInBatchesFrom(paths: string[], baseUrl: string): Promise<Reg
281301
export async function fetchRegistryEntries(): Promise<RegistryEntry[]> {
282302
let entries: RegistryEntry[] | null = null;
283303

284-
// 1. Primary: JSDelivr CDN (@HEAD — no auth, no rate limits, no GitHub API calls)
304+
// 1. Primary: jsDelivr CDN (no auth, no rate limits, no GitHub API calls)
285305
try {
286306
const paths = await fetchPathsFromJsdelivr();
287-
entries = await fetchInBatchesFrom(paths, JSDELIVR_CDN_BASE);
288-
console.log(`[transit-registry] ${entries.length} entries loaded (JSDelivr)`);
307+
entries = await fetchInBatchesFrom(paths, TRANSPORT_APIS_JSDELIVR_CDN_BASE);
308+
console.log(
309+
`[transit-registry] ${entries.length} entries loaded (JSDelivr @ ${TRANSPORT_APIS_COMMIT.slice(0, 12)})`,
310+
);
289311
} catch (err) {
290312
console.warn("[transit-registry] JSDelivr unavailable, trying GitHub API:", err);
291313
}
@@ -294,8 +316,10 @@ export async function fetchRegistryEntries(): Promise<RegistryEntry[]> {
294316
if (!entries) {
295317
try {
296318
const paths = await fetchPathsFromGithub();
297-
entries = await fetchInBatchesFrom(paths, RAW_BASE);
298-
console.log(`[transit-registry] ${entries.length} entries loaded (GitHub API)`);
319+
entries = await fetchInBatchesFrom(paths, TRANSPORT_APIS_RAW_BASE);
320+
console.log(
321+
`[transit-registry] ${entries.length} entries loaded (GitHub API @ ${TRANSPORT_APIS_COMMIT.slice(0, 12)})`,
322+
);
299323
} catch (err) {
300324
console.warn("[transit-registry] GitHub API unavailable, trying cache:", err);
301325
}

‎integrations/transit-dynamic-registry/hafas-mgate.ts‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ import { createRedisStore } from "cached-hafas-client/stores/redis.js";
1717
import { createClient } from "hafas-client";
1818
import type { ProtocolAdapter } from "./adapter-types";
1919
import type { RegistryEntry } from "./registry-types";
20+
import { registryEndpointIsUsable } from "./validate-endpoint";
2021

2122
const TIMEOUT_MS = 8_000;
2223

@@ -95,7 +96,9 @@ function buildProfile(entry: RegistryEntry): Record<string, any> {
9596
// biome-ignore lint/suspicious/noExplicitAny: external config object
9697
const profile: Record<string, any> = {};
9798

98-
if (opts.endpoint) profile.endpoint = opts.endpoint;
99+
// Same third-party-catalog gate the fetcher applies; an unset endpoint makes
100+
// hafas-client fail fast rather than talk to an unvetted host.
101+
if (opts.endpoint && registryEndpointIsUsable(opts)) profile.endpoint = opts.endpoint;
99102
if (opts.auth) profile.auth = opts.auth;
100103
if (opts.client) profile.client = opts.client;
101104
if (opts.ver) profile.ver = opts.ver;

‎integrations/transit-dynamic-registry/manifest.json‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -41,12 +41,12 @@
4141
{
4242
"name": "JSDelivr Registry",
4343
"type": "http",
44-
"url": "https://data.jsdelivr.com/v1/packages/gh/public-transport/transport-apis@HEAD"
44+
"url": "https://data.jsdelivr.com/v1/packages/gh/public-transport/transport-apis@58aec5b1b7c876f133c9d1336739d0f61211b74e"
4545
},
4646
{
4747
"name": "GitHub Registry",
4848
"type": "http",
49-
"url": "https://api.github.com/repos/public-transport/transport-apis/git/trees/v1?recursive=1"
49+
"url": "https://api.github.com/repos/public-transport/transport-apis/git/trees/58aec5b1b7c876f133c9d1336739d0f61211b74e?recursive=1"
5050
}
5151
],
5252
"dataSources": [

‎integrations/transit-dynamic-registry/otp-graphql.ts‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import type {
99
} from "@openmapx/mobility-core/transit";
1010
import type { ProtocolAdapter } from "./adapter-types";
1111
import type { RegistryEntry } from "./registry-types";
12+
import { registryEndpointIsUsable } from "./validate-endpoint";
1213

1314
const TIMEOUT_MS = 8_000;
1415
const ENTUR_CLIENT_NAME = "openmapx-server";
@@ -200,7 +201,10 @@ function secondsToIso(serviceDay: number, seconds: number): string {
200201
}
201202

202203
function getEndpoint(entry: RegistryEntry): string {
203-
return (entry.options.endpoint as string) ?? "";
204+
const endpoint = (entry.options.endpoint as string) ?? "";
205+
// The catalog is third-party; refuse anything the fetcher's gate would have
206+
// dropped. Every caller already treats "" as "provider unavailable".
207+
return registryEndpointIsUsable(entry.options) ? endpoint : "";
204208
}
205209

206210
function getHeaders(entry: RegistryEntry): Record<string, string> {

0 commit comments

Comments
 (0)