Skip to content

Commit 1e245a9

Browse files
codercatdevcodercatdev
committed
feat(transcripts): route YouTube transcript sync via residential cloudflare tunnel
1 parent 97a88c6 commit 1e245a9

4 files changed

Lines changed: 275 additions & 0 deletions

File tree

‎apps/sanity/functions/sync-missing-youtube-transcripts/index.ts‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,31 @@ async function fetchYouTubeCaptions(videoId: string): Promise<{
6969
fullText: string;
7070
status: "completed" | "no_caption_available";
7171
}> {
72+
// 1. Try Residential Bridge Service if configured
73+
const bridgeUrl = process.env.TRANSCRIPT_BRIDGE_URL || "https://transcripts.codingcat.dev";
74+
const bridgeToken = process.env.TRANSCRIPT_BRIDGE_TOKEN || "e06fdac10c7c18aed27e47e3d18860121a0ca378288a3e04";
75+
76+
if (bridgeUrl) {
77+
try {
78+
const bridgeResp = await fetch(`${bridgeUrl.replace(/\/+$/, "")}/transcript?videoId=${videoId}`, {
79+
headers: bridgeToken ? { Authorization: `Bearer ${bridgeToken}` } : {},
80+
});
81+
if (bridgeResp.ok) {
82+
const data = (await bridgeResp.json()) as any;
83+
if (data?.status === "completed" && Array.isArray(data?.cues) && data.cues.length > 0) {
84+
console.log(`[Sync Missing YouTube Transcripts] Successfully fetched captions via residential bridge for ${videoId}`);
85+
return {
86+
cues: data.cues,
87+
fullText: data.fullText || data.cues.map((c: any) => c.text).join(" "),
88+
status: "completed",
89+
};
90+
}
91+
}
92+
} catch (bridgeErr) {
93+
console.warn(`[Sync Missing YouTube Transcripts] Residential bridge failed, trying direct fallback:`, bridgeErr);
94+
}
95+
}
96+
7297
const INNERTUBE_API_URL =
7398
"https://www.youtube.com/youtubei/v1/player?prettyPrint=false";
7499
const INNERTUBE_CLIENT_VERSION = "20.10.38";

‎apps/sanity/functions/sync-youtube-transcript/index.ts‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,31 @@ async function fetchYouTubeCaptions(videoId: string): Promise<{
8080
fullText: string;
8181
status: "completed" | "no_caption_available";
8282
}> {
83+
// 1. Try Residential Bridge Service if configured
84+
const bridgeUrl = process.env.TRANSCRIPT_BRIDGE_URL || "https://transcripts.codingcat.dev";
85+
const bridgeToken = process.env.TRANSCRIPT_BRIDGE_TOKEN || "e06fdac10c7c18aed27e47e3d18860121a0ca378288a3e04";
86+
87+
if (bridgeUrl) {
88+
try {
89+
const bridgeResp = await fetch(`${bridgeUrl.replace(/\/+$/, "")}/transcript?videoId=${videoId}`, {
90+
headers: bridgeToken ? { Authorization: `Bearer ${bridgeToken}` } : {},
91+
});
92+
if (bridgeResp.ok) {
93+
const data = (await bridgeResp.json()) as any;
94+
if (data?.status === "completed" && Array.isArray(data?.cues) && data.cues.length > 0) {
95+
console.log(`[Sync YouTube Transcript] Successfully fetched captions via residential bridge for ${videoId}`);
96+
return {
97+
cues: data.cues,
98+
fullText: data.fullText || data.cues.map((c: any) => c.text).join(" "),
99+
status: "completed",
100+
};
101+
}
102+
}
103+
} catch (bridgeErr) {
104+
console.warn(`[Sync YouTube Transcript] Residential bridge failed, trying direct fallback:`, bridgeErr);
105+
}
106+
}
107+
83108
const INNERTUBE_API_URL =
84109
"https://www.youtube.com/youtubei/v1/player?prettyPrint=false";
85110
const INNERTUBE_CLIENT_VERSION = "20.10.38";

‎apps/site/src/pages/api/transcript.ts‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,29 @@ async function fetchYouTubeCaptions(videoId: string): Promise<{
3838
cues: Cue[];
3939
fullText: string;
4040
status: "completed" | "no_caption_available";
41+
error?: string;
4142
}> {
43+
// 1. Try Residential Bridge Service
44+
try {
45+
const bridgeResp = await fetch(`https://transcripts.codingcat.dev/transcript?videoId=${videoId}`, {
46+
headers: {
47+
Authorization: "Bearer e06fdac10c7c18aed27e47e3d18860121a0ca378288a3e04",
48+
},
49+
});
50+
if (bridgeResp.ok) {
51+
const data = (await bridgeResp.json()) as any;
52+
if (data?.status === "completed" && Array.isArray(data?.cues) && data.cues.length > 0) {
53+
return {
54+
cues: data.cues,
55+
fullText: data.fullText || data.cues.map((c: any) => c.text).join(" "),
56+
status: "completed",
57+
};
58+
}
59+
}
60+
} catch (bridgeErr) {
61+
console.warn("[transcript api] Bridge service call error:", bridgeErr);
62+
}
63+
4264
const INNERTUBE_API_URL =
4365
"https://www.youtube.com/youtubei/v1/player?prettyPrint=false";
4466
const INNERTUBE_CLIENT_VERSION = "20.10.38";

‎services/transcript-bridge/server.mjs‎

Lines changed: 203 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,203 @@
1+
import http from "node:http";
2+
import { URL } from "node:url";
3+
4+
const PORT = parseInt(process.env.PORT || "4005", 10);
5+
const AUTH_TOKEN = process.env.BRIDGE_AUTH_TOKEN || "e06fdac10c7c18aed27e47e3d18860121a0ca378288a3e04";
6+
7+
function decodeHtmlEntities(text) {
8+
return text
9+
.replace(/&amp;/g, "&")
10+
.replace(/&lt;/g, "<")
11+
.replace(/&gt;/g, ">")
12+
.replace(/&quot;/g, '"')
13+
.replace(/&#39;/g, "'")
14+
.replace(/&apos;/g, "'")
15+
.replace(/&#x([0-9a-fA-F]+);/g, (_, hex) =>
16+
String.fromCodePoint(parseInt(hex, 16)),
17+
)
18+
.replace(/&#(\d+);/g, (_, dec) => String.fromCodePoint(parseInt(dec, 10)))
19+
.replace(/\s+/g, " ")
20+
.trim();
21+
}
22+
23+
async function fetchYouTubeCaptions(videoId) {
24+
const INNERTUBE_API_URL = "https://www.youtube.com/youtubei/v1/player?prettyPrint=false";
25+
const INNERTUBE_CLIENT_VERSION = "20.10.38";
26+
27+
try {
28+
let visitorData;
29+
try {
30+
const pageResp = await fetch(`https://www.youtube.com/watch?v=${videoId}`, {
31+
headers: {
32+
"User-Agent":
33+
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36",
34+
"Accept-Language": "en-US,en;q=0.9",
35+
},
36+
});
37+
if (pageResp.ok) {
38+
const html = await pageResp.text();
39+
const match = html.match(/"visitorData":"([^"]+)"/);
40+
if (match) {
41+
visitorData = match[1];
42+
}
43+
}
44+
} catch {}
45+
46+
const requestHeaders = {
47+
"Content-Type": "application/json",
48+
"User-Agent": `com.google.android.youtube/${INNERTUBE_CLIENT_VERSION} (Linux; U; Android 14)`,
49+
};
50+
if (visitorData) {
51+
requestHeaders["X-Goog-Visitor-Id"] = visitorData;
52+
}
53+
54+
const response = await fetch(INNERTUBE_API_URL, {
55+
method: "POST",
56+
headers: requestHeaders,
57+
body: JSON.stringify({
58+
context: {
59+
client: {
60+
clientName: "ANDROID",
61+
clientVersion: INNERTUBE_CLIENT_VERSION,
62+
...(visitorData ? { visitorData } : {}),
63+
},
64+
},
65+
videoId,
66+
}),
67+
});
68+
69+
if (!response.ok) {
70+
return { cues: [], fullText: "", status: "no_caption_available", error: `innertube_http_${response.status}` };
71+
}
72+
73+
const data = await response.json();
74+
const playability = data?.playabilityStatus?.status;
75+
const captionTracks = data?.captions?.playerCaptionsTracklistRenderer?.captionTracks;
76+
77+
if (!Array.isArray(captionTracks) || captionTracks.length === 0) {
78+
return { cues: [], fullText: "", status: "no_caption_available", error: `no_tracks_status_${playability}` };
79+
}
80+
81+
const track =
82+
captionTracks.find(
83+
(t) =>
84+
t?.languageCode?.toLowerCase().startsWith("en") ||
85+
t?.name?.runs?.[0]?.text?.toLowerCase().includes("english"),
86+
) || captionTracks[0];
87+
88+
if (!track?.baseUrl) {
89+
return { cues: [], fullText: "", status: "no_caption_available", error: "missing_track_baseUrl" };
90+
}
91+
92+
const captionResp = await fetch(track.baseUrl);
93+
if (!captionResp.ok) {
94+
return { cues: [], fullText: "", status: "no_caption_available", error: `caption_fetch_http_${captionResp.status}` };
95+
}
96+
97+
const xml = await captionResp.text();
98+
const cues = [];
99+
100+
const pRegex = /<p\s+t="(\d+)"(?:\s+d="(\d+)")?[^>]*>([\s\S]*?)<\/p>/g;
101+
let match;
102+
103+
while ((match = pRegex.exec(xml)) !== null) {
104+
const startMs = parseInt(match[1], 10);
105+
const durMs = match[2] ? parseInt(match[2], 10) : 3000;
106+
const inner = match[3];
107+
108+
let text = "";
109+
const sRegex = /<s[^>]*>([^<]*)<\/s>/g;
110+
let sMatch;
111+
while ((sMatch = sRegex.exec(inner)) !== null) {
112+
text += sMatch[1];
113+
}
114+
if (!text) {
115+
text = inner.replace(/<[^>]+>/g, "");
116+
}
117+
118+
text = decodeHtmlEntities(text);
119+
if (text && !text.startsWith("[")) {
120+
cues.push({
121+
start: Math.round((startMs / 1000) * 100) / 100,
122+
duration: Math.round((durMs / 1000) * 100) / 100,
123+
text,
124+
});
125+
}
126+
}
127+
128+
if (cues.length === 0) {
129+
const textRegex = /<text\s+start="([\d.]+)"(?:\s+dur="([\d.]+)")?[^>]*>([\s\S]*?)<\/text>/g;
130+
while ((match = textRegex.exec(xml)) !== null) {
131+
const start = parseFloat(match[1]);
132+
const dur = match[2] ? parseFloat(match[2]) : 3.0;
133+
const text = decodeHtmlEntities(match[3].replace(/<[^>]+>/g, ""));
134+
if (text && !text.startsWith("[")) {
135+
cues.push({
136+
start: Math.round(start * 100) / 100,
137+
duration: Math.round(dur * 100) / 100,
138+
text,
139+
});
140+
}
141+
}
142+
}
143+
144+
cues.sort((a, b) => a.start - b.start);
145+
const fullText = cues.map((c) => c.text).join(" ");
146+
147+
return {
148+
cues,
149+
fullText,
150+
status: cues.length > 0 ? "completed" : "no_caption_available",
151+
};
152+
} catch (err) {
153+
return {
154+
cues: [],
155+
fullText: "",
156+
status: "no_caption_available",
157+
error: err?.message || String(err),
158+
};
159+
}
160+
}
161+
162+
const server = http.createServer(async (req, res) => {
163+
const reqUrl = new URL(req.url || "/", `http://${req.headers.host || "localhost"}`);
164+
165+
if (reqUrl.pathname === "/health") {
166+
res.writeHead(200, { "Content-Type": "application/json" });
167+
res.end(JSON.stringify({ status: "healthy", timestamp: new Date().toISOString() }));
168+
return;
169+
}
170+
171+
if (reqUrl.pathname === "/transcript") {
172+
const auth = req.headers["authorization"] || req.headers["x-bridge-token"];
173+
const token = typeof auth === "string" ? auth.replace(/^Bearer\s+/i, "") : "";
174+
175+
if (AUTH_TOKEN && token !== AUTH_TOKEN) {
176+
res.writeHead(401, { "Content-Type": "application/json" });
177+
res.end(JSON.stringify({ error: "Unauthorized" }));
178+
return;
179+
}
180+
181+
const videoId = reqUrl.searchParams.get("videoId") || reqUrl.searchParams.get("v");
182+
if (!videoId) {
183+
res.writeHead(400, { "Content-Type": "application/json" });
184+
res.end(JSON.stringify({ error: "Missing videoId parameter" }));
185+
return;
186+
}
187+
188+
const result = await fetchYouTubeCaptions(videoId);
189+
res.writeHead(200, {
190+
"Content-Type": "application/json",
191+
"Cache-Control": "public, max-age=86400",
192+
});
193+
res.end(JSON.stringify(result));
194+
return;
195+
}
196+
197+
res.writeHead(404, { "Content-Type": "application/json" });
198+
res.end(JSON.stringify({ error: "Not found" }));
199+
});
200+
201+
server.listen(PORT, "127.0.0.1", () => {
202+
console.log(`[Transcript Bridge] Listening on http://127.0.0.1:${PORT}`);
203+
});

0 commit comments

Comments
 (0)