Skip to content

Commit 35ca735

Browse files
codercatdevcodercatdev
committed
fix(sanity,ci): deduplicate syndications, preserve strong asset/syndication refs, and sync active image binaries to dev
1 parent d8c576b commit 35ca735

2 files changed

Lines changed: 295 additions & 40 deletions

File tree

‎.github/workflows/nightly-dataset-sync.yml‎

Lines changed: 231 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -51,38 +51,133 @@ jobs:
5151
gzip -k -f /tmp/production.ndjson
5252
wc -l /tmp/production.ndjson
5353
54-
echo "3. Importing documents into 'dev' dataset via Sanity Mutate API..."
54+
echo "3. Importing documents and referenced image binaries into 'dev' dataset..."
5555
cat << 'EOF' > /tmp/sync.cjs
5656
const fs = require("fs");
5757
const projectId = process.env.SANITY_STUDIO_PROJECT_ID || "hfh83o0w";
5858
const tokens = [process.env.SANITY_API_WRITE_TOKEN, process.env.SANITY_AUTH_TOKEN].filter(Boolean);
5959
const lines = fs.readFileSync("/tmp/production.ndjson", "utf8").trim().split("\n").filter(Boolean);
6060
const allDocs = lines.map((l) => JSON.parse(l));
61+
62+
const assetDocs = allDocs.filter((d) => d._type === "sanity.imageAsset" || d._type === "sanity.fileAsset");
63+
const imageAssetMap = new Map(
64+
allDocs.filter((d) => d._type === "sanity.imageAsset").map((d) => [d._id, d])
65+
);
6166
const contentDocs = allDocs.filter((d) => d._type !== "sanity.imageAsset" && d._type !== "sanity.fileAsset");
62-
const contentIds = new Set(contentDocs.map((d) => d._id));
67+
68+
// Collect all known IDs (including assets and syndication.* targets) so valid strong references are never weakened
69+
const allTargetIds = new Set(allDocs.map((d) => d._id));
70+
const syndicationStubs = new Map();
71+
72+
for (const doc of contentDocs) {
73+
if (Array.isArray(doc.syndications)) {
74+
for (const item of doc.syndications) {
75+
if (item && typeof item._ref === "string" && item._ref.startsWith("syndication.")) {
76+
allTargetIds.add(item._ref);
77+
if (!contentDocs.some((d) => d._id === item._ref)) {
78+
const platform = item._ref.endsWith(".devto") ? "devto" : "hashnode";
79+
const parentId = doc._id.replace(/^drafts\./, "");
80+
syndicationStubs.set(item._ref, {
81+
_id: item._ref,
82+
_type: "syndication",
83+
platform,
84+
status: "simulated_complete",
85+
externalUrl: `https://${platform === "devto" ? "dev.to" : "hashnode.com"}/codingcatdev`,
86+
syndicatedAt: doc._updatedAt || new Date().toISOString(),
87+
target: { _type: "reference", _ref: parentId, _weak: true },
88+
});
89+
}
90+
}
91+
}
92+
}
93+
}
94+
95+
// Track any production documents that have duplicate syndications keys so we can clean them up in production too
96+
const prodDeduplicateMutations = [];
97+
for (const doc of contentDocs) {
98+
if (Array.isArray(doc.syndications) && doc.syndications.length > 0) {
99+
const seenRefs = new Set();
100+
const seenKeys = new Set();
101+
const deduped = [];
102+
let changed = false;
103+
104+
for (const item of doc.syndications) {
105+
if (!item || typeof item._ref !== "string") continue;
106+
if (seenRefs.has(item._ref)) {
107+
changed = true;
108+
continue;
109+
}
110+
seenRefs.add(item._ref);
111+
const detKey = `syn_${item._ref.replace(/[^a-zA-Z0-9]/g, "_")}`;
112+
const key = item._key && !seenKeys.has(item._key) ? item._key : detKey;
113+
if (key !== item._key || item._weak) {
114+
changed = true;
115+
}
116+
seenKeys.add(key);
117+
deduped.push({
118+
_key: key,
119+
_type: "reference",
120+
_ref: item._ref,
121+
});
122+
}
123+
124+
if (changed) {
125+
doc.syndications = deduped;
126+
prodDeduplicateMutations.push({
127+
patch: {
128+
id: doc._id,
129+
set: { syndications: deduped },
130+
},
131+
});
132+
}
133+
}
134+
}
63135
64136
function sanitizeDoc(obj) {
65137
if (!obj || typeof obj !== "object") return obj;
66-
if (Array.isArray(obj)) return obj.map(sanitizeDoc);
138+
if (Array.isArray(obj)) {
139+
const mapped = obj.map(sanitizeDoc);
140+
const usedKeys = new Set();
141+
const deduped = [];
142+
for (const item of mapped) {
143+
if (item && typeof item === "object" && typeof item._key === "string") {
144+
if (usedKeys.has(item._key)) {
145+
if (typeof item._ref === "string") {
146+
// Duplicate reference key in array: skip duplicate entry
147+
continue;
148+
}
149+
item._key = `${item._key}_${usedKeys.size}`;
150+
}
151+
usedKeys.add(item._key);
152+
}
153+
deduped.push(item);
154+
}
155+
return deduped;
156+
}
67157
const copy = {};
68158
for (const [k, v] of Object.entries(obj)) {
69159
if (k === "_rev") continue;
70160
copy[k] = sanitizeDoc(v);
71161
}
72-
if (typeof copy._ref === "string" && !contentIds.has(copy._ref)) {
73-
copy._weak = true;
162+
if (typeof copy._ref === "string") {
163+
if (allTargetIds.has(copy._ref)) {
164+
delete copy._weak;
165+
delete copy._strengthenOnPublish;
166+
} else {
167+
copy._weak = true;
168+
}
74169
}
75170
return copy;
76171
}
77172
78173
const sanitized = contentDocs.map(sanitizeDoc);
79-
console.log(`Prepared ${sanitized.length} content documents (from ${allDocs.length} total lines) for dataset "dev"...`);
174+
console.log(`Prepared ${sanitized.length} content documents and ${syndicationStubs.size} syndication stubs for dataset "dev"...`);
80175
81176
let activeToken = tokens[0];
82-
async function mutateBatch(mutations) {
177+
async function mutateBatch(mutations, dataset = "dev") {
83178
for (let attempt = 0; attempt < 5; attempt++) {
84179
const token = activeToken || tokens[attempt % tokens.length];
85-
const res = await fetch(`https://${projectId}.api.sanity.io/v2021-06-07/data/mutate/dev?visibility=deferred&autoGenerateArrayKeys=true`, {
180+
const res = await fetch(`https://${projectId}.api.sanity.io/v2021-06-07/data/mutate/${dataset}?visibility=deferred&autoGenerateArrayKeys=true`, {
86181
method: "POST",
87182
headers: {
88183
Authorization: `Bearer ${token}`,
@@ -105,20 +200,20 @@ jobs:
105200
await new Promise((r) => setTimeout(r, delay));
106201
continue;
107202
}
108-
throw new Error(`Mutate batch failed: HTTP ${res.status}: ${errText}`);
203+
throw new Error(`Mutate batch (${dataset}) failed: HTTP ${res.status}: ${errText}`);
109204
}
110-
throw new Error("Mutate batch failed after 5 retries.");
205+
throw new Error(`Mutate batch (${dataset}) failed after 5 retries.`);
111206
}
112207
113-
async function sendChunked(items, label) {
208+
async function sendChunked(items, label, op = "createOrReplace") {
114209
let batch = [];
115210
let batchBytes = 0;
116211
let sent = 0;
117212
const MAX_ITEMS = 200;
118213
const MAX_BYTES = 2500000;
119214
120215
for (const item of items) {
121-
const mutation = { createOrReplace: item };
216+
const mutation = { [op]: item };
122217
const itemBytes = Buffer.byteLength(JSON.stringify(mutation), "utf8");
123218
if (batch.length > 0 && (batch.length >= MAX_ITEMS || batchBytes + itemBytes > MAX_BYTES)) {
124219
await mutateBatch(batch);
@@ -139,20 +234,135 @@ jobs:
139234
console.log(`[${label}] Completed (${sent} / ${items.length}).`);
140235
}
141236
237+
async function uploadMissingDevImages() {
238+
const q = encodeURIComponent(`*[_type == "sanity.imageAsset" && path match "images/${projectId}/dev/*"]._id`);
239+
let devUploadedIds = new Set();
240+
for (const token of tokens) {
241+
const res = await fetch(`https://${projectId}.api.sanity.io/v2025-02-19/data/query/dev?query=${q}`, {
242+
headers: { Authorization: `Bearer ${token}` },
243+
});
244+
if (res.ok) {
245+
const json = await res.json();
246+
devUploadedIds = new Set(json.result || []);
247+
break;
248+
}
249+
}
250+
console.log(`Found ${devUploadedIds.size} image binaries already in 'dev' CDN bucket.`);
251+
252+
const activeTypes = new Set(["post", "podcast", "guest", "author", "sponsor", "page", "settings", "podcastType"]);
253+
const priorityRefs = new Set();
254+
const inlineRefs = new Set();
255+
256+
function collectRefs(obj) {
257+
if (!obj || typeof obj !== "object") return;
258+
if (Array.isArray(obj)) return obj.forEach(collectRefs);
259+
if (typeof obj._ref === "string" && obj._ref.startsWith("image-")) {
260+
inlineRefs.add(obj._ref);
261+
}
262+
for (const v of Object.values(obj)) collectRefs(v);
263+
}
264+
265+
for (const doc of contentDocs) {
266+
if (!activeTypes.has(doc._type)) continue;
267+
if (doc.coverImage?.asset?._ref) priorityRefs.add(doc.coverImage.asset._ref);
268+
if (doc.photo?.asset?._ref) priorityRefs.add(doc.photo.asset._ref);
269+
if (doc.ogImage?.asset?._ref) priorityRefs.add(doc.ogImage.asset._ref);
270+
collectRefs(doc);
271+
}
272+
273+
const orderedRefs = [...new Set([...priorityRefs, ...inlineRefs])].filter(
274+
(id) => !devUploadedIds.has(id) && imageAssetMap.has(id)
275+
);
276+
console.log(`Uploading ${orderedRefs.length} missing active image binaries to 'dev' dataset...`);
277+
278+
let uploaded = 0;
279+
let idx = 0;
280+
const CONCURRENCY = 20;
281+
282+
async function worker() {
283+
while (idx < orderedRefs.length) {
284+
const currentIdx = idx++;
285+
const assetId = orderedRefs[currentIdx];
286+
const assetDoc = imageAssetMap.get(assetId);
287+
if (!assetDoc?.url) continue;
288+
289+
for (let attempt = 0; attempt < 4; attempt++) {
290+
try {
291+
const imgRes = await fetch(assetDoc.url);
292+
if (!imgRes.ok) break;
293+
const buf = Buffer.from(await imgRes.arrayBuffer());
294+
const token = activeToken || tokens[attempt % tokens.length];
295+
const filename = encodeURIComponent(assetDoc.originalFilename || `${assetId}.${assetDoc.extension || "jpg"}`);
296+
const upRes = await fetch(`https://${projectId}.api.sanity.io/v2021-06-07/assets/images/dev?filename=${filename}`, {
297+
method: "POST",
298+
headers: {
299+
Authorization: `Bearer ${token}`,
300+
"Content-Type": assetDoc.mimeType || imgRes.headers.get("content-type") || "image/jpeg",
301+
},
302+
body: buf,
303+
});
304+
if (upRes.ok) {
305+
activeToken = token;
306+
devUploadedIds.add(assetId);
307+
uploaded++;
308+
if (uploaded % 100 === 0 || uploaded === orderedRefs.length) {
309+
console.log(`[Image Binaries] Uploaded ${uploaded} / ${orderedRefs.length}...`);
310+
}
311+
break;
312+
}
313+
if (upRes.status === 401 || upRes.status === 403) {
314+
activeToken = tokens.find((t) => t !== token) || token;
315+
continue;
316+
}
317+
if (upRes.status === 429 || upRes.status >= 500) {
318+
await new Promise((r) => setTimeout(r, 500 * Math.pow(2, attempt)));
319+
continue;
320+
}
321+
break;
322+
} catch (err) {
323+
await new Promise((r) => setTimeout(r, 500));
324+
}
325+
}
326+
}
327+
}
328+
329+
await Promise.all(Array.from({ length: CONCURRENCY }, () => worker()));
330+
console.log(`[Image Binaries] Finished uploading ${uploaded} image binaries to 'dev'.`);
331+
return devUploadedIds;
332+
}
333+
142334
(async () => {
143-
const stubs = sanitized.map((d) => ({ _id: d._id, _type: d._type }));
144-
await sendChunked(stubs, "Pass 1: Stubs");
145-
await sendChunked(sanitized, "Pass 2: Content Docs");
335+
if (prodDeduplicateMutations.length > 0) {
336+
console.log(`Cleaning up ${prodDeduplicateMutations.length} duplicate syndications arrays in production...`);
337+
try {
338+
await mutateBatch(prodDeduplicateMutations, "production");
339+
console.log("Production syndications arrays deduplicated.");
340+
} catch (prodErr) {
341+
console.warn("Could not patch production syndications:", prodErr.message);
342+
}
343+
}
344+
345+
// Upload missing active image binaries to dev (creates real sanity.imageAsset docs with dev bucket paths)
346+
const devUploadedIds = await uploadMissingDevImages();
146347
147-
const assetDocs = allDocs
148-
.filter((d) => d._type === "sanity.imageAsset" || d._type === "sanity.fileAsset")
348+
// Import remaining asset metadata docs (only those not already uploaded as dev binaries) so all asset refs resolve
349+
const remainingAssetDocs = assetDocs
350+
.filter((d) => !devUploadedIds.has(d._id))
149351
.map(({ _rev, ...rest }) => rest);
150-
try {
151-
await sendChunked(assetDocs, "Pass 3: Asset Docs");
152-
} catch (assetErr) {
153-
console.warn("Note: Asset metadata docs skipped (served via CDN):", assetErr.message);
352+
if (remainingAssetDocs.length > 0) {
353+
await sendChunked(remainingAssetDocs, "Pass 1a: Remaining Asset Metadata", "createIfNotExists");
154354
}
155355
356+
// Create content + syndication stubs so all internal document references resolve as strong references
357+
const stubs = [
358+
...sanitized.map((d) => ({ _id: d._id, _type: d._type })),
359+
...Array.from(syndicationStubs.values()),
360+
];
361+
await sendChunked(stubs, "Pass 1b: Document & Syndication Stubs", "createIfNotExists");
362+
363+
// Populate full content documents (with strong coverImage.asset & syndications references and unique _keys)
364+
await sendChunked(sanitized, "Pass 2: Content Docs", "createOrReplace");
365+
156366
console.log("Dataset sync completed successfully!");
157367
})().catch((err) => {
158368
console.error(err);

0 commit comments

Comments
 (0)