|
| 1 | +#!/usr/bin/env node |
| 2 | +/** |
| 3 | + * Railway cron / local runner for catalog indexing. |
| 4 | + * |
| 5 | + * Reuses the same pure indexer functions as Trigger.dev tasks and /api/cron. |
| 6 | + * Must exit when finished so Railway cron can schedule the next run. |
| 7 | + * |
| 8 | + * Usage: |
| 9 | + * node scripts/run-indexer.js [task] |
| 10 | + * INDEXER_TASK=plugins node scripts/run-indexer.js |
| 11 | + * npm run index:cron -- stats |
| 12 | + * |
| 13 | + * Tasks: |
| 14 | + * mcp | marketplaces | plugins | skills | stats | markdown | scheduled | all |
| 15 | + * |
| 16 | + * Default (scheduled): day-rotated primary task + skills.sh + markdown reindex. |
| 17 | + */ |
| 18 | +const path = require('path') |
| 19 | + |
| 20 | +const webuiRoot = path.join(__dirname, '..') |
| 21 | + |
| 22 | +// Load .env.local locally; on Railway vars are already injected. |
| 23 | +try { |
| 24 | + require('@next/env').loadEnvConfig(webuiRoot) |
| 25 | +} catch { |
| 26 | + // @next/env may be unavailable in a minimal install; Railway supplies env. |
| 27 | +} |
| 28 | + |
| 29 | +const jiti = require('jiti')(__filename, { alias: { '@': webuiRoot } }) |
| 30 | +const { indexMCPServers, syncMCPServerStats } = jiti( |
| 31 | + '../lib/indexer/mcp-server-indexer.ts' |
| 32 | +) |
| 33 | +const { indexMarketplaces } = jiti('../lib/indexer/marketplace-indexer.ts') |
| 34 | +const { indexPlugins } = jiti('../lib/indexer/plugin-indexer.ts') |
| 35 | +const { indexSkillsFromSkillsSh } = jiti('../lib/indexer/skills-sh-indexer.ts') |
| 36 | + |
| 37 | +const APP_BASE_URL = process.env.APP_BASE_URL || 'https://buildwithclaude.com' |
| 38 | + |
| 39 | +const TASK_NAMES = { |
| 40 | + mcp: 'MCP servers', |
| 41 | + marketplaces: 'Marketplaces', |
| 42 | + plugins: 'Plugins', |
| 43 | + skills: 'skills.sh', |
| 44 | + stats: 'MCP server stats', |
| 45 | + markdown: 'Deploy-static content reindex', |
| 46 | + scheduled: 'Scheduled (day-rotated + skills + markdown)', |
| 47 | + all: 'All indexing tasks', |
| 48 | +} |
| 49 | + |
| 50 | +/** @type {Record<number, keyof typeof TASK_NAMES | null>} */ |
| 51 | +const DAY_TO_TASK = { |
| 52 | + 0: 'stats', |
| 53 | + 1: 'mcp', |
| 54 | + 2: 'marketplaces', |
| 55 | + 3: 'plugins', |
| 56 | + 4: 'stats', |
| 57 | + 5: 'marketplaces', |
| 58 | + 6: 'plugins', |
| 59 | +} |
| 60 | + |
| 61 | +/** |
| 62 | + * Ask the web app to refresh Meilisearch for the given types. |
| 63 | + * Never throws — search hiccups must not fail the DB sync. |
| 64 | + * @param {string[]} types |
| 65 | + */ |
| 66 | +async function reindexSearch(types) { |
| 67 | + const adminToken = process.env.ADMIN_API_TOKEN |
| 68 | + if (!adminToken) { |
| 69 | + console.warn('[search] ADMIN_API_TOKEN not set; skipping reindex') |
| 70 | + return |
| 71 | + } |
| 72 | + for (const type of types) { |
| 73 | + try { |
| 74 | + const res = await fetch( |
| 75 | + `${APP_BASE_URL}/api/admin/reindex-search?mode=type&type=${type}`, |
| 76 | + { |
| 77 | + method: 'POST', |
| 78 | + headers: { Authorization: `Bearer ${adminToken}` }, |
| 79 | + } |
| 80 | + ) |
| 81 | + console.log(`[search] reindex ${type} -> HTTP ${res.status}`) |
| 82 | + } catch (error) { |
| 83 | + console.error(`[search] reindex ${type} failed:`, error) |
| 84 | + } |
| 85 | + } |
| 86 | +} |
| 87 | + |
| 88 | +/** |
| 89 | + * @param {string} task |
| 90 | + * @returns {Promise<unknown>} |
| 91 | + */ |
| 92 | +async function runPrimaryTask(task) { |
| 93 | + switch (task) { |
| 94 | + case 'mcp': { |
| 95 | + const result = await indexMCPServers() |
| 96 | + await reindexSearch(['mcp-server']) |
| 97 | + return result |
| 98 | + } |
| 99 | + case 'marketplaces': { |
| 100 | + const result = await indexMarketplaces() |
| 101 | + await reindexSearch(['marketplace']) |
| 102 | + return result |
| 103 | + } |
| 104 | + case 'plugins': { |
| 105 | + const result = await indexPlugins() |
| 106 | + await reindexSearch(['plugin', 'skill']) |
| 107 | + return result |
| 108 | + } |
| 109 | + case 'skills': { |
| 110 | + const result = await indexSkillsFromSkillsSh() |
| 111 | + await reindexSearch(['skill']) |
| 112 | + return result |
| 113 | + } |
| 114 | + case 'stats': { |
| 115 | + const result = await syncMCPServerStats() |
| 116 | + await reindexSearch(['mcp-server']) |
| 117 | + return result |
| 118 | + } |
| 119 | + case 'markdown': { |
| 120 | + const reindexed = ['subagent', 'command', 'hook', 'plugin', 'skill'] |
| 121 | + await reindexSearch(reindexed) |
| 122 | + return { reindexed } |
| 123 | + } |
| 124 | + default: |
| 125 | + throw new Error(`Unknown task: ${task}`) |
| 126 | + } |
| 127 | +} |
| 128 | + |
| 129 | +async function main() { |
| 130 | + // CLI argv overrides INDEXER_TASK so one-off tests work while cron keeps scheduled. |
| 131 | + const arg = process.argv[2] || process.env.INDEXER_TASK || 'scheduled' |
| 132 | + const task = arg.toLowerCase() |
| 133 | + |
| 134 | + if (!(task in TASK_NAMES) && task !== 'scheduled' && task !== 'all') { |
| 135 | + console.error( |
| 136 | + `Unknown task "${task}". Valid: ${Object.keys(TASK_NAMES).join(', ')}` |
| 137 | + ) |
| 138 | + process.exit(1) |
| 139 | + } |
| 140 | + |
| 141 | + const startTime = Date.now() |
| 142 | + console.log(`[indexer] Starting task="${task}" (${TASK_NAMES[task] || task})`) |
| 143 | + |
| 144 | + /** @type {Record<string, unknown>} */ |
| 145 | + const results = {} |
| 146 | + |
| 147 | + if (task === 'all') { |
| 148 | + for (const name of [ |
| 149 | + 'mcp', |
| 150 | + 'marketplaces', |
| 151 | + 'plugins', |
| 152 | + 'skills', |
| 153 | + 'stats', |
| 154 | + 'markdown', |
| 155 | + ]) { |
| 156 | + console.log(`[indexer] --- ${name} ---`) |
| 157 | + results[name] = await runPrimaryTask(name) |
| 158 | + } |
| 159 | + } else if (task === 'scheduled') { |
| 160 | + const dayOfWeek = new Date().getUTCDay() |
| 161 | + const primary = DAY_TO_TASK[dayOfWeek] |
| 162 | + if (primary) { |
| 163 | + console.log( |
| 164 | + `[indexer] Day ${dayOfWeek}: primary=${primary} (${TASK_NAMES[primary]})` |
| 165 | + ) |
| 166 | + results.primary = primary |
| 167 | + results[primary] = await runPrimaryTask(primary) |
| 168 | + } else { |
| 169 | + console.log(`[indexer] Day ${dayOfWeek}: no primary indexing scheduled`) |
| 170 | + results.primary = null |
| 171 | + } |
| 172 | + |
| 173 | + console.log('[indexer] --- skills (daily) ---') |
| 174 | + results.skills = await runPrimaryTask('skills') |
| 175 | + |
| 176 | + console.log('[indexer] --- markdown (daily) ---') |
| 177 | + results.markdown = await runPrimaryTask('markdown') |
| 178 | + } else { |
| 179 | + results[task] = await runPrimaryTask(task) |
| 180 | + } |
| 181 | + |
| 182 | + const durationMs = Date.now() - startTime |
| 183 | + console.log(`[indexer] Completed in ${durationMs}ms`) |
| 184 | + console.log('[indexer] Result:', JSON.stringify(results, null, 2)) |
| 185 | +} |
| 186 | + |
| 187 | +main() |
| 188 | + .then(() => process.exit(0)) |
| 189 | + .catch((err) => { |
| 190 | + console.error('[indexer] Failed:', err) |
| 191 | + process.exit(1) |
| 192 | + }) |
0 commit comments