From 89fffa0cec1b322ccd042432f242686c74ef9793 Mon Sep 17 00:00:00 2001 From: Johl Brown Date: Fri, 15 May 2026 23:13:14 +1000 Subject: [PATCH 1/2] feat: add structured archive export --- .env.example | 13 +- README.md | 35 ++- package.json | 1 + src/download-import/cli.ts | 80 ++++++ .../perplexity-download-bundler.ts | 242 ++++++++++++++++++ src/export/file-writer.ts | 94 ++++++- src/export/sanitizer.ts | 22 +- src/scraper/browser.ts | 22 +- src/scraper/conversation-extractor.ts | 242 +++++++++++++++++- src/scraper/worker-pool.ts | 91 ++++++- src/utils/config.ts | 23 +- test/setup.ts | 4 +- test/unit/config.unit.test.ts | 52 ++++ test/unit/conversation-extractor.unit.test.ts | 147 +++++++++++ test/unit/file-writer.unit.test.ts | 83 ++++++ .../perplexity-download-bundler.unit.test.ts | 57 +++++ test/unit/sanitizer.unit.test.ts | 9 +- 17 files changed, 1160 insertions(+), 57 deletions(-) create mode 100644 src/download-import/cli.ts create mode 100644 src/download-import/perplexity-download-bundler.ts create mode 100644 test/unit/config.unit.test.ts create mode 100644 test/unit/conversation-extractor.unit.test.ts create mode 100644 test/unit/file-writer.unit.test.ts create mode 100644 test/unit/perplexity-download-bundler.unit.test.ts diff --git a/.env.example b/.env.example index 0cbfdc3..896d6e7 100644 --- a/.env.example +++ b/.env.example @@ -2,13 +2,18 @@ AUTH_STORAGE_PATH=.storage/auth.json # Scraping behavior -WAIT_MODE=fixed +WAIT_MODE=static RATE_LIMIT_MS=3000 PARALLEL_WORKERS=2 CHECKPOINT_SAVE_INTERVAL=10 -# Vector search -ENABLE_VECTOR_SEARCH=true +# Export formats +EXPORT_STRUCTURED_JSON=true +EXPORT_MARKDOWN=false +STRUCTURED_EXPORT_DIR=exports + +# Vector search (requires Markdown exports and Ollama) +ENABLE_VECTOR_SEARCH=false # AI services GEMINI_API_KEY= @@ -23,4 +28,4 @@ VECTOR_INDEX_PATH=.storage/vector-index # Browser behavior # HEADLESS can be 'true', 'false', or 'new' -HEADLESS=true +HEADLESS=false diff --git a/README.md b/README.md index 37ed959..2dae6d0 100644 --- a/README.md +++ b/README.md @@ -35,14 +35,16 @@ ## Introduction -This tool is designed to externalize your Perplexity.ai conversation history into organized, semantically searchable Markdown files. It facilitates the emergence of a personal knowledge base powered by local AI, bridging the gap between ephemeral inquiry and structured knowledge. +This tool is designed to externalize your Perplexity.ai conversation history into structured JSON archives suitable for canonical SQLite archival, full-text search, vector indexing, and downstream tools such as MyChatArchive/ITIR. Markdown and vector search remain available as optional sidecars for local reading and semantic exploration. ## Key Features - **Parallelized Extraction**: Leverages Playwright to extract multiple conversation threads simultaneously for high-velocity data retrieval. - **Architectural Resilience**: Automatically restores browser contexts and retries operations, ensuring continuity amidst environmental instability. -- **Advanced RAG (Retrieval-Augmented Generation)**: Engage in a cognitive dialogue with your history. The system employs intent analysis to synthesize broad summaries or pinpoint specific technical insights. -- **Semantic Vector Search**: Move beyond keyword matching. Locate information based on conceptual depth and semantic relevance. +- **Structured Archive Output**: Emits `itir.perplexity.thread.v1` JSON artifacts with normalized messages, stable source IDs, metadata, and captured API data for downstream SQLite/archive ingest. +- **Optional Markdown Sidecars**: Preserve the previous human-readable Markdown export path when `EXPORT_MARKDOWN=true`. +- **Optional RAG (Retrieval-Augmented Generation)**: Engage in a cognitive dialogue with your history when vector search is enabled. +- **Optional Semantic Vector Search**: Move beyond keyword matching with Markdown sidecars, Ollama, and Vectra enabled. - **Persistent State Tracking**: Frequent checkpoints allow the system to resume progress after any interruption. - **Interactive Synthesis (REPL)**: A streamlined command-line interface for human-system synergy. @@ -101,11 +103,14 @@ cp .env.example .env ### Key Environment Variables -- **HEADLESS**: Set to `false` in your `.env` file. **Note:** Headless mode (`true`) is currently non-functional due to Cloudflare Turnstile protection on Perplexity.ai. Using headful mode allows you to complete any challenges manually if they appear. +- **HEADLESS**: Defaults to `false`. **Note:** Headless mode (`true`) is currently non-functional due to Cloudflare Turnstile protection on Perplexity.ai. Using headful mode allows you to complete any challenges manually if they appear. - **OLLAMA_URL**: Access point for your local AI engine (default: http://localhost:11434). - **OLLAMA_MODEL**: Cognitive model for RAG synthesis (e.g., deepseek-r1). - **OLLAMA_EMBED_MODEL**: Model for generating vector representations (e.g., nomic-embed-text). -- **ENABLE_VECTOR_SEARCH**: Set to `true` to activate semantic and RAG layers. +- **EXPORT_STRUCTURED_JSON**: Defaults to `true`. Writes canonical `itir.perplexity.thread.v1` JSON artifacts for downstream archive ingest. +- **STRUCTURED_EXPORT_DIR**: Defaults to `EXPORT_DIR`. Set this to separate canonical JSON archives from sidecar files. +- **EXPORT_MARKDOWN**: Defaults to `false`. Set to `true` to also write the previous Markdown files. +- **ENABLE_VECTOR_SEARCH**: Defaults to `false`. Set to `true` to activate semantic and RAG layers. Current vector indexing reads Markdown exports, so enable `EXPORT_MARKDOWN=true` before rebuilding the vector index. ## Usage Guide @@ -120,6 +125,24 @@ npm run dev - **Start scraper (Library)**: Initiates extraction. Authenticate manually if required. - **Note**: Due to the complexity of Perplexity's API and potential network fluctuations, it may be necessary to **run the scraper multiple times** to ensure all conversations are fully gathered. The system uses checkpoints to resume where it left off. +- **Canonical archive**: The primary export is structured JSON. Treat Markdown and vector indexes as optional sidecars that can be regenerated from canonical thread/message records. +- **SQLite/MyChatArchive ingest**: After exporting, tools such as `chat-export-structurer` can ingest the structured JSON into a canonical SQLite archive: + ```bash + python src/ingest.py \ + --in /path/to/perplexity-ai-export/exports \ + --format perplexity \ + --account perplexity \ + --source-id perplexity_auto + ``` +- **Bundle downloaded Perplexity Markdown**: If Perplexity's API only returns the first page of a long thread, place the downloaded `.md` chunks in a local folder and run: + ```bash + npm run bundle:perplexity-downloads -- \ + --input /path/to/downloaded/perplexity-markdown \ + --title-prefix "Thread title prefix" \ + --thread-id "" \ + --out exports-downloads/thread.download.itir.perplexity.json + ``` + Then ingest that JSON with `chat-export-structurer --format perplexity --account perplexity` so the recovered turns attach to the same canonical Perplexity thread. - **Search conversations**: Interface with your history using various modes: - **Auto**: Heuristic selection between semantic and exact search. - **Semantic**: Fuzzy matching via high-dimensional vector space. @@ -160,4 +183,4 @@ npm run test:unit # Execute integration-level verifications npm run test:integration -``` \ No newline at end of file +``` diff --git a/package.json b/package.json index a124b92..166abdf 100644 --- a/package.json +++ b/package.json @@ -7,6 +7,7 @@ "start": "node dist/index.js", "build": "tsc", "build:exe": "node scripts/build-exe.js && npm run format", + "bundle:perplexity-downloads": "tsx src/download-import/cli.ts", "release": "release-it", "type-check": "tsc --noEmit", "toc": "node scripts/update-toc.js", diff --git a/src/download-import/cli.ts b/src/download-import/cli.ts new file mode 100644 index 0000000..03109be --- /dev/null +++ b/src/download-import/cli.ts @@ -0,0 +1,80 @@ +import { bundlePerplexityDownloads } from './perplexity-download-bundler.js' + +function parseArgs(argv: string[]): { + inputs: string[] + titlePrefix: string + outPath: string + threadId?: string + title?: string +} { + const inputs: string[] = [] + let titlePrefix = '' + let outPath = 'exports-downloads/perplexity-download.itir.perplexity.json' + let threadId: string | undefined + let title: string | undefined + + for (let i = 0; i < argv.length; i++) { + const arg = argv[i] + const next = argv[i + 1] + if (arg === '--input' && next) { + inputs.push(next) + i++ + } else if (arg === '--title-prefix' && next) { + titlePrefix = next + i++ + } else if (arg === '--out' && next) { + outPath = next + i++ + } else if (arg === '--thread-id' && next) { + threadId = next + i++ + } else if (arg === '--title' && next) { + title = next + i++ + } else if (arg === '--help') { + printHelp() + process.exit(0) + } + } + + if (inputs.length === 0) { + throw new Error('Provide at least one --input file or directory.') + } + + if (!titlePrefix) { + throw new Error('Provide --title-prefix so unrelated Markdown files are not bundled.') + } + + return { + inputs, + titlePrefix, + outPath, + threadId, + title, + } +} + +function printHelp(): void { + console.log(`Usage: + npm run bundle:perplexity-downloads -- [options] + +Options: + --input File or directory to scan. Repeatable. + --title-prefix Download filename prefix to match. + --out Output .itir.perplexity.json path. + --thread-id Stable source thread id for the bundle. + --title Archive thread title. +`) +} + +try { + const summary = bundlePerplexityDownloads(parseArgs(process.argv.slice(2))) + console.log(`Wrote ${summary.outPath}`) + console.log(`Source files: ${summary.sourceFiles}`) + console.log(`Parsed turns: ${summary.parsedTurns}`) + console.log(`Unique turns: ${summary.uniqueTurns}`) + console.log(`Messages: ${summary.messages}`) +} catch (error) { + console.error(error instanceof Error ? error.message : String(error)) + process.exit(1) +} diff --git a/src/download-import/perplexity-download-bundler.ts b/src/download-import/perplexity-download-bundler.ts new file mode 100644 index 0000000..25c08f2 --- /dev/null +++ b/src/download-import/perplexity-download-bundler.ts @@ -0,0 +1,242 @@ +import { existsSync, mkdirSync, readdirSync, readFileSync, statSync, writeFileSync } from 'node:fs' +import { basename, dirname, extname, join, resolve } from 'node:path' +import { createHash } from 'node:crypto' + +export interface BundleOptions { + inputs: string[] + titlePrefix: string + outPath: string + threadId?: string + title?: string +} + +interface SourceDocument { + path: string + kind: 'markdown' + ordinal: number + text: string +} + +interface ParsedTurn { + question: string + answer: string + sourcePath: string + sourceOrdinal: number + segmentIndex: number +} + +export interface BundleSummary { + outPath: string + sourceFiles: number + parsedTurns: number + uniqueTurns: number + messages: number +} + +const DEFAULT_THREAD_ID = 'perplexity-download-bundle' + +export function bundlePerplexityDownloads(options: BundleOptions): BundleSummary { + const sourceDocuments = discoverMarkdownSources(options.inputs, options.titlePrefix) + if (sourceDocuments.length === 0) { + throw new Error(`No Markdown downloads matched title prefix: ${options.titlePrefix}`) + } + + const parsedTurns = sourceDocuments.flatMap((source) => parseMarkdownTurns(source)) + const uniqueTurns = dedupeTurns(parsedTurns) + const threadId = options.threadId ?? DEFAULT_THREAD_ID + const title = + options.title ?? sourceDocuments[0]?.text.match(/^#\s+(.+)$/m)?.[1] ?? options.titlePrefix + const exportedAt = new Date().toISOString() + + const messages = uniqueTurns.flatMap((turn, turnIndex) => [ + { + role: 'user', + content: turn.question, + source_message_id: `${threadId}:download:${turnIndex + 1}:user`, + created_at: exportedAt, + turn_index: turnIndex, + provenance: { + source: 'perplexity_download_markdown', + source_path: turn.sourcePath, + source_ordinal: turn.sourceOrdinal, + segment_index: turn.segmentIndex, + }, + }, + { + role: 'assistant', + content: turn.answer, + source_message_id: `${threadId}:download:${turnIndex + 1}:assistant`, + created_at: exportedAt, + turn_index: turnIndex, + provenance: { + source: 'perplexity_download_markdown', + source_path: turn.sourcePath, + source_ordinal: turn.sourceOrdinal, + segment_index: turn.segmentIndex, + }, + }, + ]) + + const bundle = { + schema: 'itir.perplexity.thread.v1', + source: 'perplexity_download_bundle', + source_thread_id: threadId, + url: '', + title, + space: 'Downloaded Perplexity', + updated_at: exportedAt, + exported_at: exportedAt, + messages, + raw: { + source_files: sourceDocuments.map((source) => ({ + path: source.path, + kind: source.kind, + ordinal: source.ordinal, + bytes: Buffer.byteLength(source.text, 'utf8'), + })), + parsed_turns: parsedTurns.length, + unique_turns: uniqueTurns.length, + }, + } + + const absoluteOutPath = resolve(options.outPath) + const outDir = dirname(absoluteOutPath) + if (!existsSync(outDir)) { + mkdirSync(outDir, { recursive: true }) + } + writeFileSync(absoluteOutPath, JSON.stringify(bundle, null, 2), 'utf8') + + return { + outPath: absoluteOutPath, + sourceFiles: sourceDocuments.length, + parsedTurns: parsedTurns.length, + uniqueTurns: uniqueTurns.length, + messages: messages.length, + } +} + +export function discoverMarkdownSources(inputs: string[], titlePrefix: string): SourceDocument[] { + const normalizedPrefix = normalizeTitlePrefix(titlePrefix) + const candidates = inputs.flatMap((input) => collectMarkdownFiles(resolve(input))) + + return candidates + .filter((path) => + normalizeTitlePrefix(basename(path, extname(path))).startsWith(normalizedPrefix) + ) + .map((path) => ({ + path, + kind: 'markdown' as const, + ordinal: extractOrdinal(path), + text: readFileSync(path, 'utf8'), + })) + .sort((a, b) => a.ordinal - b.ordinal || a.path.localeCompare(b.path)) +} + +export function parseMarkdownTurns(source: SourceDocument): ParsedTurn[] { + const body = stripPerplexityChrome(source.text) + const chunks = body + .split(/\n-{3,}\n/g) + .map((chunk) => chunk.trim()) + .filter(Boolean) + + const turns: ParsedTurn[] = [] + for (let segmentIndex = 0; segmentIndex < chunks.length; segmentIndex++) { + const parsed = parseMarkdownChunk(chunks[segmentIndex] ?? '') + if (!parsed) continue + turns.push({ + ...parsed, + sourcePath: source.path, + sourceOrdinal: source.ordinal, + segmentIndex, + }) + } + return turns +} + +function parseMarkdownChunk(chunk: string): Pick | null { + const heading = chunk.match(/^#\s+(.+?)(?:\n|$)/s) + if (!heading) return null + + const question = cleanText(heading[1] ?? '') + const answer = cleanText(chunk.slice(heading[0].length)) + if (!question || !answer) return null + return { question, answer } +} + +function dedupeTurns(turns: ParsedTurn[]): ParsedTurn[] { + const seen = new Set() + const unique: ParsedTurn[] = [] + + for (const turn of turns) { + const key = hashTurn(turn) + if (seen.has(key)) continue + seen.add(key) + unique.push(turn) + } + return unique +} + +function hashTurn(turn: Pick): string { + return createHash('sha1') + .update(normalizeForDedupe(turn.question)) + .update('\0') + .update(normalizeForDedupe(turn.answer)) + .digest('hex') +} + +function stripPerplexityChrome(text: string): string { + return text + .replace(/]*pplx-full-logo[^>]*>\s*/gi, '') + .replace(/[\s\S]*?<\/span>/gi, '') + .replace(/
[\s\S]*?<\/div>/gi, '') + .trim() +} + +function cleanText(text: string): string { + return text + .replace(/\r\n/g, '\n') + .replace(/[ \t]+\n/g, '\n') + .trim() +} + +function normalizeForDedupe(text: string): string { + return cleanText(text).replace(/\s+/g, ' ').toLowerCase() +} + +function normalizeTitlePrefix(text: string): string { + return text + .replace(/\(\d+\)$/g, '') + .replace(/-\d+$/g, '') + .replace(/[^a-z0-9]+/gi, ' ') + .replace(/\s+/g, ' ') + .trim() + .toLowerCase() +} + +function collectMarkdownFiles(path: string): string[] { + if (!existsSync(path)) return [] + const stats = statSync(path) + if (stats.isFile()) { + return extname(path).toLowerCase() === '.md' ? [path] : [] + } + if (!stats.isDirectory()) return [] + + return readdirSync(path) + .map((entry) => join(path, entry)) + .filter((entryPath) => { + try { + return statSync(entryPath).isFile() && extname(entryPath).toLowerCase() === '.md' + } catch (_error) { + return false + } + }) +} + +function extractOrdinal(path: string): number { + const name = basename(path, extname(path)) + const parenMatch = name.match(/\((\d+)\)$/) + if (parenMatch?.[1]) return Number(parenMatch[1]) + const dashMatch = name.match(/-(\d+)$/) + if (dashMatch?.[1]) return Number(dashMatch[1]) + return 0 +} diff --git a/src/export/file-writer.ts b/src/export/file-writer.ts index 62170e2..48f0464 100644 --- a/src/export/file-writer.ts +++ b/src/export/file-writer.ts @@ -4,6 +4,12 @@ import { config } from '../utils/config.js' import type { ExtractedConversation } from '../scraper/conversation-extractor.js' import { sanitizeFilename, sanitizeSpaceName } from './sanitizer.js' +export interface WrittenConversationFiles { + primaryPath: string + structuredJsonPath?: string + markdownPath?: string +} + export class FileWriter { static readonly WriteError = class extends Error { constructor(message: string) { @@ -16,21 +22,38 @@ export class FileWriter { this.ensureRootExportDirectoryExists() } - write(conversation: ExtractedConversation): string { + write(conversation: ExtractedConversation): WrittenConversationFiles { try { - const destinationFilePath = this.constructDestinationFilePath(conversation) - const markdownContent = this.formatConversationAsMarkdown(conversation) + if (!config.exportStructuredJson && !config.exportMarkdown) { + throw new FileWriter.WriteError( + 'No export formats enabled. Enable EXPORT_STRUCTURED_JSON or EXPORT_MARKDOWN.' + ) + } - const spaceSpecificDirectory = join( - config.exportDir, - sanitizeSpaceName(conversation.spaceName) - ) - if (!existsSync(spaceSpecificDirectory)) { - mkdirSync(spaceSpecificDirectory, { recursive: true }) + const writtenFiles: Partial = {} + + if (config.exportStructuredJson) { + const structuredJsonPath = this.constructStructuredJsonPath(conversation) + this.ensureSpaceDirectoryExists(config.structuredExportDir, conversation.spaceName) + writeFileSync( + structuredJsonPath, + JSON.stringify(this.formatConversationAsStructuredJson(conversation), null, 2), + 'utf-8' + ) + writtenFiles.structuredJsonPath = structuredJsonPath + writtenFiles.primaryPath = structuredJsonPath } - writeFileSync(destinationFilePath, markdownContent, 'utf-8') - return destinationFilePath + if (config.exportMarkdown) { + const markdownPath = this.constructMarkdownPath(conversation) + const markdownContent = this.formatConversationAsMarkdown(conversation) + this.ensureSpaceDirectoryExists(config.exportDir, conversation.spaceName) + writeFileSync(markdownPath, markdownContent, 'utf-8') + writtenFiles.markdownPath = markdownPath + writtenFiles.primaryPath ??= markdownPath + } + + return writtenFiles as WrittenConversationFiles } catch (error) { throw new FileWriter.WriteError( `Failed to write conversation ${conversation.id}: ${error instanceof Error ? error.message : String(error)}` @@ -42,15 +65,62 @@ export class FileWriter { if (!existsSync(config.exportDir)) { mkdirSync(config.exportDir, { recursive: true }) } + if (!existsSync(config.structuredExportDir)) { + mkdirSync(config.structuredExportDir, { recursive: true }) + } + } + + private ensureSpaceDirectoryExists(rootDirectory: string, spaceName: string): void { + const spaceSpecificDirectory = join(rootDirectory, sanitizeSpaceName(spaceName)) + if (!existsSync(spaceSpecificDirectory)) { + mkdirSync(spaceSpecificDirectory, { recursive: true }) + } } - private constructDestinationFilePath(conversation: ExtractedConversation): string { + private constructMarkdownPath(conversation: ExtractedConversation): string { const safeSpaceName = sanitizeSpaceName(conversation.spaceName) const safeFileTitle = sanitizeFilename(conversation.title) const fileNameWithId = `${safeFileTitle} (${conversation.id}).md` return join(config.exportDir, safeSpaceName, fileNameWithId) } + private constructStructuredJsonPath(conversation: ExtractedConversation): string { + const safeSpaceName = sanitizeSpaceName(conversation.spaceName) + const safeFileTitle = sanitizeFilename(conversation.title) + const fileNameWithId = `${safeFileTitle} (${conversation.id}).itir.perplexity.json` + return join(config.structuredExportDir, safeSpaceName, fileNameWithId) + } + + private formatConversationAsStructuredJson(conversation: ExtractedConversation): unknown { + return { + schema: 'itir.perplexity.thread.v1', + source: 'perplexity', + source_thread_id: conversation.id, + url: conversation.url, + title: conversation.title, + space: conversation.spaceName, + updated_at: conversation.timestamp.toISOString(), + exported_at: new Date().toISOString(), + messages: conversation.messages.map((message) => ({ + role: message.role, + content: message.content, + source_message_id: message.id, + created_at: conversation.timestamp.toISOString(), + turn_index: message.entryIndex, + provenance: { + thread_id: conversation.id, + entry_index: message.entryIndex, + message_index: message.index, + }, + })), + markdown: conversation.content, + raw: { + api_response: conversation.rawApiResponse, + entries: conversation.rawEntries, + }, + } + } + private formatConversationAsMarkdown(conversation: ExtractedConversation): string { const header = `# ${conversation.title}\n\n` const metadata = diff --git a/src/export/sanitizer.ts b/src/export/sanitizer.ts index 60a6ce5..a6ec509 100644 --- a/src/export/sanitizer.ts +++ b/src/export/sanitizer.ts @@ -2,13 +2,13 @@ import sanitize from 'sanitize-filename' export function sanitizeFilename(filename: string): string { const illegalCharacterReplacement = '_' - const maximumFilenameLength = 100 + const maximumFilenameByteLength = 80 - return sanitize(filename, { + const sanitizedFilename = sanitize(filename, { replacement: illegalCharacterReplacement, - }) - .replace(/\s+/g, '_') - .substring(0, maximumFilenameLength) + }).replace(/\s+/g, '_') + + return truncateUtf8(sanitizedFilename, maximumFilenameByteLength) } export function sanitizeSpaceName(spaceName: string): string { @@ -18,3 +18,15 @@ export function sanitizeSpaceName(spaceName: string): string { export function sanitizeMarkdownContent(raw: string): string { return raw || '' } + +function truncateUtf8(value: string, maximumByteLength: number): string { + let output = '' + for (const character of value) { + const next = output + character + if (Buffer.byteLength(next, 'utf-8') > maximumByteLength) { + break + } + output = next + } + return output +} diff --git a/src/scraper/browser.ts b/src/scraper/browser.ts index 07dbe9e..519971f 100644 --- a/src/scraper/browser.ts +++ b/src/scraper/browser.ts @@ -193,19 +193,29 @@ export class BrowserManager { private async verifyLoginStatus(page: Page): Promise { await page.waitForTimeout(1000).catch(() => {}) await page.waitForLoadState('networkidle', { timeout: 5000 }).catch(() => {}) - const currentUrl = page.url() - const authenticatedUrlPaths = ['/settings', '/library', '/collections', '/account/details'] - if (authenticatedUrlPaths.some((path) => currentUrl.includes(path))) { + const userMenuElementCount = await page + .locator('[data-testid="user-menu"]') + .count() + .catch(() => 0) + + if (userMenuElementCount > 0) { return true } - const userMenuElementCount = await page - .locator('[data-testid="user-menu"]') + const visibleLoginControlCount = await page + .locator('button, a') + .filter({ hasText: /^(log in|login|sign in|sign up)$/i }) .count() .catch(() => 0) - return userMenuElementCount > 0 + if (visibleLoginControlCount > 0) { + return false + } + + const currentUrl = page.url() + const authenticatedUrlPaths = ['/settings', '/library', '/collections', '/account/details'] + return authenticatedUrlPaths.some((path) => currentUrl.includes(path)) } private async persistAuthenticationState(): Promise { diff --git a/src/scraper/conversation-extractor.ts b/src/scraper/conversation-extractor.ts index bf342b7..6e24558 100644 --- a/src/scraper/conversation-extractor.ts +++ b/src/scraper/conversation-extractor.ts @@ -6,9 +6,26 @@ import { z } from 'zod' export interface ExtractedConversation { id: string title: string + url: string spaceName: string timestamp: Date content: string + messages: ExtractedConversationMessage[] + rawApiResponse?: unknown + rawEntries: unknown[] +} + +export interface ExtractedConversationMessage { + id: string + role: 'user' | 'assistant' + content: string + index: number + entryIndex: number +} + +interface CapturedApiResponse { + data: any + responseUrl: string } export class ConversationExtractor { @@ -108,17 +125,23 @@ export class ConversationExtractor { ) } - const apiDataPromise = this.captureConversationApiResponse(page) + const threadId = this.extractIdFromUrl(url) + const apiDataPromise = this.captureConversationApiResponse(page, threadId) try { await this.navigateToConversationUrl(page, url) await waitStrategy.afterScroll(page) - const apiData = await apiDataPromise - if (!apiData) { + const apiCapture = await apiDataPromise + if (!apiCapture) { throw new ConversationExtractor.NoDataError('API response timeout or not found') } + const apiData = await this.fetchAllConversationPages( + page, + apiCapture.data, + apiCapture.responseUrl + ) const parsed = this.parseConversationData(apiData, url) if (!parsed) { throw new ConversationExtractor.ParsingError('Failed to parse conversation data') @@ -148,7 +171,10 @@ export class ConversationExtractor { } } - private captureConversationApiResponse(page: Page): Promise { + private captureConversationApiResponse( + page: Page, + threadId: string + ): Promise { let resolved = false return new Promise((resolve) => { @@ -164,7 +190,7 @@ export class ConversationExtractor { if (resolved) return const url = response.url() - if (!url.includes('/rest/thread/') || url.includes('list_ask_threads')) return + if (!this.isThreadDetailApiResponse(url, threadId)) return logger.info(`Found matching thread API response: ${url}`) @@ -184,7 +210,7 @@ export class ConversationExtractor { clearTimeout(timeout) resolved = true - resolve(json) + resolve({ data: json, responseUrl: url }) } catch (_error) { if (resolved) return logger.error(`Failed to parse JSON from thread API: ${_error}`) @@ -193,6 +219,145 @@ export class ConversationExtractor { }) } + private async fetchAllConversationPages( + page: Page, + firstPageData: any, + firstPageUrl: string + ): Promise { + if (!this.shouldFetchNextPage(firstPageData)) { + return firstPageData + } + + const firstEntries = this.ensureEntriesFormat(firstPageData) + const allEntries = [...firstEntries] + let currentPageData = firstPageData + const maxPages = 100 + const seenEntryKeys = new Set(firstEntries.map((entry) => this.getEntryIdentity(entry))) + + for ( + let pageIndex = 1; + pageIndex < maxPages && this.shouldFetchNextPage(currentPageData); + pageIndex++ + ) { + const nextCursor = this.getNextCursor(currentPageData) + if (!nextCursor) { + logger.warn('Thread API reported another page but did not provide a next_cursor') + break + } + + const nextPageData = await this.fetchConversationPageAfterCursor( + page, + firstPageUrl, + nextCursor + ) + if (!nextPageData) { + logger.warn('Could not fetch additional conversation page; using partial thread data') + break + } + + const nextEntries = this.ensureEntriesFormat(nextPageData) + if (nextEntries.length === 0) { + break + } + + const newEntries = nextEntries.filter((entry) => { + const key = this.getEntryIdentity(entry) + if (seenEntryKeys.has(key)) return false + seenEntryKeys.add(key) + return true + }) + if (newEntries.length === 0) { + logger.warn('Conversation pagination returned only duplicate entries; stopping pagination') + break + } + + allEntries.push(...newEntries) + currentPageData = nextPageData + } + + if (allEntries.length === firstEntries.length) { + return firstPageData + } + + logger.info(`Fetched ${allEntries.length} thread entries across paginated API responses`) + return { + ...firstPageData, + entries: allEntries, + has_next_page: this.shouldFetchNextPage(currentPageData), + next_cursor: currentPageData?.next_cursor, + } + } + + private shouldFetchNextPage(data: any): boolean { + return !!( + data && + typeof data === 'object' && + data.has_next_page === true && + Array.isArray(data.entries) && + data.entries.length > 0 + ) + } + + private getNextCursor(data: any): string | null { + return typeof data?.next_cursor === 'string' && data.next_cursor.length > 0 + ? data.next_cursor + : null + } + + private getEntryIdentity(entry: any): string { + for (const key of ['uuid', 'frontend_uuid', 'entry_uuid']) { + const value = entry?.[key] + if (typeof value === 'string' && value.length > 0) { + return `${key}:${value}` + } + } + + const createdAt = entry?.entry_created_datetime ?? entry?.created_at ?? entry?.updated_datetime + const query = typeof entry?.query_str === 'string' ? entry.query_str : '' + return `fallback:${createdAt ?? ''}:${query}` + } + + private async fetchConversationPageAfterCursor( + page: Page, + firstPageUrl: string, + cursor: string + ): Promise { + try { + return await page.evaluate( + async ({ firstPageUrl, cursor }) => { + const nextUrl = new URL(firstPageUrl) + nextUrl.searchParams.delete('offset') + nextUrl.searchParams.set('from_first', 'false') + nextUrl.searchParams.set('cursor', cursor) + const response = await fetch(nextUrl.toString(), { + method: 'GET', + credentials: 'include', + headers: { Accept: 'application/json' }, + }) + if (!response.ok) { + return null + } + return response.json() + }, + { firstPageUrl, cursor } + ) + } catch (_error) { + return null + } + } + + private isThreadDetailApiResponse(responseUrl: string, threadId: string): boolean { + if (!threadId || threadId === 'unknown') return false + + try { + const parsedUrl = new URL(responseUrl) + const expectedPath = `/rest/thread/${threadId}` + return parsedUrl.hostname.endsWith('perplexity.ai') && parsedUrl.pathname === expectedPath + } catch (_error) { + return false + } + } + private async navigateToConversationUrl(page: Page, url: string): Promise { const response = await page.goto(url, { waitUntil: 'domcontentloaded', @@ -244,13 +409,24 @@ export class ConversationExtractor { firstEntry.collection_info?.title ?? data.collection_info?.title ?? 'General' const timestamp = this.extractTimestamp(firstEntry, data) const content = this.convertEntriesToMarkdown(validEntries, title) + const messages = this.normalizeEntriesToMessages(validEntries, title) - if (!content) { + if (!content && messages.length === 0) { logger.warn(`Thread has empty content after formatting: ${url}`) return null } - return { id, title, spaceName, timestamp, content } + return { + id, + title, + url, + spaceName, + timestamp, + content, + messages, + rawApiResponse: data, + rawEntries: validEntries, + } } catch (_error) { logger.error('Failed to parse conversation data.') return null @@ -313,4 +489,54 @@ export class ConversationExtractor { return markdown.trim() } + + private normalizeEntriesToMessages( + entries: any[], + threadTitle: string + ): ExtractedConversationMessage[] { + const messages: ExtractedConversationMessage[] = [] + + for (let entryIndex = 0; entryIndex < entries.length; entryIndex++) { + const entry = entries[entryIndex] + const question = this.extractQuestionText(entry, threadTitle, entryIndex) + const answer = this.extractAnswerText(entry) + + if (question) { + messages.push({ + id: `${entryIndex + 1}-user`, + role: 'user', + content: question, + index: messages.length, + entryIndex, + }) + } + + if (answer) { + messages.push({ + id: `${entryIndex + 1}-assistant`, + role: 'assistant', + content: answer, + index: messages.length, + entryIndex, + }) + } + } + + return messages + } + + private extractQuestionText(entry: any, threadTitle: string, entryIndex: number): string { + if (entry.query_str) return entry.query_str + return entryIndex === 0 ? threadTitle : 'Follow-up' + } + + private extractAnswerText(entry: any): string { + return (entry.blocks ?? []) + .map((block: any) => block.markdown_block?.answer) + .filter( + (answer: unknown): answer is string => typeof answer === 'string' && answer.length > 0 + ) + .join('\n\n') + .trim() + } } diff --git a/src/scraper/worker-pool.ts b/src/scraper/worker-pool.ts index 4ddd601..09ac76b 100644 --- a/src/scraper/worker-pool.ts +++ b/src/scraper/worker-pool.ts @@ -227,10 +227,10 @@ export class WorkerPool { return } - const savedFilePath = this.conversationFileWriter.write(extractedData) - await this.verifySavedMarkdownFile(savedFilePath, extractedData) + const savedFiles = this.conversationFileWriter.write(extractedData) + await this.verifySavedFiles(savedFiles, extractedData) - this.logConversationProcessingSuccess(worker, savedFilePath) + this.logConversationProcessingSuccess(worker, savedFiles.primaryPath) this.processingStats.succeeded++ this.progressCheckpointManager.markProcessed(conversation.url) } catch (_error) { @@ -265,32 +265,103 @@ export class WorkerPool { logger.success(`Worker ${worker.id} saved: ${filepath}`) } - private async verifySavedMarkdownFile( - filepath: string, + private async verifySavedFiles( + savedFiles: { primaryPath: string; structuredJsonPath?: string; markdownPath?: string }, extracted: ExtractedConversation ): Promise { - const validationErrorMessage = this.performFileIntegrityChecks(filepath, extracted) + const validationErrorMessage = this.performExportIntegrityChecks(savedFiles, extracted) if (validationErrorMessage) { throw new WorkerPool.FileValidationError(validationErrorMessage) } } - private performFileIntegrityChecks( + private performExportIntegrityChecks( + savedFiles: { primaryPath: string; structuredJsonPath?: string; markdownPath?: string }, + extracted: ExtractedConversation + ): string | null { + try { + if (config.exportStructuredJson) { + if (!savedFiles.structuredJsonPath) { + return 'Structured JSON path missing after write' + } + + const structuredJsonError = this.performStructuredJsonIntegrityChecks( + savedFiles.structuredJsonPath, + extracted + ) + if (structuredJsonError) return structuredJsonError + } + + if (config.exportMarkdown) { + if (!savedFiles.markdownPath) { + return 'Markdown path missing after write' + } + + const markdownError = this.performMarkdownFileIntegrityChecks( + savedFiles.markdownPath, + extracted + ) + if (markdownError) return markdownError + } + + return null + } catch (_error) { + const errorMessage = _error instanceof Error ? _error.message : String(_error) + return `Validation exception: ${errorMessage}` + } + } + + private performStructuredJsonIntegrityChecks( + filepath: string, + extracted: ExtractedConversation + ): string | null { + if (!existsSync(filepath)) { + return 'Structured JSON file not found after write' + } + + const fileStats = statSync(filepath) + if (fileStats.size === 0) { + return 'Structured JSON file is empty' + } + + const fileContent = readFileSync(filepath, 'utf-8') + const parsed = JSON.parse(fileContent) as Record + + if (parsed['schema'] !== 'itir.perplexity.thread.v1') { + return 'Structured JSON schema mismatch' + } + + if (parsed['source_thread_id'] !== extracted.id) { + return 'Structured JSON source_thread_id mismatch' + } + + if (parsed['title'] !== extracted.title) { + return 'Structured JSON title mismatch' + } + + if (!Array.isArray(parsed['messages']) || parsed['messages'].length === 0) { + return 'Structured JSON messages missing or empty' + } + + return null + } + + private performMarkdownFileIntegrityChecks( filepath: string, extracted: ExtractedConversation ): string | null { try { if (!existsSync(filepath)) { - return 'File not found after write' + return 'Markdown file not found after write' } const fileStats = statSync(filepath) if (fileStats.size === 0) { - return 'File is empty' + return 'Markdown file is empty' } if (fileStats.size < 50) { - return `File too small (${fileStats.size} bytes)` + return `Markdown file too small (${fileStats.size} bytes)` } const fileContent = readFileSync(filepath, 'utf-8') diff --git a/src/utils/config.ts b/src/utils/config.ts index 030c6de..40ea453 100644 --- a/src/utils/config.ts +++ b/src/utils/config.ts @@ -13,6 +13,15 @@ const configSchema = z.object({ parallelWorkers: z.number().int().min(1).max(20), checkpointSaveInterval: z.number().int().positive(), exportDir: z.string().min(1), + structuredExportDir: z.string().min(1), + exportStructuredJson: z + .string() + .optional() + .transform((v) => v === undefined || v === 'true'), + exportMarkdown: z + .string() + .optional() + .transform((v) => v === 'true'), checkpointPath: z.string().min(1), vectorIndexPath: z.string().min(1), ollamaUrl: z.string().url(), @@ -34,8 +43,8 @@ function parseEnvConfig(): Config { const defaultParallelWorkers = '5' const defaultCheckpointInterval = '10' - const rawHeadless = process.env['HEADLESS'] ?? 'true' - let headlessValue: boolean | 'new' = true + const rawHeadless = process.env['HEADLESS'] ?? 'false' + let headlessValue: boolean | 'new' = false if (rawHeadless === 'false') { headlessValue = false } else if (rawHeadless === 'new') { @@ -44,7 +53,7 @@ function parseEnvConfig(): Config { const rawConfig = { authStoragePath: process.env['AUTH_STORAGE_PATH'] ?? join('.storage', 'auth.json'), - waitMode: process.env['WAIT_MODE'] ?? 'dynamic', + waitMode: process.env['WAIT_MODE'] ?? 'static', rateLimitMs: parseInt(process.env['RATE_LIMIT_MS'] ?? defaultRateLimitMs, 10), parallelWorkers: parseInt(process.env['PARALLEL_WORKERS'] ?? defaultParallelWorkers, 10), checkpointSaveInterval: parseInt( @@ -52,6 +61,10 @@ function parseEnvConfig(): Config { 10 ), exportDir: process.env['EXPORT_DIR'] ?? 'exports', + structuredExportDir: + process.env['STRUCTURED_EXPORT_DIR'] ?? process.env['EXPORT_DIR'] ?? 'exports', + exportStructuredJson: process.env['EXPORT_STRUCTURED_JSON'], + exportMarkdown: process.env['EXPORT_MARKDOWN'], checkpointPath: process.env['CHECKPOINT_PATH'] ?? join('.storage', 'checkpoint.json'), vectorIndexPath: process.env['VECTOR_INDEX_PATH'] ?? join('.storage', 'vector-index'), ollamaUrl: process.env['OLLAMA_URL'] ?? defaultOllamaUrl, @@ -97,3 +110,7 @@ ensureDirectory(config.vectorIndexPath) if (!existsSync(config.exportDir)) { mkdirSync(config.exportDir, { recursive: true }) } + +if (!existsSync(config.structuredExportDir)) { + mkdirSync(config.structuredExportDir, { recursive: true }) +} diff --git a/test/setup.ts b/test/setup.ts index d9c83f6..e80239e 100644 --- a/test/setup.ts +++ b/test/setup.ts @@ -5,11 +5,11 @@ let sharedBrowserInstance: Browser beforeAll(async () => { try { - sharedBrowserInstance = await chromium.launch({ headless: true }) + sharedBrowserInstance = await chromium.launch({ headless: true, timeout: 3000 }) } catch (_error) { console.warn('Could not launch browser in setup.ts, some tests might fail if they require it.') } -}) +}, 5000) afterAll(async () => { if (sharedBrowserInstance) { diff --git a/test/unit/config.unit.test.ts b/test/unit/config.unit.test.ts new file mode 100644 index 0000000..e9d7ae4 --- /dev/null +++ b/test/unit/config.unit.test.ts @@ -0,0 +1,52 @@ +import { mkdtempSync } from 'node:fs' +import { join } from 'node:path' +import { tmpdir } from 'node:os' +import { describe, expect, it, vi } from 'vitest' + +function stubStorageEnv(root: string): void { + vi.stubEnv('AUTH_STORAGE_PATH', join(root, '.storage', 'auth.json')) + vi.stubEnv('CHECKPOINT_PATH', join(root, '.storage', 'checkpoint.json')) + vi.stubEnv('VECTOR_INDEX_PATH', join(root, '.storage', 'vector-index')) + vi.stubEnv('EXPORT_DIR', join(root, 'exports')) +} + +async function importFreshConfig( + root: string +): Promise { + vi.resetModules() + vi.unstubAllEnvs() + stubStorageEnv(root) + return import('../../src/utils/config.js') +} + +describe('config', () => { + it('defaults structured JSON on and optional sidecars off', async () => { + const root = mkdtempSync(join(tmpdir(), 'perplexity-config-')) + const { config } = await importFreshConfig(root) + + expect(config.waitMode).toBe('static') + expect(config.exportStructuredJson).toBe(true) + expect(config.exportMarkdown).toBe(false) + expect(config.structuredExportDir).toBe(config.exportDir) + expect(config.enableVectorSearch).toBe(false) + expect(config.headless).toBe(false) + }) + + it('honors explicit export and vector settings', async () => { + const root = mkdtempSync(join(tmpdir(), 'perplexity-config-')) + vi.resetModules() + vi.unstubAllEnvs() + stubStorageEnv(root) + vi.stubEnv('STRUCTURED_EXPORT_DIR', join(root, 'structured')) + vi.stubEnv('EXPORT_STRUCTURED_JSON', 'false') + vi.stubEnv('EXPORT_MARKDOWN', 'true') + vi.stubEnv('ENABLE_VECTOR_SEARCH', 'true') + + const { config } = await import('../../src/utils/config.js') + + expect(config.structuredExportDir).toBe(join(root, 'structured')) + expect(config.exportStructuredJson).toBe(false) + expect(config.exportMarkdown).toBe(true) + expect(config.enableVectorSearch).toBe(true) + }) +}) diff --git a/test/unit/conversation-extractor.unit.test.ts b/test/unit/conversation-extractor.unit.test.ts new file mode 100644 index 0000000..1e1e1f5 --- /dev/null +++ b/test/unit/conversation-extractor.unit.test.ts @@ -0,0 +1,147 @@ +import { describe, expect, it } from 'vitest' +import { ConversationExtractor } from '../../src/scraper/conversation-extractor.js' + +describe('ConversationExtractor normalization', () => { + it('only accepts the target thread detail API response', () => { + const extractor = new ConversationExtractor({} as any) + + expect( + (extractor as any).isThreadDetailApiResponse( + 'https://www.perplexity.ai/rest/thread/thread-123?version=2.18', + 'thread-123' + ) + ).toBe(true) + expect( + (extractor as any).isThreadDetailApiResponse( + 'https://www.perplexity.ai/rest/thread/list_recent?version=2.18', + 'thread-123' + ) + ).toBe(false) + expect( + (extractor as any).isThreadDetailApiResponse( + 'https://www.perplexity.ai/rest/thread/other-thread?version=2.18', + 'thread-123' + ) + ).toBe(false) + }) + + it('normalizes Perplexity entries into ITIR-ready user and assistant messages', () => { + const extractor = new ConversationExtractor({} as any) + const apiResponse = { + entries: [ + { + thread_title: 'Thread title', + collection_info: { title: 'Research' }, + updated_datetime: '2026-05-15T10:00:00.000Z', + query_str: 'What is the plan?', + blocks: [ + { markdown_block: { answer: 'First answer.' } }, + { markdown_block: { answer: 'Second answer.' } }, + ], + }, + { + query_str: 'Follow-up question', + blocks: [{ markdown_block: { answer: 'Follow-up answer.' } }], + }, + ], + } + + const parsed = (extractor as any).parseConversationData( + apiResponse, + 'https://www.perplexity.ai/search/thread-123' + ) + + expect(parsed).not.toBeNull() + expect(parsed.id).toBe('thread-123') + expect(parsed.url).toBe('https://www.perplexity.ai/search/thread-123') + expect(parsed.spaceName).toBe('Research') + expect(parsed.messages).toEqual([ + { + id: '1-user', + role: 'user', + content: 'What is the plan?', + index: 0, + entryIndex: 0, + }, + { + id: '1-assistant', + role: 'assistant', + content: 'First answer.\n\nSecond answer.', + index: 1, + entryIndex: 0, + }, + { + id: '2-user', + role: 'user', + content: 'Follow-up question', + index: 2, + entryIndex: 1, + }, + { + id: '2-assistant', + role: 'assistant', + content: 'Follow-up answer.', + index: 3, + entryIndex: 1, + }, + ]) + expect(parsed.rawApiResponse).toBe(apiResponse) + expect(parsed.rawEntries).toEqual(apiResponse.entries) + }) + + it('combines paginated Perplexity API entries', async () => { + const extractor = new ConversationExtractor({} as any) + const firstPage = { + has_next_page: true, + next_cursor: 'cursor-1', + entries: [{ query_str: 'First', blocks: [{ markdown_block: { answer: 'Answer 1' } }] }], + } + const secondPage = { + has_next_page: false, + entries: [{ query_str: 'Second', blocks: [{ markdown_block: { answer: 'Answer 2' } }] }], + } + const page = { + evaluate: async (_fn: unknown, args: { cursor: string }) => { + expect(args.cursor).toBe('cursor-1') + return secondPage + }, + } + + const combined = await (extractor as any).fetchAllConversationPages( + page, + firstPage, + 'https://www.perplexity.ai/rest/thread/thread-123?offset=0&limit=10' + ) + + expect(combined.entries).toHaveLength(2) + expect(combined.entries[0].query_str).toBe('First') + expect(combined.entries[1].query_str).toBe('Second') + expect(combined.has_next_page).toBe(false) + }) + + it('stops paginating when Perplexity replays the same page', async () => { + const extractor = new ConversationExtractor({} as any) + const firstPage = { + has_next_page: true, + next_cursor: 'cursor-1', + entries: [ + { + uuid: 'entry-1', + query_str: 'First', + blocks: [{ markdown_block: { answer: 'Answer 1' } }], + }, + ], + } + const page = { + evaluate: async () => firstPage, + } + + const combined = await (extractor as any).fetchAllConversationPages( + page, + firstPage, + 'https://www.perplexity.ai/rest/thread/thread-123?offset=0&limit=10' + ) + + expect(combined).toBe(firstPage) + }) +}) diff --git a/test/unit/file-writer.unit.test.ts b/test/unit/file-writer.unit.test.ts new file mode 100644 index 0000000..ff2709b --- /dev/null +++ b/test/unit/file-writer.unit.test.ts @@ -0,0 +1,83 @@ +import { existsSync, mkdtempSync, readFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { beforeEach, describe, expect, it } from 'vitest' +import { FileWriter } from '../../src/export/file-writer.js' +import type { ExtractedConversation } from '../../src/scraper/conversation-extractor.js' +import { config } from '../../src/utils/config.js' + +function sampleConversation(): ExtractedConversation { + return { + id: 'thread-123', + title: 'Test Thread', + url: 'https://www.perplexity.ai/search/thread-123', + spaceName: 'Research Space', + timestamp: new Date('2026-05-15T10:00:00.000Z'), + content: '## What is the plan?\n\nA structured export.\n\n---', + messages: [ + { + id: '1-user', + role: 'user', + content: 'What is the plan?', + index: 0, + entryIndex: 0, + }, + { + id: '1-assistant', + role: 'assistant', + content: 'A structured export.', + index: 1, + entryIndex: 0, + }, + ], + rawApiResponse: { entries: [{ query_str: 'What is the plan?' }] }, + rawEntries: [{ query_str: 'What is the plan?' }], + } +} + +describe('FileWriter', () => { + let root: string + + beforeEach(() => { + root = mkdtempSync(join(tmpdir(), 'perplexity-writer-')) + config.exportDir = join(root, 'markdown') + config.structuredExportDir = join(root, 'structured') + config.exportStructuredJson = true + config.exportMarkdown = false + }) + + it('writes structured ITIR JSON by default without Markdown', () => { + const writer = new FileWriter() + const written = writer.write(sampleConversation()) + + expect(written.primaryPath).toBe(written.structuredJsonPath) + expect(written.markdownPath).toBeUndefined() + expect(written.structuredJsonPath).toBeTruthy() + expect(existsSync(written.structuredJsonPath!)).toBe(true) + + const artifact = JSON.parse(readFileSync(written.structuredJsonPath!, 'utf-8')) + expect(artifact.schema).toBe('itir.perplexity.thread.v1') + expect(written.structuredJsonPath).toContain('.itir.perplexity.json') + expect(artifact.source_thread_id).toBe('thread-123') + expect(artifact.space).toBe('Research Space') + expect(artifact.url).toBe('https://www.perplexity.ai/search/thread-123') + expect(artifact.messages).toHaveLength(2) + expect(artifact.messages[0].source_message_id).toBe('1-user') + expect(artifact.raw.entries).toEqual([{ query_str: 'What is the plan?' }]) + }) + + it('preserves Markdown output when enabled', () => { + config.exportMarkdown = true + const writer = new FileWriter() + const written = writer.write(sampleConversation()) + + expect(written.structuredJsonPath).toBeTruthy() + expect(written.markdownPath).toBeTruthy() + expect(existsSync(written.markdownPath!)).toBe(true) + + const markdown = readFileSync(written.markdownPath!, 'utf-8') + expect(markdown).toContain('# Test Thread') + expect(markdown).toContain('**Space:** Research Space') + expect(markdown).toContain('## What is the plan?') + }) +}) diff --git a/test/unit/perplexity-download-bundler.unit.test.ts b/test/unit/perplexity-download-bundler.unit.test.ts new file mode 100644 index 0000000..8d6f86d --- /dev/null +++ b/test/unit/perplexity-download-bundler.unit.test.ts @@ -0,0 +1,57 @@ +import { mkdtempSync, readFileSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { describe, expect, it } from 'vitest' +import { + bundlePerplexityDownloads, + discoverMarkdownSources, + parseMarkdownTurns, +} from '../../src/download-import/perplexity-download-bundler.js' + +describe('Perplexity download bundler', () => { + it('discovers numbered Markdown exports in ordinal order', () => { + const dir = mkdtempSync(join(tmpdir(), 'pplx-downloads-')) + writeFileSync(join(dir, 'Example Perplexity Thread(2).md'), '# two\n\nb') + writeFileSync(join(dir, 'Example Perplexity Thread.md'), '# zero\n\nb') + writeFileSync(join(dir, 'Other.md'), '# other\n\nb') + + const sources = discoverMarkdownSources([dir], 'Example Perplexity Thread') + + expect(sources.map((source) => source.ordinal)).toEqual([0, 2]) + }) + + it('parses Perplexity Markdown sections as user and assistant turns', () => { + const turns = parseMarkdownTurns({ + path: '/tmp/source.md', + kind: 'markdown', + ordinal: 0, + text: '\n\n# first question\n\nfirst answer\n\n---\n\n# second question\n\nsecond answer\n', + }) + + expect(turns).toMatchObject([ + { question: 'first question', answer: 'first answer' }, + { question: 'second question', answer: 'second answer' }, + ]) + }) + + it('writes a deduped ITIR Perplexity JSON bundle', () => { + const dir = mkdtempSync(join(tmpdir(), 'pplx-downloads-')) + const outPath = join(dir, 'bundle.itir.perplexity.json') + const content = '# q\n\na\n\n---\n\n# q\n\na\n' + writeFileSync(join(dir, 'Example Perplexity Thread.md'), content) + + const summary = bundlePerplexityDownloads({ + inputs: [dir], + titlePrefix: 'Example Perplexity Thread', + outPath, + threadId: 'test-thread', + }) + const bundle = JSON.parse(readFileSync(outPath, 'utf8')) + + expect(summary.parsedTurns).toBe(2) + expect(summary.uniqueTurns).toBe(1) + expect(bundle.schema).toBe('itir.perplexity.thread.v1') + expect(bundle.messages).toHaveLength(2) + expect(bundle.messages[0].source_message_id).toBe('test-thread:download:1:user') + }) +}) diff --git a/test/unit/sanitizer.unit.test.ts b/test/unit/sanitizer.unit.test.ts index a2c3af7..aec8af5 100644 --- a/test/unit/sanitizer.unit.test.ts +++ b/test/unit/sanitizer.unit.test.ts @@ -21,7 +21,14 @@ describe('sanitizeFilename', () => { const excessivelyLongName = 'a'.repeat(200) const truncatedName = sanitizeFilename(excessivelyLongName) - expect(truncatedName.length).toBeLessThanOrEqual(100) + expect(Buffer.byteLength(truncatedName, 'utf-8')).toBeLessThanOrEqual(80) + }) + + it('should truncate multibyte filenames by UTF-8 bytes', () => { + const excessivelyLongName = '─'.repeat(200) + const truncatedName = sanitizeFilename(excessivelyLongName) + + expect(Buffer.byteLength(truncatedName, 'utf-8')).toBeLessThanOrEqual(80) }) it('should handle problematic filenames gracefully', () => { From 93cd3cf04b15352b245195496875a976bb0b4f48 Mon Sep 17 00:00:00 2001 From: Johl Brown Date: Mon, 8 Jun 2026 03:19:07 +1000 Subject: [PATCH 2/2] feat: add Perplexity thread artifact capture --- CHANGELOG.md | 12 + README.md | 9 +- package.json | 1 + sea-config.json | 2 +- src/export-thread.ts | 106 +++ src/export/file-writer.ts | 28 +- src/repl/commands.ts | 4 +- src/scraper/browser.ts | 68 +- src/scraper/conversation-extractor.ts | 636 ++++++++++++++++-- test/unit/conversation-extractor.unit.test.ts | 95 ++- test/unit/file-writer.unit.test.ts | 28 + 11 files changed, 897 insertions(+), 92 deletions(-) create mode 100644 src/export-thread.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 0c73ced..ae56347 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,17 @@ # Changelog +## Unreleased + +### Features + +- Added live artifact capture for Perplexity thread exports. Browser response + images/binaries are downloaded to a configurable artifact directory and + emitted in structured `itir.perplexity.thread.v1` JSON as top-level + `artifacts` refs with local path, source URL, MIME type, size, and SHA-256. +- Relaxed saved-auth validation so existing signed-in browser storage is reused + instead of forcing repeated MFA/SSO when the settings-page probe is slow or + inconclusive. + # 1.1.0 (2026-03-15) ### Bug Fixes diff --git a/README.md b/README.md index 2dae6d0..f569137 100644 --- a/README.md +++ b/README.md @@ -103,13 +103,15 @@ cp .env.example .env ### Key Environment Variables -- **HEADLESS**: Defaults to `false`. **Note:** Headless mode (`true`) is currently non-functional due to Cloudflare Turnstile protection on Perplexity.ai. Using headful mode allows you to complete any challenges manually if they appear. +- **HEADLESS**: Defaults to `false` for interactive use. Headless mode can work for authenticated resolver-style pulls after browser/session state is valid. Use headful mode when a manual login or challenge must be completed. - **OLLAMA_URL**: Access point for your local AI engine (default: http://localhost:11434). - **OLLAMA_MODEL**: Cognitive model for RAG synthesis (e.g., deepseek-r1). - **OLLAMA_EMBED_MODEL**: Model for generating vector representations (e.g., nomic-embed-text). - **EXPORT_STRUCTURED_JSON**: Defaults to `true`. Writes canonical `itir.perplexity.thread.v1` JSON artifacts for downstream archive ingest. - **STRUCTURED_EXPORT_DIR**: Defaults to `EXPORT_DIR`. Set this to separate canonical JSON archives from sidecar files. +- **PERPLEXITY_ARTIFACT_DIR**: Optional base directory for downloaded binary artifacts captured during live thread extraction. Defaults to `CHAT_ARCHIVE_ARTIFACT_DIR` or `/home/c/chat_archive_artifacts/perplexity`. - **EXPORT_MARKDOWN**: Defaults to `false`. Set to `true` to also write the previous Markdown files. +- **PERPLEXITY_SCROLL_MODE**: Defaults to `step` for full-thread safety. `end` is a fast tail probe that can miss middle virtual-scroll pages; `hybrid` mostly steps with occasional end probes. - **ENABLE_VECTOR_SEARCH**: Defaults to `false`. Set to `true` to activate semantic and RAG layers. Current vector indexing reads Markdown exports, so enable `EXPORT_MARKDOWN=true` before rebuilding the vector index. ## Usage Guide @@ -126,6 +128,7 @@ npm run dev - **Start scraper (Library)**: Initiates extraction. Authenticate manually if required. - **Note**: Due to the complexity of Perplexity's API and potential network fluctuations, it may be necessary to **run the scraper multiple times** to ensure all conversations are fully gathered. The system uses checkpoints to resume where it left off. - **Canonical archive**: The primary export is structured JSON. Treat Markdown and vector indexes as optional sidecars that can be regenerated from canonical thread/message records. +- **Artifact capture**: Structured exports may include a top-level `artifacts` array with generated images/files captured from browser responses. Binaries are written outside the JSON archive, normally under `/home/c/chat_archive_artifacts/perplexity//`, while the JSON records local paths, source URLs, sizes, and hashes for `chat-export-structurer` to index and hyperlink. - **SQLite/MyChatArchive ingest**: After exporting, tools such as `chat-export-structurer` can ingest the structured JSON into a canonical SQLite archive: ```bash python src/ingest.py \ @@ -134,7 +137,7 @@ npm run dev --account perplexity \ --source-id perplexity_auto ``` -- **Bundle downloaded Perplexity Markdown**: If Perplexity's API only returns the first page of a long thread, place the downloaded `.md` chunks in a local folder and run: +- **Bundle downloaded Perplexity Markdown**: Perplexity's own export/download button can be seriously incomplete, especially for long threads. Treat downloaded `.md` files as recovery evidence, not canonical truth. If you need to preserve them with provenance, place the downloaded chunks in a local folder and run: ```bash npm run bundle:perplexity-downloads -- \ --input /path/to/downloaded/perplexity-markdown \ @@ -142,7 +145,7 @@ npm run dev --thread-id "" \ --out exports-downloads/thread.download.itir.perplexity.json ``` - Then ingest that JSON with `chat-export-structurer --format perplexity --account perplexity` so the recovered turns attach to the same canonical Perplexity thread. + Then ingest that JSON with `chat-export-structurer --format perplexity --account perplexity` so the recovered turns attach to the same Perplexity thread. Prefer a verified full app-API capture when available. - **Search conversations**: Interface with your history using various modes: - **Auto**: Heuristic selection between semantic and exact search. - **Semantic**: Fuzzy matching via high-dimensional vector space. diff --git a/package.json b/package.json index 166abdf..3041347 100644 --- a/package.json +++ b/package.json @@ -7,6 +7,7 @@ "start": "node dist/index.js", "build": "tsc", "build:exe": "node scripts/build-exe.js && npm run format", + "export:thread": "tsx src/export-thread.ts", "bundle:perplexity-downloads": "tsx src/download-import/cli.ts", "release": "release-it", "type-check": "tsc --noEmit", diff --git a/sea-config.json b/sea-config.json index 7767d77..8d77841 100644 --- a/sea-config.json +++ b/sea-config.json @@ -2,4 +2,4 @@ "main": "dist/bundle.cjs", "output": "dist/sea-prep.blob", "disableSentinel": false -} \ No newline at end of file +} diff --git a/src/export-thread.ts b/src/export-thread.ts new file mode 100644 index 0000000..db76613 --- /dev/null +++ b/src/export-thread.ts @@ -0,0 +1,106 @@ +import { resolve } from 'node:path' +import { BrowserManager } from './scraper/browser.js' +import { ConversationExtractor } from './scraper/conversation-extractor.js' +import { FileWriter } from './export/file-writer.js' +import { logger } from './utils/logger.js' + +interface CliOptions { + url: string + out?: string + json: boolean +} + +function usage(): string { + return [ + 'Usage: npm run export:thread -- --url [--out ] [--json]', + '', + 'Exports one Perplexity thread as itir.perplexity.thread.v1 JSON.', + ].join('\n') +} + +function parseArgs(argv: string[]): CliOptions { + const options: Partial = { json: false } + + for (let index = 0; index < argv.length; index++) { + const arg = argv[index] + if (arg === '--url') { + const value = argv[++index] + if (!value) throw new Error('--url requires a value') + options.url = value + } else if (arg === '--out') { + const value = argv[++index] + if (!value) throw new Error('--out requires a value') + options.out = value + } else if (arg === '--json') { + options.json = true + } else if (arg === '--help' || arg === '-h') { + console.log(usage()) + process.exit(0) + } else { + throw new Error(`Unknown argument: ${arg}`) + } + } + + if (!options.url) { + throw new Error('--url is required') + } + + return options as CliOptions +} + +async function main(): Promise { + let options: CliOptions + try { + options = parseArgs(process.argv.slice(2)) + } catch (error) { + console.error(error instanceof Error ? error.message : String(error)) + console.error(usage()) + process.exitCode = 2 + return + } + + const browserManager = new BrowserManager() + try { + const page = await browserManager.launch() + const extractor = new ConversationExtractor(page.context()) + const conversation = await extractor.extract(options.url) + const writer = new FileWriter() + const written = options.out + ? writer.writeStructuredJsonToPath(conversation, resolve(options.out)) + : writer.write(conversation) + + const payload = { + ok: true, + source: 'perplexity', + source_thread_id: conversation.id, + title: conversation.title, + output_path: written.structuredJsonPath ?? written.primaryPath, + markdown_path: written.markdownPath, + message_count: conversation.messages.length, + } + + if (options.json) { + console.log(JSON.stringify(payload, null, 2)) + } else { + logger.success(`Exported ${conversation.id} to ${payload.output_path}`) + } + } catch (error) { + const message = error instanceof Error ? error.message : String(error) + const payload = { + ok: false, + error: message, + hint: 'Perplexity export requires a usable Playwright browser session. If login or Cloudflare blocks this run, refresh the saved auth state in perplexity-ai-export.', + } + if (options.json) { + console.error(JSON.stringify(payload, null, 2)) + } else { + logger.error(payload.error) + logger.error(payload.hint) + } + process.exitCode = 1 + } finally { + await browserManager.close() + } +} + +main() diff --git a/src/export/file-writer.ts b/src/export/file-writer.ts index 48f0464..e3ade7f 100644 --- a/src/export/file-writer.ts +++ b/src/export/file-writer.ts @@ -1,4 +1,4 @@ -import { join } from 'node:path' +import { dirname, join } from 'node:path' import { writeFileSync, existsSync, mkdirSync } from 'node:fs' import { config } from '../utils/config.js' import type { ExtractedConversation } from '../scraper/conversation-extractor.js' @@ -61,6 +61,31 @@ export class FileWriter { } } + writeStructuredJsonToPath( + conversation: ExtractedConversation, + structuredJsonPath: string + ): WrittenConversationFiles { + try { + const directory = dirname(structuredJsonPath) + if (!existsSync(directory)) { + mkdirSync(directory, { recursive: true }) + } + writeFileSync( + structuredJsonPath, + JSON.stringify(this.formatConversationAsStructuredJson(conversation), null, 2), + 'utf-8' + ) + return { + primaryPath: structuredJsonPath, + structuredJsonPath, + } + } catch (error) { + throw new FileWriter.WriteError( + `Failed to write conversation ${conversation.id} to ${structuredJsonPath}: ${error instanceof Error ? error.message : String(error)}` + ) + } + } + private ensureRootExportDirectoryExists(): void { if (!existsSync(config.exportDir)) { mkdirSync(config.exportDir, { recursive: true }) @@ -113,6 +138,7 @@ export class FileWriter { message_index: message.index, }, })), + artifacts: conversation.artifacts, markdown: conversation.content, raw: { api_response: conversation.rawApiResponse, diff --git a/src/repl/commands.ts b/src/repl/commands.ts index 9a56314..e991458 100644 --- a/src/repl/commands.ts +++ b/src/repl/commands.ts @@ -88,7 +88,9 @@ export class CommandHandler { await this.conversationSearchOrchestrator.validateVectorSearch() } catch (_error) { if (searchMode === 'auto') { - logger.warn('Ollama is not available (required for semantic features). Falling back to Exact Text search (ripgrep).') + logger.warn( + 'Ollama is not available (required for semantic features). Falling back to Exact Text search (ripgrep).' + ) searchMode = 'rg' } else { const errorMessage = _error instanceof Error ? _error.message : String(_error) diff --git a/src/scraper/browser.ts b/src/scraper/browser.ts index 519971f..1f5c72e 100644 --- a/src/scraper/browser.ts +++ b/src/scraper/browser.ts @@ -1,5 +1,5 @@ import { chromium, type Browser, type BrowserContext, type Page } from '@playwright/test' -import { readFileSync, writeFileSync, existsSync, statSync } from 'node:fs' +import { readFileSync, writeFileSync, existsSync } from 'node:fs' import { config } from '../utils/config.js' import { logger } from '../utils/logger.js' import { confirm } from '@inquirer/prompts' @@ -39,10 +39,12 @@ export class BrowserManager { async launch(): Promise { try { - const isSavedAuthValid = this.checkIfSavedAuthenticationIsFresh(config.authStoragePath) + const hasSavedAuth = existsSync(config.authStoragePath) - if (isSavedAuthValid) { - // Try starting in requested headless mode directly + if (hasSavedAuth) { + // Try the persisted browser state first. Perplexity sessions can remain + // valid after the local file is older than a day, and discarding them + // forces unnecessary SSO/MFA loops. await this.launchBrowser(config.headless) await this.initializeBrowserContext() await this.navigateToSettingsPage() @@ -54,9 +56,9 @@ export class BrowserManager { } logger.warn( - 'Saved authentication expired or invalid. Restarting in headful mode for login...' + 'Saved authentication could not be verified on settings; trying target URL anyway.' ) - await this.close() + return this.getActivePage() } // Need login: launch headful @@ -94,7 +96,7 @@ export class BrowserManager { try { this.browserInstance = await chromium.launch({ headless: headless === 'new' ? true : headless, - args: ['--disable-blink-features=AutomationControlled'], + args: ['--disable-blink-features=AutomationControlled', '--disable-gpu'], }) } catch (_error) { throw new BrowserManager.BrowserLaunchError( @@ -106,9 +108,7 @@ export class BrowserManager { private async initializeBrowserContext(): Promise { if (!this.browserInstance) throw new BrowserManager.ContextError('Browser not initialized') - const isSavedAuthValid = this.checkIfSavedAuthenticationIsFresh(config.authStoragePath) - - if (isSavedAuthValid) { + if (existsSync(config.authStoragePath)) { logger.info('Loading saved authentication state...') try { const storageStateData = JSON.parse(readFileSync(config.authStoragePath, 'utf-8')) @@ -120,25 +120,10 @@ export class BrowserManager { this.activeContext = await this.browserInstance.newContext() } } else { - if (existsSync(config.authStoragePath)) { - logger.info('Saved authentication is older than 1 day, discarding.') - } this.activeContext = await this.browserInstance.newContext() } } - private checkIfSavedAuthenticationIsFresh(path: string): boolean { - if (!existsSync(path)) return false - try { - const fileStats = statSync(path) - const fileAgeInMs = Date.now() - fileStats.mtimeMs - const twentyFourHoursInMs = 24 * 60 * 60 * 1000 - return fileAgeInMs < twentyFourHoursInMs - } catch (_error) { - return false - } - } - private async navigateToSettingsPage(): Promise { if (!this.activeContext) { throw new BrowserManager.NavigationError('No browser context available') @@ -175,9 +160,18 @@ export class BrowserManager { }) const perplexitySettingsUrl = 'https://www.perplexity.ai/settings' - await this.activePage.goto(perplexitySettingsUrl, { - waitUntil: 'networkidle', - }) + await this.activePage + .goto(perplexitySettingsUrl, { + waitUntil: 'domcontentloaded', + timeout: 10000, + }) + .catch((error) => { + logger.warn( + `Settings recheck navigation did not fully settle: ${ + error instanceof Error ? error.message : String(error) + }` + ) + }) const isLoginSuccessfulNow = await this.verifyLoginStatus(this.activePage) if (!isLoginSuccessfulNow) { @@ -214,8 +208,22 @@ export class BrowserManager { } const currentUrl = page.url() - const authenticatedUrlPaths = ['/settings', '/library', '/collections', '/account/details'] - return authenticatedUrlPaths.some((path) => currentUrl.includes(path)) + if (currentUrl.includes('/account/details')) { + return true + } + + if (currentUrl.includes('/settings')) { + const settingsText = await page + .locator('body') + .innerText({ timeout: 2000 }) + .catch(() => '') + return ( + visibleLoginControlCount === 0 || + /account|profile|subscription|settings/i.test(settingsText) + ) + } + + return false } private async persistAuthenticationState(): Promise { diff --git a/src/scraper/conversation-extractor.ts b/src/scraper/conversation-extractor.ts index 6e24558..9ca0943 100644 --- a/src/scraper/conversation-extractor.ts +++ b/src/scraper/conversation-extractor.ts @@ -2,6 +2,9 @@ import type { BrowserContext, Page, Response } from '@playwright/test' import { waitStrategy } from '../utils/wait-strategy.js' import { logger } from '../utils/logger.js' import { z } from 'zod' +import { createHash } from 'node:crypto' +import { existsSync, mkdirSync, writeFileSync } from 'node:fs' +import { basename, extname, join } from 'node:path' export interface ExtractedConversation { id: string @@ -13,6 +16,7 @@ export interface ExtractedConversation { messages: ExtractedConversationMessage[] rawApiResponse?: unknown rawEntries: unknown[] + artifacts: ExtractedConversationArtifact[] } export interface ExtractedConversationMessage { @@ -23,11 +27,52 @@ export interface ExtractedConversationMessage { entryIndex: number } +export interface ExtractedConversationArtifact { + artifact_id: string + kind: 'image' | 'binary' + mime_type: string + source_url: string + local_path: string + size_bytes: number + sha256: string +} + interface CapturedApiResponse { data: any responseUrl: string } +interface ThreadApiResponseCollector { + captures: CapturedApiResponse[] + waitForFirst(timeoutMs?: number): Promise + dispose(): void +} + +interface ArtifactCollector { + artifacts: ExtractedConversationArtifact[] + dispose(): void +} + +type ScrollMode = 'step' | 'end' | 'hybrid' + +interface ThreadLoadDiagnostics { + completed: boolean + partial: boolean + scrollMode: ScrollMode + possibleGap: boolean + gapReasons: string[] + passes: number + stablePasses: number + capturedResponseCount: number + uniqueEntryCount: number + uniqueCursorCount: number + lastApiHasNextPage: boolean | null + lastScrollHeight: number + lastBodyTextLength: number + maxPasses: number + requiredStablePasses: number +} + export class ConversationExtractor { private static readonly BlockSchema = z.object({ intended_usage: z.string().optional(), @@ -64,7 +109,6 @@ export class ConversationExtractor { this.name = 'ExtractionError' } } - static readonly NavigationError = class extends Error { constructor(message: string) { super(message) @@ -126,32 +170,35 @@ export class ConversationExtractor { } const threadId = this.extractIdFromUrl(url) - const apiDataPromise = this.captureConversationApiResponse(page, threadId) + const apiCollector = this.createThreadApiResponseCollector(page, threadId) + const artifactCollector = this.createArtifactCollector(page, threadId) try { await this.navigateToConversationUrl(page, url) - await waitStrategy.afterScroll(page) - const apiCapture = await apiDataPromise + const apiCapture = await apiCollector.waitForFirst() if (!apiCapture) { throw new ConversationExtractor.NoDataError('API response timeout or not found') } - const apiData = await this.fetchAllConversationPages( - page, - apiCapture.data, - apiCapture.responseUrl + const loadDiagnostics = await this.driveThreadInfiniteLoader(page, apiCollector) + const apiData = this.mergeCapturedConversationApiResponses( + apiCollector.captures, + loadDiagnostics ) const parsed = this.parseConversationData(apiData, url) if (!parsed) { throw new ConversationExtractor.ParsingError('Failed to parse conversation data') } + parsed.artifacts = artifactCollector.artifacts return parsed } catch (_error) { if (_error instanceof Error) throw _error throw new ConversationExtractor.ExtractionError(String(_error)) } finally { + apiCollector.dispose() + artifactCollector.dispose() if (page) { await page.close().catch((e) => { logger.warn(`Failed to close page: ${e}`) @@ -160,6 +207,108 @@ export class ConversationExtractor { } } + private createArtifactCollector(page: Page, threadId: string): ArtifactCollector { + const artifacts: ExtractedConversationArtifact[] = [] + const seen = new Set() + const artifactRoot = this.getArtifactRoot(threadId) + + const responseHandler = async (response: Response): Promise => { + const url = response.url() + const headers = response.headers() + const mimeType = (headers['content-type'] ?? '').split(';')[0]?.trim().toLowerCase() ?? '' + const kind = this.classifyArtifactResponse(url, mimeType) + if (!kind) return + if (seen.has(url)) return + seen.add(url) + + try { + const body = await response.body() + if (body.length === 0) return + + const sha256 = createHash('sha256').update(body).digest('hex') + if (artifacts.some((artifact) => artifact.sha256 === sha256)) return + + if (!existsSync(artifactRoot)) { + mkdirSync(artifactRoot, { recursive: true }) + } + + const extension = this.artifactExtension(url, mimeType) + const artifactId = `${String(artifacts.length + 1).padStart(4, '0')}-${sha256.slice(0, 16)}` + const localPath = join(artifactRoot, `${artifactId}${extension}`) + writeFileSync(localPath, body) + artifacts.push({ + artifact_id: artifactId, + kind, + mime_type: mimeType || 'application/octet-stream', + source_url: url, + local_path: localPath, + size_bytes: body.length, + sha256, + }) + logger.info(`Captured artifact ${artifactId}${extension} (${body.length} bytes)`) + } catch (_error) { + logger.warn( + `Failed to capture artifact response: ${ + _error instanceof Error ? _error.message : String(_error) + }` + ) + } + } + + page.on('response', responseHandler) + + return { + artifacts, + dispose: () => page.off('response', responseHandler), + } + } + + private getArtifactRoot(threadId: string): string { + const configuredRoot = + process.env['PERPLEXITY_ARTIFACT_DIR'] ?? + process.env['CHAT_ARCHIVE_ARTIFACT_DIR'] ?? + '/home/c/chat_archive_artifacts/perplexity' + return join(configuredRoot, threadId) + } + + private classifyArtifactResponse( + url: string, + mimeType: string + ): ExtractedConversationArtifact['kind'] | null { + if (mimeType.startsWith('image/')) return 'image' + if (/\.(png|jpe?g|webp|gif|svg)(?:[?#]|$)/i.test(url)) return 'image' + if ( + mimeType === 'application/octet-stream' && + /\.(png|jpe?g|webp|gif|svg|pdf|zip)(?:[?#]|$)/i.test(url) + ) { + return 'binary' + } + return null + } + + private artifactExtension(url: string, mimeType: string): string { + const mimeExtensions: Record = { + 'image/png': '.png', + 'image/jpeg': '.jpg', + 'image/webp': '.webp', + 'image/gif': '.gif', + 'image/avif': '.avif', + 'image/svg+xml': '.svg', + 'application/pdf': '.pdf', + 'application/zip': '.zip', + } + if (mimeExtensions[mimeType]) return mimeExtensions[mimeType] + + try { + const parsed = new URL(url) + const name = basename(parsed.pathname) + const extension = extname(name) + return extension && extension.length <= 12 ? extension : '.bin' + } catch (_error) { + return '.bin' + } + } + private async ensureContextIsAlive(): Promise { if (!this.context) { throw new ConversationExtractor.ExtractionError('Browser context is missing') @@ -171,55 +320,362 @@ export class ConversationExtractor { } } - private captureConversationApiResponse( + private createThreadApiResponseCollector( page: Page, threadId: string - ): Promise { - let resolved = false - - return new Promise((resolve) => { - const timeout = setTimeout(() => { - if (!resolved) { - logger.warn('API response timeout – resolving with null') - resolved = true - resolve(null) - } - }, 30000) + ): ThreadApiResponseCollector { + const captures: CapturedApiResponse[] = [] + const firstWaiters: Array<(capture: CapturedApiResponse | null) => void> = [] - page.on('response', async (response: Response) => { - if (resolved) return + const responseHandler = async (response: Response): Promise => { + const url = response.url() + if (!this.isThreadDetailApiResponse(url, threadId)) return - const url = response.url() - if (!this.isThreadDetailApiResponse(url, threadId)) return + logger.info(`Captured thread API response: ${url}`) - logger.info(`Found matching thread API response: ${url}`) + if (page.isClosed()) { + logger.warn('Page is closed – cannot read response body') + return + } - if (page.isClosed()) { - logger.warn('Page is closed – cannot read response body') - return + try { + const json = await response.json() + + const parseResult = ConversationExtractor.ApiResponseSchema.safeParse(json) + if (!parseResult.success) { + logger.warn(`API response validation failed: ${parseResult.error.message}`) + if (process.env['PERPLEXITY_DEBUG_RAW_RESPONSE'] === 'true') { + writeFileSync( + `/tmp/perplexity-thread-${threadId}.raw.json`, + JSON.stringify(json, null, 2), + 'utf-8' + ) + } } - try { - const json = await response.json() - if (resolved) return + const capture = { data: json, responseUrl: url } + captures.push(capture) + while (firstWaiters.length > 0) { + firstWaiters.shift()?.(capture) + } + } catch (_error) { + logger.error(`Failed to parse JSON from thread API: ${_error}`) + } + } - const parseResult = ConversationExtractor.ApiResponseSchema.safeParse(json) - if (!parseResult.success) { - logger.warn(`API response validation failed: ${parseResult.error.message}`) + page.on('response', responseHandler) + + return { + captures, + waitForFirst: (timeoutMs = 30000) => + new Promise((resolve) => { + if (captures.length > 0) { + resolve(captures[0]!) + return } + const timeout = setTimeout(() => { + logger.warn('API response timeout – resolving with null') + const waiterIndex = firstWaiters.indexOf(resolve) + if (waiterIndex >= 0) firstWaiters.splice(waiterIndex, 1) + resolve(null) + }, timeoutMs) + firstWaiters.push((capture) => { + clearTimeout(timeout) + resolve(capture) + }) + }), + dispose: () => { + page.off('response', responseHandler) + while (firstWaiters.length > 0) { + firstWaiters.shift()?.(null) + } + }, + } + } + + private async driveThreadInfiniteLoader( + page: Page, + apiCollector: ThreadApiResponseCollector + ): Promise { + await waitStrategy.afterScroll(page) + + const maxPasses = this.getPositiveIntegerEnv('PERPLEXITY_MAX_SCROLL_PASSES', 260) + const requiredStablePasses = this.getPositiveIntegerEnv('PERPLEXITY_STABLE_SCROLL_PASSES', 8) + const scrollMode = this.getScrollMode() + let stablePasses = 0 + let previousEntryCount = -1 + let previousResponseCount = -1 + let previousScrollHeight = -1 + let previousBodyTextLength = -1 + let lastScrollHeight = 0 + let lastBodyTextLength = 0 + let completed = false + let pass = 0 + + for (pass = 1; pass <= maxPasses; pass++) { + await this.clickShowMoreControls(page) + const scrollState = await this.scrollThreadContainerTowardEnd(page, scrollMode, pass) + await this.waitForApiCollectorToSettle( + apiCollector, + this.getPositiveIntegerEnv('PERPLEXITY_RESPONSE_QUIET_MS', 700), + this.getPositiveIntegerEnv('PERPLEXITY_RESPONSE_MAX_WAIT_MS', 5000) + ) + + const uniqueEntryCount = this.countUniqueCapturedEntries(apiCollector.captures) + const responseCount = apiCollector.captures.length + lastScrollHeight = scrollState.scrollHeight + lastBodyTextLength = scrollState.bodyTextLength + + const isStable = + uniqueEntryCount === previousEntryCount && + responseCount === previousResponseCount && + lastScrollHeight === previousScrollHeight && + lastBodyTextLength === previousBodyTextLength && + scrollState.atEnd + + stablePasses = isStable ? stablePasses + 1 : 0 - clearTimeout(timeout) - resolved = true - resolve({ data: json, responseUrl: url }) + if (pass === 1 || pass % 10 === 0 || !isStable) { + logger.info( + `Perplexity loader pass ${pass}: entries=${uniqueEntryCount}, responses=${responseCount}, scroll=${scrollState.scrollTop}/${scrollState.scrollHeight}, stable=${stablePasses}/${requiredStablePasses}` + ) + } + + if (stablePasses >= requiredStablePasses) { + completed = true + break + } + + previousEntryCount = uniqueEntryCount + previousResponseCount = responseCount + previousScrollHeight = lastScrollHeight + previousBodyTextLength = lastBodyTextLength + } + + const uniqueEntryCount = this.countUniqueCapturedEntries(apiCollector.captures) + const lastApiData = apiCollector.captures.at(-1)?.data + const lastApiHasNextPage = + typeof lastApiData?.has_next_page === 'boolean' ? lastApiData.has_next_page : null + const coverage = this.assessCapturedPageContinuity(apiCollector.captures, scrollMode) + completed = completed && lastApiHasNextPage !== true + const partial = !completed || coverage.possibleGap + + return { + completed: !partial, + partial, + scrollMode, + possibleGap: coverage.possibleGap, + gapReasons: coverage.gapReasons, + passes: pass, + stablePasses, + capturedResponseCount: apiCollector.captures.length, + uniqueEntryCount, + uniqueCursorCount: this.countUniqueCapturedCursors(apiCollector.captures), + lastApiHasNextPage, + lastScrollHeight, + lastBodyTextLength, + maxPasses, + requiredStablePasses, + } + } + + private async clickShowMoreControls(page: Page): Promise { + const showMore = page.getByRole('button', { name: /show more/i }).first() + if ((await showMore.count().catch(() => 0)) === 0) return + await showMore.click({ timeout: 1000 }).catch(() => {}) + } + + private async scrollThreadContainerTowardEnd( + page: Page, + scrollMode: ScrollMode, + pass: number + ): Promise<{ + found: boolean + scrollTop: number + scrollHeight: number + clientHeight: number + atEnd: boolean + bodyTextLength: number + }> { + return await page.evaluate( + ({ scrollMode, pass }) => { + const explicit = document.querySelector('.scrollable-container') + const scrollables = Array.from(document.querySelectorAll('body, main, div')) + .filter((element) => element.scrollHeight > element.clientHeight + 40) + .sort((a, b) => b.scrollHeight - a.scrollHeight) + const target = explicit ?? scrollables[0] ?? document.scrollingElement + + if (!target) { + return { + found: false, + scrollTop: 0, + scrollHeight: 0, + clientHeight: 0, + atEnd: true, + bodyTextLength: document.body?.innerText?.length ?? 0, + } + } + + const before = target.scrollTop + const step = Math.max(target.clientHeight * 6, 4800) + const effectiveMode = scrollMode === 'hybrid' && pass % 6 === 0 ? 'end' : 'step' + target.scrollTop = + effectiveMode === 'step' + ? Math.min(target.scrollHeight, before + step) + : target.scrollHeight + target.dispatchEvent(new Event('scroll', { bubbles: true })) + window.dispatchEvent(new Event('scroll')) + + const atEnd = target.scrollTop + target.clientHeight >= target.scrollHeight - 16 + return { + found: true, + scrollTop: target.scrollTop, + scrollHeight: target.scrollHeight, + clientHeight: target.clientHeight, + atEnd, + bodyTextLength: document.body?.innerText?.length ?? 0, + } + }, + { scrollMode, pass } + ) + } + + private async waitForApiCollectorToSettle( + apiCollector: ThreadApiResponseCollector, + quietMs: number, + maxMs: number + ): Promise { + const startedAt = Date.now() + let lastCount = apiCollector.captures.length + let stableSince = Date.now() + + while (Date.now() - startedAt < maxMs) { + await new Promise((resolve) => setTimeout(resolve, 250)) + const currentCount = apiCollector.captures.length + if (currentCount !== lastCount) { + lastCount = currentCount + stableSince = Date.now() + } + if (Date.now() - stableSince >= quietMs) { + return + } + } + } + + private mergeCapturedConversationApiResponses( + captures: CapturedApiResponse[], + diagnostics: ThreadLoadDiagnostics + ): any { + const firstData = captures[0]?.data + const lastData = captures.at(-1)?.data + const base = firstData && !Array.isArray(firstData) ? { ...firstData } : {} + const entries: any[] = [] + const seen = new Set() + + for (const capture of captures) { + for (const entry of this.ensureEntriesFormat(capture.data)) { + const key = this.getEntryIdentity(entry) + if (seen.has(key)) continue + seen.add(key) + entries.push(entry) + } + } + + return { + ...base, + entries, + has_next_page: diagnostics.partial, + source_has_next_page: + typeof lastData?.has_next_page === 'boolean' ? lastData.has_next_page : undefined, + next_cursor: diagnostics.partial ? lastData?.next_cursor : null, + captured_response_urls: captures.map((capture) => capture.responseUrl), + extraction: diagnostics, + } + } + + private getPositiveIntegerEnv(name: string, fallback: number): number { + const rawValue = process.env[name] + if (!rawValue) return fallback + + const parsed = Number.parseInt(rawValue, 10) + return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback + } + + private getScrollMode(): ScrollMode { + const rawValue = (process.env['PERPLEXITY_SCROLL_MODE'] ?? 'step').toLowerCase() + return rawValue === 'end' || rawValue === 'hybrid' || rawValue === 'step' ? rawValue : 'step' + } + + private assessCapturedPageContinuity( + captures: CapturedApiResponse[], + scrollMode: ScrollMode + ): { possibleGap: boolean; gapReasons: string[] } { + const gapReasons: string[] = [] + + if (scrollMode === 'end' && captures.length > 1) { + gapReasons.push('end-scroll mode can jump across virtual-scroll loader boundaries') + } + + const offsetPages = captures + .map((capture) => { + try { + const parsedUrl = new URL(capture.responseUrl) + const offset = parsedUrl.searchParams.get('offset') + return offset === null + ? null + : { + offset: Number.parseInt(offset, 10), + entryCount: this.ensureEntriesFormat(capture.data).length, + } } catch (_error) { - if (resolved) return - logger.error(`Failed to parse JSON from thread API: ${_error}`) + return null } }) - }) + .filter( + (page): page is { offset: number; entryCount: number } => + page !== null && Number.isFinite(page.offset) && page.offset >= 0 + ) + .sort((a, b) => a.offset - b.offset) + + for (let index = 1; index < offsetPages.length; index++) { + const previous = offsetPages[index - 1]! + const current = offsetPages[index]! + if (current.offset > previous.offset + previous.entryCount) { + gapReasons.push( + `offset gap between ${previous.offset} and ${current.offset}; middle API page may be missing` + ) + break + } + } + + return { possibleGap: gapReasons.length > 0, gapReasons } } - private async fetchAllConversationPages( + private countUniqueCapturedEntries(captures: CapturedApiResponse[]): number { + const seen = new Set() + for (const capture of captures) { + for (const entry of this.ensureEntriesFormat(capture.data)) { + seen.add(this.getEntryIdentity(entry)) + } + } + return seen.size + } + + private countUniqueCapturedCursors(captures: CapturedApiResponse[]): number { + const cursors = new Set() + for (const capture of captures) { + try { + const parsedUrl = new URL(capture.responseUrl) + const cursor = parsedUrl.searchParams.get('cursor') + if (cursor) cursors.add(cursor) + } catch (_error) { + // Ignore malformed diagnostic URLs. + } + } + return cursors.size + } + + async fetchAllConversationPages( page: Page, firstPageData: any, firstPageUrl: string @@ -245,14 +701,17 @@ export class ConversationExtractor { break } - const nextPageData = await this.fetchConversationPageAfterCursor( - page, - firstPageUrl, - nextCursor - ) + let nextPageData = await this.fetchConversationPageAfterCursor(page, firstPageUrl, nextCursor) if (!nextPageData) { - logger.warn('Could not fetch additional conversation page; using partial thread data') - break + nextPageData = await this.fetchConversationPageAtOffset( + page, + firstPageUrl, + allEntries.length + ) + if (!nextPageData) { + logger.warn('Could not fetch additional conversation page; using partial thread data') + break + } } const nextEntries = this.ensureEntriesFormat(nextPageData) @@ -267,8 +726,33 @@ export class ConversationExtractor { return true }) if (newEntries.length === 0) { - logger.warn('Conversation pagination returned only duplicate entries; stopping pagination') - break + nextPageData = await this.fetchConversationPageAtOffset( + page, + firstPageUrl, + allEntries.length + ) + if (!nextPageData) { + logger.warn( + 'Conversation pagination returned only duplicate entries; stopping pagination' + ) + break + } + const offsetEntries = this.ensureEntriesFormat(nextPageData) + const offsetNewEntries = offsetEntries.filter((entry) => { + const key = this.getEntryIdentity(entry) + if (seenEntryKeys.has(key)) return false + seenEntryKeys.add(key) + return true + }) + if (offsetNewEntries.length === 0) { + logger.warn( + 'Conversation pagination returned only duplicate entries; stopping pagination' + ) + break + } + allEntries.push(...offsetNewEntries) + currentPageData = nextPageData + continue } allEntries.push(...newEntries) @@ -346,6 +830,41 @@ export class ConversationExtractor { } } + private async fetchConversationPageAtOffset( + page: Page, + firstPageUrl: string, + offset: number + ): Promise { + try { + return await page.evaluate( + async ({ firstPageUrl, offset }) => { + const candidateUrls = [true, false].map((fromFirst) => { + const nextUrl = new URL(firstPageUrl) + nextUrl.searchParams.delete('cursor') + nextUrl.searchParams.set('offset', String(offset)) + nextUrl.searchParams.set('from_first', String(fromFirst)) + return nextUrl.toString() + }) + + for (const url of candidateUrls) { + const response = await fetch(url, { + method: 'GET', + credentials: 'include', + headers: { Accept: 'application/json' }, + }) + if (response.ok) { + return response.json() + } + } + return null + }, + { firstPageUrl, offset } + ) + } catch (_error) { + return null + } + } + private isThreadDetailApiResponse(responseUrl: string, threadId: string): boolean { if (!threadId || threadId === 'unknown') return false @@ -426,6 +945,7 @@ export class ConversationExtractor { messages, rawApiResponse: data, rawEntries: validEntries, + artifacts: [], } } catch (_error) { logger.error('Failed to parse conversation data.') @@ -503,7 +1023,7 @@ export class ConversationExtractor { if (question) { messages.push({ - id: `${entryIndex + 1}-user`, + id: `${this.getStableEntryMessagePrefix(entry, entryIndex)}:user`, role: 'user', content: question, index: messages.length, @@ -513,7 +1033,7 @@ export class ConversationExtractor { if (answer) { messages.push({ - id: `${entryIndex + 1}-assistant`, + id: `${this.getStableEntryMessagePrefix(entry, entryIndex)}:assistant`, role: 'assistant', content: answer, index: messages.length, @@ -525,6 +1045,18 @@ export class ConversationExtractor { return messages } + private getStableEntryMessagePrefix(entry: any, entryIndex: number): string { + const identity = this.getEntryIdentity(entry) + if (!identity.startsWith('fallback:')) { + return identity + } + + const question = typeof entry?.query_str === 'string' ? entry.query_str : '' + const answer = this.extractAnswerText(entry) + const digest = createHash('sha1').update(`${question}\n\n${answer}`).digest('hex').slice(0, 16) + return `entry:${entryIndex + 1}:${digest}` + } + private extractQuestionText(entry: any, threadTitle: string, entryIndex: number): string { if (entry.query_str) return entry.query_str return entryIndex === 0 ? threadTitle : 'Follow-up' diff --git a/test/unit/conversation-extractor.unit.test.ts b/test/unit/conversation-extractor.unit.test.ts index 1e1e1f5..e60607b 100644 --- a/test/unit/conversation-extractor.unit.test.ts +++ b/test/unit/conversation-extractor.unit.test.ts @@ -57,28 +57,28 @@ describe('ConversationExtractor normalization', () => { expect(parsed.spaceName).toBe('Research') expect(parsed.messages).toEqual([ { - id: '1-user', + id: 'entry:1:cce8ffe9a9c36031:user', role: 'user', content: 'What is the plan?', index: 0, entryIndex: 0, }, { - id: '1-assistant', + id: 'entry:1:cce8ffe9a9c36031:assistant', role: 'assistant', content: 'First answer.\n\nSecond answer.', index: 1, entryIndex: 0, }, { - id: '2-user', + id: 'entry:2:391ccd0ce2a5d71a:user', role: 'user', content: 'Follow-up question', index: 2, entryIndex: 1, }, { - id: '2-assistant', + id: 'entry:2:391ccd0ce2a5d71a:assistant', role: 'assistant', content: 'Follow-up answer.', index: 3, @@ -144,4 +144,91 @@ describe('ConversationExtractor normalization', () => { expect(combined).toBe(firstPage) }) + + it('falls back to offset pagination when cursor pagination replays the first page', async () => { + const extractor = new ConversationExtractor({} as any) + const firstPage = { + has_next_page: true, + next_cursor: 'cursor-1', + entries: [ + { + query_str: 'First', + blocks: [{ markdown_block: { answer: 'Answer 1' } }], + }, + ], + } + const secondPage = { + has_next_page: false, + entries: [ + { + query_str: 'Second', + blocks: [{ markdown_block: { answer: 'Answer 2' } }], + }, + ], + } + let calls = 0 + const page = { + evaluate: async (_fn: unknown, args: { cursor?: string; offset?: number }) => { + calls += 1 + if (args.cursor) return firstPage + expect(args.offset).toBe(1) + return secondPage + }, + } + + const combined = await (extractor as any).fetchAllConversationPages( + page, + firstPage, + 'https://www.perplexity.ai/rest/thread/thread-123?offset=0&limit=10' + ) + + expect(calls).toBe(2) + expect(combined.entries).toHaveLength(2) + expect(combined.entries[1].query_str).toBe('Second') + expect(combined.has_next_page).toBe(false) + }) + + it('defaults to stepped Perplexity scroll mode for full-load safety', () => { + const extractor = new ConversationExtractor({} as any) + const previous = process.env['PERPLEXITY_SCROLL_MODE'] + delete process.env['PERPLEXITY_SCROLL_MODE'] + + try { + expect((extractor as any).getScrollMode()).toBe('step') + process.env['PERPLEXITY_SCROLL_MODE'] = 'end' + expect((extractor as any).getScrollMode()).toBe('end') + process.env['PERPLEXITY_SCROLL_MODE'] = 'hybrid' + expect((extractor as any).getScrollMode()).toBe('hybrid') + process.env['PERPLEXITY_SCROLL_MODE'] = 'bogus' + expect((extractor as any).getScrollMode()).toBe('step') + } finally { + if (previous === undefined) { + delete process.env['PERPLEXITY_SCROLL_MODE'] + } else { + process.env['PERPLEXITY_SCROLL_MODE'] = previous + } + } + }) + + it('marks end-scroll captures and offset jumps as possible page gaps', () => { + const extractor = new ConversationExtractor({} as any) + const captures = [ + { + responseUrl: 'https://www.perplexity.ai/rest/thread/thread-123?offset=0&limit=2', + data: { entries: [{ uuid: 'a' }, { uuid: 'b' }] }, + }, + { + responseUrl: 'https://www.perplexity.ai/rest/thread/thread-123?offset=5&limit=2', + data: { entries: [{ uuid: 'f' }] }, + }, + ] + + const stepped = (extractor as any).assessCapturedPageContinuity(captures, 'step') + expect(stepped.possibleGap).toBe(true) + expect(stepped.gapReasons.join('\n')).toContain('offset gap') + + const end = (extractor as any).assessCapturedPageContinuity(captures, 'end') + expect(end.possibleGap).toBe(true) + expect(end.gapReasons.join('\n')).toContain('end-scroll mode') + }) }) diff --git a/test/unit/file-writer.unit.test.ts b/test/unit/file-writer.unit.test.ts index ff2709b..f6550fa 100644 --- a/test/unit/file-writer.unit.test.ts +++ b/test/unit/file-writer.unit.test.ts @@ -30,6 +30,17 @@ function sampleConversation(): ExtractedConversation { entryIndex: 0, }, ], + artifacts: [ + { + artifact_id: '0001-deadbeef', + kind: 'image', + mime_type: 'image/png', + source_url: 'https://example.test/image.png', + local_path: '/home/c/chat_archive_artifacts/perplexity/thread-123/0001-deadbeef.png', + size_bytes: 1234, + sha256: 'deadbeef', + }, + ], rawApiResponse: { entries: [{ query_str: 'What is the plan?' }] }, rawEntries: [{ query_str: 'What is the plan?' }], } @@ -63,6 +74,8 @@ describe('FileWriter', () => { expect(artifact.url).toBe('https://www.perplexity.ai/search/thread-123') expect(artifact.messages).toHaveLength(2) expect(artifact.messages[0].source_message_id).toBe('1-user') + expect(artifact.artifacts).toHaveLength(1) + expect(artifact.artifacts[0].local_path).toContain('chat_archive_artifacts') expect(artifact.raw.entries).toEqual([{ query_str: 'What is the plan?' }]) }) @@ -80,4 +93,19 @@ describe('FileWriter', () => { expect(markdown).toContain('**Space:** Research Space') expect(markdown).toContain('## What is the plan?') }) + + it('writes structured ITIR JSON to an explicit resolver path', () => { + const writer = new FileWriter() + const outPath = join(root, 'resolver', 'thread-123.itir.perplexity.json') + const written = writer.writeStructuredJsonToPath(sampleConversation(), outPath) + + expect(written.primaryPath).toBe(outPath) + expect(written.structuredJsonPath).toBe(outPath) + expect(written.markdownPath).toBeUndefined() + expect(existsSync(outPath)).toBe(true) + + const artifact = JSON.parse(readFileSync(outPath, 'utf-8')) + expect(artifact.schema).toBe('itir.perplexity.thread.v1') + expect(artifact.source_thread_id).toBe('thread-123') + }) })