feat: add structured archive export

.env.example

@@ -2,13 +2,18 @@
 AUTH_STORAGE_PATH=.storage/auth.json
 
 # Scraping behavior
-WAIT_MODE=fixed
+WAIT_MODE=static
 RATE_LIMIT_MS=3000
 PARALLEL_WORKERS=2
 CHECKPOINT_SAVE_INTERVAL=10
 
-# Vector search
-ENABLE_VECTOR_SEARCH=true
+# Export formats
+EXPORT_STRUCTURED_JSON=true
+EXPORT_MARKDOWN=false
+STRUCTURED_EXPORT_DIR=exports
+
+# Vector search (requires Markdown exports and Ollama)
+ENABLE_VECTOR_SEARCH=false
 
 # AI services
 GEMINI_API_KEY=
@@ -23,4 +28,4 @@ VECTOR_INDEX_PATH=.storage/vector-index
 
 # Browser behavior
 # HEADLESS can be 'true', 'false', or 'new'
-HEADLESS=true
+HEADLESS=false

CHANGELOG.md

@@ -1,5 +1,17 @@
 # Changelog
 
+## Unreleased
+
+### Features
+
+- Added live artifact capture for Perplexity thread exports. Browser response
+  images/binaries are downloaded to a configurable artifact directory and
+  emitted in structured `itir.perplexity.thread.v1` JSON as top-level
+  `artifacts` refs with local path, source URL, MIME type, size, and SHA-256.
+- Relaxed saved-auth validation so existing signed-in browser storage is reused
+  instead of forcing repeated MFA/SSO when the settings-page probe is slow or
+  inconclusive.
+
 # 1.1.0 (2026-03-15)
 
 ### Bug Fixes

README.md

@@ -35,14 +35,16 @@
 
 ## Introduction
 
-This tool is designed to externalize your Perplexity.ai conversation history into organized, semantically searchable Markdown files. It facilitates the emergence of a personal knowledge base powered by local AI, bridging the gap between ephemeral inquiry and structured knowledge.
+This tool is designed to externalize your Perplexity.ai conversation history into structured JSON archives suitable for canonical SQLite archival, full-text search, vector indexing, and downstream tools such as MyChatArchive/ITIR. Markdown and vector search remain available as optional sidecars for local reading and semantic exploration.
 
 ## Key Features
 
 - **Parallelized Extraction**: Leverages Playwright to extract multiple conversation threads simultaneously for high-velocity data retrieval.
 - **Architectural Resilience**: Automatically restores browser contexts and retries operations, ensuring continuity amidst environmental instability.
-- **Advanced RAG (Retrieval-Augmented Generation)**: Engage in a cognitive dialogue with your history. The system employs intent analysis to synthesize broad summaries or pinpoint specific technical insights.
-- **Semantic Vector Search**: Move beyond keyword matching. Locate information based on conceptual depth and semantic relevance.
+- **Structured Archive Output**: Emits `itir.perplexity.thread.v1` JSON artifacts with normalized messages, stable source IDs, metadata, and captured API data for downstream SQLite/archive ingest.
+- **Optional Markdown Sidecars**: Preserve the previous human-readable Markdown export path when `EXPORT_MARKDOWN=true`.
+- **Optional RAG (Retrieval-Augmented Generation)**: Engage in a cognitive dialogue with your history when vector search is enabled.
+- **Optional Semantic Vector Search**: Move beyond keyword matching with Markdown sidecars, Ollama, and Vectra enabled.
 - **Persistent State Tracking**: Frequent checkpoints allow the system to resume progress after any interruption.
 - **Interactive Synthesis (REPL)**: A streamlined command-line interface for human-system synergy.
 
@@ -101,11 +103,16 @@ cp .env.example .env
 
 ### Key Environment Variables
 
-- **HEADLESS**: Set to `false` in your `.env` file. **Note:** Headless mode (`true`) is currently non-functional due to Cloudflare Turnstile protection on Perplexity.ai. Using headful mode allows you to complete any challenges manually if they appear.
+- **HEADLESS**: Defaults to `false` for interactive use. Headless mode can work for authenticated resolver-style pulls after browser/session state is valid. Use headful mode when a manual login or challenge must be completed.
 - **OLLAMA_URL**: Access point for your local AI engine (default: http://localhost:11434).
 - **OLLAMA_MODEL**: Cognitive model for RAG synthesis (e.g., deepseek-r1).
 - **OLLAMA_EMBED_MODEL**: Model for generating vector representations (e.g., nomic-embed-text).
-- **ENABLE_VECTOR_SEARCH**: Set to `true` to activate semantic and RAG layers.
+- **EXPORT_STRUCTURED_JSON**: Defaults to `true`. Writes canonical `itir.perplexity.thread.v1` JSON artifacts for downstream archive ingest.
+- **STRUCTURED_EXPORT_DIR**: Defaults to `EXPORT_DIR`. Set this to separate canonical JSON archives from sidecar files.
+- **PERPLEXITY_ARTIFACT_DIR**: Optional base directory for downloaded binary artifacts captured during live thread extraction. Defaults to `CHAT_ARCHIVE_ARTIFACT_DIR` or `/home/c/chat_archive_artifacts/perplexity`.
+- **EXPORT_MARKDOWN**: Defaults to `false`. Set to `true` to also write the previous Markdown files.
+- **PERPLEXITY_SCROLL_MODE**: Defaults to `step` for full-thread safety. `end` is a fast tail probe that can miss middle virtual-scroll pages; `hybrid` mostly steps with occasional end probes.
+- **ENABLE_VECTOR_SEARCH**: Defaults to `false`. Set to `true` to activate semantic and RAG layers. Current vector indexing reads Markdown exports, so enable `EXPORT_MARKDOWN=true` before rebuilding the vector index.
 
 ## Usage Guide
 
@@ -120,6 +127,25 @@ npm run dev
 
 - **Start scraper (Library)**: Initiates extraction. Authenticate manually if required.
   - **Note**: Due to the complexity of Perplexity's API and potential network fluctuations, it may be necessary to **run the scraper multiple times** to ensure all conversations are fully gathered. The system uses checkpoints to resume where it left off.
+- **Canonical archive**: The primary export is structured JSON. Treat Markdown and vector indexes as optional sidecars that can be regenerated from canonical thread/message records.
+- **Artifact capture**: Structured exports may include a top-level `artifacts` array with generated images/files captured from browser responses. Binaries are written outside the JSON archive, normally under `/home/c/chat_archive_artifacts/perplexity/<thread-id>/`, while the JSON records local paths, source URLs, sizes, and hashes for `chat-export-structurer` to index and hyperlink.
+- **SQLite/MyChatArchive ingest**: After exporting, tools such as `chat-export-structurer` can ingest the structured JSON into a canonical SQLite archive:
+  ```bash
+  python src/ingest.py \
+    --in /path/to/perplexity-ai-export/exports \
+    --format perplexity \
+    --account perplexity \
+    --source-id perplexity_auto
+  ```
+- **Bundle downloaded Perplexity Markdown**: Perplexity's own export/download button can be seriously incomplete, especially for long threads. Treat downloaded `.md` files as recovery evidence, not canonical truth. If you need to preserve them with provenance, place the downloaded chunks in a local folder and run:
+  ```bash
+  npm run bundle:perplexity-downloads -- \
+    --input /path/to/downloaded/perplexity-markdown \
+    --title-prefix "Thread title prefix" \
+    --thread-id "<perplexity-thread-uuid>" \
+    --out exports-downloads/thread.download.itir.perplexity.json
+  ```
+  Then ingest that JSON with `chat-export-structurer --format perplexity --account perplexity` so the recovered turns attach to the same Perplexity thread. Prefer a verified full app-API capture when available.
 - **Search conversations**: Interface with your history using various modes:
   - **Auto**: Heuristic selection between semantic and exact search.
   - **Semantic**: Fuzzy matching via high-dimensional vector space.
@@ -160,4 +186,4 @@ npm run test:unit
 
 # Execute integration-level verifications
 npm run test:integration
-```
+```

package.json

@@ -7,6 +7,8 @@
     "start": "node dist/index.js",
     "build": "tsc",
     "build:exe": "node scripts/build-exe.js && npm run format",
+    "export:thread": "tsx src/export-thread.ts",
+    "bundle:perplexity-downloads": "tsx src/download-import/cli.ts",
     "release": "release-it",
     "type-check": "tsc --noEmit",
     "toc": "node scripts/update-toc.js",

sea-config.json

@@ -2,4 +2,4 @@
   "main": "dist/bundle.cjs",
   "output": "dist/sea-prep.blob",
   "disableSentinel": false
-}
+}

src/download-import/cli.ts

@@ -0,0 +1,80 @@
+import { bundlePerplexityDownloads } from './perplexity-download-bundler.js'
+
+function parseArgs(argv: string[]): {
+  inputs: string[]
+  titlePrefix: string
+  outPath: string
+  threadId?: string
+  title?: string
+} {
+  const inputs: string[] = []
+  let titlePrefix = ''
+  let outPath = 'exports-downloads/perplexity-download.itir.perplexity.json'
+  let threadId: string | undefined
+  let title: string | undefined
+
+  for (let i = 0; i < argv.length; i++) {
+    const arg = argv[i]
+    const next = argv[i + 1]
+    if (arg === '--input' && next) {
+      inputs.push(next)
+      i++
+    } else if (arg === '--title-prefix' && next) {
+      titlePrefix = next
+      i++
+    } else if (arg === '--out' && next) {
+      outPath = next
+      i++
+    } else if (arg === '--thread-id' && next) {
+      threadId = next
+      i++
+    } else if (arg === '--title' && next) {
+      title = next
+      i++
+    } else if (arg === '--help') {
+      printHelp()
+      process.exit(0)
+    }
+  }
+
+  if (inputs.length === 0) {
+    throw new Error('Provide at least one --input file or directory.')
+  }
+
+  if (!titlePrefix) {
+    throw new Error('Provide --title-prefix so unrelated Markdown files are not bundled.')
+  }
+
+  return {
+    inputs,
+    titlePrefix,
+    outPath,
+    threadId,
+    title,
+  }
+}
+
+function printHelp(): void {
+  console.log(`Usage:
+  npm run bundle:perplexity-downloads -- [options]
+
+Options:
+  --input <path>          File or directory to scan. Repeatable.
+  --title-prefix <text>   Download filename prefix to match.
+  --out <path>            Output .itir.perplexity.json path.
+  --thread-id <id>        Stable source thread id for the bundle.
+  --title <text>          Archive thread title.
+`)
+}
+
+try {
+  const summary = bundlePerplexityDownloads(parseArgs(process.argv.slice(2)))
+  console.log(`Wrote ${summary.outPath}`)
+  console.log(`Source files: ${summary.sourceFiles}`)
+  console.log(`Parsed turns: ${summary.parsedTurns}`)
+  console.log(`Unique turns: ${summary.uniqueTurns}`)
+  console.log(`Messages: ${summary.messages}`)
+} catch (error) {
+  console.error(error instanceof Error ? error.message : String(error))
+  process.exit(1)
+}