milestone(jina-two-profiles): source-kind routing so academic sources keep their metadata — Fixes #78

The Jina parser was blog-only: it read OpenGraph keys and missed the
Highwire/Dublin-Core/PRISM metadata that scholarly pages use, so their date +
publisher fell through empty and the operator hand-filled them.

Now jina.ts has two profiles and fuzzy routing:
- detectProfile() reads which convention is present and returns a profile
  (structured | opengraph) + a source_kind (academic-paper | article |
  company-landing | web-page), recorded on the source.
- extractBib() is a pure, profile-aware, forceProfile-overridable extractor;
  every field resolves across an ordered alias list (first hit wins), so a
  field missing under one convention still resolves under another.
- fetchViaJina() accepts forceProfile — the foundation for a human re-route
  button (UI + re-extract capability tracked as a fast-follow).

Authors stay an array; "Last, First" is never comma-split. Verified with a
vitest suite whose academic fixture is the REAL metadata captured live from a
Springer article — 11 tests + the existing corpus-files suite, all green;
tsc --noEmit clean. Reaches augment.didi.sh on the next redeploy.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_018UYTYu4MAFZ7iyr2VTo2kq

Author: mpstaton

services/content-ingest/src/jina.ts

@@ -5,6 +5,14 @@
 
 const JINA_BASE = 'https://r.jina.ai/';
 
+// Metadata comes back under different conventions depending on what KIND of
+// page it is — a scholarly article hides everything under Highwire/Dublin-Core/
+// PRISM keys (citation_*, dc.*, prism.*); a blog/news/company page uses
+// OpenGraph (og:*, article:*). So there are two PROFILES and fuzzy routing
+// picks one; forceProfile lets a human re-route when detection is wrong.
+export type ParserProfile = 'structured' | 'opengraph';
+export type SourceKind = 'academic-paper' | 'article' | 'company-landing' | 'web-page';
+
 export type JinaResult =
   | {
       ok: true;
@@ -15,12 +23,15 @@ export type JinaResult =
     }
   | { ok: false; error: string; status?: number };
 
-export async function fetchViaJina(url: string, opts: { noCache?: boolean } = {}): Promise<JinaResult> {
+export async function fetchViaJina(
+  url: string,
+  opts: { noCache?: boolean; forceProfile?: ParserProfile } = {},
+): Promise<JinaResult> {
   const RETRIES = 3;
   let backoffMs = 2000;
   let lastErr: { ok: false; error: string; status?: number } | null = null;
   for (let attempt = 0; attempt < RETRIES; attempt += 1) {
-    const result = await jinaFetchOnce(url, opts.noCache);
+    const result = await jinaFetchOnce(url, opts.noCache, opts.forceProfile);
     if (result.ok) return result;
     if (result.status !== 429) return result;
     lastErr = result;
@@ -32,7 +43,7 @@ export async function fetchViaJina(url: string, opts: { noCache?: boolean } = {}
   return lastErr ?? { ok: false, error: 'jina fetch failed after retries' };
 }
 
-async function jinaFetchOnce(url: string, noCache = false): Promise<JinaResult> {
+async function jinaFetchOnce(url: string, noCache = false, forceProfile?: ParserProfile): Promise<JinaResult> {
   const fetched_at = new Date().toISOString();
   const apiKey = process.env.JINA_API_KEY;
   const headers: Record<string, string> = { Accept: 'application/json' };
@@ -60,37 +71,28 @@ async function jinaFetchOnce(url: string, noCache = false): Promise<JinaResult>
     parsed = null;
   }
 
-  // JSON path (the normal case) — rich metadata available.
+  // JSON path (the normal case) — rich metadata available, routed by profile.
   if (parsed?.data) {
     const data = parsed.data;
-    const meta = (data.metadata && typeof data.metadata === 'object' ? data.metadata : {}) as Record<string, unknown>;
     const markdown = typeof data.content === 'string' ? data.content : '';
     if (!markdown.trim()) return { ok: false, error: 'Jina returned empty content' };
-    const title = firstStr(data.title) ?? extractTitle(markdown, url);
-    const extra: Record<string, unknown> = { jina_status: res.status, content_length_bytes: markdown.length };
-
-    const publishedRaw = firstStr(data.publishedTime, meta['article:published_time'], meta['article:modified_time'], meta.date);
-    if (publishedRaw) {
-      const iso = normalizeToISO(publishedRaw);
-      if (iso) extra.published_at = iso;
-    }
-    const authors = normalizeAuthors(meta.author, meta['article:author'], meta['dc.creator'], data.author);
-    if (authors.length) extra.authors = authors;
-    const publisher = firstStr(meta['og:site_name'], data.publisher) ?? hostnameOf(url);
-    if (publisher) extra.publisher = publisher;
-    const description = firstStr(data.description, meta.description);
-    if (description) extra.description = description;
-    const language = firstStr(meta.lang, meta.language, data.lang);
-    if (language) extra.language = language;
-
+    const { title, extra } = extractBib(data, url, { forceProfile });
+    extra.jina_status = res.status;
+    extra.content_length_bytes = markdown.length;
     return { ok: true, markdown, title, fetched_at, extra };
   }
 
   // Fallback: non-JSON body (some upstreams / error shapes) — parse the legacy
-  // key:value preamble so we still degrade gracefully.
+  // key:value preamble so we still degrade gracefully. Always the opengraph
+  // profile, kind web-page (no structured metadata is available this way).
   const markdown = body;
   const title = extractTitle(markdown, url);
-  const extra: Record<string, unknown> = { jina_status: res.status, content_length_bytes: markdown.length };
+  const extra: Record<string, unknown> = {
+    jina_status: res.status,
+    content_length_bytes: markdown.length,
+    source_kind: 'web-page' as SourceKind,
+    parser_profile: 'opengraph' as ParserProfile,
+  };
   const preamble = parsePreamble(markdown);
   const publishedTime = preamble['Published Time'] ?? preamble['published_time'];
   if (publishedTime) {
@@ -105,6 +107,119 @@ async function jinaFetchOnce(url: string, noCache = false): Promise<JinaResult>
   return { ok: true, markdown, title, fetched_at, extra };
 }
 
+// ── Profile routing ────────────────────────────────────────────────────────
+
+function metaOf(data: Record<string, unknown>): Record<string, unknown> {
+  return (data.metadata && typeof data.metadata === 'object' ? data.metadata : {}) as Record<string, unknown>;
+}
+
+function present(meta: Record<string, unknown>, key: string): boolean {
+  const v = meta[key];
+  if (v == null || v === '') return false;
+  if (Array.isArray(v)) return v.length > 0;
+  return true;
+}
+
+// Fuzzy-match the source kind from which metadata convention is present, and
+// map it to a parser profile. Academic tells (a DOI or citation_* journal
+// keys) are the strongest signal; then OpenGraph article/website; else a
+// plain web page.
+export function detectProfile(data: Record<string, unknown>): { profile: ParserProfile; kind: SourceKind } {
+  const meta = metaOf(data);
+  const ogType = firstStr(meta['og:type']);
+  if (present(meta, 'citation_title') || present(meta, 'citation_doi') || present(meta, 'DOI') || present(meta, 'citation_journal_title')) {
+    return { profile: 'structured', kind: 'academic-paper' };
+  }
+  if (ogType === 'article' || present(meta, 'article:published_time') || present(meta, 'article:author')) {
+    return { profile: 'opengraph', kind: 'article' };
+  }
+  if (ogType === 'website' || (present(meta, 'og:site_name') && !present(meta, 'article:published_time'))) {
+    return { profile: 'opengraph', kind: 'company-landing' };
+  }
+  return { profile: 'opengraph', kind: 'web-page' };
+}
+
+// Pure, testable, profile-aware bibliographic extraction. Each field resolves
+// across an ordered alias list (first hit wins), so a field missing under one
+// convention still resolves under another. forceProfile overrides routing (the
+// manual re-route path).
+export function extractBib(
+  data: Record<string, unknown>,
+  url: string,
+  opts: { forceProfile?: ParserProfile } = {},
+): { title: string; extra: Record<string, unknown> } {
+  const meta = metaOf(data);
+  const routed = detectProfile(data);
+  const profile = opts.forceProfile ?? routed.profile;
+  const kind: SourceKind = opts.forceProfile
+    ? (opts.forceProfile === 'structured' ? 'academic-paper' : routed.kind)
+    : routed.kind;
+
+  const markdown = typeof data.content === 'string' ? data.content : '';
+
+  // Per-profile candidate lists. `data.*` are Jina's top-level convenience
+  // fields (often undefined); the rest are pass-through meta tags.
+  const titleCands =
+    profile === 'structured'
+      ? [data.title, meta['citation_title'], meta['dc.title'], meta['og:title']]
+      : [data.title, meta['og:title'], meta['twitter:title'], meta['dc.title']];
+
+  const authorCands =
+    profile === 'structured'
+      ? [meta['citation_author'], meta['dc.creator'], data.author]
+      : [meta['author'], meta['article:author'], data.author, meta['dc.creator']];
+
+  const dateCands =
+    profile === 'structured'
+      ? [meta['dc.date'], meta['prism.publicationDate'], meta['citation_online_date'], meta['citation_publication_date'], meta['citation_cover_date'], data.publishedTime]
+      : [data.publishedTime, meta['article:published_time'], meta['article:modified_time'], meta['date'], meta['dc.date']];
+
+  const publisherCands =
+    profile === 'structured'
+      ? [meta['citation_publisher'], meta['dc.publisher'], meta['prism.publicationName'], meta['citation_journal_title'], meta['og:site_name']]
+      : [meta['og:site_name'], data.publisher, meta['dc.publisher']];
+
+  const descCands =
+    profile === 'structured'
+      ? [meta['dc.description'], data.description, meta['description'], meta['og:description']]
+      : [data.description, meta['og:description'], meta['description'], meta['twitter:description']];
+
+  const langCands =
+    profile === 'structured'
+      ? [meta['citation_language'], meta['dc.language'], meta['lang'], meta['language']]
+      : [meta['lang'], meta['language'], data.lang];
+
+  const title = firstStr(...titleCands) ?? extractTitle(markdown, url);
+
+  const extra: Record<string, unknown> = { source_kind: kind, parser_profile: profile };
+
+  const publishedRaw = firstStr(...dateCands);
+  if (publishedRaw) {
+    const iso = normalizeToISO(publishedRaw);
+    if (iso) extra.published_at = iso;
+  }
+  const authors = normalizeAuthors(...authorCands);
+  if (authors.length) extra.authors = authors;
+  const publisher = firstStr(...publisherCands) ?? hostnameOf(url);
+  if (publisher) extra.publisher = publisher;
+  const description = firstStr(...descCands);
+  if (description) extra.description = description;
+  const language = firstStr(...langCands);
+  if (language) extra.language = language;
+
+  // Bonus identifiers worth carrying when the structured profile has them.
+  if (profile === 'structured') {
+    const doi = firstStr(meta['citation_doi'], meta['DOI'], meta['prism.doi']);
+    if (doi) extra.doi = doi.replace(/^doi:/i, '');
+    const journal = firstStr(meta['citation_journal_title'], meta['prism.publicationName']);
+    if (journal) extra.journal = journal;
+  }
+
+  return { title, extra };
+}
+
+// ── helpers ──────────────────────────────────────────────────────────────
+
 function firstStr(...vals: unknown[]): string | undefined {
   for (const v of vals) {
     if (typeof v === 'string' && v.trim()) return v.trim();
@@ -114,7 +229,8 @@ function firstStr(...vals: unknown[]): string | undefined {
 
 // Jina returns author as a STRING for one author but an ARRAY for several.
 // Normalize to a string[] (one author → one-element array), taking the first
-// key that yields anything and stripping a leading "By ".
+// key that yields anything and stripping a leading "By ". Never comma-splits a
+// single string — "Pal, Soumen" (Last, First) is one author, not two.
 function normalizeAuthors(...vals: unknown[]): string[] {
   for (const v of vals) {
     let list: string[] = [];
@@ -156,9 +272,9 @@ function parsePreamble(markdown: string): Record<string, string> {
 }
 
 // Jina passes through whatever the upstream meta tag carried. Coerce
-// the common cases (ISO already, RFC 2822, "YYYY-MM-DD") to ISO 8601.
-// Returns null when Date parsing yields NaN — better to drop than to
-// stamp garbage into the frontmatter.
+// the common cases (ISO already, RFC 2822, "YYYY-MM-DD", "YYYY/MM/DD") to
+// ISO 8601. Returns null when Date parsing yields NaN — better to drop than
+// to stamp garbage into the frontmatter.
 function normalizeToISO(raw: string): string | null {
   if (/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}/.test(raw)) {
     const d = new Date(raw);

services/content-ingest/test/jina-extract.test.ts

@@ -0,0 +1,117 @@
+// Jina metadata extraction — two profiles + fuzzy routing.
+// Issue: context-v/issues/Jina-Metadata-Parser-Is-Blog-Only-Needs-Two-Profiles-And-Routing.md
+//
+// Pure extraction, no network: feeds representative Jina `data` objects (the
+// academic fixture is the REAL metadata block captured live from a Springer
+// article) through detectProfile + extractBib and asserts the routing and the
+// resolved bibliographic fields.
+
+import { describe, expect, test } from 'vitest';
+import { detectProfile, extractBib } from '../src/jina';
+
+// Real metadata captured live from r.jina.ai for
+// https://link.springer.com/article/10.1007/s12033-023-00765-4
+const academic = {
+  title: 'Quantum Computing in the Next-Generation Computational Biology Landscape: From Protein Folding to Molecular Dynamics',
+  content: '# Quantum Computing …\n\nbody text',
+  // Jina's top-level convenience fields come back undefined for scholarly pages:
+  publishedTime: undefined,
+  author: undefined,
+  metadata: {
+    citation_title: 'Quantum Computing in the Next-Generation Computational Biology Landscape: From Protein Folding to Molecular Dynamics',
+    citation_author: ['Pal, Soumen', 'Bhattacharya, Manojit', 'Lee, Sang-Soo', 'Chakraborty, Chiranjib'],
+    'dc.creator': ['Pal, Soumen', 'Bhattacharya, Manojit', 'Lee, Sang-Soo', 'Chakraborty, Chiranjib'],
+    citation_publisher: 'Springer US',
+    'dc.publisher': 'Springer',
+    'dc.date': '2023-05-27',
+    'prism.publicationDate': '2023-05-27',
+    citation_publication_date: '2024/02',
+    citation_journal_title: 'Molecular Biotechnology',
+    citation_doi: '10.1007/s12033-023-00765-4',
+    DOI: '10.1007/s12033-023-00765-4',
+    'og:site_name': 'SpringerLink',
+    citation_language: 'en',
+  },
+};
+
+const blog = {
+  title: 'Why We Rebuilt Our Pipeline',
+  content: '# Why We Rebuilt Our Pipeline\n\nbody',
+  metadata: {
+    'og:type': 'article',
+    'article:published_time': '2025-01-15T10:00:00Z',
+    'article:author': 'Jane Doe',
+    'og:site_name': 'Cool Engineering Blog',
+    'og:description': 'A story about our rebuild.',
+  },
+};
+
+const company = {
+  title: 'Acme — Industrial Robotics',
+  content: '# Acme\n\nWe build robots.',
+  metadata: {
+    'og:type': 'website',
+    'og:site_name': 'Acme Inc',
+    'og:description': 'Industrial robotics for the modern factory.',
+  },
+};
+
+const URL_ACADEMIC = 'https://link.springer.com/article/10.1007/s12033-023-00765-4';
+
+describe('detectProfile — routes by source kind', () => {
+  test('academic article → structured / academic-paper', () => {
+    expect(detectProfile(academic)).toEqual({ profile: 'structured', kind: 'academic-paper' });
+  });
+  test('blog post → opengraph / article', () => {
+    expect(detectProfile(blog)).toEqual({ profile: 'opengraph', kind: 'article' });
+  });
+  test('company landing → opengraph / company-landing', () => {
+    expect(detectProfile(company)).toEqual({ profile: 'opengraph', kind: 'company-landing' });
+  });
+});
+
+describe('extractBib — structured profile (academic)', () => {
+  const { title, extra } = extractBib(academic, URL_ACADEMIC);
+  test('title from citation_title', () => {
+    expect(title).toContain('Quantum Computing in the Next-Generation');
+  });
+  test('all four authors, as an array, un-split', () => {
+    expect(extra.authors).toEqual(['Pal, Soumen', 'Bhattacharya, Manojit', 'Lee, Sang-Soo', 'Chakraborty, Chiranjib']);
+  });
+  test('publisher prefers citation_publisher over og:site_name', () => {
+    expect(extra.publisher).toBe('Springer US');
+  });
+  test('date resolves from dc.date (the field the old parser missed)', () => {
+    expect(String(extra.published_at)).toMatch(/^2023-05-27/);
+  });
+  test('carries kind, profile, doi, journal', () => {
+    expect(extra.source_kind).toBe('academic-paper');
+    expect(extra.parser_profile).toBe('structured');
+    expect(extra.doi).toBe('10.1007/s12033-023-00765-4');
+    expect(extra.journal).toBe('Molecular Biotechnology');
+  });
+});
+
+describe('extractBib — opengraph profile', () => {
+  test('blog: author + date + publisher', () => {
+    const { extra } = extractBib(blog, 'https://blog.example.com/rebuild');
+    expect(extra.authors).toEqual(['Jane Doe']);
+    expect(String(extra.published_at)).toMatch(/^2025-01-15/);
+    expect(extra.publisher).toBe('Cool Engineering Blog');
+    expect(extra.source_kind).toBe('article');
+  });
+  test('company landing: publisher + description, NO date (none exists)', () => {
+    const { extra } = extractBib(company, 'https://acme.example.com');
+    expect(extra.publisher).toBe('Acme Inc');
+    expect(extra.description).toBe('Industrial robotics for the modern factory.');
+    expect(extra.published_at).toBeUndefined();
+    expect(extra.source_kind).toBe('company-landing');
+  });
+});
+
+describe('forceProfile — the manual re-route override', () => {
+  test('forcing structured overrides the routed opengraph profile', () => {
+    const { extra } = extractBib(blog, 'https://blog.example.com/rebuild', { forceProfile: 'structured' });
+    expect(extra.parser_profile).toBe('structured');
+  });
+});