Tavily news source (#42)

* tavily news

* feat(sources): add tavily news source

* fix(tavily-news): log unexpected exceptions; drop stale docstring numbers

* wiring: opt-in tavily-news source plus --only-source filter

* plan: mark Task 2 complete

* polish(tavily-news): drop unused gate field from _SourceSpec

* polish(tavily-news): idiomatic pass

* polish(tavily-news): cache YAML, dedupe urlparse, type _SourceSpec by class

* polish(tavily-news): strip and humanize comments

* cleanups and back

* trims

Author: vaporif

.gitignore

@@ -112,5 +112,6 @@ slopmortem.local.toml
 tests/fixtures/cassettes/*.live.yaml
 post_mortems/
 journal.sqlite
+backups/
 
 .tldextract-cache/

docs/plans/2026-05-06-tavily-news-source.md

@@ -107,6 +107,7 @@
           git
           git-lfs
           curl
+          zip
         ];
 
         env = {

docs/specs/2026-05-06-tavily-news-source-design.md

@@ -163,6 +163,39 @@ init-env:
 
     echo "wrote $ENV_FILE"
 
+# Snapshot ingested state into a single zip under backups/. Stops Qdrant
+# briefly (only if it's running) so the storage volume is copied at rest,
+# then restarts it. Captures data/qdrant/, journal.sqlite, and post_mortems/
+# — the same surface `just nuke` wipes.
+backup:
+    #!/usr/bin/env bash
+    set -euo pipefail
+    mkdir -p backups
+    stamp=$(date +%Y%m%d-%H%M%S)
+    out="backups/slopmortem-backup-${stamp}.zip"
+
+    qdrant_running=0
+    if docker compose ps --status running --services 2>/dev/null | grep -qx qdrant; then
+        qdrant_running=1
+        echo "→ stopping qdrant for a consistent snapshot"
+        docker compose stop qdrant >/dev/null
+    fi
+
+    trap '[ "$qdrant_running" = "1" ] && docker compose start qdrant >/dev/null || true' EXIT
+
+    targets=()
+    [ -d data/qdrant ]   && targets+=(data/qdrant)
+    [ -f journal.sqlite ] && targets+=(journal.sqlite)
+    [ -d post_mortems ]   && targets+=(post_mortems)
+
+    if [ ${#targets[@]} -eq 0 ]; then
+        echo "nothing to back up (no qdrant, journal, or post_mortems present)"
+        exit 0
+    fi
+
+    zip -qr "$out" "${targets[@]}"
+    echo "wrote $out ($(du -h "$out" | cut -f1))"
+
 # Wipe all ingested state: stop Qdrant, delete its storage volume, drop
 # the merge journal, and remove the post_mortems tree. Prompts before
 # touching anything. Run before a fresh `just ingest` when you want to

flake.nix

@@ -11,7 +11,9 @@
 
 import contextlib
 import functools
+import os
 import sys
+from dataclasses import dataclass
 from pathlib import Path
 from typing import TYPE_CHECKING, Annotated, cast
 
@@ -38,6 +40,7 @@
     CuratedSource,
     HNAlgoliaSource,
     TavilyEnricher,
+    TavilyNewsSource,
     WaybackEnricher,
 )
 from slopmortem.ingest import INGEST_PHASE_LABELS, IngestPhase, IngestResult, ingest
@@ -97,6 +100,58 @@ def ingest_cmd(  # noqa: PLR0913 - every flag mirrors the spec; user types kwarg
     tavily_enrich: Annotated[
         bool, typer.Option("--tavily-enrich", help="Enable the Tavily enricher.")
     ] = False,
+    enable_tavily_news: Annotated[
+        bool,
+        typer.Option(
+            "--enable-tavily-news",
+            help=(
+                "Enable the Tavily news shutdown-event source. "
+                "Requires TAVILY_API_KEY. Bodies are returned inline; "
+                "no Tavily-extract hop is implied."
+            ),
+        ),
+    ] = False,
+    tavily_news_start_year: Annotated[
+        int | None,
+        typer.Option(
+            "--tavily-news-start-year",
+            help="Override year_range.start for the Tavily news source.",
+        ),
+    ] = None,
+    tavily_news_end_year: Annotated[
+        int | None,
+        typer.Option(
+            "--tavily-news-end-year",
+            help="Override year_range.end for the Tavily news source. Defaults to current year.",
+        ),
+    ] = None,
+    tavily_news_max_emit: Annotated[
+        int | None,
+        typer.Option(
+            "--tavily-news-max-emit",
+            help="Override the Tavily news source's max_emit cap.",
+        ),
+    ] = None,
+    tavily_news_search_depth: Annotated[
+        str | None,
+        typer.Option(
+            "--tavily-news-search-depth",
+            help=(
+                "Override search_depth for the Tavily news source: "
+                "basic (1 credit) or advanced (2)."
+            ),
+        ),
+    ] = None,
+    only_source: Annotated[
+        str | None,
+        typer.Option(
+            "--only-source",
+            help=(
+                "Run only the named source, auto-enabling its --enable-* flag if any. "
+                "Accepts source identifiers (curated, hn_algolia, crunchbase_csv, tavily_news)."
+            ),
+        ),
+    ] = None,
     post_mortems_root: Annotated[
         Path,
         typer.Option(
@@ -128,6 +183,12 @@ def ingest_cmd(  # noqa: PLR0913 - every flag mirrors the spec; user types kwarg
             crunchbase_csv=crunchbase_csv,
             enrich_wayback=enrich_wayback,
             tavily_enrich=tavily_enrich,
+            enable_tavily_news=enable_tavily_news,
+            tavily_news_start_year=tavily_news_start_year,
+            tavily_news_end_year=tavily_news_end_year,
+            tavily_news_max_emit=tavily_news_max_emit,
+            tavily_news_search_depth=tavily_news_search_depth,
+            only_source=only_source,
             post_mortems_root=post_mortems_root,
             limit=limit,
         )
@@ -176,7 +237,7 @@ async def _run_reconcile(config: Config, post_mortems_root: Path) -> None:
 
 
 @observe(name="cli.ingest")
-async def _run_ingest(  # noqa: PLR0913, C901 - the ingest CLI surface is wide.
+async def _run_ingest(  # noqa: PLR0913, PLR0912, PLR0915, C901 - the ingest CLI surface is wide.
     *,
     dry_run: bool,
     force: bool,
@@ -187,6 +248,12 @@ async def _run_ingest(  # noqa: PLR0913, C901 - the ingest CLI surface is wide.
     crunchbase_csv: Path | None,
     enrich_wayback: bool,
     tavily_enrich: bool,
+    enable_tavily_news: bool,
+    tavily_news_start_year: int | None,
+    tavily_news_end_year: int | None,
+    tavily_news_max_emit: int | None,
+    tavily_news_search_depth: str | None,
+    only_source: str | None,
     post_mortems_root: Path,
 ) -> None:
     config = load_config()
@@ -214,6 +281,30 @@ async def _run_ingest(  # noqa: PLR0913, C901 - the ingest CLI surface is wide.
         await _run_reconcile(config, post_mortems_root)
         return
 
+    if only_source is not None:
+        if only_source not in _SOURCE_REGISTRY:
+            valid = ", ".join(sorted(_SOURCE_REGISTRY))
+            msg = f"--only-source: unknown source {only_source!r}. Valid: {valid}."
+            raise typer.BadParameter(msg)
+        spec = _SOURCE_REGISTRY[only_source]
+        # Each opt-in source's --enable-* flag needs an explicit branch here —
+        # Python's keyword-only parameter binding can't be table-driven without
+        # ``locals()`` tricks. Add one when introducing a new opt-in source.
+        if spec.source_class is TavilyNewsSource:
+            enable_tavily_news = True
+        # crunchbase_csv is gated by a path argument, not a boolean — require it explicitly.
+        if spec.source_class is CrunchbaseCsvSource and crunchbase_csv is None:
+            msg = "--only-source crunchbase_csv requires --crunchbase-csv PATH."
+            raise typer.BadParameter(msg)
+
+    if enable_tavily_news and not os.environ.get("TAVILY_API_KEY"):
+        msg = (
+            "--enable-tavily-news requires TAVILY_API_KEY: the Tavily news source "
+            "calls /search and pulls article bodies via include_raw_content. "
+            "Set TAVILY_API_KEY in .env or unset --enable-tavily-news."
+        )
+        raise typer.BadParameter(msg)
+
     llm, embedder, corpus, budget, journal, classifier = await _build_ingest_deps(
         config, post_mortems_root, dry_run=dry_run
     )
@@ -227,6 +318,25 @@ async def _run_ingest(  # noqa: PLR0913, C901 - the ingest CLI surface is wide.
     ]
     if crunchbase_csv is not None:
         sources.append(CrunchbaseCsvSource(csv_path=crunchbase_csv))
+    if enable_tavily_news:
+        sources.append(
+            TavilyNewsSource(
+                start_year=tavily_news_start_year,
+                end_year=tavily_news_end_year,
+                max_emit=tavily_news_max_emit,
+                search_depth=tavily_news_search_depth,
+            )
+        )
+
+    if only_source is not None:
+        wanted_class = _SOURCE_REGISTRY[only_source].source_class
+        sources = [s for s in sources if isinstance(s, wanted_class)]
+        if not sources:
+            msg = (
+                f"--only-source {only_source!r}: source enabled but not constructed; "
+                "check that its prerequisites (e.g. --crunchbase-csv path) are present."
+            )
+            raise typer.BadParameter(msg)
 
     enrichers: list[Enricher] = []
     if enrich_wayback:
@@ -277,6 +387,19 @@ async def _run_ingest(  # noqa: PLR0913, C901 - the ingest CLI surface is wide.
         typer.echo(f"slopmortem ingest result: {result} cost=${budget.spent_usd:.4f}")
 
 
+@dataclass(frozen=True)
+class _SourceSpec:
+    source_class: type[Source]
+
+
+_SOURCE_REGISTRY: dict[str, _SourceSpec] = {
+    "curated": _SourceSpec(source_class=CuratedSource),
+    "hn_algolia": _SourceSpec(source_class=HNAlgoliaSource),
+    "crunchbase_csv": _SourceSpec(source_class=CrunchbaseCsvSource),
+    "tavily_news": _SourceSpec(source_class=TavilyNewsSource),
+}
+
+
 def _default_curated_yaml() -> Path:
     return Path(__file__).parent.parent / "corpus" / "sources" / "curated" / "post_mortems_v0.yml"
 

justfile

@@ -8,6 +8,7 @@
 from slopmortem.corpus.sources.curated import CuratedSource as CuratedSource
 from slopmortem.corpus.sources.hn_algolia import HNAlgoliaSource as HNAlgoliaSource
 from slopmortem.corpus.sources.tavily import TavilyEnricher as TavilyEnricher
+from slopmortem.corpus.sources.tavily_news import TavilyNewsSource as TavilyNewsSource
 from slopmortem.corpus.sources.wayback import WaybackEnricher as WaybackEnricher
 
 __all__ = [
@@ -17,5 +18,6 @@
     "HNAlgoliaSource",
     "Source",
     "TavilyEnricher",
+    "TavilyNewsSource",
     "WaybackEnricher",
 ]

slopmortem/cli/_ingest_cmd.py

@@ -7,3 +7,4 @@
 SOURCE_CURATED: Final = "curated"
 SOURCE_HN_ALGOLIA: Final = "hn_algolia"
 SOURCE_CRUNCHBASE_CSV: Final = "crunchbase_csv"
+SOURCE_TAVILY_NEWS: Final = "tavily_news"

slopmortem/corpus/sources/__init__.py

@@ -0,0 +1,12 @@
+# Aggregator / syndicated-content rehosts. Suffix match: a host matches when
+# it equals the listed domain or ends with `.<domain>`. Match is host-only
+# and case-insensitive (canonicalisation lowercases hosts upstream).
+#
+# Seed list intentionally small. Easy to grow if filler reappears at scores
+# above min_score.
+- bundle.app          # observed mirroring TechCrunch in 2026-05-06 probe
+- flipboard.com
+- feedly.com
+- smartnews.com
+- inoreader.com
+- news.google.com