crawler: seed root, log start accession, single-transaction bulk insert

Fixes surfaced by large-scale crawls (e.g. ENA umbrella projects like
PRJNA9506 that fan out to thousands of child projects):

- Discovery is run-driven, so a project/study with no runs discovered
  nothing — not even itself — and persisted an empty database. Discovery.withRoot
  seeds the root into the bucket its prefix implies (PRJ... -> BioProject,
  SRP/ERP/DRP... -> Study) so the root record always persists.
- Add a Started crawl event emitted before discovery, so the first log
  line identifies the root accession instead of only learning it at 'done'.
- Make Sql.withTransaction reentrant (a nested call joins the active
  transaction rather than throwing — SQLite has no nested transactions) and
  wrap the crawler's whole insert pass in one transaction, collapsing one
  commit per record into a single commit. disableForeignKeys stays outside it.

Adds regression tests for root seeding, childless-project persistence,
Started-first ordering, and the reentrant-transaction contract.

Author: kMutagene

RELEASE_NOTES.md

@@ -1,3 +1,11 @@
+### 0.2.1 (2026-07-09)
+
+Crawler fixes surfaced by large-scale crawls (incl. ENA umbrella projects such as `PRJNA9506`, which fans out to thousands of child projects).
+
+- **BioFSharp.INSDC.Crawler** — a project/study with no sequencing runs now persists its own record: discovery is run-driven, so a childless root previously discovered nothing (not even itself). `Discovery.withRoot` seeds the root into the bucket its accession prefix implies (`PRJ…` → BioProject, `SRP`/`ERP`/`DRP…` → Study).
+- **BioFSharp.INSDC.Crawler** — new `Started` crawl event, emitted before discovery, so the first log line identifies the root accession rather than only learning it at `done —`.
+- **BioFSharp.INSDC.SQLite / Crawler** — bulk insert: `Sql.withTransaction` is now reentrant (a nested call joins the active transaction rather than throwing, since SQLite has no nested transactions), letting the crawler persist an entire crawl in a single transaction instead of one commit per record — the difference between a trickle and a bulk load on crawls of hundreds of thousands of runs.
+
 ### 0.2.0 (2026-07-09)
 
 Expands the suite from two packages to five: `BioFSharp.FileFormats.INSDC` and

src/BioFSharp.INSDC.Crawler/Crawler.fs

@@ -89,6 +89,7 @@ module private Fetch =
     /// carries the relationships persistence needs to thread foreign keys).
     let crawlCore (options: CrawlOptions) (rootAccession: string) : Async<CrawlResult * DiscoveredSet> =
         async {
+            options.Log(Started rootAccession)
             let! discovered = Discovery.discoverAsync options rootAccession
 
             options.Log(
@@ -128,6 +129,11 @@ module private Persist =
     open Microsoft.Data.Sqlite
     open BioFSharp.INSDC.SQLite
 
+    // Just the transaction helper — a module abbreviation rather than `open
+    // ...Internal`, whose `Schema` would shadow the public `Schema.init` used
+    // below.
+    module Sql = BioFSharp.INSDC.SQLite.Internal.Sql
+
     /// Opens (creating the file if needed) a SQLite connection for the crawl.
     /// Foreign-key enforcement is turned OFF for the insert pass in `persist`
     /// (the bundled schema's `PRAGMA foreign_keys = ON` would otherwise reject
@@ -199,6 +205,14 @@ module private Persist =
         // -> run) is guaranteed by the insertion order below instead.
         disableForeignKeys connection
 
+        // One surrounding transaction for the whole insert pass. Each entity
+        // `insert` opens its own `Sql.withTransaction`, but that is reentrant
+        // and joins this one, so the ~half-million per-record commits collapse
+        // into a single commit at the end — the difference between a trickle and
+        // a bulk load. `disableForeignKeys` stays *outside* it: `PRAGMA
+        // foreign_keys` is a no-op while a transaction is open.
+        Sql.withTransaction connection (fun _tx ->
+
         let hashSet (accessions: string seq) =
             System.Collections.Generic.HashSet<string>(accessions)
 
@@ -288,7 +302,7 @@ module private Persist =
                     rIns <- rIns + 1
                 | _ -> options.Log(Failed(sprintf "persist Run %s" run.Accession, "no stored parent experiment"))
 
-        options.Log(Persisted("Run", rIns, rSkip))
+        options.Log(Persisted("Run", rIns, rSkip)))
 
 /// The public crawler API. Two surfaces: return the connected records, or
 /// persist them (and the connectivity relations) into a SQLite database.

src/BioFSharp.INSDC.Crawler/Discovery.fs

@@ -164,11 +164,33 @@ module Discovery =
                 RunToExperiment = mapOf (fun r -> r.RunAccession) (fun r -> r.ExperimentAccession)
             }
 
+    /// Ensures `rootAccession` is present in the discovered set even when the
+    /// `read_run` report never named it. Discovery is entirely run-driven, so a
+    /// project or study with no runs yields a header-only report and an empty
+    /// set — without this the root's own record would never be fetched or
+    /// stored. The root is added to the bucket its accession prefix implies
+    /// (`PRJ...` -> BioProject, `SRP`/`ERP`/`DRP...` -> Study); an unrecognized
+    /// prefix (or one already present) leaves the set untouched.
+    let withRoot (rootAccession: string) (set: DiscoveredSet) : DiscoveredSet =
+        let hasPrefix (prefixes: string list) =
+            prefixes
+            |> List.exists (fun p -> rootAccession.StartsWith(p, StringComparison.OrdinalIgnoreCase))
+
+        let addDistinct (xs: string list) =
+            if List.contains rootAccession xs then xs else xs @ [ rootAccession ]
+
+        if String.IsNullOrWhiteSpace rootAccession then set
+        elif hasPrefix [ "PRJ" ] then { set with BioProjects = addDistinct set.BioProjects }
+        elif hasPrefix [ "SRP"; "ERP"; "DRP" ] then { set with Studies = addDistinct set.Studies }
+        else set
+
     /// Fetches and parses the `filereport` for `rootAccession` using `options`
-    /// (its Portal base URL, fetch function, retry count, and log sink).
+    /// (its Portal base URL, fetch function, retry count, and log sink), then
+    /// seeds the root itself (`withRoot`) so a childless project/study still
+    /// yields its own record.
     let discoverAsync (options: CrawlOptions) (rootAccession: string) : Async<DiscoveredSet> =
         async {
             let url = Endpoints.portalFileReport options.PortalBaseUrl rootAccession
             let! tsv = Internal.Http.withRetry options.Retries options.Log options.Fetch url
-            return parse tsv
+            return parse tsv |> withRoot rootAccession
         }

src/BioFSharp.INSDC.Crawler/Log.fs

@@ -6,6 +6,9 @@ open System
 /// through `CrawlOptions.Log`; the built-in `Log.console` sink renders them to
 /// stdout, and a custom sink can forward them to any logger (e.g. an `ILogger`).
 type CrawlEvent =
+    /// The crawl is starting — carries the root accession it was invoked with.
+    /// Emitted before discovery so the very first log line identifies the crawl.
+    | Started of accession: string
     /// Discovery finished — a per-entity count of distinct accessions found.
     | Discovered of counts: Map<string, int>
     /// About to fetch `count` records of the named entity kind.
@@ -29,6 +32,7 @@ module Log =
     /// Renders `event` as a single human-readable line (no timestamp/prefix).
     let format (event: CrawlEvent) : string =
         match event with
+        | Started accession -> sprintf "start — %s" accession
         | Discovered counts ->
             let parts =
                 counts

src/BioFSharp.INSDC.SQLite/Internal/Sql.fs

@@ -68,15 +68,37 @@ module Sql =
         use reader = cmd.ExecuteReader()
         if reader.Read() then Some (project reader) else None
 
+    /// The transaction currently active on `connection`, or `null` if none.
+    /// `SqliteConnection` exposes no public accessor for it, but a freshly
+    /// created command auto-binds its `Transaction` to the connection's active
+    /// one (and leaves it `null` when there is none), so a throwaway command
+    /// reveals it.
+    let private activeTransaction (connection: SqliteConnection) : SqliteTransaction =
+        use probe = connection.CreateCommand()
+        probe.Transaction
+
     /// Runs `work` inside an explicit transaction; commits on success, rolls
     /// back on any exception, and rethrows. Use this around the per-entity
     /// `insert` calls so a partial deconstruction never leaves orphan rows.
+    ///
+    /// Reentrant: SQLite has no nested transactions (a second `BeginTransaction`
+    /// throws), so when a transaction is already active on `connection` this
+    /// joins it — running `work` inline and leaving the commit/rollback to the
+    /// outermost owner. That lets a bulk caller wrap many per-entity `insert`s
+    /// (each of which calls `withTransaction` itself) in one surrounding
+    /// transaction, so the whole batch commits once instead of fsync-ing per
+    /// record.
     let withTransaction (connection: SqliteConnection) (work: SqliteTransaction -> 'a) : 'a =
-        use tx = connection.BeginTransaction()
-        try
-            let result = work tx
-            tx.Commit()
-            result
-        with _ ->
-            tx.Rollback()
-            reraise ()
+        let active = activeTransaction connection
+
+        if isNull active then
+            use tx = connection.BeginTransaction()
+            try
+                let result = work tx
+                tx.Commit()
+                result
+            with _ ->
+                tx.Rollback()
+                reraise ()
+        else
+            work active

tests/BioFSharp.INSDC.Tests/Tests.fs

@@ -1249,6 +1249,106 @@ type CrawlerTests() =
             if File.Exists dbPath then
                 File.Delete dbPath
 
+    [<Fact>]
+    member _.``Discovery.withRoot seeds a childless root into the bucket its prefix implies`` () =
+        // A childless project/study yields a header-only report -> the empty set.
+        let empty = Discovery.parse "run_accession\texperiment_accession\n"
+        Assert.Empty empty.BioProjects
+        Assert.Empty empty.Studies
+
+        // The root is seeded so its own record is still fetched: PRJ... -> project,
+        // SRP/ERP/DRP... -> study.
+        Assert.Equal<string[]>([| "PRJNA999" |], Discovery.withRoot "PRJNA999" empty |> fun s -> List.toArray s.BioProjects)
+        Assert.Empty((Discovery.withRoot "PRJNA999" empty).Studies)
+        Assert.Equal<string[]>([| "SRP123456" |], Discovery.withRoot "SRP123456" empty |> fun s -> List.toArray s.Studies)
+
+        // An already-present root is not duplicated; an unrecognized prefix
+        // (e.g. a run accession) leaves every bucket untouched.
+        let parsed = Discovery.parse (TestFiles.fixtureText "crawl-PRJDB5192.filereport.tsv")
+        Assert.Equal<string[]>([| "PRJDB5192" |], Discovery.withRoot "PRJDB5192" parsed |> fun s -> List.toArray s.BioProjects)
+        Assert.Equal<string[]>(List.toArray parsed.BioProjects, Discovery.withRoot "DRR999" parsed |> fun s -> List.toArray s.BioProjects)
+
+    [<Fact>]
+    member _.``crawlToSqlite persists a childless project (no runs) via root seeding`` () =
+        // ENA returns a header-only filereport for a project with no runs, so
+        // discovery finds nothing to relate. The root must still be stored.
+        let headerOnly =
+            "run_accession\texperiment_accession\tsample_accession\tstudy_accession\t\
+             secondary_study_accession\tfastq_ftp\tfastq_md5\tfastq_bytes\n"
+
+        let stub (url: string) : Async<string> =
+            async {
+                return
+                    if url.Contains "filereport" then headerOnly
+                    elif url.Contains "PRJDB5192" then TestFiles.fixtureText "PRJDB5192.xml"
+                    else failwithf "unexpected crawl URL: %s" url
+            }
+
+        let options = { CrawlOptions.Default with Fetch = stub; Log = Log.silent; ThrottleMs = 0 }
+        let dbPath = Path.Combine(Path.GetTempPath(), $"{Guid.NewGuid():N}.sqlite")
+
+        try
+            Crawler.crawlToSqliteWithAsync options "PRJDB5192" dbPath |> Async.RunSynchronously
+
+            (
+                use connection = new Microsoft.Data.Sqlite.SqliteConnection($"Data Source={dbPath}")
+                connection.Open()
+
+                Assert.True(
+                    (BioFSharp.INSDC.SQLite.BioProject.tryGet connection "PRJDB5192").IsSome,
+                    "childless BioProject was not persisted")
+
+                // No runs, so the connectivity table is legitimately empty.
+                Assert.Empty(BioFSharp.INSDC.SQLite.AccessionRelations.listAccessions connection |> Seq.toList)
+            )
+        finally
+            Microsoft.Data.Sqlite.SqliteConnection.ClearAllPools() |> ignore
+
+            if File.Exists dbPath then
+                File.Delete dbPath
+
+    [<Fact>]
+    member _.``crawl emits a Started event carrying the root accession as its first event`` () =
+        let events = System.Collections.Generic.List<CrawlEvent>()
+        let options = { CrawlerFixtures.options with Log = events.Add }
+
+        Crawler.crawlWithAsync options "PRJDB5192" |> Async.RunSynchronously |> ignore
+
+        match Seq.tryHead events with
+        | Some (Started accession) -> Assert.Equal("PRJDB5192", accession)
+        | other -> Assert.Fail(sprintf "expected a Started event first, got %A" other)
+
+    [<Fact>]
+    member _.``Sql.withTransaction is reentrant — a nested call joins the outer transaction`` () =
+        use connection = new Microsoft.Data.Sqlite.SqliteConnection("Data Source=:memory:")
+        connection.Open()
+        let exec sql = BioFSharp.INSDC.SQLite.Internal.Sql.execNonQuery connection sql [] |> ignore
+        let count () =
+            BioFSharp.INSDC.SQLite.Internal.Sql.queryAll connection "SELECT COUNT(*) FROM t;" [] (fun r -> r.GetInt32 0)
+            |> List.head
+
+        exec "CREATE TABLE t (v INTEGER);"
+
+        // A nested withTransaction must not throw (SQLite has no nested
+        // transactions) — it joins the outer one, so both inserts commit together.
+        BioFSharp.INSDC.SQLite.Internal.Sql.withTransaction connection (fun _ ->
+            exec "INSERT INTO t VALUES (1);"
+            BioFSharp.INSDC.SQLite.Internal.Sql.withTransaction connection (fun _ ->
+                exec "INSERT INTO t VALUES (2);"))
+        Assert.Equal(2, count ())
+
+        // When the outer rolls back, the joined-inner write is discarded too —
+        // proof the inner did not commit independently.
+        try
+            BioFSharp.INSDC.SQLite.Internal.Sql.withTransaction connection (fun _ ->
+                exec "INSERT INTO t VALUES (3);"
+                BioFSharp.INSDC.SQLite.Internal.Sql.withTransaction connection (fun _ ->
+                    exec "INSERT INTO t VALUES (4);")
+                failwith "boom")
+        with _ -> ()
+
+        Assert.Equal(2, count ())
+
     [<Fact>]
     member _.``LIVE crawl of a small public project (opt-in via INSDC_LIVE_TESTS=1)`` () =
         // Off by default (AGENTS.md forbids network at test time). Set