Skip to content

Commit 6d3640c

Browse files
committed
feat: account supplementary F1 inputs
1 parent efd1501 commit 6d3640c

9 files changed

Lines changed: 716 additions & 17 deletions

File tree

‎plans/implementation.md‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -351,19 +351,21 @@ version rather than depending on a mutable sibling checkout.
351351

352352
## Phase 5 — Additive semantic enrichment, complete F1 accounting, and unified diagnostics
353353

354-
> **Status: IN PROGRESS (2026-08-27).** Begin with the neutral additive mapper,
355-
> the PolyglotSSSOM claim adapter, and an end-to-end BioProject/Study fixture using
356-
> the reviewed Phase 4 base mapping. Extend the same accounting model across all
357-
> supported F1 inputs after that boundary is stable.
354+
> **Status: COMPLETE (2026-08-27).** The neutral additive mapper, PolyglotSSSOM
355+
> claim adapter, and occurrence-level accounting now cover every supported F1
356+
> input without adding a parallel provenance artifact.
358357
>
359358
> **Implemented slices:** the shared diagnostic and field-accounting core, neutral
360359
> additive mapper, PolyglotSSSOM adapter, and artifact-qualified accounting for all
361360
> eight supported INSDC XML entities are implemented. Their tests resolve source
362361
> XML XPointers and emitted ArcIR JSON fragments, including the source-assertion
363362
> handoff to an additive base mapping, the connected BioSample-to-Experiment-to-Run
364363
> chain, analysis files and references, submission agents, and receipt
365-
> acknowledgements. Accounting for supplementary paper/count inputs is still
366-
> outstanding.
364+
> acknowledgements. Supplementary JATS front matter uses XPointer designations;
365+
> count-header cells use W3C Data Position Selectors over the exact immutable
366+
> bytes, so CSV, TSV, and materialized archive entries share one resolvable path.
367+
> Every emitted output is qualified against and resolved from a persisted ArcIR
368+
> state, and identical inputs produce deterministic reports and graph fragments.
367369
368370
### One result and diagnostic model
369371

‎src/BioFSharp.INSDC.ArcIR/BioFSharp.INSDC.ArcIR.fsproj‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
<Compile Include="Ingest\Paper.fs" />
3535
<Compile Include="Ingest\CountData.fs" />
3636
<Compile Include="Ingest\Readers.fs" />
37+
<Compile Include="Ingest\Accounting.fs" />
3738
<Compile Include="Ingest\Ingest.fs" />
3839
<None Include="..\..\docs\img\icon.png" Pack="true" PackagePath="" />
3940
<None Include="README.md" Pack="true" PackagePath="" />

‎src/BioFSharp.INSDC.ArcIR/Ingest/Accounting.fs‎

Lines changed: 509 additions & 0 deletions
Large diffs are not rendered by default.

‎src/BioFSharp.INSDC.ArcIR/Ingest/CountData.fs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ open BioFSharp.ArcIR
1010
[<RequireQualifiedAccess>]
1111
module CountData =
1212

13-
let private fileId (file: ResourceFile) = "count:" + file.Name
13+
let internal fileId (file: ResourceFile) = "count:" + file.Name
1414

1515
/// Convert one count file into a `ConversionResult`: a `countMatrix` Resource node plus a `countColumn`
1616
/// fragment (with `hasColumn` and `producesData` edges) for each run-accession column in its header.

‎src/BioFSharp.INSDC.ArcIR/Ingest/Ingest.fs‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
namespace BioFSharp.INSDC.ArcIR
22

3+
open System.IO
34
open BioFSharp.ArcIR
45

56
/// The ingest facade: fold supplementary sources (a paper, a count-data archive) into ArcIR fragments and
@@ -17,10 +18,42 @@ module Ingest =
1718
/// Convert one already-read count file into fragments.
1819
let countData (countFile: CountFile) : ConversionResult = CountData.convert countFile
1920

21+
/// Convert and account for a JATS paper from its exact immutable artifact bytes.
22+
let paperWithAccounting
23+
(artifact: ArtifactRevision)
24+
(fileName: string)
25+
(bytes: byte array)
26+
(relatedAccessions: string list)
27+
: AccountedConversion =
28+
IngestAccounting.paper artifact fileName bytes relatedAccessions
29+
30+
/// Convert and account for a delimited count table from its exact immutable artifact bytes.
31+
let countDataWithAccounting
32+
(artifact: ArtifactRevision)
33+
(fileName: string)
34+
(bytes: byte array)
35+
: AccountedConversion =
36+
IngestAccounting.countData artifact fileName bytes
37+
2038
/// Read a paper from a JATS XML file and convert it, linking it to the given dataset accession(s).
2139
let paperFromJats (path: string) (relatedAccessions: string list) : ConversionResult =
2240
Paper.convert (IngestReaders.readJats path) (IngestReaders.describeFile path) relatedAccessions
2341

42+
/// Read a JATS paper, verify it against the declared artifact revision, convert it, and account for its front matter.
43+
let paperFromJatsWithAccounting
44+
(artifact: ArtifactRevision)
45+
(path: string)
46+
(relatedAccessions: string list)
47+
: AccountedConversion =
48+
paperWithAccounting artifact (Path.GetFileName path) (File.ReadAllBytes path) relatedAccessions
49+
50+
/// Read a count table, verify it against the declared artifact revision, convert it, and account for its header cells.
51+
let countDataFromFileWithAccounting
52+
(artifact: ArtifactRevision)
53+
(path: string)
54+
: AccountedConversion =
55+
countDataWithAccounting artifact (Path.GetFileName path) (File.ReadAllBytes path)
56+
2457
/// Read a count-data zip archive and convert each tabular entry into fragments.
2558
let countDataFromArchive (path: string) : ConversionResult list =
2659
IngestReaders.readCountArchive path |> CountData.convertMany

‎src/BioFSharp.INSDC.ArcIR/Ingest/Paper.fs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -20,18 +20,18 @@ module Paper =
2020

2121
/// The paper Resource node id: `doi:<doi>` when a DOI is present (stable across file copies), else
2222
/// `paper:<name>`.
23-
let private paperId (meta: PaperMetadata) (file: ResourceFile) : string =
23+
let internal paperId (meta: PaperMetadata) (file: ResourceFile) : string =
2424
match present meta.Doi with
2525
| Some doi -> "doi:" + doi.Trim().ToLowerInvariant()
2626
| None -> "paper:" + file.Name
2727

2828
/// An author's dedup id: `agent:<email>`, else `agent:<orcid>`, else `agent:<name>`; `None` if all blank.
29-
let private authorId (a: PaperAuthor) : string option =
29+
let internal authorId (a: PaperAuthor) : string option =
3030
[ a.Email; a.Orcid; a.Name ]
3131
|> List.tryPick present
3232
|> Option.map (fun key -> "agent:" + key.Trim().ToLowerInvariant())
3333

34-
let private authorFragment (paperId: string) (a: PaperAuthor) : (ArcObject * ArcRelation) option =
34+
let internal authorFragment (paperId: string) (a: PaperAuthor) : (ArcObject * ArcRelation) option =
3535
authorId a
3636
|> Option.map (fun authorNodeId ->
3737
let props =

‎src/BioFSharp.INSDC.ArcIR/Ingest/Readers.fs‎

Lines changed: 43 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ open System
44
open System.IO
55
open System.IO.Compression
66
open System.Security.Cryptography
7+
open System.Text
78
open System.Text.RegularExpressions
89
open System.Xml.Linq
910

@@ -37,6 +38,19 @@ module IngestReaders =
3738
let private countResourceFile (name: string) (byteSize: int64) : ResourceFile =
3839
{ Name = name; ByteSize = Some byteSize; Checksum = None; MediaType = mediaType name }
3940

41+
let private sha256 (bytes: byte array) =
42+
use sha = SHA256.Create()
43+
"sha256:" + (BitConverter.ToString(sha.ComputeHash bytes).Replace("-", "").ToLowerInvariant())
44+
45+
let internal describeBytes (name: string) (bytes: byte array) : ResourceFile =
46+
if isNull bytes then
47+
nullArg (nameof bytes)
48+
49+
{ Name = name
50+
ByteSize = Some(int64 bytes.LongLength)
51+
Checksum = Some(sha256 bytes)
52+
MediaType = mediaType name }
53+
4054
// ---- files ----
4155

4256
/// Describe a file as a `ResourceFile`: name, byte size, a `sha256:<hex>` checksum, and a media type
@@ -67,11 +81,8 @@ module IngestReaders =
6781
| s when String.IsNullOrWhiteSpace s -> None
6882
| s -> Some(s.Trim())
6983

70-
/// Read paper-level metadata from a JATS XML article: title, DOI, journal, and the authors
71-
/// (name, email, affiliation, ORCID). Elements are matched by local name, so namespaced and
72-
/// non-namespaced JATS both work.
73-
let readJats (path: string) : PaperMetadata =
74-
let root = XDocument.Load(path: string).Root
84+
let private readJatsDocument (document: XDocument) : PaperMetadata =
85+
let root = document.Root
7586
let descendants (name: string) = root.Descendants() |> Seq.filter (fun e -> e.Name.LocalName = name)
7687
let firstText name = descendants name |> Seq.tryHead |> Option.bind (fun e -> text e.Value)
7788

@@ -116,6 +127,19 @@ module IngestReaders =
116127

117128
{ Title = firstText "article-title"; Doi = doi; Journal = firstText "journal-title"; Authors = authors }
118129

130+
let internal readJatsBytes (bytes: byte array) : PaperMetadata =
131+
if isNull bytes then
132+
nullArg (nameof bytes)
133+
134+
use stream = new MemoryStream(bytes, false)
135+
XDocument.Load stream |> readJatsDocument
136+
137+
/// Read paper-level metadata from a JATS XML article: title, DOI, journal, and the authors
138+
/// (name, email, affiliation, ORCID). Elements are matched by local name, so namespaced and
139+
/// non-namespaced JATS both work.
140+
let readJats (path: string) : PaperMetadata =
141+
XDocument.Load(path: string) |> readJatsDocument
142+
119143
// ---- count data ----
120144

121145
/// Parse a count-matrix header line into its run-accession columns, each with its 1-based position
@@ -131,6 +155,20 @@ module IngestReaders =
131155
|> Array.choose (fun (index, cell) -> if runAccession.IsMatch cell then Some { Index = index; RunAccession = cell } else None)
132156
|> List.ofArray
133157

158+
let internal readCountBytes (name: string) (bytes: byte array) : CountFile =
159+
if isNull bytes then
160+
nullArg (nameof bytes)
161+
162+
use stream = new MemoryStream(bytes, false)
163+
use reader = new StreamReader(stream, Encoding.UTF8, true)
164+
165+
let header =
166+
match reader.ReadLine() with
167+
| null -> ""
168+
| line -> line
169+
170+
{ File = countResourceFile name (int64 bytes.LongLength); Columns = parseHeader header }
171+
134172
/// Read one loose count file: its metadata plus the run-accession columns parsed from its header.
135173
let readCountFile (path: string) : CountFile =
136174
let info = FileInfo path

‎src/BioFSharp.INSDC.ArcIR/README.md‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,11 @@ Run, Analysis, Submission, and Receipt, plus supplementary papers and count data
1919

2020
The graph can be rendered as text, GraphML, or a self-contained HTML page. These
2121
are derived inspection tools; the embedded HTML page is not the intended final
22-
workbench architecture. Deterministic canonical persistence and revision-bound
23-
fragment addressing are planned separately.
22+
workbench architecture. `*WithAccounting` F1 functions bind exact source
23+
occurrences to typed locations in the resulting graph: INSDC and JATS XML use
24+
XPointer, while count-header cells use W3C byte-position selectors. The source
25+
artifact bytes are digest-checked, and emitted locations can be qualified against
26+
the canonical persisted ArcIR state with `FieldAccounting.qualifyEmitted`.
2427

2528
Built on
2629
[`BioFSharp.IO.INSDC`](https://www.nuget.org/packages/BioFSharp.IO.INSDC).

‎tests/BioFSharp.INSDC.Tests/F1AccountingTests.fs‎

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -405,3 +405,116 @@ type F1AccountingTests() =
405405

406406
find "/RECEIPT/ACTIONS[1]" accountedReceipt.Accounting
407407
|> expectUnsupported accountedReceipt.Accounting)
408+
409+
[<Fact>]
410+
member _.``JATS front matter accounting resolves source and ArcIR output designations`` () =
411+
let path = TestFiles.fixture "paper-PRJDB5192.jats.xml"
412+
let bytes = File.ReadAllBytes path
413+
let xml = Encoding.UTF8.GetString bytes
414+
let revision = ArtifactRevision.ofBytes path None bytes
415+
let accounted = Ingest.paperWithAccounting revision (Path.GetFileName path) bytes []
416+
let repeated = Ingest.paperWithAccounting revision (Path.GetFileName path) bytes []
417+
let graph = INSDC.build [ accounted.Conversion ]
418+
let sourceDocument = XPointer.entityDoc xml
419+
let outputLocations = ArcIRJson.locations graph |> Set.ofSeq
420+
421+
Assert.Equal(accounted, repeated)
422+
Assert.NotEmpty accounted.Accounting.Entries
423+
Assert.Empty accounted.Accounting.Diagnostics.Diagnostics
424+
425+
for entry in accounted.Accounting.Entries do
426+
Assert.Equal(revision, entry.Input.Artifact)
427+
Assert.Equal(F1Accounting.XPointerConformsTo, entry.Input.Selector.ConformsTo)
428+
Assert.True(
429+
XPointer.resolve sourceDocument entry.Input.Selector.Value |> Option.isSome,
430+
$"JATS selector did not resolve: {entry.Input.Selector.Value}"
431+
)
432+
433+
match entry.Outcome with
434+
| FieldAccountingOutcome.Emitted outputs ->
435+
Assert.NotEmpty outputs
436+
for output in outputs do
437+
Assert.Contains(output, outputLocations)
438+
| FieldAccountingOutcome.Ignored reason -> Assert.False(String.IsNullOrWhiteSpace reason)
439+
| outcome -> failwithf "Expected emitted or reviewed-ignored JATS metadata, got %A" outcome
440+
441+
withTempDirectory (fun directory ->
442+
assertQualifiedOutputsResolve directory "paper-state.arcir.json" graph accounted.Accounting
443+
|> ignore)
444+
445+
let title =
446+
accounted.Accounting.Entries
447+
|> List.find (fun entry -> entry.Input.Selector.Value.Contains("article-title"))
448+
449+
match title.Outcome with
450+
| FieldAccountingOutcome.Emitted outputs ->
451+
Assert.True(outputs |> List.exists (function ArcJsonLocation.ObjectAnnotationValue _ -> true | _ -> false))
452+
| outcome -> failwithf "Expected the JATS title to be emitted, got %A" outcome
453+
454+
[<Fact>]
455+
member _.``count header accounting uses byte-position cells and resolves every ArcIR output`` () =
456+
let path = TestFiles.fixture "counts-PRJDB5192.tsv"
457+
let bytes = File.ReadAllBytes path
458+
let revision = ArtifactRevision.ofBytes path None bytes
459+
let accounted = Ingest.countDataWithAccounting revision (Path.GetFileName path) bytes
460+
let repeated = Ingest.countDataWithAccounting revision (Path.GetFileName path) bytes
461+
let graph = INSDC.build [ accounted.Conversion ]
462+
let outputLocations = ArcIRJson.locations graph |> Set.ofSeq
463+
let selectorPattern =
464+
"^#selector\\(type=DataPositionSelector,start=(\\d+),end=(\\d+)\\)$"
465+
466+
let selectedValue (entry: FieldAccountingEntry) =
467+
let matched =
468+
Text.RegularExpressions.Regex.Match(entry.Input.Selector.Value, selectorPattern)
469+
470+
Assert.True(matched.Success, $"Invalid data-position selector: {entry.Input.Selector.Value}")
471+
let startPosition = Int32.Parse matched.Groups.[1].Value
472+
let endPosition = Int32.Parse matched.Groups.[2].Value
473+
Assert.InRange(startPosition, 0, bytes.Length)
474+
Assert.InRange(endPosition, startPosition, bytes.Length)
475+
Encoding.UTF8.GetString(bytes, startPosition, endPosition - startPosition)
476+
477+
Assert.Equal(accounted, repeated)
478+
Assert.Equal(3, accounted.Accounting.Entries.Length)
479+
Assert.Empty accounted.Accounting.Diagnostics.Diagnostics
480+
481+
for entry in accounted.Accounting.Entries do
482+
Assert.Equal(revision, entry.Input.Artifact)
483+
Assert.Equal(IngestAccounting.DataPositionSelectorConformsTo, entry.Input.Selector.ConformsTo)
484+
Assert.False(String.IsNullOrWhiteSpace(selectedValue entry))
485+
486+
match entry.Outcome with
487+
| FieldAccountingOutcome.Emitted outputs ->
488+
Assert.NotEmpty outputs
489+
for output in outputs do
490+
Assert.Contains(output, outputLocations)
491+
| FieldAccountingOutcome.Ignored reason -> Assert.False(String.IsNullOrWhiteSpace reason)
492+
| outcome -> failwithf "Expected emitted or reviewed-ignored count header cell, got %A" outcome
493+
494+
let featureColumn =
495+
accounted.Accounting.Entries
496+
|> List.find (fun entry -> selectedValue entry = "gene_id")
497+
498+
match featureColumn.Outcome with
499+
| FieldAccountingOutcome.Ignored _ -> ()
500+
| outcome -> failwithf "Expected the feature identifier column to be intentionally ignored, got %A" outcome
501+
502+
let runColumn =
503+
accounted.Accounting.Entries
504+
|> List.find (fun entry -> selectedValue entry = "DRR072834")
505+
506+
match runColumn.Outcome with
507+
| FieldAccountingOutcome.Emitted outputs ->
508+
Assert.True(outputs |> List.exists (function ArcJsonLocation.Relation _ -> true | _ -> false))
509+
| outcome -> failwithf "Expected the run header to emit the column and linkage, got %A" outcome
510+
511+
withTempDirectory (fun directory ->
512+
assertQualifiedOutputsResolve directory "count-state.arcir.json" graph accounted.Accounting
513+
|> ignore)
514+
515+
let mismatched = Array.copy bytes
516+
mismatched.[0] <- mismatched.[0] ^^^ 1uy
517+
518+
Assert.Throws<ArgumentException>(fun () ->
519+
Ingest.countDataWithAccounting revision (Path.GetFileName path) mismatched |> ignore)
520+
|> ignore

0 commit comments

Comments
 (0)