Skip to content

Commit 5d97169

Browse files
committed
Complete Phase 1 repository stabilization
1 parent fe54894 commit 5d97169

353 files changed

Lines changed: 3271 additions & 2193 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.config/dotnet-tools.json‎

Lines changed: 1 addition & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -2,13 +2,6 @@
22
"version": 1,
33
"isRoot": true,
44
"tools": {
5-
"fsdocs-tool": {
6-
"version": "20.0.1",
7-
"commands": [
8-
"fsdocs"
9-
],
10-
"rollForward": false
11-
},
125
"dotnet-xscgen": {
136
"version": "3.0.1270",
147
"commands": [
@@ -17,4 +10,4 @@
1710
"rollForward": false
1811
}
1912
}
20-
}
13+
}

‎.github/workflows/deploy-docs.yml‎

Lines changed: 0 additions & 31 deletions
This file was deleted.

‎README.md‎

Lines changed: 94 additions & 282 deletions
Large diffs are not rendered by default.

‎RELEASE_NOTES.md‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,13 @@
1+
### 0.3.0 (Unreleased)
2+
3+
Phase 1 repository stabilization.
4+
5+
- Added dependency-vulnerability and generated-artifact drift gates to the full FAKE test path; generators now produce deterministic committed output.
6+
- Added a versioned, forward-migrated SQLite schema with explicit foreign-key modes, transactional public writes, and exact fixture round-trip coverage for all five stored entities.
7+
- Hardened crawler cancellation, bounded retries, strict partial-failure handling, upstream parsing, atomic writes, validation, and resume behavior while preserving injectable offline fetch seams.
8+
- Split the test suite into focused IO, ArcIR, SQLite, crawler, and crawler-hardening modules; refreshed package-boundary and ArcIR terminology documentation; removed the stale local fsdocs publication path.
9+
- Removed the unused structural-decompilation overlay from the proof-of-concept ArcIR package; its explicit converters remain the current mapping surface.
10+
111
### 0.2.1 (2026-07-09)
212

313
Crawler fixes surfaced by large-scale crawls (incl. ENA umbrella projects such as `PRJNA9506`, which fans out to thousands of child projects).

‎build/BasicTasks.fs‎

Lines changed: 55 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
11
module BasicTasks
22

3+
open System
4+
open System.IO
5+
open System.Text
36
open BlackFox.Fake
47
open Fake.IO
58
open Fake.DotNet
@@ -27,21 +30,65 @@ let clean = BuildTask.create "Clean" [] {
2730
// -i l : map xs:integer to System.Int64
2831
// --tnsf <file> : substitute generated type names per schemas/typename-substitutions.txt
2932
// (strips the Type infix from top-level + child types; cleans Type* enum prefixes)
30-
let regenerateInsdcTypes = BuildTask.create "regenerateInsdcTypes" [] {
31-
let schemasDir = "src/BioFSharp.FileFormats.INSDC/schemas"
32-
let generatedDir = "src/BioFSharp.FileFormats.INSDC/Generated"
33-
let substitutionFile = schemasDir + "/typename-substitutions.txt"
34-
Shell.cleanDir generatedDir
33+
let private schemasDir = "src/BioFSharp.FileFormats.INSDC/schemas"
34+
let private generatedDir = "src/BioFSharp.FileFormats.INSDC/Generated"
35+
let private substitutionFile = schemasDir + "/typename-substitutions.txt"
36+
37+
let private schemaFiles () =
38+
Directory.GetFiles(schemasDir, "*.xsd")
39+
|> Array.map (fun path -> path.Replace('\\', '/'))
40+
|> Array.sortWith (fun left right -> String.CompareOrdinal(left, right))
41+
42+
let private canonicalGeneratorCommand () =
43+
let inputs = schemaFiles () |> String.concat " "
44+
sprintf
45+
"// xscgen --separateFiles -n BioFSharp.FileFormats.INSDC -0 -i l --tnsf %s -o %s %s"
46+
substitutionFile
47+
generatedDir
48+
inputs
49+
50+
let private normalizeGeneratedFiles (outputDir: string) =
51+
let canonicalCommand = canonicalGeneratorCommand ()
52+
let utf8NoBom = UTF8Encoding(false)
53+
54+
for path in Directory.GetFiles(outputDir, "*.cs", SearchOption.AllDirectories) do
55+
let normalized =
56+
File.ReadAllLines(path)
57+
|> Array.map (fun line ->
58+
if line.StartsWith("// xscgen ", StringComparison.Ordinal) then
59+
canonicalCommand
60+
else
61+
line)
62+
|> String.concat "\n"
63+
64+
File.WriteAllText(path, normalized + "\n", utf8NoBom)
65+
66+
/// Restores the repository-local tools pinned in `.config/dotnet-tools.json`.
67+
/// Generator targets depend on this so they also work on a clean CI runner.
68+
let restoreTools = BuildTask.create "RestoreTools" [] {
69+
let result = DotNet.exec id "tool" "restore"
70+
if not result.OK then
71+
failwithf "dotnet tool restore failed (exit %d): %A" result.ExitCode result.Errors
72+
}
73+
74+
/// Runs the pinned xscgen tool into `outputDir` and normalizes generator
75+
/// metadata and line endings so output is byte-reproducible across machines.
76+
let generateInsdcTypesTo (outputDir: string) =
77+
Shell.cleanDir outputDir
3578
let schemaArgs =
36-
!! (schemasDir + "/*.xsd")
37-
|> Seq.map (fun p -> sprintf "\"%s\"" p)
79+
schemaFiles ()
80+
|> Seq.map (sprintf "\"%s\"")
3881
|> String.concat " "
3982
let args =
4083
sprintf "--separateFiles -n BioFSharp.FileFormats.INSDC -0 -i l --tnsf \"%s\" -o \"%s\" %s"
41-
substitutionFile generatedDir schemaArgs
84+
substitutionFile outputDir schemaArgs
4285
let result = DotNet.exec id "xscgen" args
4386
if not result.OK then
4487
failwithf "dotnet xscgen failed (exit %d): %A" result.ExitCode result.Errors
88+
normalizeGeneratedFiles outputDir
89+
90+
let regenerateInsdcTypes = BuildTask.create "regenerateInsdcTypes" [ restoreTools ] {
91+
generateInsdcTypesTo generatedDir
4592
}
4693

4794

@@ -62,4 +109,3 @@ let buildSolution =
62109
|> DotNet.Options.withCustomParams (Some "-tl")
63110
)
64111
}
65-

‎build/Build.fs‎

Lines changed: 12 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -14,31 +14,32 @@ initializeContext()
1414
open BasicTasks
1515
open FragmentSelectorTasks
1616
open StructuralOntologyTasks
17+
open GeneratedArtifactTasks
18+
open DependencyAuditTasks
1719
open TestTasks
1820
open PackageTasks
19-
open DocumentationTasks
2021
open ReleaseTasks
2122
open ReleaseFromNotesTask
2223

23-
/// Full release of nuget package, git tag, and documentation for the stable version.
24+
/// Full release of NuGet packages and the stable git tag.
2425
let _release =
2526
BuildTask.createEmpty
2627
"Release"
27-
[clean; buildSolution; runTests; pack; buildDocs; createTag; publishNuget; releaseDocs]
28+
[clean; buildSolution; runTests; pack; createTag; publishNuget]
2829

29-
/// Full release of nuget package, git tag, and documentation for the prerelease version.
30+
/// Full release of NuGet packages and the prerelease git tag.
3031
let _preRelease =
3132
BuildTask.createEmpty
3233
"PreRelease"
33-
[setPrereleaseTag; clean; buildSolution; runTests; packPrerelease; buildDocsPrerelease; createPrereleaseTag; publishNugetPrerelease; prereleaseDocs]
34+
[setPrereleaseTag; clean; buildSolution; runTests; packPrerelease; createPrereleaseTag; publishNugetPrerelease]
3435

35-
/// Full release of nuget package for the prerelease version.
36+
/// Legacy alias for `Release`, retained after local docs publishing was removed.
3637
let _releaseNoDocs =
3738
BuildTask.createEmpty
3839
"ReleaseNoDocs"
3940
[clean; buildSolution; runTests; pack; createTag; publishNuget;]
4041

41-
/// Full release of nuget package for the prerelease version.
42+
/// Legacy alias for `PreRelease`, retained after local docs publishing was removed.
4243
let _preReleaseNoDocs =
4344
BuildTask.createEmpty
4445
"PreReleaseNoDocs"
@@ -56,6 +57,10 @@ let _generateFragmentSelectors = generateFragmentSelectors
5657
// Same forced-init reason: register `generateStructuralOntology` for the CLI.
5758
let _generateStructuralOntology = generateStructuralOntology
5859

60+
// Register the Phase 1 verification gates for direct CLI use.
61+
let _verifyGeneratedArtifacts = verifyGeneratedArtifacts
62+
let _dependencyAudit = dependencyAudit
63+
5964
[<EntryPoint>]
6065
let main args =
6166
runOrDefault buildSolution args

‎build/DependencyAuditTasks.fs‎

Lines changed: 174 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,174 @@
1+
module DependencyAuditTasks
2+
3+
open System
4+
open System.Diagnostics
5+
open System.IO
6+
open System.Text.Json
7+
8+
open BlackFox.Fake
9+
10+
open BasicTasks
11+
open ProjectInfo
12+
13+
type private Vulnerability =
14+
{ Project: string
15+
Package: string
16+
Version: string
17+
Severity: string
18+
AdvisoryUrl: string }
19+
20+
type private Suppression =
21+
{ Project: string
22+
Package: string
23+
AdvisoryUrl: string
24+
Expires: DateTime
25+
Reason: string }
26+
27+
let private tryProperty (name: string) (element: JsonElement) =
28+
let mutable value = Unchecked.defaultof<JsonElement>
29+
if element.TryGetProperty(name, &value) then Some value else None
30+
31+
let private requiredString name element =
32+
match tryProperty name element with
33+
| Some value when value.ValueKind = JsonValueKind.String -> value.GetString()
34+
| _ -> failwithf "Dependency-audit suppression is missing string property '%s'." name
35+
36+
let private repositoryPath path =
37+
let absolute = Path.GetFullPath(path)
38+
Path.GetRelativePath(Environment.CurrentDirectory, absolute).Replace('\\', '/')
39+
40+
let private readSuppressions () =
41+
let path = "build/dependency-audit-suppressions.json"
42+
use document = JsonDocument.Parse(File.ReadAllText(path))
43+
44+
document.RootElement.GetProperty("suppressions").EnumerateArray()
45+
|> Seq.map (fun item ->
46+
let expiresText = requiredString "expires" item
47+
let mutable expires = DateTime.MinValue
48+
49+
if not (DateTime.TryParseExact(expiresText, "yyyy-MM-dd", null, Globalization.DateTimeStyles.None, &expires)) then
50+
failwithf "Dependency-audit suppression expiry '%s' must use yyyy-MM-dd." expiresText
51+
52+
let suppression =
53+
{ Project = requiredString "project" item
54+
Package = requiredString "package" item
55+
AdvisoryUrl = requiredString "advisoryUrl" item
56+
Expires = expires
57+
Reason = requiredString "reason" item }
58+
59+
if String.IsNullOrWhiteSpace suppression.Reason then
60+
failwith "Dependency-audit suppressions require a non-empty review reason."
61+
62+
suppression)
63+
|> List.ofSeq
64+
65+
let private runAudit () =
66+
let startInfo = ProcessStartInfo()
67+
startInfo.FileName <- "dotnet"
68+
startInfo.UseShellExecute <- false
69+
startInfo.RedirectStandardOutput <- true
70+
startInfo.RedirectStandardError <- true
71+
72+
[ "list"
73+
solutionFile
74+
"package"
75+
"--vulnerable"
76+
"--include-transitive"
77+
"--format"
78+
"json"
79+
"--no-restore" ]
80+
|> List.iter startInfo.ArgumentList.Add
81+
82+
use auditProcess = Process.Start(startInfo)
83+
let stdout = auditProcess.StandardOutput.ReadToEndAsync()
84+
let stderr = auditProcess.StandardError.ReadToEndAsync()
85+
auditProcess.WaitForExit()
86+
let output = stdout.Result
87+
let errors = stderr.Result
88+
89+
if auditProcess.ExitCode <> 0 then
90+
failwithf "dotnet dependency audit failed (exit %d):\n%s\n%s" auditProcess.ExitCode output errors
91+
92+
use document = JsonDocument.Parse(output)
93+
94+
[ for project in document.RootElement.GetProperty("projects").EnumerateArray() do
95+
let projectPath = project.GetProperty("path").GetString() |> repositoryPath
96+
97+
match tryProperty "frameworks" project with
98+
| None -> ()
99+
| Some frameworks ->
100+
for framework in frameworks.EnumerateArray() do
101+
for packageGroup in [ "topLevelPackages"; "transitivePackages" ] do
102+
match tryProperty packageGroup framework with
103+
| None -> ()
104+
| Some packages ->
105+
for package in packages.EnumerateArray() do
106+
match tryProperty "vulnerabilities" package with
107+
| None -> ()
108+
| Some vulnerabilities ->
109+
for vulnerability in vulnerabilities.EnumerateArray() do
110+
yield
111+
{ Project = projectPath
112+
Package = package.GetProperty("id").GetString()
113+
Version = package.GetProperty("resolvedVersion").GetString()
114+
Severity = vulnerability.GetProperty("severity").GetString()
115+
AdvisoryUrl = vulnerability.GetProperty("advisoryurl").GetString() } ]
116+
117+
let private matches (suppression: Suppression) (vulnerability: Vulnerability) =
118+
String.Equals(suppression.Project, vulnerability.Project, StringComparison.OrdinalIgnoreCase)
119+
&& String.Equals(suppression.Package, vulnerability.Package, StringComparison.OrdinalIgnoreCase)
120+
&& String.Equals(suppression.AdvisoryUrl, vulnerability.AdvisoryUrl, StringComparison.OrdinalIgnoreCase)
121+
122+
/// Audits direct and transitive NuGet packages. Every reported advisory fails
123+
/// the build unless an exact project/package/advisory suppression with a review
124+
/// reason and unexpired date exists; stale or unused suppressions also fail.
125+
let dependencyAudit =
126+
BuildTask.create "DependencyAudit" [ buildSolution ] {
127+
let today = DateTime.UtcNow.Date
128+
let suppressions = readSuppressions ()
129+
let expired = suppressions |> List.filter (fun item -> item.Expires < today)
130+
131+
if not (List.isEmpty expired) then
132+
expired
133+
|> List.map (fun item -> sprintf "%s / %s / %s" item.Project item.Package item.AdvisoryUrl)
134+
|> String.concat "\n"
135+
|> failwithf "Dependency-audit suppressions have expired:\n%s"
136+
137+
let vulnerabilities = runAudit ()
138+
let unsuppressed = vulnerabilities |> List.filter (fun item -> suppressions |> List.exists (fun s -> matches s item) |> not)
139+
let unused = suppressions |> List.filter (fun item -> vulnerabilities |> List.exists (matches item) |> not)
140+
141+
if not (List.isEmpty unused) then
142+
unused
143+
|> List.map (fun item -> sprintf "%s / %s / %s" item.Project item.Package item.AdvisoryUrl)
144+
|> String.concat "\n"
145+
|> failwithf "Dependency-audit suppressions no longer match an advisory; remove them:\n%s"
146+
147+
for vulnerability in vulnerabilities do
148+
match suppressions |> List.tryFind (fun suppression -> matches suppression vulnerability) with
149+
| Some suppression ->
150+
printfn
151+
"SUPPRESSED %s %s %s (%s; expires %s): %s"
152+
vulnerability.Package
153+
vulnerability.Version
154+
vulnerability.AdvisoryUrl
155+
vulnerability.Project
156+
(suppression.Expires.ToString("yyyy-MM-dd"))
157+
suppression.Reason
158+
| None -> ()
159+
160+
if not (List.isEmpty unsuppressed) then
161+
unsuppressed
162+
|> List.map (fun item ->
163+
sprintf
164+
"%s %s [%s] in %s — %s"
165+
item.Package
166+
item.Version
167+
item.Severity
168+
item.Project
169+
item.AdvisoryUrl)
170+
|> String.concat "\n"
171+
|> failwithf "Unsuppressed vulnerable dependencies:\n%s"
172+
173+
printfn "Dependency audit passed (%d reviewed suppression(s))." suppressions.Length
174+
}

0 commit comments

Comments
 (0)