|
| 1 | +Set-StrictMode -Version Latest |
| 2 | +$ErrorActionPreference = 'Stop' |
| 3 | + |
| 4 | +Describe 'Invoke-OfflineRealHistoryCorpusEvaluation.ps1' -Tag 'Unit' { |
| 5 | + BeforeAll { |
| 6 | + $script:RepoRoot = (Resolve-Path (Join-Path $PSScriptRoot '..')).Path |
| 7 | + $script:EvaluateScript = Join-Path $script:RepoRoot 'tools' 'Invoke-OfflineRealHistoryCorpusEvaluation.ps1' |
| 8 | + if (-not (Test-Path -LiteralPath $script:EvaluateScript -PathType Leaf)) { |
| 9 | + throw "Invoke-OfflineRealHistoryCorpusEvaluation.ps1 not found at $script:EvaluateScript" |
| 10 | + } |
| 11 | + |
| 12 | + $script:CorpusPath = Join-Path $script:RepoRoot 'fixtures' 'real-history' 'offline-corpus.normalized.json' |
| 13 | + $script:ResolveEvaluationOutputPath = { |
| 14 | + param([Parameter(Mandatory)][string]$PathValue) |
| 15 | + |
| 16 | + if ([System.IO.Path]::IsPathRooted($PathValue)) { |
| 17 | + return $PathValue |
| 18 | + } |
| 19 | + |
| 20 | + return (Join-Path $script:RepoRoot ($PathValue -replace '/', '\')) |
| 21 | + } |
| 22 | + } |
| 23 | + |
| 24 | + It 'passes against the checked-in corpus and records report coverage checks' { |
| 25 | + $resultsRoot = Join-Path $TestDrive 'offline-corpus-eval' |
| 26 | + $runOutput = & pwsh -NoLogo -NoProfile -File $script:EvaluateScript ` |
| 27 | + -ResultsRoot $resultsRoot 2>&1 |
| 28 | + $LASTEXITCODE | Should -Be 0 -Because (($runOutput | ForEach-Object { [string]$_ }) -join [Environment]::NewLine) |
| 29 | + |
| 30 | + $reportPath = Join-Path $resultsRoot 'offline-real-history-corpus-evaluation.json' |
| 31 | + $reportPath | Should -Exist |
| 32 | + |
| 33 | + $report = Get-Content -LiteralPath $reportPath -Raw | ConvertFrom-Json -Depth 12 |
| 34 | + $report.schema | Should -Be 'vi-history/offline-real-history-corpus-evaluation@v1' |
| 35 | + $report.overallStatus | Should -Be 'ok' |
| 36 | + |
| 37 | + $target = @($report.targets | Where-Object { [string]$_.id -eq 'icon-editor-settings-init' } | Select-Object -First 1) |
| 38 | + $target | Should -Not -BeNullOrEmpty |
| 39 | + $target.status | Should -Be 'ok' |
| 40 | + $target.expected.coverageClass | Should -Be 'catalog-partial' |
| 41 | + $target.expected.modeSensitivity | Should -Be 'single-mode-observed' |
| 42 | + @($target.expected.outcomeLabels) | Should -Be @('clean', 'signal-diff') |
| 43 | + |
| 44 | + $target.observed.markdown.coverageClass | Should -BeTrue |
| 45 | + $target.observed.markdown.modeSensitivity | Should -BeTrue |
| 46 | + $target.observed.markdown.outcomeLabels | Should -BeTrue |
| 47 | + $target.observed.html.coverageClass | Should -BeTrue |
| 48 | + $target.observed.html.modeSensitivity | Should -BeTrue |
| 49 | + $target.observed.html.outcomeLabels | Should -BeTrue |
| 50 | + $target.observed.stepSummary.coverageClass | Should -BeTrue |
| 51 | + $target.observed.stepSummary.modeSensitivity | Should -BeTrue |
| 52 | + $target.observed.stepSummary.outcomeLabels | Should -BeTrue |
| 53 | + |
| 54 | + $markdownPath = & $script:ResolveEvaluationOutputPath ([string]$target.outputs.markdownPath) |
| 55 | + $htmlPath = & $script:ResolveEvaluationOutputPath ([string]$target.outputs.htmlPath) |
| 56 | + $stepSummaryPath = & $script:ResolveEvaluationOutputPath ([string]$target.outputs.stepSummaryPath) |
| 57 | + $markdownPath | Should -Exist |
| 58 | + $htmlPath | Should -Exist |
| 59 | + $stepSummaryPath | Should -Exist |
| 60 | + |
| 61 | + $markdown = Get-Content -LiteralPath $markdownPath -Raw |
| 62 | + $markdown | Should -Match '## Observed interpretation' |
| 63 | + $markdown | Should -Match '\| Coverage Class \| `catalog-partial` \|' |
| 64 | + $markdown | Should -Match '\| Mode Sensitivity \| `single-mode-observed` \|' |
| 65 | + $markdown | Should -Match '\| Outcome Labels \| `clean`, `signal-diff` \|' |
| 66 | + } |
| 67 | + |
| 68 | + It 'detects drift when corpus expectations no longer match the rendered report' { |
| 69 | + $corpus = Get-Content -LiteralPath $script:CorpusPath -Raw | ConvertFrom-Json -Depth 20 |
| 70 | + $corpusTarget = @($corpus.targets | Where-Object { [string]$_.id -eq 'icon-editor-settings-init' } | Select-Object -First 1) |
| 71 | + $corpusTarget | Should -Not -BeNullOrEmpty |
| 72 | + $corpusTarget.annotations.coverageClass = 'catalog-aligned' |
| 73 | + $driftCorpusPath = Join-Path $TestDrive 'offline-corpus.drift.json' |
| 74 | + $corpus | ConvertTo-Json -Depth 20 | Set-Content -LiteralPath $driftCorpusPath -Encoding utf8 |
| 75 | + |
| 76 | + $resultsRoot = Join-Path $TestDrive 'offline-corpus-drift' |
| 77 | + $runOutput = & pwsh -NoLogo -NoProfile -File $script:EvaluateScript ` |
| 78 | + -CorpusPath $driftCorpusPath ` |
| 79 | + -ResultsRoot $resultsRoot ` |
| 80 | + -SkipSchemaValidation 2>&1 |
| 81 | + $LASTEXITCODE | Should -Not -Be 0 |
| 82 | + |
| 83 | + $outputText = ($runOutput | ForEach-Object { [string]$_ }) -join [Environment]::NewLine |
| 84 | + $outputText | Should -Match 'detected report drift' |
| 85 | + |
| 86 | + $reportPath = Join-Path $resultsRoot 'offline-real-history-corpus-evaluation.json' |
| 87 | + $reportPath | Should -Exist |
| 88 | + $report = Get-Content -LiteralPath $reportPath -Raw | ConvertFrom-Json -Depth 12 |
| 89 | + $report.overallStatus | Should -Be 'drift' |
| 90 | + |
| 91 | + $target = @($report.targets | Where-Object { [string]$_.id -eq 'icon-editor-settings-init' } | Select-Object -First 1) |
| 92 | + $target | Should -Not -BeNullOrEmpty |
| 93 | + $target.status | Should -Be 'drift' |
| 94 | + $target.observed.markdown.coverageClass | Should -BeFalse |
| 95 | + ((@($target.notes)) -join [Environment]::NewLine) | Should -Match 'coverageClass' |
| 96 | + } |
| 97 | +} |
0 commit comments