Validate-ConformanceTestReport.ps1

[CmdletBinding()]
param(
    [Parameter(Mandatory)]
    [ValidateScript({ Test-Path -LiteralPath $_ -PathType Leaf })]
    [string] $AssemblyPath,

    [Parameter(Mandatory)]
    [ValidateScript({ Test-Path -LiteralPath $_ -PathType Leaf })]
    [string] $ReportPath
)

Set-StrictMode -Version Latest
$ErrorActionPreference = "Stop"

$attributeTypeName = "Expressif.Testing.Conformance.ConformanceAttribute"
$caseIdentifierPropertyName = "Case-Identifier"
$assemblyFile = (Resolve-Path -LiteralPath $AssemblyPath).Path
$reportFile = (Resolve-Path -LiteralPath $ReportPath).Path
$assemblyDirectory = Split-Path -Parent $assemblyFile
$originalDirectory = [Environment]::CurrentDirectory
$originalConsoleOut = [Console]::Out

try {
    [Environment]::CurrentDirectory = $assemblyDirectory
    [Console]::SetOut([System.IO.TextWriter]::Null)
    $assembly = [System.Reflection.Assembly]::LoadFrom($assemblyFile)
    $dataSourceType = $assembly.GetType(
        "Expressif.Testing.Conformance.ConformanceTestCaseDataSource",
        $true
    )
    $getCases = $dataSourceType.GetMethod("GetCases")

    $expectedCases = @{}
    $bindingFlags = [System.Reflection.BindingFlags]::Instance -bor
        [System.Reflection.BindingFlags]::Static -bor
        [System.Reflection.BindingFlags]::Public -bor
        [System.Reflection.BindingFlags]::NonPublic

    foreach ($type in $assembly.GetTypes() | Sort-Object FullName) {
        $reflectedType = $type.psobject.BaseObject
        foreach ($method in $type.GetMethods($bindingFlags) | Sort-Object Name) {
            $isConformanceAnchor = @(
                $method.GetCustomAttributesData() |
                Where-Object { $_.AttributeType.FullName -eq $attributeTypeName }
            ).Count -gt 0

            if (-not $isConformanceAnchor) {
                continue
            }

            $anchor = "$($reflectedType.FullName).$($method.Name)"
            foreach ($testCaseData in $getCases.Invoke($null, @($reflectedType, $method.Name))) {
                $caseId = [string] $testCaseData.Properties[$caseIdentifierPropertyName]
                if ([string]::IsNullOrWhiteSpace($caseId)) {
                    throw "Conformance anchor '$anchor' produced a case without a Case-Identifier."
                }

                if ($expectedCases.Contains($caseId)) {
                    throw (
                        "Conformance case '$caseId' is produced by both " +
                        "'$($expectedCases[$caseId])' and '$anchor'."
                    )
                }

                $expectedCases[$caseId] = $anchor
            }
        }
    }
}
finally {
    [Console]::SetOut($originalConsoleOut)
    [Environment]::CurrentDirectory = $originalDirectory
}

[xml] $xml = Get-Content -LiteralPath $reportFile -Raw -Encoding UTF8
$reportedCases = @{}

foreach ($property in $xml.SelectNodes(
    "//*[local-name()='test-case']/*[local-name()='properties']/*[local-name()='property']"
)) {
    if ($property.GetAttribute("name") -ne $caseIdentifierPropertyName) {
        continue
    }

    $caseId = $property.GetAttribute("value").Trim()
    if (-not [string]::IsNullOrWhiteSpace($caseId)) {
        $reportedCases[$caseId] = $true
    }
}

$missingCases = @(
    $expectedCases.Keys |
    Where-Object { -not $reportedCases.Contains($_) } |
    Sort-Object
)

if ($missingCases.Count -gt 0) {
    $details = $missingCases |
        ForEach-Object { "$_ ($($expectedCases[$_]))" }

    throw (
        "$($missingCases.Count) case(s) generated by [Conformance] anchors are absent " +
        "from '$reportFile': $($details -join ', ')."
    )
}

Write-Host (
    "Conformance test report validation passed: {0} anchor case(s) are present." -f
    $expectedCases.Count
)