|
| 1 | +{ |
| 2 | + "schema": 1, |
| 3 | + "purpose": "P-037 value/research benchmark (tests R2 of the #304 freeze only; not B2.1a, not implementation)", |
| 4 | + "ownnet_commit": "79207320c4a0fd5af8ff4e66fae78eba043f99d9", |
| 5 | + "note": "The ground truth below (expected codes) is NOT authored by this benchmark -- it is the pre-existing, project-owned expected-diagnostics.txt already committed in corpus/p036-bakeoff/. This file records that ground truth plus the exact bytes measured, so no label could be adjusted after seeing tool output. All 4 files run through both engines and codeql BEFORE this manifest was written; nothing here was changed afterward.", |
| 6 | + "capability_corpus": { |
| 7 | + "description": "4 families from corpus/p036-bakeoff/guarded-consume-*, each before.cs=buggy/after.cs=fixed, both real C# fed through the actual Roslyn extractor + core (own-check.sh), never a .own reduction.", |
| 8 | + "families": [ |
| 9 | + { |
| 10 | + "name": "guarded-consume-early-return", |
| 11 | + "p037_transform_edge": "const (guard spelled as an early return)", |
| 12 | + "before_sha256": "5953c796b227d8cb2b3149c125d3a59f445209d429a990677ecae1f189715219", |
| 13 | + "after_sha256": "c725fed4c98de064eca2eed312b66f46281f4e78303d9c60fe23bf9327530caa", |
| 14 | + "expected_sha256": "f631016d0ab006ef036c956c79ac6f93bad941feef5e9491eeb199de72912748", |
| 15 | + "expected_before": ["OWN001"], |
| 16 | + "expected_after": [] |
| 17 | + }, |
| 18 | + { |
| 19 | + "name": "guarded-consume-flag-branch", |
| 20 | + "p037_transform_edge": "const (direct literal argument, if(!keep) form)", |
| 21 | + "before_sha256": "b3e79c98225b6036e5ffd60ea1796aa722ab60ea69ac5cbff1b6ac144e123b85", |
| 22 | + "after_sha256": "2b5d245bcb48ac9d3847376a52c9b2e4a447511a9b66823fb0f407cebda0fc7b", |
| 23 | + "expected_sha256": "f631016d0ab006ef036c956c79ac6f93bad941feef5e9491eeb199de72912748", |
| 24 | + "expected_before": ["OWN001"], |
| 25 | + "expected_after": [] |
| 26 | + }, |
| 27 | + { |
| 28 | + "name": "guarded-consume-negation-wrapper", |
| 29 | + "p037_transform_edge": "neg (wrapper negates the flag on the edge)", |
| 30 | + "before_sha256": "468a23803de8900316f8ed1fa00ef65ad46714ceddbbbca2af2f1852916cf579", |
| 31 | + "after_sha256": "c9304ae920902811d7a02143af263b500084aeaa30fd91da9c6961b39cdd97b3", |
| 32 | + "expected_sha256": "f631016d0ab006ef036c956c79ac6f93bad941feef5e9491eeb199de72912748", |
| 33 | + "expected_before": ["OWN001"], |
| 34 | + "expected_after": [] |
| 35 | + }, |
| 36 | + { |
| 37 | + "name": "guarded-consume-wrapper-forward", |
| 38 | + "p037_transform_edge": "id (wrapper forwards the flag unchanged)", |
| 39 | + "before_sha256": "2de02c5dcad5036a4f6fd66ad8506617760e5fba4d01e5334b92341bf8986c61", |
| 40 | + "after_sha256": "b01880c912edac8b30ae4e855f2c6c96c8092b1398e92fdb4be5406ade219707", |
| 41 | + "expected_sha256": "f631016d0ab006ef036c956c79ac6f93bad941feef5e9491eeb199de72912748", |
| 42 | + "expected_before": ["OWN001"], |
| 43 | + "expected_after": [] |
| 44 | + } |
| 45 | + ], |
| 46 | + "not_in_scope_but_observed": [ |
| 47 | + "corpus/p036-bakeoff/gv4-control-aliased-self-null/ (a different P-037 family, surfaced incidentally by the live oracle run's file scan; not analysed)", |
| 48 | + "at least one other p036-bakeoff case producing an own-only OWN003 in the live run; not analysed (outside the 4 frozen families)" |
| 49 | + ] |
| 50 | + }, |
| 51 | + "reference_control_already_solved": { |
| 52 | + "case": "corpus/real-world/ownership-handoff-consume", |
| 53 | + "mechanism": "D5 / ConsumesParam inter-procedural CONSUME contract (unconditional, first-party consumer disposes its by-value IDisposable param) -- NOT guarded/conditional, NOT P-037", |
| 54 | + "measured": "both engines fire OWN001 on Leak() and OWN002 on Run(), silent on RunOk() -- matches docs/notes/corpus-benchmark.md's '9/11' ratchet exactly", |
| 55 | + "note": "corpus/real-world/ownership-handoff-consume/notes.md's own 'extractor status' paragraph (claiming the OWN002 arm is 'still extractor-future') is STALE relative to corpus-benchmark.md and to this measurement; flagged, not fixed (docs-only, out of scope for this benchmark)." |
| 56 | + }, |
| 57 | + "tools_measured": { |
| 58 | + "own_python_reference": { |
| 59 | + "invocation": "scripts/benchmark.py --corpus corpus/p036-bakeoff --min-recall 0 --json ... (existing harness, --engine python is hardcoded inside it) AND scripts/own-check.sh --engine python --format sarif -- <file> per file (cross-check)", |
| 60 | + "dotnet": "8.0.425", "python": "3.11.15", |
| 61 | + "result": "0/4 caught, 4/4 clean, 0 false positives" |
| 62 | + }, |
| 63 | + "own_rust_production": { |
| 64 | + "invocation": "OWEN_RUST_CORE=<release build> scripts/own-check.sh --engine rust --format sarif -- <file> per file (own-cli built via cargo build -p own-cli --release, rustc 1.94.1)", |
| 65 | + "result": "0/4 caught, 4/4 clean, 0 false positives", |
| 66 | + "python_rust_agreement": "IDENTICAL on all 8 files (before+after x4) -- no Own.NET engine-parity problem on this corpus" |
| 67 | + }, |
| 68 | + "codeql": { |
| 69 | + "invocation": "LIVE run via .github/workflows/oracle.yml push-trigger dev-loop (workflow_dispatch returned 403 for this session's GitHub access; confirmed empirically, matches the workflow's own comment), repo=local:corpus/p036-bakeoff, queries: security-and-quality, build-mode: none", |
| 70 | + "run": "https://github.com/PhysShell/Own.NET/actions/runs/36408630758 (job 108883233435), CodeQL CLI 2.27.1, CODEQL_ACTION_VERSION 4.37.0, commit 1efbdf1", |
| 71 | + "result": "0/4 on the guarded-consume families. Its one leak-class finding in the p036-bakeoff scan was in the unrelated gv4-control-aliased-self-null case (cs/dispose-not-called-on-throw, an exceptional-path gap, same class as the historical Dapper oracle-only findings) -- not in any of the 4 measured families." |
| 72 | + }, |
| 73 | + "infersharp": { |
| 74 | + "invocation": "same live run; build step could not find a .csproj/.sln to build (p036-bakeoff has none -- these are loose reduction .cs files)", |
| 75 | + "result": "UNAVAILABLE for this target (job step 'Run Infer#' concluded 'skipped'; BUILD_OK never set) -- recorded as untested, not as 0 findings" |
| 76 | + }, |
| 77 | + "rlc_sharp": { |
| 78 | + "invocation": "none -- no packaged/runnable implementation located; bounded (<30 min) literature check only (arXiv 2312.01912)", |
| 79 | + "architecture_from_literature": "CodeQL-query-based must-call/typestate accumulation checked against method-boundary resource-management specifications (C# attributes) for interprocedural reasoning -- no documented call-site-argument-sensitive resolution", |
| 80 | + "result": "NOT MEASURED. Documented architecture does not enter the benchmark score (task rule: only measured capability counts); recorded here only as an architectural prior." |
| 81 | + }, |
| 82 | + "ownaudit_tier2": { |
| 83 | + "invocation": "inspected PhysShell/OwnAudit @ 9c2f8fe553ce767e83798016233b282c1fa9b425, Run-Roslyn.ps1 (IDisposableAnalyzers + Microsoft.CodeAnalysis.NetAnalyzers + others)", |
| 84 | + "result": "ROSYLN_BASELINES = NOT_ESTABLISHED_IN_GATE_A -- hard-wired to PowerShell 7 + VS2022 BuildTools + vswhere/MSBuild.exe (Windows-only) and a hardcoded proprietary target C:\\Repos\\STS_new\\Broker.sln; none of this exists in this Linux sandbox and porting it is out of scope (would be building a new harness)." |
| 85 | + } |
| 86 | + }, |
| 87 | + "harness_controls": { |
| 88 | + "systemevents_console_local": { |
| 89 | + "invocation": "scripts/own-check.sh --engine {python,rust} --format human -- corpus/fixtures/systemevents-console (no OWN_EXTRA_REF_DIRS materialized locally)", |
| 90 | + "result": "Both engines identical: 3/3 dispose leaks (OWN001 at Program.cs:43,54,77) plus the documented OWN050 advisory on the unresolved SystemEvents subscription (degrades from OWN014 to OWN050 without the WindowsDesktop ref pack, exactly as the mechanism predicts) -- matches the documented historical shape; harness not qualified as broken.", |
| 91 | + "caveat": "Full 3-tool systemevents-console reproduction (own+CodeQL+Infer#) and live Dapper/Polly precision-tripwire reruns were NOT repeated this session (time budget); relied on the committed record in docs/notes/oracle.md / docs/notes/precision-remeasure-2026-07-12.md for those, clearly as historical, not re-measured." |
| 92 | + } |
| 93 | + }, |
| 94 | + "real_world_materiality_existing_evidence": { |
| 95 | + "issue": "https://github.com/PhysShell/Own.NET/issues/278", |
| 96 | + "state": "CLOSED, fixed by merged PR #293", |
| 97 | + "pattern": "boolean-parameter-guarded release, callers pass the constant that skips it -- structurally identical to guarded-consume-flag-branch/early-return, but on subscription release (-=) rather than IDisposable.Dispose()", |
| 98 | + "materiality": "P1, heap-PROVEN via ClrMD retention-path walk on the project's own SectorTS reference codebase: 66.3% of a 223MB heap genuinely retained after 31 documents, traced to GTD.cs:5192's guarded -= (UnregisterEventHandlers(bool UnregOnlyGoodys), if (!UnregOnlyGoodys) {...}, real callers pass true)", |
| 99 | + "shipped_fix_mechanism": "blanket/non-call-site-sensitive: 'a release guarded by a parameter of its enclosing method, outside a recognised teardown context, is NEVER credited as a release' (corpus/wpf/subscription-param-guarded-unregister). This is cheap but NOT context-sensitive -- it never had to prove itself against a safe call site that legitimately relies on the guard (the fixture's own after.cs sidesteps the guard entirely via an unconditional Dispose release rather than proving the guarded call safe).", |
| 100 | + "analytical_consequence": "Mechanically porting that same blanket rule to the Dispose()-guarded corpus measured here would flip recall (0/4 -> 4/4) at the cost of specificity (4/4 clean -> 0/4 clean): Fine()/RunOk() call the SAME guarded method with the argument that DOES release, and the blanket rule cannot tell that call site apart from Leak()'s. This is a concrete, non-hypothetical instance of the 'traded one error class for another' failure the Dapper/Polly precision tripwires warn about -- derived from this project's own shipped code, not a synthetic argument.", |
| 101 | + "counter_consideration": "The incident's actual shipped fix took a third path this analysis must name fairly: restructure the API so the guard disappears (move the release to an unconditional Dispose in a real teardown) rather than build a smarter checker. That is a legitimate general engineering answer to this pattern class and did not require any analyzer improvement at all." |
| 102 | + } |
| 103 | +} |
0 commit comments