Skip to content

Commit 44bd40a

Browse files
juntakijuntaki
committed
feat: relational-operator schemata lowering, MutantBench-Swift benchmark harness, schemata proof-chain v3, runner performance work (result cache, simulator pool, worker affinity, coverage cache), batch-attribution correctness fixes
1 parent bbb65d0 commit 44bd40a

432 files changed

Lines changed: 75975 additions & 0 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/ci.yml‎

Lines changed: 221 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,221 @@
1+
name: CI
2+
3+
on:
4+
push:
5+
branches: [main]
6+
pull_request:
7+
8+
permissions:
9+
contents: read
10+
pull-requests: write
11+
issues: write
12+
13+
concurrency:
14+
group: ${{ github.workflow }}-${{ github.ref }}
15+
cancel-in-progress: true
16+
17+
jobs:
18+
# Fastest feedback of all: static analysis needs no build. SwiftLint and
19+
# SwiftFormat run against the whole tree using the repo's own configs
20+
# (.swiftlint.yml / .swiftformat), which were tuned against this
21+
# codebase's actual style rather than stock defaults.
22+
#
23+
# `swiftlint lint` runs `--strict --baseline .swiftlint-baseline.json`:
24+
# a handful of complexity/length rules (function_body_length,
25+
# type_body_length, cyclomatic_complexity, file_length, large_tuple) fire
26+
# against pre-existing debt — concentrated in, but not limited to, a few
27+
# deliberately large orchestration files (see .swiftlint.yml's comments
28+
# for the full breakdown, including which CLI command handlers are
29+
# baselined too) — that aren't worth a risky structural refactor just to
30+
# silence a linter. The baseline freezes exactly those pre-existing
31+
# violations by identity (rule + file + line), so `--strict` still fails
32+
# the build on any *new* violation anywhere, including a new one in an
33+
# already-baselined file. Shrink the baseline over time by fixing an
34+
# entry and regenerating with
35+
# `swiftlint lint --write-baseline .swiftlint-baseline.json`; never grow
36+
# it to paper over new debt.
37+
#
38+
# SwiftLint is pinned to an exact version (matching the version the
39+
# baseline was generated with) rather than `brew install`'s floating
40+
# latest: baseline identity and violation detection are both
41+
# SwiftLint-version-sensitive, so an unpinned upgrade could silently
42+
# change what the baseline recognizes, in either direction. Bump
43+
# SWIFTLINT_VERSION and regenerate .swiftlint-baseline.json together, as
44+
# one deliberate change, not independently.
45+
lint:
46+
name: Lint & format check
47+
runs-on: macos-15
48+
env:
49+
SWIFTLINT_VERSION: "0.63.2"
50+
steps:
51+
- uses: actions/checkout@v4
52+
- name: Install SwiftFormat
53+
run: brew install swiftformat
54+
- name: Install pinned SwiftLint
55+
run: |
56+
curl -fsSL -o portable_swiftlint.zip \
57+
"https://github.com/realm/SwiftLint/releases/download/${SWIFTLINT_VERSION}/portable_swiftlint.zip"
58+
unzip -o portable_swiftlint.zip swiftlint
59+
chmod +x swiftlint
60+
sudo mv swiftlint /usr/local/bin/swiftlint
61+
rm portable_swiftlint.zip
62+
installed_version="$(swiftlint version)"
63+
if [ "$installed_version" != "$SWIFTLINT_VERSION" ]; then
64+
echo "::error::Installed SwiftLint $installed_version does not match pinned SWIFTLINT_VERSION $SWIFTLINT_VERSION"
65+
exit 1
66+
fi
67+
- name: SwiftLint
68+
run: swiftlint lint --strict --config .swiftlint.yml --baseline .swiftlint-baseline.json Sources Tests
69+
- name: SwiftFormat (check only)
70+
run: swiftformat --lint --config .swiftformat .
71+
72+
# Fast feedback: everything that does not build another project.
73+
unit:
74+
name: Unit tests
75+
runs-on: macos-15
76+
steps:
77+
- uses: actions/checkout@v4
78+
- name: Toolchain
79+
run: swift --version && xcodebuild -version
80+
- name: Build
81+
shell: bash
82+
run: |
83+
set -o pipefail
84+
swift build --build-tests 2>&1 | tee build.log
85+
- name: Test
86+
shell: bash
87+
run: |
88+
set -o pipefail
89+
swift test 2>&1 | tee test.log
90+
- name: Upload unit failure logs
91+
if: failure()
92+
uses: actions/upload-artifact@v4
93+
with:
94+
name: unit-failure-logs
95+
if-no-files-found: ignore
96+
path: |
97+
build.log
98+
test.log
99+
- name: Publish unit failure excerpt
100+
if: failure() && github.event_name == 'pull_request'
101+
uses: actions/github-script@v7
102+
with:
103+
script: |
104+
const fs = require('fs');
105+
const marker = '<!-- mutantkit-ci-unit-failure -->';
106+
const readTail = (path) => {
107+
if (!fs.existsSync(path)) return '';
108+
const lines = fs.readFileSync(path, 'utf8').split('\n');
109+
return lines.slice(-180).join('\n');
110+
};
111+
const build = readTail('build.log');
112+
const test = readTail('test.log');
113+
const excerpt = (test || build || 'No captured build/test log was available.').slice(-30000);
114+
const body = `${marker}\n### Latest unit CI failure\n\n\`\`\`text\n${excerpt}\n\`\`\``;
115+
const { owner, repo } = context.repo;
116+
const issue_number = context.issue.number;
117+
const comments = await github.paginate(github.rest.issues.listComments, { owner, repo, issue_number });
118+
const existing = comments.find(c => c.body && c.body.includes(marker));
119+
if (existing) {
120+
await github.rest.issues.updateComment({ owner, repo, comment_id: existing.id, body });
121+
} else {
122+
await github.rest.issues.createComment({ owner, repo, issue_number, body });
123+
}
124+
125+
# The suites that build and mutate real projects. Slow, and the only thing that
126+
# proves the tool works rather than merely compiles: every wiring bug this
127+
# project has had — a sandbox handed the wrong excludes, xcodebuild pointed at
128+
# unmutated sources, concurrent mutants fighting over one simulator — was
129+
# invisible to the unit tests and produced a confident, wrong score.
130+
acceptance:
131+
name: Acceptance (${{ matrix.fixture }})
132+
runs-on: macos-15
133+
needs: unit
134+
strategy:
135+
# Never cancel siblings: which fixtures fail together is a diagnosis.
136+
fail-fast: false
137+
matrix:
138+
include:
139+
- fixture: swift-package
140+
filter: SwiftPackageMacOSAcceptanceTests
141+
simulator: "0"
142+
- fixture: swift-package-coverage
143+
filter: SwiftPackageMacOSCoverageAcceptanceTests
144+
simulator: "0"
145+
- fixture: shard-merge
146+
filter: ShardMergeAcceptanceTests
147+
simulator: "0"
148+
- fixture: swift-package-ios
149+
filter: SwiftPackageIOSAcceptanceTests
150+
simulator: "1"
151+
- fixture: xcode-project
152+
filter: XcodeProjectAcceptanceTests
153+
simulator: "1"
154+
- fixture: xcode-workspace
155+
filter: XcodeWorkspaceAcceptanceTests
156+
simulator: "1"
157+
- fixture: xcode-app-debug-dylib
158+
filter: XcodeAppDebugDylibAcceptanceTests
159+
simulator: "1"
160+
- fixture: xcode-unlinked-source
161+
filter: XcodeUnlinkedSourceAcceptanceTests
162+
simulator: "1"
163+
- fixture: cli-commands
164+
filter: CLICommandsAcceptanceTests
165+
simulator: "0"
166+
- fixture: process-supervision
167+
filter: ProcessSupervisionAcceptanceTests
168+
simulator: "0"
169+
# The four below all build the same real .xcodeproj through the
170+
# batching/incremental/coverage-selection paths that a 100-mutant
171+
# benchmark against a real external project found broken while
172+
# every unit test (all fakes, no real xcodebuild invocation or
173+
# .xctestrun) stayed green. They are the whole reason this job
174+
# exists rather than trusting `unit` alone.
175+
- fixture: xcode-batch-testing
176+
filter: XcodeBatchTestingAcceptanceTests
177+
simulator: "1"
178+
- fixture: xcode-batch-testing-ui-target
179+
filter: XcodeBatchTestingUITargetAcceptanceTests
180+
simulator: "1"
181+
- fixture: xcode-coverage-selection
182+
filter: XcodeCoverageSelectionAcceptanceTests
183+
simulator: "1"
184+
- fixture: xcode-incremental-batch-testing
185+
filter: XcodeIncrementalBatchTestingAcceptanceTests
186+
simulator: "1"
187+
steps:
188+
- uses: actions/checkout@v4
189+
- name: Toolchain
190+
run: swift --version && xcodebuild -version
191+
192+
# The suites pick whichever iPhone this machine actually has rather than
193+
# pinning a model, so this is context for a failure, not a gate.
194+
- name: Available simulators
195+
if: matrix.simulator == '1'
196+
run: xcrun simctl list devices available | grep -i iphone || true
197+
198+
- name: Build
199+
shell: bash
200+
run: |
201+
set -o pipefail
202+
swift build --build-tests 2>&1 | tee build.log
203+
204+
- name: Acceptance
205+
shell: bash
206+
env:
207+
MUTANTKIT_ACCEPTANCE: "1"
208+
MUTANTKIT_ACCEPTANCE_SIMULATOR: ${{ matrix.simulator }}
209+
run: |
210+
set -o pipefail
211+
swift test --filter ${{ matrix.filter }} 2>&1 | tee acceptance.log
212+
213+
- name: Upload acceptance failure logs
214+
if: failure()
215+
uses: actions/upload-artifact@v4
216+
with:
217+
name: acceptance-${{ matrix.fixture }}-failure-logs
218+
if-no-files-found: ignore
219+
path: |
220+
build.log
221+
acceptance.log

‎.gitignore‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
# Note what is deliberately NOT here:
2+
#
3+
# Package.resolved tracked on purpose — see "Dependency lock" below
4+
# Fixtures/**/*.xcodeproj tracked on purpose, so acceptance fixtures build
5+
# without xcodegen installed
6+
#
7+
# Rule of thumb for anything added later: ignore what a build or a run produces,
8+
# track what a build or a run needs.
9+
10+
# ── Swift / SwiftPM ───────────────────────────────────────────────────────────
11+
.build/
12+
.swiftpm/
13+
*.o
14+
*.dylib
15+
16+
# ── Dependency lock ───────────────────────────────────────────────────────────
17+
# Package.resolved is intentionally NOT ignored. Do not add it.
18+
#
19+
# This package's product is an executable, so SwiftPM's convention is to commit
20+
# the lock — but there is a sharper reason here. SwiftSyntax decides where a
21+
# node's trivia ends, which decides the UTF-8 byte range a mutation is anchored
22+
# to, which is an input to its Mutation ID. A floating dependency therefore means
23+
# two machines can derive different IDs from identical source, and every promise
24+
# this tool makes about reproducing a plan quietly stops holding. The pinned
25+
# version is recorded in each plan's ToolchainFingerprint; the lock is what makes
26+
# that recording mean something.
27+
28+
# ── Xcode ─────────────────────────────────────────────────────────────────────
29+
DerivedData/
30+
build/
31+
*.xcresult
32+
# Per-user editor state that lives inside an otherwise-tracked .xcodeproj.
33+
# Shared schemes (xcshareddata/) are tracked — the fixtures need them.
34+
xcuserdata/
35+
*.xcuserstate
36+
*.xcscmblueprint
37+
*.moved-aside
38+
39+
# ── Coverage / profiling ──────────────────────────────────────────────────────
40+
*.profraw
41+
*.profdata
42+
43+
# ── MutantKit's own output ────────────────────────────────────────────────────
44+
# Sandboxes, run locks, coverage/result caches, checkpoints, artifacts and
45+
# rendered reports. Both names are ignored: `.mutantkit` is current and
46+
# `.mutare` is the prior tool name kept for any mixed-version transition.
47+
# (RunContextProbe.gitState also strips these from its digest independently,
48+
# so the digest does not rely on a project having these rules — but ignoring
49+
# them here keeps `git status` clean too.)
50+
.mutantkit/
51+
.mutare/
52+
# Artifacts from the default output paths of `plan`, `shard`, `run` and `merge`,
53+
# anchored to the repository root so a plan.json committed as a test fixture is
54+
# not swallowed by the same rule.
55+
/plan.json
56+
/plan.*.json
57+
/report.json
58+
/report.html
59+
/stryker-report.json
60+
/summary.md
61+
/results*.json
62+
63+
# ── Reference clones ──────────────────────────────────────────────────────────
64+
# Independent git repositories cloned for investigation. They are not part of the
65+
# build, are not distributed, and cannot be committed here without vendoring
66+
# someone else's history. Each keeps its own licence — see THIRD_PARTY_NOTICES
67+
# section 4.
68+
#
69+
# To restore them in a fresh clone:
70+
# mkdir -p reference && cd reference
71+
# git clone https://github.com/muter-mutation-testing/muter.git
72+
# git clone https://github.com/ericodx/swift-mutation-testing.git
73+
reference/
74+
75+
# ── Editors / OS ──────────────────────────────────────────────────────────────
76+
.vscode/
77+
.idea/
78+
*.swp
79+
*~
80+
.DS_Store

‎.swiftformat‎

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
# SwiftFormat configuration for MutantKit.
2+
#
3+
# Chosen over Apple's toolchain `swift format`: that tool's default style
4+
# (2-space indent) reformats essentially every indented line in this
5+
# codebase, which already uses 4-space indent consistently. SwiftFormat's
6+
# defaults, by contrast, left all but one of a representative sample of
7+
# large files (MutationRunner.swift, XcodeBuildAdapter.swift,
8+
# CISummaryReporter.swift, ConsoleReporter.swift, HistoryCommand.swift,
9+
# ConfigurationValidation.swift) untouched or near-untouched — the house
10+
# style already matches its defaults far more closely, so adopting it is a
11+
# small, low-risk diff instead of a full-codebase reindent.
12+
#
13+
# The rules disabled below are ones whose *default* behavior fights a
14+
# pattern this codebase uses on purpose, discovered by diffing SwiftFormat's
15+
# output against the existing source rather than guessing:
16+
--swiftversion 6.0
17+
--exclude .build,Fixtures,Research,reference
18+
19+
--disable wrapIfStatementBodies,wrapLoopBodies,wrapFunctionBodies,wrapPropertyBodies,wrapIfExpressionBodies
20+
# Concise one-liners (`if x { return y }`, `func bold(_ t: String) -> String { wrap(t, "1") }`,
21+
# `for x in xs { f(x) }`) are used deliberately for small guard/wrapper
22+
# functions (see Sources/Reporting/ConsoleReporter.swift). The default
23+
# would expand every one of these onto 3+ lines.
24+
25+
--disable unusedArguments
26+
# Rewrites unused named parameters to `_` (e.g. `artifact: BuildArtifact` ->
27+
# `artifact _: BuildArtifact`). Semantically inert, but it's a signature
28+
# edit with no formatting content, and not worth the diff noise on first
29+
# adoption.
30+
31+
--disable preferKeyPath
32+
# Rewrites `.map { $0.foo }` to `.map(\.foo)`. A style preference, not a
33+
# formatting fix; left for the team to opt into deliberately rather than
34+
# have a formatter silently switch idioms.
35+
36+
--disable hoistAwait, hoistTry
37+
# Reorders `try`/`await` to the front of an expression (e.g.
38+
# `((try await x) ?? y)` -> `try await (x ?? y)`). Behavior-preserving but
39+
# a real token-order rewrite, not whitespace; left alone.
40+
41+
--disable redundantSelf
42+
# The codebase intentionally keeps explicit `self.` in some async/Task
43+
# contexts for capture clarity; the default ("remove") would strip it
44+
# everywhere it's not strictly required, which is a much bigger and more
45+
# opinionated diff than this adoption should make in one pass.
46+
47+
--disable wrapMultilineStatementBraces
48+
# Diffing showed this codebase's dominant convention (measured: only 8
49+
# lone-brace lines out of 18.5k) is to keep the opening brace attached to
50+
# the last line of a wrapped condition, e.g.:
51+
# if let x = a,
52+
# let y = b {
53+
# Leaving this rule off lets the plain (non-Allman) `braces` rule do that
54+
# — and it also normalizes the 8 outlier spots that already had the brace
55+
# on its own line, which is a legitimate consistency fix.
56+
57+
--disable conditionalAssignment
58+
# Rewrites `if/else` assignment blocks into `let x = if ... { } else { }`
59+
# expressions. A structural modernization, not a formatting fix.
60+
61+
--disable redundantThrows
62+
# Would strip `throws` from test functions the tool can prove never throw.
63+
# Changes a declared signature; left for the team to do deliberately.
64+
65+
--disable noForceUnwrapInTests
66+
# Would rewrite `foo!` in test files into `try #require(foo)` / `try
67+
# XCTUnwrap(foo)`, which also adds `throws` to the enclosing test function.
68+
# Too structural for an automated first pass.
69+
70+
--disable redundantReturn
71+
# Would remove `return` from switch-statement branches that are a
72+
# function's sole statement (implicit-return-from-switch, SE-0380). The
73+
# codebase doesn't use that idiom anywhere yet, so applying it only in the
74+
# ~8 spots the tool happens to touch would introduce a brand-new pattern
75+
# inconsistently rather than fix one.
76+
77+
--disable swiftTestingTestCaseNames
78+
# Would rename `@Test`/`@Suite` function identifiers themselves (not just
79+
# their display strings). Left alone; a naming choice, not formatting.
80+
81+
--disable docComments
82+
# Would convert `//` rationale comments into `///` doc comments (and vice
83+
# versa) purely because they sit directly above a declaration. This
84+
# codebase deliberately keeps that line: `///` documents the public
85+
# contract, `//` explains internal reasoning, even when both happen to
86+
# precede a declaration (e.g. Sources/MutationModel/MutationResult.swift's
87+
# legacy-decoding notes). Converting on proximity alone would blur that
88+
# distinction.
89+
90+
--trailing-commas never
91+
# Measured: 2101 multi-line collection/argument closings have no trailing
92+
# comma vs. 34 that do. "Never" matches the dominant convention and
93+
# normalizes the 34 outliers.

0 commit comments

Comments
 (0)