Skip to content

Commit 65fc0bb

Browse files
DeusDataDeusData
authored
Merge branch 'main' into fix/510-enclosing-gitignore-not-inherited
2 parents 282fbfc + da3ea36 commit 65fc0bb

252 files changed

Lines changed: 32824 additions & 3410 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/_memwaste.yml‎

Lines changed: 183 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,183 @@
1+
# Reusable: the waste sanitizer (memory + CPU waste), REPORT-ONLY.
2+
#
3+
# What runs (scripts/memwaste.sh, memwaste-scaling.py, memwaste-ratchet.py):
4+
# mode=pr Linux: the event lane (memory waste + CPU work) indexing this
5+
# repository at the checked-out commit, and the CPU scaling lane
6+
# (src/ replicated 2x against 4x, per-function counter ratios).
7+
# mode=full the same, plus the event lane on macOS and Windows and the access
8+
# lane (read/write verdicts per allocation) on Linux.
9+
#
10+
# The corpus is the repository itself: pinned by the commit, no network, and a
11+
# real mix of C, C++, Python, JavaScript and vendored code.
12+
#
13+
# REPORT-ONLY: every job uploads its report and prints the ratchet comparison
14+
# against scripts/memwaste-baseline.json, but nothing here can turn a PR red --
15+
# the ratchet runs without --enforce and the PR's ci-ok gate does not depend on
16+
# these jobs. A job fails only when the MEASUREMENT is unsound (a table
17+
# overflowed, frees the layer never saw): that is a defect of the tool, and a
18+
# red job is how it gets noticed. Switching the ratchet to --enforce is a
19+
# separate, deliberate change once the numbers have been stable across reruns.
20+
name: Memwaste
21+
22+
on:
23+
workflow_call:
24+
inputs:
25+
mode:
26+
description: "pr (Linux event + scaling) or full (all platforms, access lane)"
27+
type: string
28+
default: pr
29+
30+
permissions:
31+
contents: read
32+
33+
jobs:
34+
event-linux:
35+
runs-on: ubuntu-latest
36+
timeout-minutes: 60
37+
steps:
38+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
39+
- name: Install deps
40+
run: sudo apt-get update && sudo apt-get install -y zlib1g-dev python3 clang llvm
41+
- name: Event lane on this repository
42+
run: |
43+
# `cmd | tee` reports TEE's status: without this a failing lane
44+
# (a missing exec bit, a build error) passes the step and only
45+
# surfaces as the ratchet not finding its JSON.
46+
set -o pipefail
47+
mkdir -p "$RUNNER_TEMP/memwaste-event"
48+
scripts/memwaste.sh "$GITHUB_WORKSPACE" --top 40 --out "$RUNNER_TEMP/memwaste-event" \
49+
| tee "$RUNNER_TEMP/memwaste-event/report.txt"
50+
- name: Ratchet (report-only)
51+
run: python3 scripts/memwaste-ratchet.py --out "$RUNNER_TEMP/memwaste-event" --venue ${{ inputs.mode }}-linux
52+
- name: Upload report
53+
if: ${{ always() }}
54+
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
55+
with:
56+
name: memwaste-event-linux-${{ inputs.mode }}
57+
path: |
58+
${{ runner.temp }}/memwaste-event/report.txt
59+
${{ runner.temp }}/memwaste-event/memwaste.json
60+
${{ runner.temp }}/memwaste-event/index.json
61+
if-no-files-found: warn
62+
63+
scaling-linux:
64+
runs-on: ubuntu-latest
65+
timeout-minutes: 60
66+
steps:
67+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
68+
- name: Install deps
69+
run: sudo apt-get update && sudo apt-get install -y zlib1g-dev python3 clang llvm
70+
- name: CPU scaling lane (src/ at 2x against 4x)
71+
# Report-only: the script exits 1 when it flags a super-linear function.
72+
continue-on-error: true
73+
run: |
74+
# `cmd | tee` reports TEE's status: without this a failing lane
75+
# (a missing exec bit, a build error) passes the step and only
76+
# surfaces as the ratchet not finding its JSON.
77+
set -o pipefail
78+
mkdir -p "$RUNNER_TEMP/memwaste-scaling"
79+
python3 scripts/memwaste-scaling.py --corpus src --replicas 2 \
80+
--out "$RUNNER_TEMP/memwaste-scaling" | tee "$RUNNER_TEMP/memwaste-scaling/report.txt"
81+
- name: Upload report
82+
if: ${{ always() }}
83+
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
84+
with:
85+
name: memwaste-scaling-linux-${{ inputs.mode }}
86+
path: ${{ runner.temp }}/memwaste-scaling/report.txt
87+
if-no-files-found: warn
88+
89+
access-linux:
90+
if: ${{ inputs.mode == 'full' }}
91+
runs-on: ubuntu-latest
92+
timeout-minutes: 120
93+
steps:
94+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
95+
- name: Install deps
96+
run: sudo apt-get update && sudo apt-get install -y zlib1g-dev python3 clang llvm
97+
- name: Access lane on this repository
98+
run: |
99+
# `cmd | tee` reports TEE's status: without this a failing lane
100+
# (a missing exec bit, a build error) passes the step and only
101+
# surfaces as the ratchet not finding its JSON.
102+
set -o pipefail
103+
mkdir -p "$RUNNER_TEMP/memwaste-access"
104+
scripts/memwaste.sh "$GITHUB_WORKSPACE" --lane access --top 40 --sort untouched \
105+
--out "$RUNNER_TEMP/memwaste-access" | tee "$RUNNER_TEMP/memwaste-access/report.txt"
106+
- name: Upload report
107+
if: ${{ always() }}
108+
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
109+
with:
110+
name: memwaste-access-linux-${{ inputs.mode }}
111+
path: |
112+
${{ runner.temp }}/memwaste-access/report.txt
113+
${{ runner.temp }}/memwaste-access/memwaste.json
114+
if-no-files-found: warn
115+
116+
event-macos:
117+
if: ${{ inputs.mode == 'full' }}
118+
runs-on: macos-14
119+
timeout-minutes: 60
120+
steps:
121+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
122+
- name: Event lane on this repository
123+
run: |
124+
# `cmd | tee` reports TEE's status: without this a failing lane
125+
# (a missing exec bit, a build error) passes the step and only
126+
# surfaces as the ratchet not finding its JSON.
127+
set -o pipefail
128+
mkdir -p "$RUNNER_TEMP/memwaste-event"
129+
scripts/memwaste.sh "$GITHUB_WORKSPACE" --top 40 --out "$RUNNER_TEMP/memwaste-event" \
130+
| tee "$RUNNER_TEMP/memwaste-event/report.txt"
131+
- name: Ratchet (report-only)
132+
run: python3 scripts/memwaste-ratchet.py --out "$RUNNER_TEMP/memwaste-event" --venue full-macos
133+
- name: Upload report
134+
if: ${{ always() }}
135+
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
136+
with:
137+
name: memwaste-event-macos-${{ inputs.mode }}
138+
path: |
139+
${{ runner.temp }}/memwaste-event/report.txt
140+
${{ runner.temp }}/memwaste-event/memwaste.json
141+
if-no-files-found: warn
142+
143+
event-windows:
144+
if: ${{ inputs.mode == 'full' }}
145+
runs-on: windows-latest
146+
timeout-minutes: 90
147+
steps:
148+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
149+
- uses: msys2/setup-msys2@66cd2cce69caa17b53920067426061ca1de3a884 # v2
150+
with:
151+
msystem: CLANG64
152+
path-type: inherit
153+
install: >-
154+
mingw-w64-clang-x86_64-clang
155+
mingw-w64-clang-x86_64-llvm
156+
mingw-w64-clang-x86_64-zlib
157+
mingw-w64-clang-x86_64-python3
158+
make
159+
git
160+
coreutils
161+
- name: Event lane on this repository
162+
shell: msys2 {0}
163+
run: |
164+
# `cmd | tee` reports TEE's status: without this a failing lane
165+
# (a missing exec bit, a build error) passes the step and only
166+
# surfaces as the ratchet not finding its JSON.
167+
set -o pipefail
168+
out="$(cygpath -u "$RUNNER_TEMP")/memwaste-event"
169+
mkdir -p "$out"
170+
CC=clang CXX=clang++ scripts/memwaste.sh "$(cygpath -u "$GITHUB_WORKSPACE")" --top 40 --out "$out" \
171+
| tee "$out/report.txt"
172+
- name: Ratchet (report-only)
173+
shell: msys2 {0}
174+
run: python3 scripts/memwaste-ratchet.py --out "$(cygpath -u "$RUNNER_TEMP")/memwaste-event" --venue full-windows
175+
- name: Upload report
176+
if: ${{ always() }}
177+
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
178+
with:
179+
name: memwaste-event-windows-${{ inputs.mode }}
180+
path: |
181+
${{ runner.temp }}/memwaste-event/report.txt
182+
${{ runner.temp }}/memwaste-event/memwaste.json
183+
if-no-files-found: warn

‎.github/workflows/_security.yml‎

Lines changed: 42 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,14 @@ jobs:
4141
codeql-gate:
4242
runs-on: ubuntu-latest
4343
timeout-minutes: 240
44+
# Without security-events:read this job inherits the workflow's
45+
# contents:read, the code-scanning alert API answers 403, and the gate
46+
# below reported "0 alerts" on every run it has ever made. actions:read is
47+
# what the CodeQL-run poll above needs for the same reason.
48+
permissions:
49+
contents: read
50+
security-events: read
51+
actions: read
4452
steps:
4553
- name: Wait for CodeQL on current commit (max 150 min)
4654
env:
@@ -77,11 +85,43 @@ jobs:
7785
env:
7886
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
7987
run: |
88+
# FAIL CLOSED. The previous form was `gh api ... 2>/dev/null || echo "0"`,
89+
# which looks defensive and is the opposite. gh writes the API error
90+
# BODY to stdout, so a 403 made ALERTS the string
91+
# {"message":"Resource not accessible by integration",...,"status":"403"}0
92+
# every `[` comparison then failed as non-integer, the `if` took its
93+
# false branch, and the step printed "CodeQL gate passed (0 alerts)"
94+
# and exited 0. That is exactly how a security gate should NEVER fail:
95+
# silently, in the direction of "everything is fine". An unreadable
96+
# alert list is now an error, not a zero.
97+
# NOTE: this runs in a command substitution, so it must `return`, not
98+
# `exit` -- an `exit` here would only end the subshell and let the
99+
# caller continue with an empty count, recreating the very bug this
100+
# replaces. The callers below check the status explicitly.
101+
count_open_alerts() {
102+
local body status
103+
body=$(gh api 'repos/${{ github.repository }}/code-scanning/alerts?state=open' \
104+
--jq 'length' 2>/dev/null)
105+
status=$?
106+
if [ $status -ne 0 ]; then
107+
echo "BLOCKED: cannot read code scanning alerts (gh exit $status)." >&2
108+
echo " Refusing to treat an unreadable alert list as zero." >&2
109+
echo " Most likely cause: the job lacks security-events:read." >&2
110+
return 1
111+
fi
112+
case "$body" in
113+
''|*[!0-9]*)
114+
echo "BLOCKED: alert count was not a number: ${body:-<empty>}" >&2
115+
return 1
116+
;;
117+
esac
118+
printf '%s' "$body"
119+
}
80120
echo "Waiting 60s for alert API to settle..."
81121
sleep 60
82-
ALERTS=$(gh api 'repos/${{ github.repository }}/code-scanning/alerts?state=open' --jq 'length' 2>/dev/null || echo "0")
122+
ALERTS=$(count_open_alerts) || exit 1
83123
sleep 15
84-
ALERTS2=$(gh api 'repos/${{ github.repository }}/code-scanning/alerts?state=open' --jq 'length' 2>/dev/null || echo "0")
124+
ALERTS2=$(count_open_alerts) || exit 1
85125
[ "$ALERTS" -lt "$ALERTS2" ] && ALERTS=$ALERTS2
86126
if [ "$ALERTS" -gt 0 ]; then
87127
echo "BLOCKED: $ALERTS open alert(s)"

‎.github/workflows/_test.yml‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -107,6 +107,32 @@ jobs:
107107
if: startsWith(matrix.os, 'ubuntu')
108108
run: sudo apt-get update && sudo apt-get install -y zlib1g-dev ccache
109109

110+
# Ubuntu 23.10+ ships kernel.apparmor_restrict_unprivileged_userns=1, which
111+
# blocks unshare(CLONE_NEWUSER) for unprivileged users. Without this, the
112+
# #1830 single-uid-userns smoke test SKIPS on every 24.04 leg, and its only
113+
# real-namespace coverage lived on the two ubuntu-22.04 legs -- which are
114+
# broad-matrix only (never PR CI) and which GitHub retires on 2027-04-17.
115+
# A deliberate relaxation of a security check was therefore guarded by a
116+
# test that ran in no PR and was on a clock. Hosted runners have
117+
# passwordless sudo, so one sysctl puts it on the CORE matrix instead.
118+
# Best-effort: a runner that refuses simply returns to skipping.
119+
- name: Permit unprivileged user namespaces (Ubuntu)
120+
if: startsWith(matrix.os, 'ubuntu')
121+
run: |
122+
sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 || \
123+
echo "could not lift the userns restriction; namespace tests will skip"
124+
# Then SAY whether it worked. The runner prints per-shard aggregates
125+
# ("N passed, M skipped") and never individual test names, so a
126+
# namespace test that silently returns to skipping is invisible in a
127+
# green log -- the same false assurance that let #1830 ship with a
128+
# test that never executed. This line is the evidence. Probed exactly
129+
# as the test does it: unprivileged, no sudo.
130+
if unshare -U -r true 2>/dev/null; then
131+
echo "CBM_USERNS_AVAILABLE=yes"
132+
else
133+
echo "CBM_USERNS_AVAILABLE=no (namespace-dependent tests will skip)"
134+
fi
135+
110136
- name: Install ccache (macOS)
111137
if: startsWith(matrix.os, 'macos')
112138
run: command -v ccache >/dev/null 2>&1 || brew install ccache

‎.github/workflows/codeql.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ jobs:
2929
run: sudo apt-get update && sudo apt-get install -y zlib1g-dev
3030

3131
- name: Initialize CodeQL
32-
uses: github/codeql-action/init@cdf488f595d80d6e07e03d4674febd5ab45fa938 # v4.37.9
32+
uses: github/codeql-action/init@b96794f015dfd88f77b49b1c93e0fa7110f94c63 # v4.38.0
3333
with:
3434
languages: c-cpp
3535
build-mode: manual
@@ -38,6 +38,6 @@ jobs:
3838
run: scripts/build.sh
3939

4040
- name: Perform CodeQL Analysis
41-
uses: github/codeql-action/analyze@cdf488f595d80d6e07e03d4674febd5ab45fa938 # v4.37.9
41+
uses: github/codeql-action/analyze@b96794f015dfd88f77b49b1c93e0fa7110f94c63 # v4.38.0
4242
with:
4343
category: "/language:c-cpp"

‎.github/workflows/dry-run.yml‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,10 @@ on:
4545
description: 'Skip candidate scan; mark unscanned and default to stripped'
4646
type: boolean
4747
default: false
48+
skip_memwaste:
49+
description: 'Skip the waste sanitizer (report-only) and the fuzz smoke'
50+
type: boolean
51+
default: false
4852
soak_level:
4953
description: 'Soak: full (quick+asan), quick (10min), none'
5054
type: choice
@@ -58,6 +62,13 @@ jobs:
5862
# ── Security (independent island — does not block main pipeline) ──
5963
security:
6064
uses: ./.github/workflows/_security.yml
65+
# The called job requests security-events:read to count code-scanning
66+
# alerts; a reusable workflow cannot request more than its caller grants,
67+
# so withholding this here fails the whole run at startup.
68+
permissions:
69+
contents: read
70+
security-events: read
71+
actions: read
6172
secrets: inherit
6273

6374
# ── Lint (cppcheck + clang-format) ────────────────────────────
@@ -100,6 +111,38 @@ jobs:
100111
with:
101112
broad_platforms: true
102113

114+
# ── Waste sanitizer, REPORT-ONLY (independent island) ─────────
115+
# Event lane on Linux, macOS and Windows, access lane and the per-function
116+
# scaling lane on Linux. Uploads reports; fails only when the measurement is
117+
# unsound (a defect of the tool itself).
118+
memwaste:
119+
if: ${{ inputs.skip_memwaste != true }}
120+
uses: ./.github/workflows/_memwaste.yml
121+
with:
122+
mode: full
123+
124+
# ── Fuzz smoke (independent island) ───────────────────────────
125+
# A fixed number of executions from a fixed seed per target, so the verdict is
126+
# a function of the commit (O9). A crash here is a real defect in a parser
127+
# that reads untrusted input; its reproducer is uploaded.
128+
fuzz:
129+
if: ${{ inputs.skip_memwaste != true }}
130+
runs-on: ubuntu-latest
131+
timeout-minutes: 90
132+
steps:
133+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
134+
- name: Install deps
135+
run: sudo apt-get update && sudo apt-get install -y zlib1g-dev clang llvm libclang-rt-dev
136+
- name: Fuzz smoke (extract, cypher, config)
137+
run: scripts/fuzz.sh all --runs 20000 --seed 1
138+
- name: Upload reproducers
139+
if: ${{ failure() }}
140+
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
141+
with:
142+
name: fuzz-artifacts
143+
path: build/fuzz/artifacts-*/
144+
if-no-files-found: ignore
145+
103146
# ── Soak tests (optional, parallel with smoke) ────────────────
104147
soak:
105148
if: ${{ inputs.soak_level != 'none' && !cancelled() && needs.build.result != 'failure' && needs.build.result != 'skipped' }}

‎.github/workflows/nightly-soak.yml‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,27 @@ permissions:
3535
contents: read
3636

3737
jobs:
38+
# Fuzz exploration: new inputs are the point here, so it runs for a time
39+
# budget instead of a fixed count. NON-GATING like the soak; a crash uploads
40+
# its reproducer, which becomes a fixed-seed regression input once fixed.
41+
fuzz:
42+
if: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }}
43+
runs-on: ubuntu-latest
44+
timeout-minutes: 120
45+
steps:
46+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
47+
- name: Install deps
48+
run: sudo apt-get update && sudo apt-get install -y zlib1g-dev clang llvm libclang-rt-dev
49+
- name: Fuzz exploration (30 min per target)
50+
run: scripts/fuzz.sh all --max-total-time 1800 --seed 0
51+
- name: Upload reproducers
52+
if: ${{ failure() }}
53+
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
54+
with:
55+
name: fuzz-artifacts-nightly
56+
path: build/fuzz/artifacts-*/
57+
if-no-files-found: ignore
58+
3859
soak:
3960
uses: ./.github/workflows/_soak.yml
4061
with:

0 commit comments

Comments
 (0)