Skip to content

Commit 48835ab

Browse files
Copilotfelickz
andauthored
Add Actions, Rust, and Swift CodeQL pack support (#245)
* Add Actions, Rust, and Swift pack support * Include new language packs in workspace Co-authored-by: felickz <1760475+felickz@users.noreply.github.com> --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: felickz <1760475+felickz@users.noreply.github.com>
1 parent 66b0da2 commit 48835ab

28 files changed

Lines changed: 396 additions & 49 deletions

‎.github/scripts/build-codeql-lib-versions-table.sh‎

Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -51,18 +51,21 @@ if [ -z "$CODEQL_VERSION" ]; then
5151
exit 1
5252
fi
5353

54-
LANGUAGES=(cpp csharp go java javascript python ruby)
55-
PACKTYPES=(src lib)
54+
LANGUAGES=(actions cpp csharp go java javascript python ruby rust swift)
55+
LIB_LANGUAGES=(cpp csharp go java javascript python ruby)
5656

5757
lang_label() {
5858
case "$1" in
59+
actions) echo "Actions" ;;
5960
cpp) echo "C++" ;;
6061
csharp) echo "C#" ;;
6162
go) echo "Go" ;;
6263
java) echo "Java" ;;
6364
javascript) echo "JavaScript" ;;
6465
python) echo "Python" ;;
6566
ruby) echo "Ruby" ;;
67+
rust) echo "Rust" ;;
68+
swift) echo "Swift" ;;
6669
*) echo "$1" ;;
6770
esac
6871
}
@@ -127,6 +130,14 @@ upstream_version_and_line() {
127130

128131
MISMATCH=0
129132

133+
is_lib_language() {
134+
local lang="$1"
135+
for supported in "${LIB_LANGUAGES[@]}"; do
136+
[ "$supported" == "$lang" ] && return 0
137+
done
138+
return 1
139+
}
140+
130141
echo "## CodeQL standard library & query pack versions"
131142
echo
132143
echo "Pinned CodeQL CLI/library version ([\`.codeqlversion\`](https://github.com/${REPO}/blob/${REPO_SHA}/.codeqlversion)): [\`v${CODEQL_VERSION}\`](https://github.com/github/codeql-cli-binaries/releases/tag/v${CODEQL_VERSION})"
@@ -139,7 +150,11 @@ echo "| --- | --- | --- | --- | --- | --- |"
139150
for lang in "${LANGUAGES[@]}"; do
140151
label=$(lang_label "$lang")
141152

142-
for packtype in "${PACKTYPES[@]}"; do
153+
for packtype in src lib; do
154+
if [ "$packtype" == "lib" ] && ! is_lib_language "$lang"; then
155+
echo "| $label | \`$packtype\` | - | n/a | - | n/a |"
156+
continue
157+
fi
143158
qlpackfile="${lang}/${packtype}/qlpack.yml"
144159
lockfile="${lang}/${packtype}/codeql-pack.lock.yml"
145160

‎.github/scripts/build-publish-summary.sh‎

Lines changed: 20 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -34,24 +34,30 @@ else
3434
MERGED=$(jq -s '.' "${FRAGMENTS[@]}")
3535
fi
3636

37-
LANGUAGES=(cpp csharp go java javascript python ruby)
37+
LANGUAGES=(actions cpp csharp go java javascript python ruby rust swift)
3838
TYPES=(src lib ext ext-library-sources)
39+
# Languages with a published `lib` pack - the remaining languages currently
40+
# have no shared library use cases and should not produce empty placeholder packs.
41+
LIB_LANGUAGES=(cpp csharp go java javascript python ruby)
3942
# Languages with an `ext` (model/extension) pack - matches the `extensions` job's matrix.
40-
EXT_LANGUAGES=(csharp go java python)
43+
EXT_LANGUAGES=(csharp go java python rust)
4144
# Languages with an `ext-library-sources` pack - matches the `library_sources_extensions`
4245
# job's matrix. This is a strict subset of EXT_LANGUAGES: go/python ship `ext` packs but
4346
# have no `ext-library-sources` pack, so they must not be expected to have a result for it.
4447
EXT_LIBRARY_SOURCES_LANGUAGES=(csharp java)
4548

4649
lang_label() {
4750
case "$1" in
51+
actions) echo "Actions" ;;
4852
cpp) echo "C++" ;;
4953
csharp) echo "C#" ;;
5054
go) echo "Go" ;;
5155
java) echo "Java" ;;
5256
javascript) echo "JavaScript" ;;
5357
python) echo "Python" ;;
5458
ruby) echo "Ruby" ;;
59+
rust) echo "Rust" ;;
60+
swift) echo "Swift" ;;
5561
*) echo "$1" ;;
5662
esac
5763
}
@@ -64,6 +70,14 @@ is_ext_language() {
6470
return 1
6571
}
6672

73+
is_lib_language() {
74+
local lang="$1"
75+
for l in "${LIB_LANGUAGES[@]}"; do
76+
[ "$l" == "$lang" ] && return 0
77+
done
78+
return 1
79+
}
80+
6781
is_ext_library_sources_language() {
6882
local lang="$1"
6983
for l in "${EXT_LIBRARY_SOURCES_LANGUAGES[@]}"; do
@@ -163,6 +177,10 @@ for lang in "${LANGUAGES[@]}"; do
163177
row="$row n/a |"
164178
continue
165179
fi
180+
if [ "$type" == "lib" ] && ! is_lib_language "$lang"; then
181+
row="$row n/a |"
182+
continue
183+
fi
166184
if [ "$type" == "ext-library-sources" ] && ! is_ext_library_sources_language "$lang"; then
167185
row="$row n/a |"
168186
continue

‎.github/scripts/pr-compile.sh‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -10,12 +10,12 @@ run_lenient_full_compile() {
1010
# all of them at once. Failing on that backlog would block CI-infra validation
1111
# (workflow_dispatch, or a PR that only touches .github/**) on unrelated debt.
1212
echo "[+] Compiling all queries in $LANGUAGE"
13-
codeql query compile --threads=0 --check-only "./$LANGUAGE/"
13+
codeql query compile --additional-packs "$LANGUAGE" --threads=0 --check-only "./$LANGUAGE/"
1414
}
1515

1616
run_strict_full_compile() {
1717
echo "[+] Compiling all queries in $LANGUAGE (strict)"
18-
codeql query compile --threads=0 --check-only --warnings=error "./$LANGUAGE/"
18+
codeql query compile --additional-packs "$LANGUAGE" --threads=0 --check-only --warnings=error "./$LANGUAGE/"
1919
}
2020

2121
if [[ -z "$PR_NUMBER" ]]; then
@@ -74,7 +74,7 @@ if [[ "$WORKFLOW_CHANGED" == true ]]; then
7474
continue
7575
fi
7676
echo "[+] Compiling $file (in $LANGUAGE)"
77-
codeql query compile --threads=0 --check-only --warnings=error "./$file"
77+
codeql query compile --additional-packs "$LANGUAGE" --threads=0 --check-only --warnings=error "./$file"
7878
done
7979
echo "[+] Complete"
8080
exit 0
@@ -92,7 +92,7 @@ for file in "${TOUCHED_QUERIES[@]}"; do
9292
continue
9393
fi
9494
echo "[+] Compiling $file (in $LANGUAGE)"
95-
codeql query compile --threads=0 --check-only --warnings=error "./$file"
95+
codeql query compile --additional-packs "$LANGUAGE" --threads=0 --check-only --warnings=error "./$file"
9696
done
9797

9898
echo "[+] Complete"

‎.github/scripts/pr-suites-packs.sh‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ if [[ -z "$PR_NUMBER" ]]; then
1515
if [[ -d "$LANGUAGE/suites" ]]; then
1616
while IFS= read -r -d '' suite; do
1717
echo "[+] Compiling Suite: $suite"
18-
codeql resolve queries "$suite"
18+
codeql resolve queries --additional-packs "$LANGUAGE" "$suite"
1919
done < <(find "$LANGUAGE/suites" -name '*.qls' -print0)
2020
fi
2121

@@ -30,9 +30,9 @@ if [[ -z "$PR_NUMBER" ]]; then
3030
# extension in the pack (a known CodeQL CLI bug, see
3131
# https://github.com/github/codeql/issues/20211). See CONTRIBUTING.md.
3232
if [[ "$pack_dir" != "$LANGUAGE/ext" && "$pack_dir" != "$LANGUAGE/ext-library-sources" ]]; then
33-
codeql pack install "$pack_dir"
33+
codeql pack install --additional-packs "$LANGUAGE" "$pack_dir"
3434
fi
35-
codeql pack create "$pack_dir"
35+
codeql pack create --additional-packs "$LANGUAGE" "$pack_dir"
3636
fi
3737
done
3838

@@ -48,7 +48,7 @@ for file in $(gh pr view "$PR_NUMBER" --json files --jq '.files.[].path'); do
4848
# suite folder
4949
if [[ "$file" == $LANGUAGE/suites/**.qls ]]; then
5050
echo "[+] Compiling Suite: $file"
51-
codeql resolve queries "$file"
51+
codeql resolve queries --additional-packs "$LANGUAGE" "$file"
5252

5353
# qlpack file and lock file
5454
elif [[ "$file" == $LANGUAGE/qlpack.yml ]] || [[ "$file" == $LANGUAGE/codeql-pack.lock.yml ]]; then
@@ -57,9 +57,9 @@ for file in $(gh pr view "$PR_NUMBER" --json files --jq '.files.[].path'); do
5757
fi
5858
echo "[+] Compiling Pack: $LANGUAGE"
5959
# install deps
60-
codeql pack install "$LANGUAGE"
60+
codeql pack install --additional-packs "$LANGUAGE" "$LANGUAGE"
6161
# compile / create pack
62-
codeql pack create "$LANGUAGE"
62+
codeql pack create --additional-packs "$LANGUAGE" "$LANGUAGE"
6363

6464
# if the version of the pack is changed, comment in the PR
6565
PUBLISHED_VERSION=$(gh api /orgs/githubsecuritylab/packages/container/codeql-"$LANGUAGE"-queries/versions --jq '.[0].metadata.container.tags[0]')
@@ -75,7 +75,7 @@ for file in $(gh pr view "$PR_NUMBER" --json files --jq '.files.[].path'); do
7575
fi
7676
fi
7777

78-
# Same for the libs pack
78+
# Same for the libs pack, when this language has one
7979
PUBLISHED_VERSION=$(gh api /orgs/githubsecuritylab/packages/container/codeql-"$LANGUAGE"-libs/versions --jq '.[0].metadata.container.tags[0]')
8080
CURRENT_VERSION=$(grep version "$LANGUAGE"/lib/qlpack.yml | awk '{print $2}')
8181

‎.github/workflows/ci.yml‎

Lines changed: 11 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ jobs:
2222
strategy:
2323
fail-fast: false
2424
matrix:
25-
language: [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
25+
language: [ 'actions', 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby', 'rust', 'swift' ]
2626

2727
steps:
2828
- uses: actions/checkout@v7
@@ -50,9 +50,13 @@ jobs:
5050
CODEQL_CLI_VERSION: ${{ steps.install-codeql.outputs.codeql-cli-version }}
5151
run: |
5252
gh repo clone github/codeql -- -b codeql-cli-${CODEQL_CLI_VERSION} # to make stubs available for tests
53-
codeql pack install "${{ matrix.language }}/lib"
54-
codeql pack install "${{ matrix.language }}/src"
55-
codeql pack install "${{ matrix.language }}/test"
53+
# Resolve this language's local query/library/extension packs before GHCR so
54+
# tests can use packs that have not been published yet.
55+
if [[ -f "${{ matrix.language }}/lib/qlpack.yml" ]]; then
56+
codeql pack install --additional-packs "$GITHUB_WORKSPACE/${{ matrix.language }}" "${{ matrix.language }}/lib"
57+
fi
58+
codeql pack install --additional-packs "$GITHUB_WORKSPACE/${{ matrix.language }}" "${{ matrix.language }}/src"
59+
codeql pack install --additional-packs "$GITHUB_WORKSPACE/${{ matrix.language }}" "${{ matrix.language }}/test"
5660
5761
- name: Compile Queries
5862
if: steps.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch'
@@ -168,7 +172,7 @@ jobs:
168172
# JVMs start at once, and their /tmp/hsperfdata_<user>/<pid> files can collide
169173
# ("Cannot use file ... locked by another process"), corrupting the report.
170174
# -XX:-UsePerfData stops the JVM from creating those files at all.
171-
procs.append((subprocess.Popen(["codeql", "test", "run", "-J=-XX:-UsePerfData", "--failing-exitcode=122", "--verbosity=progress", "--ram=2048", "--format=json", *extra_args], stdout=test_report_file, stderr=test_log_file), test_log_path))
175+
procs.append((subprocess.Popen(["codeql", "test", "run", "--additional-packs", language, "-J=-XX:-UsePerfData", "--failing-exitcode=122", "--verbosity=progress", "--ram=2048", "--format=json", *extra_args], stdout=test_report_file, stderr=test_log_file), test_log_path))
172176
173177
for p, test_log_path in procs:
174178
p.wait()
@@ -262,7 +266,7 @@ jobs:
262266
strategy:
263267
fail-fast: false
264268
matrix:
265-
language: [ 'csharp', 'go', 'java', 'python' ]
269+
language: [ 'csharp', 'go', 'java', 'python', 'rust' ]
266270

267271
steps:
268272
- uses: actions/checkout@v7
@@ -355,7 +359,7 @@ jobs:
355359
env:
356360
GITHUB_TOKEN: ${{ github.token }}
357361
run: |
358-
for lang in cpp csharp go java javascript python ruby; do
362+
for lang in actions cpp csharp go java javascript python ruby rust swift; do
359363
if [[ -d "${lang}/src" ]]; then
360364
codeql pack install "${lang}/src"
361365
fi

‎.github/workflows/copilot-setup-steps.yml‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -50,17 +50,18 @@ jobs:
5050
# hole before running `codeql pack install` and immediately finding/
5151
# fixing the real one-line bug.
5252
#
53-
# We install every pack (not just the language the agent will end up
53+
# We install every dependency-bearing pack (not just the language the agent will end up
5454
# touching) because we don't know ahead of time which one it'll be
5555
# asked to fix - the extra cost is a few seconds per pack, which is
5656
# trivial next to the minutes it can save. Same exclude list as
5757
# update-codeql-version.yml's `codeql pack upgrade` loop: skip
5858
# ql/hotspots (standalone tool, not a real GHCR-resolvable pack),
5959
# codeql_home (vendored CLI packs), and codeql/ + */.codeql (the test-
60-
# stubs clone below and CodeQL's own per-pack build caches).
61-
for dir in $(find . -name qlpack.yml -not -path "./ql/hotspots/*" -not -path "./codeql_home/*" -not -path "./codeql/*" -not -path "*/.codeql/*" -exec dirname {} \;); do
60+
# stubs clone below and CodeQL's own per-pack build caches). Model/extension
61+
# packs are also skipped because they must not be installed or have lockfiles.
62+
for dir in $(find . -name qlpack.yml -not -path "./ql/hotspots/*" -not -path "./codeql_home/*" -not -path "./codeql/*" -not -path "*/.codeql/*" -not -path "*/ext/*" -not -path "*/ext-library-sources/*" -exec dirname {} \;); do
6263
echo "::group::codeql pack install $dir"
63-
codeql pack install "$dir"
64+
codeql pack install --additional-packs "$(dirname "$dir")" "$dir"
6465
echo "::endgroup::"
6566
done
6667

‎.github/workflows/publish.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ jobs:
2424
strategy:
2525
fail-fast: false
2626
matrix:
27-
language: ["cpp", "csharp", "go", "java", "javascript", "python", "ruby"]
27+
language: ["actions", "cpp", "csharp", "go", "java", "javascript", "python", "ruby", "rust", "swift"]
2828

2929
steps:
3030
- uses: actions/checkout@v7
@@ -211,7 +211,7 @@ jobs:
211211
strategy:
212212
fail-fast: false
213213
matrix:
214-
language: ["csharp", "go", "java", "python"]
214+
language: ["csharp", "go", "java", "python", "rust"]
215215

216216
steps:
217217
- uses: actions/checkout@v7

‎.github/workflows/update-codeql-version.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -130,7 +130,7 @@ jobs:
130130
# GitHubSecurityLab/CodeQL-Community-Packs#206.
131131
for dir in $(find . -name qlpack.yml -not -path "./ql/hotspots/*" -not -path "./codeql_home/*" -not -path "./codeql/*" -not -path "*/.codeql/*" -not -path "*/ext/*" -not -path "*/ext-library-sources/*" -exec dirname {} \;); do
132132
echo "::group::codeql pack upgrade $dir"
133-
codeql pack upgrade "$dir"
133+
codeql pack upgrade --additional-packs "$(dirname "$dir")" "$dir"
134134
echo "::endgroup::"
135135
done
136136

‎CONTRIBUTING.md‎

Lines changed: 23 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -6,20 +6,22 @@ There is lots of useful documentation to help you write queries, ranging from in
66

77
## Submitting a new query
88

9-
If you have an idea for a query that you would like to share with other CodeQL users, please open a pull request to add it to this repository. New queries start out in a `<language>/ql/src/` directory, to which they can be merged when they meet the following requirements.
9+
If you have an idea for a query that you would like to share with other CodeQL users, please open a pull request to add it to this repository. New queries start out in a `<language>/src/` directory, to which they can be merged when they meet the following requirements.
1010

1111
1. **Directory structure**
1212

13-
There are eight language-specific query directories in this repository:
13+
Query packs are available for the following CodeQL languages:
1414

15-
* C/C++: `cpp/ql/src`
16-
* C#: `csharp/ql/src`
17-
* Go: `go/ql/src`
18-
* Java/Kotlin: `java/ql/src`
19-
* JavaScript: `javascript/ql/src`
20-
* Python: `python/ql/src`
21-
* Ruby: `ruby/ql/src`
22-
* Swift: `swift/ql/src`
15+
* GitHub Actions: `actions/src`
16+
* C/C++: `cpp/src`
17+
* C#: `csharp/src`
18+
* Go: `go/src`
19+
* Java/Kotlin: `java/src`
20+
* JavaScript/TypeScript: `javascript/src`
21+
* Python: `python/src`
22+
* Ruby: `ruby/src`
23+
* Rust: `rust/src`
24+
* Swift: `swift/src`
2325

2426
Each language-specific directory contains further subdirectories that group queries based on their `@tags` or purpose.
2527

@@ -71,7 +73,7 @@ instead of, or alongside, a query change.
7173
1. **Pick the right pack: `ext` vs `ext-library-sources`**
7274

7375
Every language with data-extension support has a `<language>/ext` pack (`csharp`/`go`/`java`/
74-
`python`). Two of those languages additionally have a second, narrower pack,
76+
`python`/`rust`). Two of those languages additionally have a second, narrower pack,
7577
`<language>/ext-library-sources` (`csharp`/`java` only). They hold different *kinds* of models,
7678
and picking the right one matters:
7779

@@ -82,7 +84,8 @@ instead of, or alongside, a query change.
8284
well-known framework API (e.g. a web framework's `getParameter()`-style methods). This is
8385
where our own [Spring R2DBC `DatabaseClient` SQL injection sink models][pr-204] live
8486
(`java/ext/manual/org.springframework.r2dbc.model.yml`) — use it as a concrete reference for
85-
both the file layout and the row shape.
87+
both the file layout and the row shape. For example, Rust models for Diesel SQL sinks belong
88+
in `rust/ext/manual/diesel.model.yml`.
8689
- **`<language>/ext-library-sources`** (published as
8790
`githubsecuritylab/codeql-<language>-library-sources`) exists only for `csharp`/`java` and
8891
holds a narrower category: `sourceModel` rows (almost always `kind: "remote"`) that flag a
@@ -96,7 +99,8 @@ instead of, or alongside, a query change.
9699

97100
**Rule of thumb: if you're modeling a dangerous sink, or how taint flows through a library
98101
method, use `ext`. Only reach for `ext-library-sources` if you're contributing a genuinely new
99-
remote-data-entry-point (source) model, and only for csharp/java.**
102+
remote-data-entry-point (source) model, and only for csharp/java. Do not add an empty
103+
`ext-library-sources` pack for other languages.**
100104

101105
2. **Row shape**
102106

@@ -135,8 +139,9 @@ instead of, or alongside, a query change.
135139
Add a test under `<language>/test/security/<CWE-id>/<your-library>/`: minimal source stubs that
136140
reproduce the vulnerable call shape, a `.ql` query that imports the *real* standard query (e.g.
137141
`SqlInjectionQuery`/`QueryInjectionFlow`, not a reimplementation of it), and a `.expected` file.
138-
Check that `<language>/test/qlpack.yml` depends on `githubsecuritylab/codeql-<language>-extensions`
139-
(and `-library-sources` where applicable) — that dependency is what makes `codeql test run`'s
142+
If the language has an `ext` pack, check that `<language>/test/qlpack.yml` depends on
143+
`githubsecuritylab/codeql-<language>-extensions` (and `-library-sources` where applicable) —
144+
that dependency is what makes `codeql test run`'s
140145
normal dependency resolution load your data extensions automatically, unlike ad-hoc `codeql
141146
database analyze`/`query run`, which need an explicit `--model-packs` flag. Add the dependency if
142147
it's missing. Confirm the test actually exercises your model (not just that it compiles) by
@@ -308,7 +313,7 @@ process for each; most of it is manual today.
308313
[`publish.yml`][publish-workflow] is organized as five jobs: four publish jobs, one per pack type
309314
(`queries` for `src`, `library` for `lib`, `extensions` for `ext`, `library_sources_extensions` for
310315
`ext-library-sources`), each matrixed over every language that has that pack type
311-
(`ext` runs for `csharp`/`go`/`java`/`python`; `ext-library-sources` only runs for `csharp`/`java`, the only languages with that pack) - plus a fifth
316+
(`src` includes Actions, Rust, and Swift; `lib` is only created where there is shared library code; `ext` runs for `csharp`/`go`/`java`/`python`/`rust`; `ext-library-sources` only runs for `csharp`/`java`, the only languages with that pack) - plus a fifth
312317
`summary` job that runs after the other four (`if: always()`, so it still runs even if one of them
313318
fails), aggregates their per-pack results into the publish-summary and CodeQL library/query pack
314319
version tables, and (on the release-cut `push` trigger only) upserts both tables into the GitHub
@@ -344,7 +349,8 @@ merge. Bumping your pack's own `version:` in a regular PR just *stages* the chan
344349
To ship a change:
345350

346351
- [ ] Make your change in the pack directory you intend to publish: `<language>/src` (queries),
347-
`<language>/lib` (library), `<language>/ext` (extensions, `csharp`/`go`/`java`/`python` only),
352+
`<language>/lib` (shared library code, only where needed), `<language>/ext` (extensions,
353+
`csharp`/`go`/`java`/`python`/`rust`),
348354
or `<language>/ext-library-sources` (extensions, `csharp`/`java` only). See [Adding a data
349355
extension](#adding-a-data-extension) above if you're not sure which extensions pack to use.
350356
- [ ] Bump `version:` in that pack's `qlpack.yml`, following [semver](https://semver.org/). Only bump

0 commit comments

Comments
 (0)