Skip to content

Commit b877f27

Browse files
henrymercerCopilot
andcommitted
Never add a per-language bundle to the toolcache
A bundle that contains a single language can also be requested directly via the `tools` input, in which case we did not choose it but must still keep it out of the toolcache, since a later job analyzing a different language could otherwise pick up an installation that is missing the extractor it needs. Recognise such bundles by their name. When one was requested explicitly, a missing bundle is an error rather than a reason to fall back, since substituting a different bundle would ignore what was asked for. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: d4c7b873-8278-4a36-b67c-6cc5ea3b2316
1 parent a113a2b commit b877f27

7 files changed

Lines changed: 222 additions & 6 deletions

File tree

‎.github/workflows/__per-language-bundle.yml‎

Lines changed: 16 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎lib/entry-points.js‎

Lines changed: 19 additions & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎pr-checks/checks/per-language-bundle.yml‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,22 @@ steps:
3333
exit 1
3434
fi
3535
done
36+
- name: Check that the bundle was not added to the toolcache
37+
env:
38+
CODEQL_PATH: ${{ steps.init.outputs.codeql-path }}
39+
run: |
40+
# A bundle that is missing most of its extractors must never be left in the toolcache, where
41+
# a later job analyzing a different language could pick it up.
42+
echo "CodeQL is at $CODEQL_PATH"
43+
if [[ "$CODEQL_PATH" == "$RUNNER_TOOL_CACHE"/* ]]; then
44+
echo "::error::The per-language bundle was added to the toolcache at $CODEQL_PATH."
45+
exit 1
46+
fi
47+
if [ -d "$RUNNER_TOOL_CACHE/CodeQL" ]; then
48+
echo "::error::The toolcache contains CodeQL, but no bundle should have been added to it."
49+
ls -R "$RUNNER_TOOL_CACHE/CodeQL"
50+
exit 1
51+
fi
3652
- uses: ./../action/analyze
3753
with:
3854
upload-database: false

‎src/per-language-bundles.test.ts‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import {
88
getPerLanguageBundleLanguage,
99
MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION,
1010
PerLanguageBundleOptions,
11+
tryGetBundleLanguageFromUrl,
1112
} from "./per-language-bundles";
1213
import { createFeatures, setupTests } from "./testing-utils";
1314
import { GitHubVariant } from "./util";
@@ -141,6 +142,53 @@ test.serial("requires the feature flag", async (t) => {
141142
t.is(await checkEligibility({}, []), undefined);
142143
});
143144

145+
test.serial("recognizes a per-language bundle from its URL", (t) => {
146+
const url = (name: string) =>
147+
`https://github.com/github/codeql-action/releases/download/codeql-bundle-v1.2.3/${name}`;
148+
149+
t.is(
150+
tryGetBundleLanguageFromUrl(url("codeql-bundle-java-linux64.tar.zst")),
151+
BuiltInLanguage.java,
152+
);
153+
t.is(
154+
tryGetBundleLanguageFromUrl(url("codeql-bundle-swift-osx64.tar.zst")),
155+
BuiltInLanguage.swift,
156+
);
157+
// We do not publish these, but should still recognize them if we ever do.
158+
t.is(
159+
tryGetBundleLanguageFromUrl(url("codeql-bundle-csharp-win64.tar.gz")),
160+
BuiltInLanguage.csharp,
161+
);
162+
// A token in the URL must not prevent us from recognizing the bundle.
163+
t.is(
164+
tryGetBundleLanguageFromUrl(
165+
`${url("codeql-bundle-ruby-linux64.tar.zst")}?token=secret`,
166+
),
167+
BuiltInLanguage.ruby,
168+
);
169+
});
170+
171+
test.serial("does not mistake other bundles for per-language ones", (t) => {
172+
const url = (name: string) =>
173+
`https://github.com/github/codeql-action/releases/download/codeql-bundle-v1.2.3/${name}`;
174+
175+
for (const name of [
176+
"codeql-bundle-linux64.tar.zst",
177+
"codeql-bundle-osx64.tar.gz",
178+
"codeql-bundle-win64.tar.zst",
179+
// The all-platform bundle.
180+
"codeql-bundle.tar.gz",
181+
// A platform we do not publish per-language bundles for, whose name also contains a hyphen.
182+
"codeql-bundle-linux-arm64.tar.zst",
183+
// Not a language we know about.
184+
"codeql-bundle-cobol-linux64.tar.zst",
185+
]) {
186+
t.is(tryGetBundleLanguageFromUrl(url(name)), undefined, name);
187+
}
188+
189+
t.is(tryGetBundleLanguageFromUrl("not a url"), undefined);
190+
});
191+
144192
function decrementPatchVersion(version: string): string {
145193
const [major, minor, patch] = version.split(".").map(Number);
146194
return `${major}.${minor}.${patch - 1}`;

‎src/per-language-bundles.ts‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,39 @@ import { GitHubVariant } from "./util";
1717
*/
1818
export const MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION = "99.99.99";
1919

20+
/**
21+
* Matches the name of a bundle that contains a single language, capturing that language.
22+
*
23+
* The language comes before the platform so that the name of a per-language bundle never has the
24+
* name of the combined bundle for the same platform as a prefix.
25+
*/
26+
const PER_LANGUAGE_BUNDLE_NAME =
27+
/^codeql-bundle-(.+)-(?:linux64|osx64|win64)\.tar\.(?:gz|zst)$/;
28+
29+
/**
30+
* Determines whether a URL points at a bundle that contains only a single language.
31+
*
32+
* We need to recognise these even when we did not choose to download one ourselves, since a bundle
33+
* that is missing most of its extractors must not be added to the toolcache no matter how we came
34+
* to be downloading it.
35+
*
36+
* @returns The language that the bundle contains, or `undefined` if the URL does not point at a
37+
* bundle for a single language.
38+
*/
39+
export function tryGetBundleLanguageFromUrl(
40+
url: string,
41+
): BuiltInLanguage | undefined {
42+
let assetName: string;
43+
try {
44+
assetName = new URL(url).pathname.split("/").pop() ?? "";
45+
} catch {
46+
return undefined;
47+
}
48+
49+
const match = assetName.match(PER_LANGUAGE_BUNDLE_NAME);
50+
return match ? parseBuiltInLanguage(match[1]) : undefined;
51+
}
52+
2053
/**
2154
* The platform of the per-language bundle we use for each language.
2255
*

‎src/setup-codeql.test.ts‎

Lines changed: 64 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -837,7 +837,7 @@ test.serial(
837837
);
838838
t.is(source.perLanguageBundle?.language, BuiltInLanguage.java);
839839
t.true(
840-
source.perLanguageBundle?.combinedBundleURL.endsWith(
840+
source.perLanguageBundle?.combinedBundleURL?.endsWith(
841841
"/codeql-bundle-linux64.tar.zst",
842842
),
843843
);
@@ -1040,6 +1040,69 @@ test.serial(
10401040
},
10411041
);
10421042

1043+
test.serial(
1044+
"setupCodeQLBundle keeps an explicitly requested per-language bundle out of the toolcache",
1045+
async (t) => {
1046+
const downloadStub = sinon.stub(setupCodeql, "downloadCodeQL").resolves({
1047+
codeqlFolder: "codeql",
1048+
statusReport: { totalDurationMs: 100 },
1049+
toolsVersion: "9.9.9",
1050+
});
1051+
1052+
await withTmpDir(async (tmpDir) => {
1053+
setupActionsVars(tmpDir, tmpDir);
1054+
const result = await setupCodeql.setupCodeQLBundle(
1055+
"https://github.com/github/codeql-action/releases/download/codeql-bundle-v9.9.9/codeql-bundle-ruby-linux64.tar.zst",
1056+
SAMPLE_DOTCOM_API_DETAILS,
1057+
tmpDir,
1058+
GitHubVariant.DOTCOM,
1059+
SAMPLE_DEFAULT_CLI_VERSION,
1060+
undefined, // rawLanguages
1061+
false, // useOverlayAwareDefaultCliVersion
1062+
createFeatures([]),
1063+
getRunnerLogger(true),
1064+
);
1065+
1066+
// Even though we did not choose this bundle, it is still missing most of its extractors.
1067+
t.true(downloadStub.calledOnce);
1068+
t.true(isPerLanguageBundleArg(downloadStub.firstCall));
1069+
t.is(
1070+
result.toolsDownloadStatusReport?.bundleLanguage,
1071+
BuiltInLanguage.ruby,
1072+
);
1073+
});
1074+
},
1075+
);
1076+
1077+
test.serial(
1078+
"setupCodeQLBundle does not substitute a bundle for an explicitly requested one that is missing",
1079+
async (t) => {
1080+
const downloadStub = sinon
1081+
.stub(setupCodeql, "downloadCodeQL")
1082+
.rejects(new util.HTTPError("Not Found", 404));
1083+
1084+
await withTmpDir(async (tmpDir) => {
1085+
setupActionsVars(tmpDir, tmpDir);
1086+
await t.throwsAsync(
1087+
setupCodeql.setupCodeQLBundle(
1088+
"https://github.com/github/codeql-action/releases/download/codeql-bundle-v9.9.9/codeql-bundle-ruby-linux64.tar.zst",
1089+
SAMPLE_DOTCOM_API_DETAILS,
1090+
tmpDir,
1091+
GitHubVariant.DOTCOM,
1092+
SAMPLE_DEFAULT_CLI_VERSION,
1093+
undefined, // rawLanguages
1094+
false, // useOverlayAwareDefaultCliVersion
1095+
createFeatures([]),
1096+
getRunnerLogger(true),
1097+
),
1098+
);
1099+
1100+
// Falling back would silently ignore the bundle that was asked for.
1101+
t.true(downloadStub.calledOnce);
1102+
});
1103+
},
1104+
);
1105+
10431106
test.serial(
10441107
"getEnabledVersionsWithOverlayBaseDatabases returns flag-enabled versions present in cache, sorted desc",
10451108
async (t) => {

‎src/setup-codeql.ts‎

Lines changed: 26 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,10 @@ import {
3333
import { BuiltInLanguage } from "./languages";
3434
import { Logger } from "./logging";
3535
import { getCodeQlVersionsForOverlayBaseDatabases } from "./overlay/caching";
36-
import { getPerLanguageBundleLanguage } from "./per-language-bundles";
36+
import {
37+
getPerLanguageBundleLanguage,
38+
tryGetBundleLanguageFromUrl,
39+
} from "./per-language-bundles";
3740
import * as tar from "./tar";
3841
import {
3942
deleteToolcacheBundles,
@@ -263,7 +266,11 @@ export type CodeQLToolsSource =
263266
*/
264267
perLanguageBundle?: {
265268
language: BuiltInLanguage;
266-
combinedBundleURL: string;
269+
/**
270+
* Absent when the bundle was requested explicitly rather than chosen by us, since in that
271+
* case we should honor the request rather than substituting a different bundle.
272+
*/
273+
combinedBundleURL?: string;
267274
};
268275
sourceType: "download";
269276
/** Human-readable description of the source of the tools for telemetry purposes. */
@@ -749,7 +756,7 @@ export async function getCodeQLSource(
749756

750757
let compressionMethod: tar.CompressionMethod;
751758
let perLanguageBundle:
752-
| { language: BuiltInLanguage; combinedBundleURL: string }
759+
| { language: BuiltInLanguage; combinedBundleURL?: string }
753760
| undefined;
754761

755762
if (!url) {
@@ -801,6 +808,16 @@ export async function getCodeQLSource(
801808
);
802809
}
803810
compressionMethod = method;
811+
812+
// The bundle was requested explicitly rather than chosen by us, but we still need to know
813+
// whether it contains a single language so that we do not add it to the toolcache.
814+
const language = tryGetBundleLanguageFromUrl(url);
815+
if (language !== undefined) {
816+
logger.info(
817+
`${url} appears to be a CodeQL bundle that contains only ${language}.`,
818+
);
819+
perLanguageBundle = { language };
820+
}
804821
}
805822

806823
if (cliVersion) {
@@ -1157,7 +1174,12 @@ async function downloadCodeQLBundle(
11571174
},
11581175
};
11591176
} catch (e) {
1160-
if (util.asHTTPError(e)?.status !== 404) {
1177+
// Without a combined bundle to fall back to, the bundle was requested explicitly, so there is
1178+
// nothing we can substitute for it.
1179+
if (
1180+
perLanguageBundle.combinedBundleURL === undefined ||
1181+
util.asHTTPError(e)?.status !== 404
1182+
) {
11611183
throw e;
11621184
}
11631185
logger.warning(

0 commit comments

Comments
 (0)