Skip to content

Commit f65587d

Browse files
henrymercerCopilot
andcommitted
Add support for per-language CodeQL bundles
Download a bundle containing only the single language being analyzed, rather than the combined bundle that contains every language, when that is both safe and beneficial. Per-language bundles are substantially smaller, so this saves download time and disk space on the runner. Eligibility is decided in one place, since it is easy for these conditions to drift apart. Per-language bundles are never added to the toolcache, because a bundle for one language must not be reused for a job that analyzes another. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: d4c7b873-8278-4a36-b67c-6cc5ea3b2316
1 parent 9d89e2d commit f65587d

13 files changed

Lines changed: 1129 additions & 113 deletions

‎lib/entry-points.js‎

Lines changed: 244 additions & 79 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎src/actions-util.ts‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -283,6 +283,22 @@ export function isSelfHostedRunner(env: Env = getEnv()) {
283283
return env.getOptional(ActionsEnvVars.RUNNER_ENVIRONMENT) === "self-hosted";
284284
}
285285

286+
/**
287+
* Whether the job is running on a runner that GitHub hosts, and which is therefore thrown away once
288+
* the job has finished.
289+
*
290+
* Unlike `isHostedRunner`, this is based on what the service reports for the job rather than on how
291+
* the runner's filesystem happens to be laid out, so it does not match self-hosted runners that are
292+
* configured to resemble hosted ones, such as those that mount a persistent volume at
293+
* `/opt/hostedtoolcache`. Code Scanning default setup gates dependency caching on the same value.
294+
*
295+
* It is deliberately conservative: if the service does not tell us, we assume the runner is not
296+
* hosted.
297+
*/
298+
export function isGitHubHostedRunner(env: Env = getEnv()) {
299+
return env.getOptional(ActionsEnvVars.RUNNER_ENVIRONMENT) === "github-hosted";
300+
}
301+
286302
/** Determines whether the workflow trigger is `dynamic`. */
287303
export function isDynamicWorkflow(env: Env = getEnv()): boolean {
288304
return getWorkflowEventName(env) === "dynamic";

‎src/feature-flags.ts‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -159,6 +159,11 @@ export enum Feature {
159159
OverlayAnalysisStatusCheck = "overlay_analysis_status_check",
160160
/** Controls whether overlay build failures on the default branch are stored in the Actions cache. */
161161
OverlayAnalysisStatusSave = "overlay_analysis_status_save",
162+
/**
163+
* Controls whether we may download a bundle containing only the single language being analysed,
164+
* rather than the combined bundle that contains every language.
165+
*/
166+
PerLanguageBundles = "per_language_bundles",
162167
QaTelemetryEnabled = "qa_telemetry_enabled",
163168
/** Routes (some) API requests through the registry proxy. */
164169
ProxyApiRequests = "proxy_api_requests",
@@ -424,6 +429,11 @@ export const featureConfig = {
424429
envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_SKIP_RESOURCE_CHECKS",
425430
minimumVersion: undefined,
426431
},
432+
[Feature.PerLanguageBundles]: {
433+
defaultValue: false,
434+
envVar: "CODEQL_ACTION_PER_LANGUAGE_BUNDLES",
435+
minimumVersion: undefined,
436+
},
427437
[Feature.QaTelemetryEnabled]: {
428438
defaultValue: false,
429439
envVar: "CODEQL_ACTION_QA_TELEMETRY",

‎src/init-action.ts‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -182,6 +182,14 @@ async function sendCompletedStatusReport(
182182
initToolsDownloadFields.tools_total_duration_ms =
183183
toolsDownloadStatusReport.totalDurationMs;
184184
}
185+
if (toolsDownloadStatusReport?.bundleLanguage !== undefined) {
186+
initToolsDownloadFields.tools_bundle_language =
187+
toolsDownloadStatusReport.bundleLanguage;
188+
}
189+
if (toolsDownloadStatusReport?.perLanguageBundleFallback !== undefined) {
190+
initToolsDownloadFields.tools_per_language_bundle_fallback =
191+
toolsDownloadStatusReport.perLanguageBundleFallback;
192+
}
185193
if (toolsFeatureFlagsValid !== undefined) {
186194
initToolsDownloadFields.tools_feature_flags_valid = toolsFeatureFlagsValid;
187195
}

‎src/per-language-bundles.test.ts‎

Lines changed: 145 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,145 @@
1+
import test from "ava";
2+
3+
import { ActionsEnvVars } from "./environment";
4+
import { Feature } from "./feature-flags";
5+
import { BuiltInLanguage } from "./languages";
6+
import { getRunnerLogger } from "./logging";
7+
import {
8+
getPerLanguageBundleLanguage,
9+
MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION,
10+
PerLanguageBundleOptions,
11+
} from "./per-language-bundles";
12+
import { createFeatures, setupTests } from "./testing-utils";
13+
import { GitHubVariant } from "./util";
14+
15+
setupTests(test);
16+
17+
/** Options for which we would use a per-language bundle. */
18+
const ELIGIBLE_OPTIONS: PerLanguageBundleOptions = {
19+
rawLanguages: ["java"],
20+
// Any version at least as new as the minimum will do.
21+
cliVersion: MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION,
22+
compressionMethod: "zstd",
23+
platform: "linux64",
24+
variant: GitHubVariant.DOTCOM,
25+
};
26+
27+
async function checkEligibility(
28+
overrides: Partial<PerLanguageBundleOptions>,
29+
enabledFeatures: Feature[] = [Feature.PerLanguageBundles],
30+
) {
31+
return getPerLanguageBundleLanguage(
32+
{ ...ELIGIBLE_OPTIONS, ...overrides },
33+
createFeatures(enabledFeatures),
34+
getRunnerLogger(true),
35+
);
36+
}
37+
38+
test.beforeEach(() => {
39+
process.env[ActionsEnvVars.RUNNER_ENVIRONMENT] = "github-hosted";
40+
});
41+
42+
test.serial(
43+
"uses a per-language bundle when everything lines up",
44+
async (t) => {
45+
t.is(await checkEligibility({}), BuiltInLanguage.java);
46+
},
47+
);
48+
49+
test.serial("resolves the language names used by default setup", async (t) => {
50+
// Code Scanning default setup passes the combined language names, which do not match the
51+
// extractor names that the bundles are published under.
52+
for (const [rawLanguage, expected] of [
53+
["java-kotlin", BuiltInLanguage.java],
54+
["kotlin", BuiltInLanguage.java],
55+
["javascript-typescript", BuiltInLanguage.javascript],
56+
["typescript", BuiltInLanguage.javascript],
57+
["c-cpp", BuiltInLanguage.cpp],
58+
["c", BuiltInLanguage.cpp],
59+
["c++", BuiltInLanguage.cpp],
60+
["c#", BuiltInLanguage.csharp],
61+
["actions", BuiltInLanguage.actions],
62+
["go", BuiltInLanguage.go],
63+
["python", BuiltInLanguage.python],
64+
["ruby", BuiltInLanguage.ruby],
65+
["rust", BuiltInLanguage.rust],
66+
] as const) {
67+
t.is(
68+
await checkEligibility({ rawLanguages: [rawLanguage] }),
69+
expected,
70+
`Expected '${rawLanguage}' to resolve to '${expected}'`,
71+
);
72+
}
73+
});
74+
75+
test.serial("uses the macOS bundle for Swift", async (t) => {
76+
t.is(
77+
await checkEligibility({ rawLanguages: ["swift"], platform: "osx64" }),
78+
BuiltInLanguage.swift,
79+
);
80+
// Swift is only published for macOS.
81+
t.is(
82+
await checkEligibility({ rawLanguages: ["swift"], platform: "linux64" }),
83+
undefined,
84+
);
85+
});
86+
87+
test.serial("only publishes non-Swift languages for Linux", async (t) => {
88+
t.is(await checkEligibility({ platform: "osx64" }), undefined);
89+
t.is(await checkEligibility({ platform: "win64" }), undefined);
90+
t.is(await checkEligibility({ platform: undefined }), undefined);
91+
});
92+
93+
test.serial("requires exactly one language", async (t) => {
94+
t.is(await checkEligibility({ rawLanguages: undefined }), undefined);
95+
t.is(await checkEligibility({ rawLanguages: [] }), undefined);
96+
t.is(await checkEligibility({ rawLanguages: ["java", "python"] }), undefined);
97+
});
98+
99+
test.serial("requires a language that CodeQL knows about", async (t) => {
100+
t.is(await checkEligibility({ rawLanguages: ["cobol"] }), undefined);
101+
});
102+
103+
test.serial("requires a zstd bundle", async (t) => {
104+
t.is(await checkEligibility({ compressionMethod: "gzip" }), undefined);
105+
});
106+
107+
test.serial("requires GitHub.com", async (t) => {
108+
// Other products resolve the combined bundle against their own instance, so asking for a
109+
// per-language bundle they do not mirror would move the download off that instance.
110+
for (const variant of [GitHubVariant.GHES, GitHubVariant.GHEC_DR]) {
111+
t.is(await checkEligibility({ variant }), undefined);
112+
}
113+
});
114+
115+
test.serial("requires a GitHub-hosted runner", async (t) => {
116+
// A self-hosted runner may have a toolcache that persists between jobs, which is worth more than
117+
// a smaller download.
118+
process.env[ActionsEnvVars.RUNNER_ENVIRONMENT] = "self-hosted";
119+
t.is(await checkEligibility({}), undefined);
120+
121+
// Self-hosted runners are routinely configured to look like hosted ones, for example by mounting
122+
// a persistent volume at `/opt/hostedtoolcache`, so we require the service to tell us explicitly.
123+
delete process.env[ActionsEnvVars.RUNNER_ENVIRONMENT];
124+
process.env["RUNNER_TOOL_CACHE"] = "/opt/hostedtoolcache";
125+
t.is(await checkEligibility({}), undefined);
126+
});
127+
128+
test.serial("requires a new enough CLI version", async (t) => {
129+
t.is(await checkEligibility({ cliVersion: undefined }), undefined);
130+
t.is(
131+
await checkEligibility({
132+
cliVersion: decrementPatchVersion(MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION),
133+
}),
134+
undefined,
135+
);
136+
});
137+
138+
test.serial("requires the feature flag", async (t) => {
139+
t.is(await checkEligibility({}, []), undefined);
140+
});
141+
142+
function decrementPatchVersion(version: string): string {
143+
const [major, minor, patch] = version.split(".").map(Number);
144+
return `${major}.${minor}.${patch - 1}`;
145+
}

‎src/per-language-bundles.ts‎

Lines changed: 150 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,150 @@
1+
import * as semver from "semver";
2+
3+
import { isGitHubHostedRunner } from "./actions-util";
4+
import { Feature, FeatureEnablement } from "./feature-flags";
5+
import { BuiltInLanguage, parseBuiltInLanguage } from "./languages";
6+
import { Logger } from "./logging";
7+
import * as tar from "./tar";
8+
import { GitHubVariant } from "./util";
9+
10+
/**
11+
* First version of the CodeQL CLI whose releases include per-language bundles.
12+
*
13+
* TODO(per-language-bundles): This is a sentinel value that no release can ever satisfy, so
14+
* per-language bundles are disabled no matter what the feature flag says. This MUST be updated to
15+
* the first CLI version that actually publishes per-language bundles before the feature can be
16+
* rolled out.
17+
*/
18+
export const MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION = "99.99.99";
19+
20+
/**
21+
* The platform of the per-language bundle we use for each language.
22+
*
23+
* We publish at most one per-language bundle per language, so a language is only eligible when the
24+
* job is running on the corresponding platform. Languages that are absent from this record have no
25+
* per-language bundle at all.
26+
*/
27+
const PER_LANGUAGE_BUNDLE_PLATFORMS: Readonly<
28+
Partial<Record<BuiltInLanguage, string>>
29+
> = {
30+
[BuiltInLanguage.actions]: "linux64",
31+
[BuiltInLanguage.cpp]: "linux64",
32+
[BuiltInLanguage.csharp]: "linux64",
33+
[BuiltInLanguage.go]: "linux64",
34+
[BuiltInLanguage.java]: "linux64",
35+
[BuiltInLanguage.javascript]: "linux64",
36+
[BuiltInLanguage.python]: "linux64",
37+
[BuiltInLanguage.ruby]: "linux64",
38+
[BuiltInLanguage.rust]: "linux64",
39+
[BuiltInLanguage.swift]: "osx64",
40+
};
41+
42+
/** Inputs that determine whether we may download a per-language bundle. */
43+
export interface PerLanguageBundleOptions {
44+
/**
45+
* The languages that were requested for analysis.
46+
*
47+
* These must have been set explicitly via the `languages` input rather than autodetected, since
48+
* autodetection needs a CodeQL instance that we do not have while we are choosing which bundle to
49+
* download.
50+
*/
51+
rawLanguages: string[] | undefined;
52+
/** The CLI version of the bundle we are about to download, if known. */
53+
cliVersion: string | undefined;
54+
/** The compression method of the bundle we are about to download. */
55+
compressionMethod: tar.CompressionMethod;
56+
/** The platform component of the bundle name, for example `linux64`. */
57+
platform: string | undefined;
58+
/** The GitHub product we are running against. */
59+
variant: GitHubVariant;
60+
}
61+
62+
/**
63+
* Determines whether we should download a bundle containing only the language being analysed
64+
* instead of the combined bundle that contains every language.
65+
*
66+
* Per-language bundles are substantially smaller than the combined bundle, so using one saves both
67+
* download time and disk space on the runner.
68+
*
69+
* @returns The language whose per-language bundle we should download, or `undefined` if we should
70+
* download the combined bundle.
71+
*/
72+
export async function getPerLanguageBundleLanguage(
73+
options: PerLanguageBundleOptions,
74+
features: FeatureEnablement,
75+
logger: Logger,
76+
): Promise<BuiltInLanguage | undefined> {
77+
const { rawLanguages, cliVersion, compressionMethod, platform, variant } =
78+
options;
79+
80+
const explain = (reason: string) => {
81+
logger.debug(`Not using a per-language CodeQL bundle since ${reason}.`);
82+
return undefined;
83+
};
84+
85+
if (rawLanguages?.length !== 1) {
86+
return explain(
87+
`exactly one language must be requested via the 'languages' input, but ${
88+
rawLanguages?.length ?? 0
89+
} were`,
90+
);
91+
}
92+
93+
const language = parseBuiltInLanguage(rawLanguages[0]);
94+
if (language === undefined) {
95+
return explain(`'${rawLanguages[0]}' is not a known CodeQL language`);
96+
}
97+
98+
if (compressionMethod !== "zstd") {
99+
// Per-language bundles are only published as zstd archives.
100+
return explain(`the bundle would be downloaded as ${compressionMethod}`);
101+
}
102+
103+
if (variant !== GitHubVariant.DOTCOM) {
104+
// Other products resolve bundles against their own instance first, and only fall back to
105+
// github.com when the instance does not have the asset. Since they are unlikely to mirror
106+
// per-language bundles, asking for one would quietly move the download off the instance.
107+
//
108+
// We cannot simply let that fail and fall back: an instance whose runners cannot reach
109+
// github.com produces a connection error rather than a 404, which is not something we fall back
110+
// from, so the job would fail outright where today it succeeds.
111+
return explain(`we are running against ${variant}`);
112+
}
113+
114+
if (!isGitHubHostedRunner()) {
115+
// We never add per-language bundles to the toolcache, since a bundle for one language must not
116+
// be reused for another. That trade-off only pays off on GitHub-hosted runners, which are
117+
// discarded after the job, so their toolcache could not have been reused anyway. A self-hosted
118+
// runner may well have a toolcache that persists between jobs, and populating it is worth more
119+
// than a smaller download.
120+
return explain("the job is not running on a GitHub-hosted runner");
121+
}
122+
123+
if (cliVersion === undefined) {
124+
return explain("the CLI version of the bundle is unknown");
125+
}
126+
127+
if (!semver.gte(cliVersion, MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION)) {
128+
return explain(
129+
`CodeQL ${cliVersion} is older than ${MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION}, which is the ` +
130+
"first version that publishes per-language bundles",
131+
);
132+
}
133+
134+
const supportedPlatform = PER_LANGUAGE_BUNDLE_PLATFORMS[language];
135+
if (supportedPlatform === undefined) {
136+
return explain(`no per-language bundle is published for ${language}`);
137+
}
138+
if (supportedPlatform !== platform) {
139+
return explain(
140+
`the ${language} bundle is only published for ${supportedPlatform}, but this job is ` +
141+
`running on ${platform ?? "an unknown platform"}`,
142+
);
143+
}
144+
145+
if (!(await features.getValue(Feature.PerLanguageBundles))) {
146+
return explain(`the ${Feature.PerLanguageBundles} feature is disabled`);
147+
}
148+
149+
return language;
150+
}

‎src/setup-codeql-action.ts‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,14 @@ async function sendCompletedStatusReport(
9393
initToolsDownloadFields.tools_total_duration_ms =
9494
toolsDownloadStatusReport.totalDurationMs;
9595
}
96+
if (toolsDownloadStatusReport?.bundleLanguage !== undefined) {
97+
initToolsDownloadFields.tools_bundle_language =
98+
toolsDownloadStatusReport.bundleLanguage;
99+
}
100+
if (toolsDownloadStatusReport?.perLanguageBundleFallback !== undefined) {
101+
initToolsDownloadFields.tools_per_language_bundle_fallback =
102+
toolsDownloadStatusReport.perLanguageBundleFallback;
103+
}
96104
if (toolsFeatureFlagsValid !== undefined) {
97105
initToolsDownloadFields.tools_feature_flags_valid = toolsFeatureFlagsValid;
98106
}

0 commit comments

Comments
 (0)