Skip to content

Commit f03d2df

Browse files
henrymercerCopilot
andcommitted
Add helpers for selecting a CodeQL bundle from a release
Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>
1 parent 6393cd4 commit f03d2df

2 files changed

Lines changed: 388 additions & 0 deletions

File tree

‎src/codeql-release.test.ts‎

Lines changed: 200 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,200 @@
1+
import * as github from "@actions/github";
2+
import test from "ava";
3+
4+
import {
5+
BundleSelectionOptions,
6+
getPublicRelease,
7+
getRelease,
8+
selectBundle,
9+
} from "./codeql-release";
10+
import { ActionsEnvVars } from "./environment";
11+
import { Feature } from "./feature-flags";
12+
import { BuiltInLanguage } from "./languages";
13+
import { BundlePlatform } from "./platform";
14+
import {
15+
createFeatures,
16+
getRecordingLogger,
17+
getTestEnv,
18+
initAllState,
19+
LoggedMessage,
20+
} from "./testing-utils";
21+
import { ConfigurationError, GitHubVariant } from "./util";
22+
23+
const TAG = "codeql-bundle-v2.27.1";
24+
const REFERENCE = {
25+
serverURL: "https://github.com",
26+
owner: "octo",
27+
repo: "tools",
28+
tagName: TAG,
29+
};
30+
const RELEASE_PAGE = `https://github.com/octo/tools/releases/tag/${TAG}`;
31+
const API_URL = "https://api.github.com";
32+
const COMBINED = "codeql-bundle-linux64.tar.zst";
33+
const JAVA = "codeql-bundle-java-linux64.tar.zst";
34+
35+
const OPTIONS: BundleSelectionOptions = {
36+
rawLanguages: ["java-kotlin"],
37+
cliVersion: "2.27.1",
38+
platform: BundlePlatform.Linux64,
39+
variant: GitHubVariant.DOTCOM,
40+
tarSupportsZstd: true,
41+
};
42+
43+
/** Serves the release tagged `TAG` with the given assets from a stubbed API. */
44+
function releaseFixture({ assetNames = [COMBINED, JAVA], status = 200 } = {}) {
45+
const apiBase = `${API_URL}/repos/octo/tools/releases`;
46+
const releaseAPIURL = `${apiBase}/tags/${TAG}`;
47+
const assets = assetNames.map((name, index) => ({
48+
name,
49+
url: `${apiBase}/assets/${1000 + index}`,
50+
}));
51+
const requests: string[] = [];
52+
const messages: LoggedMessage[] = [];
53+
const state = initAllState({
54+
env: getTestEnv({ [ActionsEnvVars.RUNNER_ENVIRONMENT]: "github-hosted" }),
55+
logger: getRecordingLogger(messages, { logToConsole: false }),
56+
features: createFeatures([Feature.PerLanguageBundles]),
57+
apiClient: github.getOctokit("123", {
58+
baseUrl: API_URL,
59+
request: {
60+
fetch: async (url) => {
61+
requests.push(String(url));
62+
if (String(url) !== releaseAPIURL) {
63+
throw new Error(`Unexpected API request: ${url}`);
64+
}
65+
return new Response(JSON.stringify({ tag_name: TAG, assets }), {
66+
status,
67+
headers: { "content-type": "application/json" },
68+
});
69+
},
70+
},
71+
}),
72+
});
73+
return {
74+
assets,
75+
messages,
76+
releaseAPIURL,
77+
requests,
78+
state,
79+
select: async (options: Partial<BundleSelectionOptions> = {}) =>
80+
selectBundle(state, await getRelease(state, REFERENCE), {
81+
...OPTIONS,
82+
...options,
83+
}),
84+
};
85+
}
86+
87+
test("selectBundle selects an eligible per-language bundle with a single release lookup", async (t) => {
88+
const fixture = releaseFixture();
89+
t.deepEqual(await fixture.select(), {
90+
bundle: {
91+
kind: "per-language",
92+
url: fixture.assets[1].url,
93+
language: BuiltInLanguage.java,
94+
combinedBundleURL: fixture.assets[0].url,
95+
},
96+
compressionMethod: "zstd",
97+
});
98+
t.deepEqual(fixture.requests, [fixture.releaseAPIURL]);
99+
});
100+
101+
test("selectBundle falls back to the combined bundle from the same release", async (t) => {
102+
const fixture = releaseFixture({ assetNames: [COMBINED] });
103+
t.deepEqual(await fixture.select(), {
104+
bundle: { kind: "combined", url: fixture.assets[0].url },
105+
compressionMethod: "zstd",
106+
perLanguageBundleFallback: true,
107+
});
108+
t.true(
109+
fixture.messages.some(
110+
(message) =>
111+
message.type === "warning" &&
112+
typeof message.message === "string" &&
113+
message.message.includes(`'java' at ${RELEASE_PAGE}`),
114+
),
115+
);
116+
});
117+
118+
test("selectBundle selects the combined bundle for jobs that aren't eligible for a per-language bundle", async (t) => {
119+
const fixture = releaseFixture();
120+
t.deepEqual(await fixture.select({ rawLanguages: ["java", "python"] }), {
121+
bundle: { kind: "combined", url: fixture.assets[0].url },
122+
compressionMethod: "zstd",
123+
});
124+
});
125+
126+
test("selectBundle uses the other compression method when the preferred one is missing", async (t) => {
127+
const gzipOnly = releaseFixture({
128+
assetNames: ["codeql-bundle-linux64.tar.gz", JAVA],
129+
});
130+
t.deepEqual(await gzipOnly.select(), {
131+
bundle: { kind: "combined", url: gzipOnly.assets[0].url },
132+
compressionMethod: "gzip",
133+
});
134+
135+
for (const [platform, options] of [
136+
[BundlePlatform.Win64, {}],
137+
[BundlePlatform.Linux64, { cliVersion: "2.18.4" }],
138+
[BundlePlatform.Linux64, { cliVersion: undefined }],
139+
[BundlePlatform.Linux64, { tarSupportsZstd: false }],
140+
] as const) {
141+
const selection = await releaseFixture({
142+
assetNames: [
143+
`codeql-bundle-${platform}.tar.zst`,
144+
`codeql-bundle-${platform}.tar.gz`,
145+
],
146+
}).select({ platform, ...options });
147+
t.is(
148+
selection.compressionMethod,
149+
"gzip",
150+
`${platform} ${JSON.stringify(options)}`,
151+
);
152+
}
153+
});
154+
155+
test("selectBundle requires a combined bundle that we can extract", async (t) => {
156+
await t.throwsAsync(releaseFixture({ assetNames: [JAVA] }).select(), {
157+
instanceOf: ConfigurationError,
158+
message: `No compatible CodeQL bundle was found in release ${RELEASE_PAGE}. Expected codeql-bundle-linux64.tar.zst or codeql-bundle-linux64.tar.gz.`,
159+
});
160+
await t.throwsAsync(
161+
releaseFixture({ assetNames: [COMBINED] }).select({
162+
tarSupportsZstd: false,
163+
}),
164+
{ instanceOf: ConfigurationError, message: /Expected [^ ]+\.tar\.gz\.$/ },
165+
);
166+
});
167+
168+
test("getRelease propagates API errors", async (t) => {
169+
const fixture = releaseFixture({ status: 404 });
170+
t.like(await t.throwsAsync(fixture.select()), { status: 404 });
171+
t.deepEqual(fixture.requests, [fixture.releaseAPIURL]);
172+
});
173+
174+
test("getPublicRelease constructs download URLs without looking up the release", async (t) => {
175+
const fixture = releaseFixture();
176+
const release = getPublicRelease({ ...REFERENCE, tagName: "nightly/v1+2" });
177+
const baseURL =
178+
"https://github.com/octo/tools/releases/download/nightly/v1%2B2";
179+
t.is(
180+
release.url,
181+
"https://github.com/octo/tools/releases/tag/nightly/v1%2B2",
182+
);
183+
t.deepEqual(
184+
await selectBundle(fixture.state, release, {
185+
...OPTIONS,
186+
cliVersion: undefined,
187+
isLatestNightly: true,
188+
}),
189+
{
190+
bundle: {
191+
kind: "per-language",
192+
url: `${baseURL}/${JAVA}`,
193+
language: BuiltInLanguage.java,
194+
combinedBundleURL: `${baseURL}/${COMBINED}`,
195+
},
196+
compressionMethod: "zstd",
197+
},
198+
);
199+
t.deepEqual(fixture.requests, []);
200+
});

‎src/codeql-release.ts‎

Lines changed: 188 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,188 @@
1+
import * as semver from "semver";
2+
3+
import { ActionState } from "./action-common";
4+
import { CodeQLBundle, getCodeQLBundleName } from "./codeql-bundle";
5+
import { CODEQL_VERSION_ZSTD_BUNDLE } from "./feature-flags";
6+
import {
7+
getPerLanguageBundleLanguage,
8+
logMissingPerLanguageBundle,
9+
} from "./per-language-bundles";
10+
import { BundlePlatform } from "./platform";
11+
import type { CompressionMethod } from "./tar";
12+
import { ConfigurationError, GitHubVariant } from "./util";
13+
14+
/** Identifies a release on a GitHub instance. */
15+
export interface CodeQLReleaseReference {
16+
serverURL: string;
17+
owner: string;
18+
repo: string;
19+
tagName: string;
20+
}
21+
22+
/** A GitHub release that contains CodeQL bundles. */
23+
export interface CodeQLRelease {
24+
/** The release's web page, for messages. */
25+
url: string;
26+
/** Returns the download URL for an asset, or `undefined` if the release doesn't have it. */
27+
getAssetURL(name: string): string | undefined;
28+
}
29+
30+
/**
31+
* Encodes a tag for use in a URL path. Slashes stay as path separators, as in GitHub's release URLs
32+
* for tags like `build/123`.
33+
*/
34+
function encodeTag(tagName: string): string {
35+
return tagName.split("/").map(encodeURIComponent).join("/");
36+
}
37+
38+
function getReleasePageURL(reference: CodeQLReleaseReference): string {
39+
const { serverURL, owner, repo, tagName } = reference;
40+
return `${serverURL}/${owner}/${repo}/releases/tag/${encodeTag(tagName)}`;
41+
}
42+
43+
/**
44+
* Looks up a release on the current GitHub instance, which works for private repositories.
45+
*
46+
* The API client determines which instance we query, so `reference.serverURL` is only used for the
47+
* release page URL in messages. The asset URLs are REST API endpoints, which accept the token and
48+
* return the file when requested with `Accept: application/octet-stream`, unlike browser download
49+
* URLs.
50+
*/
51+
export async function getRelease(
52+
{ apiClient }: ActionState<["Api"]>,
53+
reference: CodeQLReleaseReference,
54+
): Promise<CodeQLRelease> {
55+
const { owner, repo, tagName } = reference;
56+
const { data: release } = await apiClient.rest.repos.getReleaseByTag({
57+
owner,
58+
repo,
59+
tag: tagName,
60+
});
61+
return {
62+
url: getReleasePageURL(reference),
63+
getAssetURL: (name) =>
64+
release.assets.find((asset) => asset.name === name)?.url,
65+
};
66+
}
67+
68+
/**
69+
* Refers to a public release without looking it up. Every asset gets a download URL, so a missing
70+
* asset shows up as a failed download.
71+
*/
72+
export function getPublicRelease(
73+
reference: CodeQLReleaseReference,
74+
): CodeQLRelease {
75+
const { serverURL, owner, repo, tagName } = reference;
76+
return {
77+
url: getReleasePageURL(reference),
78+
getAssetURL: (name) =>
79+
`${serverURL}/${owner}/${repo}/releases/download/${encodeTag(tagName)}/${name}`,
80+
};
81+
}
82+
83+
/** Describes the job and runner that we are selecting a bundle for. */
84+
export interface BundleSelectionOptions {
85+
/** Explicit `languages` input, which determines whether a per-language bundle is eligible. */
86+
rawLanguages: string[] | undefined;
87+
/** The CLI version in the release, if known. */
88+
cliVersion: string | undefined;
89+
platform: BundlePlatform | undefined;
90+
variant: GitHubVariant;
91+
tarSupportsZstd: boolean;
92+
/** Whether the release is the latest nightly, whose CLI version we don't know yet. */
93+
isLatestNightly?: boolean;
94+
}
95+
96+
/** A bundle selected from a release. */
97+
export interface BundleSelection {
98+
bundle: CodeQLBundle;
99+
compressionMethod: CompressionMethod;
100+
/** The release lacks the eligible per-language bundle, so we selected the combined bundle. */
101+
perLanguageBundleFallback?: true;
102+
}
103+
104+
/** Returns the compression methods that we can extract, most preferred first. */
105+
function getCompressionMethods({
106+
cliVersion,
107+
isLatestNightly,
108+
platform,
109+
tarSupportsZstd,
110+
}: BundleSelectionOptions): CompressionMethod[] {
111+
if (!tarSupportsZstd) {
112+
return ["gzip"];
113+
}
114+
const preferZstd =
115+
// In testing, gzip performs better than zstd on Windows.
116+
platform !== BundlePlatform.Win64 &&
117+
// Standard bundles have zstd archives from this version, and so does the latest nightly. For a
118+
// release we looked up, gzip comes next if it lacks the zstd archive.
119+
(isLatestNightly ||
120+
(cliVersion !== undefined &&
121+
semver.gte(cliVersion, CODEQL_VERSION_ZSTD_BUNDLE)));
122+
return preferZstd ? ["zstd", "gzip"] : ["gzip", "zstd"];
123+
}
124+
125+
/**
126+
* Selects a per-language bundle if the job is eligible for one, and otherwise the combined bundle.
127+
* We only select a per-language bundle from a release that also has a combined bundle with the same
128+
* compression, so that we can fall back to it.
129+
*
130+
* If we looked up the release, we can tell that an eligible per-language bundle is missing, so we
131+
* warn and select the combined bundle straight away. A public release gives every asset a URL, so a
132+
* missing per-language bundle only shows up as a 404 when downloading, which is when we fall back.
133+
*
134+
* Throws a `ConfigurationError` if the release has no combined bundle that we can extract.
135+
*/
136+
export async function selectBundle(
137+
action: ActionState<["Logger", "ReadOnlyEnv", "FeatureFlags"]>,
138+
release: CodeQLRelease,
139+
options: BundleSelectionOptions,
140+
): Promise<BundleSelection> {
141+
const { logger } = action;
142+
const compressionMethods = getCompressionMethods(options);
143+
for (const compressionMethod of compressionMethods) {
144+
const combinedBundleName = getCodeQLBundleName(
145+
compressionMethod,
146+
options.platform,
147+
);
148+
const combinedBundleURL = release.getAssetURL(combinedBundleName);
149+
if (combinedBundleURL === undefined) {
150+
continue;
151+
}
152+
const combined: BundleSelection = {
153+
bundle: { kind: "combined", url: combinedBundleURL },
154+
compressionMethod,
155+
};
156+
157+
const language = await getPerLanguageBundleLanguage(action, {
158+
...options,
159+
compressionMethod,
160+
});
161+
if (language === undefined) {
162+
logger.info(
163+
`Selected CodeQL bundle ${combinedBundleName} from ${release.url}.`,
164+
);
165+
return combined;
166+
}
167+
const name = getCodeQLBundleName(
168+
compressionMethod,
169+
options.platform,
170+
language,
171+
);
172+
const url = release.getAssetURL(name);
173+
if (url === undefined) {
174+
logMissingPerLanguageBundle(action, language, release.url);
175+
return { ...combined, perLanguageBundleFallback: true };
176+
}
177+
logger.info(`Selected CodeQL bundle ${name} from ${release.url}.`);
178+
return {
179+
bundle: { kind: "per-language", url, language, combinedBundleURL },
180+
compressionMethod,
181+
};
182+
}
183+
throw new ConfigurationError(
184+
`No compatible CodeQL bundle was found in release ${release.url}. Expected ${compressionMethods
185+
.map((method) => getCodeQLBundleName(method, options.platform))
186+
.join(" or ")}.`,
187+
);
188+
}

0 commit comments

Comments
 (0)