Skip to content

Commit d794fa4

Browse files
henrymercerCopilot
andcommitted
Add helpers for selecting a CodeQL bundle from a release
Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>
1 parent 6ec2b18 commit d794fa4

2 files changed

Lines changed: 366 additions & 0 deletions

File tree

‎src/codeql-release.test.ts‎

Lines changed: 199 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,199 @@
1+
import * as github from "@actions/github";
2+
import test from "ava";
3+
4+
import {
5+
BundleSelectionOptions,
6+
getPublicRelease,
7+
getRelease,
8+
selectBundle,
9+
} from "./codeql-release";
10+
import { ActionsEnvVars } from "./environment";
11+
import { Feature } from "./feature-flags";
12+
import { BuiltInLanguage } from "./languages";
13+
import { BundlePlatform } from "./platform";
14+
import {
15+
createFeatures,
16+
getRecordingLogger,
17+
getTestEnv,
18+
initAllState,
19+
LoggedMessage,
20+
} from "./testing-utils";
21+
import { ConfigurationError, GitHubVariant } from "./util";
22+
23+
const TAG = "codeql-bundle-v2.27.1";
24+
const REFERENCE = {
25+
serverURL: "https://github.com",
26+
owner: "octo",
27+
repo: "tools",
28+
tagName: TAG,
29+
};
30+
const RELEASE_PAGE = `https://github.com/octo/tools/releases/tag/${TAG}`;
31+
const API_URL = "https://api.github.com";
32+
const COMBINED = "codeql-bundle-linux64.tar.zst";
33+
const JAVA = "codeql-bundle-java-linux64.tar.zst";
34+
35+
const OPTIONS: BundleSelectionOptions = {
36+
rawLanguages: ["java-kotlin"],
37+
cliVersion: "2.27.1",
38+
platform: BundlePlatform.Linux64,
39+
variant: GitHubVariant.DOTCOM,
40+
tarSupportsZstd: true,
41+
};
42+
43+
/** Serves the release tagged `TAG` with the given assets from a stubbed API. */
44+
function releaseFixture({ assetNames = [COMBINED, JAVA], status = 200 } = {}) {
45+
const apiBase = `${API_URL}/repos/octo/tools/releases`;
46+
const releaseAPIURL = `${apiBase}/tags/${TAG}`;
47+
const assets = assetNames.map((name, index) => ({
48+
name,
49+
url: `${apiBase}/assets/${1000 + index}`,
50+
}));
51+
const requests: string[] = [];
52+
const messages: LoggedMessage[] = [];
53+
const state = initAllState({
54+
env: getTestEnv({ [ActionsEnvVars.RUNNER_ENVIRONMENT]: "github-hosted" }),
55+
logger: getRecordingLogger(messages, { logToConsole: false }),
56+
features: createFeatures([Feature.PerLanguageBundles]),
57+
apiClient: github.getOctokit("123", {
58+
baseUrl: API_URL,
59+
request: {
60+
fetch: async (url) => {
61+
requests.push(String(url));
62+
if (String(url) !== releaseAPIURL) {
63+
throw new Error(`Unexpected API request: ${url}`);
64+
}
65+
return new Response(JSON.stringify({ tag_name: TAG, assets }), {
66+
status,
67+
headers: { "content-type": "application/json" },
68+
});
69+
},
70+
},
71+
}),
72+
});
73+
return {
74+
assets,
75+
messages,
76+
releaseAPIURL,
77+
requests,
78+
state,
79+
select: async (options: Partial<BundleSelectionOptions> = {}) =>
80+
selectBundle(state, await getRelease(state, REFERENCE), {
81+
...OPTIONS,
82+
...options,
83+
}),
84+
};
85+
}
86+
87+
test("selectBundle selects an eligible per-language bundle with a single release lookup", async (t) => {
88+
const fixture = releaseFixture();
89+
t.deepEqual(await fixture.select(), {
90+
bundle: {
91+
kind: "per-language",
92+
url: fixture.assets[1].url,
93+
language: BuiltInLanguage.java,
94+
combinedBundleURL: fixture.assets[0].url,
95+
},
96+
compressionMethod: "zstd",
97+
});
98+
t.deepEqual(fixture.requests, [fixture.releaseAPIURL]);
99+
});
100+
101+
test("selectBundle falls back to the combined bundle from the same release", async (t) => {
102+
const fixture = releaseFixture({ assetNames: [COMBINED] });
103+
t.deepEqual(await fixture.select(), {
104+
bundle: { kind: "combined", url: fixture.assets[0].url },
105+
compressionMethod: "zstd",
106+
perLanguageBundleFallback: true,
107+
});
108+
t.true(
109+
fixture.messages.some(
110+
(message) =>
111+
message.type === "warning" &&
112+
typeof message.message === "string" &&
113+
message.message.includes(`'java' at ${RELEASE_PAGE}`),
114+
),
115+
);
116+
});
117+
118+
test("selectBundle selects the combined bundle for jobs that aren't eligible for a per-language bundle", async (t) => {
119+
const fixture = releaseFixture();
120+
t.deepEqual(await fixture.select({ rawLanguages: ["java", "python"] }), {
121+
bundle: { kind: "combined", url: fixture.assets[0].url },
122+
compressionMethod: "zstd",
123+
});
124+
});
125+
126+
test("selectBundle uses the other compression method when the preferred one is missing", async (t) => {
127+
const gzipOnly = releaseFixture({
128+
assetNames: ["codeql-bundle-linux64.tar.gz", JAVA],
129+
});
130+
t.deepEqual(await gzipOnly.select(), {
131+
bundle: { kind: "combined", url: gzipOnly.assets[0].url },
132+
compressionMethod: "gzip",
133+
});
134+
135+
for (const [platform, options] of [
136+
[BundlePlatform.Win64, {}],
137+
[BundlePlatform.Linux64, { cliVersion: "2.18.4" }],
138+
[BundlePlatform.Linux64, { tarSupportsZstd: false }],
139+
] as const) {
140+
const selection = await releaseFixture({
141+
assetNames: [
142+
`codeql-bundle-${platform}.tar.zst`,
143+
`codeql-bundle-${platform}.tar.gz`,
144+
],
145+
}).select({ platform, ...options });
146+
t.is(
147+
selection.compressionMethod,
148+
"gzip",
149+
`${platform} ${JSON.stringify(options)}`,
150+
);
151+
}
152+
});
153+
154+
test("selectBundle requires a combined bundle that we can extract", async (t) => {
155+
await t.throwsAsync(releaseFixture({ assetNames: [JAVA] }).select(), {
156+
instanceOf: ConfigurationError,
157+
message: `No compatible CodeQL bundle was found in release ${RELEASE_PAGE}. Expected codeql-bundle-linux64.tar.zst or codeql-bundle-linux64.tar.gz.`,
158+
});
159+
await t.throwsAsync(
160+
releaseFixture({ assetNames: [COMBINED] }).select({
161+
tarSupportsZstd: false,
162+
}),
163+
{ instanceOf: ConfigurationError, message: /Expected [^ ]+\.tar\.gz\.$/ },
164+
);
165+
});
166+
167+
test("getRelease propagates API errors", async (t) => {
168+
const fixture = releaseFixture({ status: 404 });
169+
t.like(await t.throwsAsync(fixture.select()), { status: 404 });
170+
t.deepEqual(fixture.requests, [fixture.releaseAPIURL]);
171+
});
172+
173+
test("getPublicRelease constructs download URLs without looking up the release", async (t) => {
174+
const fixture = releaseFixture();
175+
const release = getPublicRelease({ ...REFERENCE, tagName: "nightly/v1+2" });
176+
const baseURL =
177+
"https://github.com/octo/tools/releases/download/nightly/v1%2B2";
178+
t.is(
179+
release.url,
180+
"https://github.com/octo/tools/releases/tag/nightly/v1%2B2",
181+
);
182+
t.deepEqual(
183+
await selectBundle(fixture.state, release, {
184+
...OPTIONS,
185+
cliVersion: undefined,
186+
isLatestNightly: true,
187+
}),
188+
{
189+
bundle: {
190+
kind: "per-language",
191+
url: `${baseURL}/${JAVA}`,
192+
language: BuiltInLanguage.java,
193+
combinedBundleURL: `${baseURL}/${COMBINED}`,
194+
},
195+
compressionMethod: "zstd",
196+
},
197+
);
198+
t.deepEqual(fixture.requests, []);
199+
});

‎src/codeql-release.ts‎

Lines changed: 167 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,167 @@
1+
import * as semver from "semver";
2+
3+
import { ActionState } from "./action-common";
4+
import { CodeQLBundle, getCodeQLBundleName } from "./codeql-bundle";
5+
import { CODEQL_VERSION_ZSTD_BUNDLE } from "./feature-flags";
6+
import {
7+
getPerLanguageBundleLanguage,
8+
logMissingPerLanguageBundle,
9+
} from "./per-language-bundles";
10+
import { BundlePlatform } from "./platform";
11+
import type { CompressionMethod } from "./tar";
12+
import { ConfigurationError, GitHubVariant } from "./util";
13+
14+
/** Identifies a release on a GitHub instance. */
15+
export interface CodeQLReleaseReference {
16+
serverURL: string;
17+
owner: string;
18+
repo: string;
19+
tagName: string;
20+
}
21+
22+
/** A GitHub release that contains CodeQL bundles. */
23+
export interface CodeQLRelease {
24+
/** The release's web page, for messages. */
25+
url: string;
26+
/** Returns the download URL for an asset, or `undefined` if the release doesn't have it. */
27+
getAssetURL(name: string): string | undefined;
28+
}
29+
30+
function encodeTag(tagName: string): string {
31+
return tagName.split("/").map(encodeURIComponent).join("/");
32+
}
33+
34+
function getReleasePageURL(reference: CodeQLReleaseReference): string {
35+
const { serverURL, owner, repo, tagName } = reference;
36+
return `${serverURL}/${owner}/${repo}/releases/tag/${encodeTag(tagName)}`;
37+
}
38+
39+
/** Looks up a release on the current GitHub instance, which works for private repositories. */
40+
export async function getRelease(
41+
{ apiClient }: ActionState<["Api"]>,
42+
reference: CodeQLReleaseReference,
43+
): Promise<CodeQLRelease> {
44+
const { owner, repo, tagName } = reference;
45+
const { data: release } = await apiClient.rest.repos.getReleaseByTag({
46+
owner,
47+
repo,
48+
tag: tagName,
49+
});
50+
return {
51+
url: getReleasePageURL(reference),
52+
getAssetURL: (name) =>
53+
release.assets.find((asset) => asset.name === name)?.url,
54+
};
55+
}
56+
57+
/**
58+
* Refers to a public release without looking it up. Every asset gets a download URL, so a missing
59+
* asset shows up as a failed download.
60+
*/
61+
export function getPublicRelease(
62+
reference: CodeQLReleaseReference,
63+
): CodeQLRelease {
64+
const { serverURL, owner, repo, tagName } = reference;
65+
return {
66+
url: getReleasePageURL(reference),
67+
getAssetURL: (name) =>
68+
`${serverURL}/${owner}/${repo}/releases/download/${encodeTag(tagName)}/${name}`,
69+
};
70+
}
71+
72+
/** Describes the job and runner that we are selecting a bundle for. */
73+
export interface BundleSelectionOptions {
74+
/** Explicit `languages` input, which determines whether a per-language bundle is eligible. */
75+
rawLanguages: string[] | undefined;
76+
/** The CLI version in the release, if known. */
77+
cliVersion: string | undefined;
78+
platform: BundlePlatform | undefined;
79+
variant: GitHubVariant;
80+
tarSupportsZstd: boolean;
81+
/** Whether the release is the latest nightly, whose CLI version we don't know yet. */
82+
isLatestNightly?: boolean;
83+
}
84+
85+
/** A bundle selected from a release. */
86+
export interface BundleSelection {
87+
bundle: CodeQLBundle;
88+
compressionMethod: CompressionMethod;
89+
/** The release lacks the eligible per-language bundle, so we selected the combined bundle. */
90+
perLanguageBundleFallback?: true;
91+
}
92+
93+
/** Returns the compression methods that we can extract, most preferred first. */
94+
function getCompressionMethods({
95+
cliVersion,
96+
platform,
97+
tarSupportsZstd,
98+
}: BundleSelectionOptions): CompressionMethod[] {
99+
if (!tarSupportsZstd) {
100+
return ["gzip"];
101+
}
102+
const preferZstd =
103+
// In testing, gzip performs better than zstd on Windows.
104+
platform !== BundlePlatform.Win64 &&
105+
(cliVersion === undefined ||
106+
semver.gte(cliVersion, CODEQL_VERSION_ZSTD_BUNDLE));
107+
return preferZstd ? ["zstd", "gzip"] : ["gzip", "zstd"];
108+
}
109+
110+
/**
111+
* Selects a per-language bundle if the job is eligible for one, and otherwise the combined bundle.
112+
* We only select a per-language bundle from a release that also has a combined bundle with the same
113+
* compression, so that we can fall back to it.
114+
*/
115+
export async function selectBundle(
116+
action: ActionState<["Logger", "ReadOnlyEnv", "FeatureFlags"]>,
117+
release: CodeQLRelease,
118+
options: BundleSelectionOptions,
119+
): Promise<BundleSelection> {
120+
const { logger } = action;
121+
const compressionMethods = getCompressionMethods(options);
122+
for (const compressionMethod of compressionMethods) {
123+
const combinedBundleName = getCodeQLBundleName(
124+
compressionMethod,
125+
options.platform,
126+
);
127+
const combinedBundleURL = release.getAssetURL(combinedBundleName);
128+
if (combinedBundleURL === undefined) {
129+
continue;
130+
}
131+
const combined: BundleSelection = {
132+
bundle: { kind: "combined", url: combinedBundleURL },
133+
compressionMethod,
134+
};
135+
136+
const language = await getPerLanguageBundleLanguage(action, {
137+
...options,
138+
compressionMethod,
139+
});
140+
if (language === undefined) {
141+
logger.info(
142+
`Selected CodeQL bundle ${combinedBundleName} from ${release.url}.`,
143+
);
144+
return combined;
145+
}
146+
const name = getCodeQLBundleName(
147+
compressionMethod,
148+
options.platform,
149+
language,
150+
);
151+
const url = release.getAssetURL(name);
152+
if (url === undefined) {
153+
logMissingPerLanguageBundle(action, language, release.url);
154+
return { ...combined, perLanguageBundleFallback: true };
155+
}
156+
logger.info(`Selected CodeQL bundle ${name} from ${release.url}.`);
157+
return {
158+
bundle: { kind: "per-language", url, language, combinedBundleURL },
159+
compressionMethod,
160+
};
161+
}
162+
throw new ConfigurationError(
163+
`No compatible CodeQL bundle was found in release ${release.url}. Expected ${compressionMethods
164+
.map((method) => getCodeQLBundleName(method, options.platform))
165+
.join(" or ")}.`,
166+
);
167+
}

0 commit comments

Comments
 (0)