Skip to content
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 43 additions & 0 deletions lib/entry-points.js

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

146 changes: 146 additions & 0 deletions src/database-upload.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,10 @@ import * as apiClient from "./api-client";
import { createStubCodeQL } from "./codeql";
import { Config } from "./config-utils";
import { cleanupAndUploadDatabases } from "./database-upload";
import { Feature } from "./feature-flags";
import * as gitUtils from "./git-utils";
import { BuiltInLanguage } from "./languages";
import { OverlayDatabaseMode } from "./overlay/overlay-database-mode";
import { RepositoryNwo } from "./repository";
import {
checkExpectedLogMessages,
Expand All @@ -24,6 +26,7 @@ import {
setupTests,
} from "./testing-utils";
import {
CleanupLevel,
GitHubVariant,
HTTPError,
initializeEnvironment,
Expand Down Expand Up @@ -335,3 +338,146 @@ test.serial("Successfully uploading a database to GHEC-DR", async (t) => {
);
});
});

test.serial(
"Records overlay and clear cleanup sizes when uploading an overlay-base database",
async (t) => {
await withTmpDir(async (tmpDir) => {
setupActionsVars(tmpDir, tmpDir);
sinon
.stub(actionsUtil, "getRequiredInput")
.withArgs("upload-database")
.returns("true");
sinon.stub(gitUtils, "isAnalyzingDefaultBranch").resolves(true);

await mockHttpRequests(201);

// Track the cleanup level passed to each cleanup so that the database
// bundle stub can write a differently-sized bundle for each level.
const cleanupLevels: CleanupLevel[] = [];
let lastCleanupLevel: CleanupLevel | undefined;
const overlaySizeBytes = 100;
const clearSizeBytes = 50;
const codeql = createStubCodeQL({
async databaseCleanupCluster(_config, cleanupLevel) {
cleanupLevels.push(cleanupLevel);
lastCleanupLevel = cleanupLevel;
},
async databaseBundle(_databasePath, outputFilePath) {
const sizeBytes =
lastCleanupLevel === CleanupLevel.Overlay
? overlaySizeBytes
: clearSizeBytes;
fs.writeFileSync(outputFilePath, "x".repeat(sizeBytes));
},
});

const config = getTestConfig(tmpDir);
config.overlayDatabaseMode = OverlayDatabaseMode.OverlayBase;

const loggedMessages: LoggedMessage[] = [];
const results = await cleanupAndUploadDatabases(
testRepoName,
codeql,
config,
testApiDetails,
createFeatures([Feature.UploadOverlayDbToApi]),
getRecordingLogger(loggedMessages),
);

// The database should be cleaned up at the `overlay` level for the upload
// and then re-cleaned at the `clear` level to measure its size.
t.deepEqual(cleanupLevels, [CleanupLevel.Overlay, CleanupLevel.Clear]);

t.is(results.length, 1);
t.is(results[0].is_overlay_base, true);
t.is(results[0].zipped_upload_size_bytes, overlaySizeBytes);
t.is(results[0].clear_cleanup_zipped_size_bytes, clearSizeBytes);
t.is(typeof results[0].clear_cleanup_measurement_duration_ms, "number");
});
},
);

test.serial(
"Does not measure clear cleanup size for a regular (non-overlay-base) upload",
async (t) => {
await withTmpDir(async (tmpDir) => {
setupActionsVars(tmpDir, tmpDir);
sinon
.stub(actionsUtil, "getRequiredInput")
.withArgs("upload-database")
.returns("true");
sinon.stub(gitUtils, "isAnalyzingDefaultBranch").resolves(true);

await mockHttpRequests(201);

const cleanupLevels: CleanupLevel[] = [];
const codeql = createStubCodeQL({
async databaseCleanupCluster(_config, cleanupLevel) {
cleanupLevels.push(cleanupLevel);
},
async databaseBundle(_databasePath, outputFilePath) {
fs.writeFileSync(outputFilePath, "");
},
});

const results = await cleanupAndUploadDatabases(
testRepoName,
codeql,
getTestConfig(tmpDir),
testApiDetails,
createFeatures([Feature.UploadOverlayDbToApi]),
getRecordingLogger([]),
);

// A regular upload is cleaned only once, at the `clear` level.
t.deepEqual(cleanupLevels, [CleanupLevel.Clear]);
t.is(results[0].is_overlay_base, false);
t.is(results[0].clear_cleanup_zipped_size_bytes, undefined);
t.is(results[0].clear_cleanup_measurement_duration_ms, undefined);
});
},
);

test.serial("Does not measure clear cleanup size in debug mode", async (t) => {
await withTmpDir(async (tmpDir) => {
setupActionsVars(tmpDir, tmpDir);
sinon
.stub(actionsUtil, "getRequiredInput")
.withArgs("upload-database")
.returns("true");
sinon.stub(gitUtils, "isAnalyzingDefaultBranch").resolves(true);

await mockHttpRequests(201);

const cleanupLevels: CleanupLevel[] = [];
const codeql = createStubCodeQL({
async databaseCleanupCluster(_config, cleanupLevel) {
cleanupLevels.push(cleanupLevel);
},
async databaseBundle(_databasePath, outputFilePath) {
fs.writeFileSync(outputFilePath, "");
},
});

const config = getTestConfig(tmpDir);
config.overlayDatabaseMode = OverlayDatabaseMode.OverlayBase;
config.debugMode = true;

const results = await cleanupAndUploadDatabases(
testRepoName,
codeql,
config,
testApiDetails,
createFeatures([Feature.UploadOverlayDbToApi]),
getRecordingLogger([]),
);

// In debug mode we clean up at the `overlay` level for the upload but skip
// the additional `clear` cleanup, to preserve the database for debugging.
t.deepEqual(cleanupLevels, [CleanupLevel.Overlay]);
t.is(results[0].is_overlay_base, true);
t.is(results[0].clear_cleanup_zipped_size_bytes, undefined);
t.is(results[0].clear_cleanup_measurement_duration_ms, undefined);
});
});
90 changes: 90 additions & 0 deletions src/database-upload.ts
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,19 @@ export interface DatabaseUploadResult {
zipped_upload_size_bytes?: number;
/** Whether the uploaded database is an overlay base. */
is_overlay_base?: boolean;
/**
* For overlay-base uploads only: the size in bytes that the zipped database
* would have been if it had been cleaned at the `clear` cleanup level instead
* of the `overlay` level.
*/
clear_cleanup_zipped_size_bytes?: number;
/**
* For overlay-base uploads only: the time in milliseconds spent measuring the
* `clear` cleanup size (cleaning up the cluster at the `clear` level and
* bundling each database). This is a cluster-wide measurement, so it is the
* same for every language in a run.
*/
clear_cleanup_measurement_duration_ms?: number;
/** Time taken to upload database in milliseconds. */
upload_duration_ms?: number;
/** If there was an error during database upload, this is its message. */
Expand Down Expand Up @@ -156,9 +169,86 @@ export async function cleanupAndUploadDatabases(
});
}
}

// When we upload an overlay-base database, we cleaned the databases at the `overlay` level, which
// retains more data than the `clear` level used for regular uploads. Measure what the zipped size
// would have been at the `clear` level too, so we can compare the storage cost of overlay-base
// databases against regular databases for the same repository.
//
// We skip this in debug mode, where the databases are preserved and uploaded as debug artifacts,
// since cleaning them up at the `clear` level would discard data that is useful for debugging.
if (shouldUploadOverlayBase && !config.debugMode) {
await withGroupAsync(
"Measuring database size at the clear cleanup level",
() => recordClearCleanupSizes(codeql, config, reports, logger),
);
}

return reports;
}

/**
* Cleans up the databases at the `clear` cleanup level and records the resulting zipped size for
* each language in `clear_cleanup_zipped_size_bytes`, along with the time spent taking the
* measurement in `clear_cleanup_measurement_duration_ms`.
*
* This mutates the entries of `reports` in place. It must run only after all overlay-base uploads
* have completed, since the `clear` cleanup discards overlay data that the uploaded database
* depends on.
Comment thread
henrymercer marked this conversation as resolved.
Resolved
*
* Failures here are non-fatal: this is telemetry-only, so we log and move on rather than failing
* the workflow.
*/
async function recordClearCleanupSizes(
codeql: CodeQL,
config: Config,
reports: DatabaseUploadResult[],
logger: Logger,
): Promise<void> {
const startTime = performance.now();
try {
try {
await codeql.databaseCleanupCluster(config, CleanupLevel.Clear);
} catch (e) {
logger.warning(
`Failed to clean up databases at the '${CleanupLevel.Clear}' level for ` +
`size measurement: ${util.getErrorMessage(e)}`,
);
return;
}

for (const language of config.languages) {
const report = reports.find((r) => r.language === language);
if (report === undefined) {
continue;
}
try {
const bundledDb = await bundleDb(config, language, codeql, language, {
includeDiagnostics: false,
});
report.clear_cleanup_zipped_size_bytes = fs.statSync(bundledDb).size;
logger.debug(
`Database for ${language} is ` +
`${report.clear_cleanup_zipped_size_bytes} bytes zipped at the ` +
`'${CleanupLevel.Clear}' cleanup level ` +
`(vs. ${report.zipped_upload_size_bytes} bytes at the ` +
`'${CleanupLevel.Overlay}' level).`,
);
Comment thread
henrymercer marked this conversation as resolved.
Resolved
} catch (e) {
logger.warning(
`Failed to measure the '${CleanupLevel.Clear}' cleanup database size ` +
`for ${language}: ${util.getErrorMessage(e)}`,
);
}
}
} finally {
const durationMs = performance.now() - startTime;
for (const report of reports) {
report.clear_cleanup_measurement_duration_ms = durationMs;
}
}
Comment thread
henrymercer marked this conversation as resolved.
Resolved
Comment thread
henrymercer marked this conversation as resolved.
Resolved
}

/**
* Uploads a bundled database to the GitHub API.
*
Expand Down
Loading