2626// node scripts/compilerBenchmarks.js prepare <checkout>
2727// node scripts/compilerBenchmarks.js measure <checkout> <out.json> [--instructions]
2828// [--project <checkout>] [--playground]
29+ // node scripts/compilerBenchmarks.js time <base checkout> <head checkout>
30+ // <out.json> [--rounds <n>]
2931// node scripts/compilerBenchmarks.js compare <base.json> <head.json>
3032// [--threshold <percent>] [--summary <file.md>]
3133//
4648// allocated words only.
4749// With `--playground`, it measures the size of the playground's compiler.
4850//
51+ // `time` builds a generated project with the build systems of two prepared
52+ // checkouts in turn, writes the wall-clock times to <out.json> and prints
53+ // their medians and the median ratio of the head's time to the base's with
54+ // a 95% confidence interval (see timeBuilds). It is the one benchmark that
55+ // measures time, run by hand to indicate the effect of a change that does
56+ // the same work sooner; CI doesn't run it.
57+ //
4958// `compare` reports the changes between two measurements as Markdown,
5059// improvements and regressions alike, one section per count, and exits with
5160// 1 when a count grew by more than the threshold. Runtime and Belt files,
@@ -419,6 +428,50 @@ export const generatedProjects = {
419428 } ,
420429} ;
421430
431+ /**
432+ * The project the wall-time benchmark builds: a long chain of modules, each
433+ * using the one before it, and a module beside each that uses it. A build
434+ * spends most of its time on the chain with most of its worker domains
435+ * idle, which is what a change to the build's scheduling would improve.
436+ *
437+ * @returns {GeneratedProject }
438+ */
439+ export function chainProject ( ) {
440+ const length = 150 ;
441+ /** @type {Record<string, string> } */
442+ const files = { "rescript.json" : projectConfig ( "bench-chain" ) } ;
443+ for ( let i = 0 ; i < length ; i ++ ) {
444+ const lines =
445+ i === 0
446+ ? [
447+ "type t = {depth: int, items: array<int>}" ,
448+ "let base = {depth: 0, items: []}" ,
449+ ]
450+ : [
451+ `type t = {depth: int, items: array<int>, prev: Chain${ i - 1 } .t}` ,
452+ `let base = {depth: ${ i } , items: [${ i } ], prev: Chain${ i - 1 } .base}` ,
453+ ] ;
454+ // Bodies with some work for the type checker and code generation.
455+ for ( let f = 0 ; f < 20 ; f ++ ) {
456+ lines . push (
457+ `let f${ f } = (r: t, x: int) =>` ,
458+ " switch mod(x, 4) {" ,
459+ ` | 0 => r.items->Array.map(y => y + x + ${ f } )->Array.length` ,
460+ ` | 1 => r.depth * ${ f + 1 } + x` ,
461+ ` | 2 => r.items->Array.reduce(0, (acc, y) => acc + y * ${ f } )` ,
462+ " | _ => -1" ,
463+ " }" ,
464+ ) ;
465+ }
466+ files [ `src/Chain${ i } .res` ] = `${ lines . join ( "\n" ) } \n` ;
467+ files [ `src/Side${ i } .res` ] = [
468+ `let total = Chain${ i } .f0(Chain${ i } .base, ${ i } ) + Chain${ i } .f1(Chain${ i } .base, 1)` ,
469+ "" ,
470+ ] . join ( "\n" ) ;
471+ }
472+ return { files, edit : `src/Chain${ length - 1 } .res` } ;
473+ }
474+
422475/**
423476 * The runtime and Belt sources of a checkout, by package.
424477 *
@@ -835,6 +888,175 @@ export function measure(checkout, { instructions, project, playground }) {
835888 return results ;
836889}
837890
891+ /**
892+ * How many times the wall-time benchmark builds its project with each
893+ * compiler, after one build with each that isn't counted.
894+ */
895+ const timingRounds = 10 ;
896+
897+ /**
898+ * How long one of the wall-time benchmark's builds may take, in
899+ * milliseconds, many times as long as a build takes, so that a build that
900+ * hangs, as it could with a deadlock in the scheduling, ends the benchmark.
901+ */
902+ const timingBuildTimeout = 2 * 60 * 1000 ;
903+
904+ /**
905+ * The wall-clock times of the wall-time benchmark's builds, in
906+ * milliseconds, by benchmark: round i built with the base's compiler and
907+ * the head's, one right after the other.
908+ *
909+ * @typedef {Record<string, { base: number[], head: number[] }> } Timings
910+ */
911+
912+ /**
913+ * Builds the wall-time benchmark's project clean with the build systems of
914+ * two checkouts in turn, on as many worker domains as the build system
915+ * chooses, and returns how long each build took. The two compilers take
916+ * turns going first, so that a machine that gets slower or faster during
917+ * the run changes both alike.
918+ *
919+ * Wall-clock time on a loaded machine varies by several percent from one
920+ * build to the next, and more over minutes, so this only indicates the
921+ * effect of a change that counting doesn't show, such as starting a module
922+ * earlier: building the same modules with idle domains doing less is
923+ * faster, but allocates and executes as much or more.
924+ *
925+ * Each checkout builds a copy of the project of its own, with its own
926+ * runtime, both under paths of the same length.
927+ *
928+ * @param {string } baseCheckout
929+ * @param {string } headCheckout
930+ * @param {number } rounds
931+ * @returns {Timings }
932+ */
933+ export function timeBuilds ( baseCheckout , headCheckout , rounds ) {
934+ const workDir = path . join ( os . tmpdir ( ) , "compiler-bench-time" ) ;
935+ fs . rmSync ( workDir , { recursive : true , force : true } ) ;
936+ try {
937+ const project = chainProject ( ) ;
938+ const sides = [
939+ { name : "base" , checkout : baseCheckout } ,
940+ { name : "head" , checkout : headCheckout } ,
941+ ] . map ( side => {
942+ const dir = path . join ( workDir , side . name ) ;
943+ writeProject ( project , dir , side . checkout ) ;
944+ return {
945+ ...side ,
946+ dir,
947+ rescript : path . join ( side . checkout , relativeBinDir , "rescript.exe" ) ,
948+ /** @type {number[] } */
949+ times : [ ] ,
950+ } ;
951+ } ) ;
952+ /** @param {(typeof sides)[number] } side */
953+ const build = side => {
954+ fs . rmSync ( path . join ( side . dir , "lib" ) , { recursive : true , force : true } ) ;
955+ const start = process . hrtime . bigint ( ) ;
956+ execFileSync ( side . rescript , [ "build" ] , {
957+ cwd : side . dir ,
958+ stdio : "pipe" ,
959+ timeout : timingBuildTimeout ,
960+ killSignal : "SIGKILL" ,
961+ } ) ;
962+ return Number ( process . hrtime . bigint ( ) - start ) / 1e6 ;
963+ } ;
964+ for ( const side of sides ) build ( side ) ;
965+ for ( let round = 0 ; round < rounds ; round ++ ) {
966+ const order = round % 2 === 0 ? sides : [ ...sides ] . reverse ( ) ;
967+ for ( const side of order ) side . times . push ( build ( side ) ) ;
968+ }
969+ return {
970+ "time/Chain/clean" : { base : sides [ 0 ] . times , head : sides [ 1 ] . times } ,
971+ } ;
972+ } finally {
973+ fs . rmSync ( workDir , { recursive : true , force : true } ) ;
974+ }
975+ }
976+
977+ /**
978+ * The median of numbers.
979+ *
980+ * @param {number[] } values
981+ */
982+ export function median ( values ) {
983+ const sorted = [ ...values ] . sort ( ( a , b ) => a - b ) ;
984+ const middle = Math . floor ( sorted . length / 2 ) ;
985+ return sorted . length % 2 === 1
986+ ? sorted [ middle ]
987+ : ( sorted [ middle - 1 ] + sorted [ middle ] ) / 2 ;
988+ }
989+
990+ /**
991+ * A pseudo-random number generator (mulberry32), so that the confidence
992+ * interval of the same times is the same in every run.
993+ *
994+ * @param {number } seed
995+ * @returns {() => number } numbers in [0, 1)
996+ */
997+ function randomNumbers ( seed ) {
998+ let state = seed >>> 0 ;
999+ return ( ) => {
1000+ state = ( state + 0x6d2b79f5 ) >>> 0 ;
1001+ let t = state ;
1002+ t = Math . imul ( t ^ ( t >>> 15 ) , t | 1 ) ;
1003+ t ^= t + Math . imul ( t ^ ( t >>> 7 ) , t | 61 ) ;
1004+ return ( ( t ^ ( t >>> 14 ) ) >>> 0 ) / 4294967296 ;
1005+ } ;
1006+ }
1007+
1008+ /**
1009+ * The median of the head's time divided by the base's time in the same
1010+ * round, with a 95% confidence interval from a percentile bootstrap: the
1011+ * medians of `resamples` samples of the rounds drawn with replacement, of
1012+ * which 2.5% are below `low` and 2.5% above `high`. Pairing the rounds
1013+ * leaves out a drift that slows both builds of a round alike.
1014+ *
1015+ * @param {number[] } base
1016+ * @param {number[] } head
1017+ * @param {number } [resamples]
1018+ * @returns {{ ratio: number, low: number, high: number, rounds: number } }
1019+ */
1020+ export function timeRatio ( base , head , resamples = 2000 ) {
1021+ if ( base . length !== head . length || base . length === 0 ) {
1022+ throw new Error ( "The base and the head need the same rounds." ) ;
1023+ }
1024+ const ratios = base . map ( ( time , i ) => head [ i ] / time ) ;
1025+ const random = randomNumbers ( 1 ) ;
1026+ const medians = Array . from ( { length : resamples } , ( ) =>
1027+ median ( ratios . map ( ( ) => ratios [ Math . floor ( random ( ) * ratios . length ) ] ) ) ,
1028+ ) . sort ( ( a , b ) => a - b ) ;
1029+ /** @param {number } q */
1030+ const quantile = q =>
1031+ medians [ Math . min ( resamples - 1 , Math . floor ( q * resamples ) ) ] ;
1032+ return {
1033+ ratio : median ( ratios ) ,
1034+ low : quantile ( 0.025 ) ,
1035+ high : quantile ( 0.975 ) ,
1036+ rounds : ratios . length ,
1037+ } ;
1038+ }
1039+
1040+ /**
1041+ * Summarizes the wall-time benchmarks as lines of text: for each, the
1042+ * base's and the head's median times, and the median of each round's head
1043+ * time over its base time, which isn't the ratio of the two medians, with
1044+ * its 95% confidence interval.
1045+ *
1046+ * @param {Timings } timings
1047+ * @returns {string[] }
1048+ */
1049+ export function renderTimings ( timings ) {
1050+ return Object . entries ( timings ) . map ( ( [ name , { base, head } ] ) => {
1051+ const { ratio, low, high, rounds } = timeRatio ( base , head ) ;
1052+ return [
1053+ `${ name } (${ rounds } ${ rounds === 1 ? "round" : "rounds" } ):` ,
1054+ `base ${ median ( base ) . toFixed ( 0 ) } ms, head ${ median ( head ) . toFixed ( 0 ) } ms (medians),` ,
1055+ `head / base ${ ratio . toFixed ( 3 ) } (95% CI ${ low . toFixed ( 3 ) } to ${ high . toFixed ( 3 ) } )` ,
1056+ ] . join ( " " ) ;
1057+ } ) ;
1058+ }
1059+
8381060/**
8391061 * @typedef {{
8401062 * name: string,
@@ -1105,7 +1327,11 @@ export function compareAll(base, head, threshold) {
11051327 ) ;
11061328}
11071329
1108- /** @param {string[] } args */
1330+ /**
1331+ * Runs a command and returns its exit status.
1332+ *
1333+ * @param {string[] } args
1334+ */
11091335function main ( args ) {
11101336 const [ command , ...rest ] = args ;
11111337 if ( command === "prepare" && rest . length === 1 ) {
@@ -1133,6 +1359,25 @@ function main(args) {
11331359 console . log ( `Measured ${ Object . keys ( results ) . length } benchmarks.` ) ;
11341360 return 0 ;
11351361 }
1362+ if ( command === "time" && rest . length >= 3 ) {
1363+ const [ baseCheckout , headCheckout , out , ...options ] = rest ;
1364+ let rounds = timingRounds ;
1365+ for ( let i = 0 ; i < options . length ; i += 2 ) {
1366+ if ( options [ i ] === "--rounds" ) rounds = Number ( options [ i + 1 ] ) ;
1367+ else throw new Error ( `Unknown option ${ options [ i ] } ` ) ;
1368+ }
1369+ if ( ! Number . isInteger ( rounds ) || rounds < 1 ) {
1370+ throw new Error ( "--rounds takes a positive integer." ) ;
1371+ }
1372+ const timings = timeBuilds (
1373+ path . resolve ( baseCheckout ) ,
1374+ path . resolve ( headCheckout ) ,
1375+ rounds ,
1376+ ) ;
1377+ fs . writeFileSync ( out , `${ JSON . stringify ( timings , null , 2 ) } \n` ) ;
1378+ for ( const line of renderTimings ( timings ) ) console . log ( line ) ;
1379+ return 0 ;
1380+ }
11361381 if ( command === "compare" && rest . length >= 2 ) {
11371382 const [ basePath , headPath , ...options ] = rest ;
11381383 let threshold = 1 ;
@@ -1154,6 +1399,8 @@ function main(args) {
11541399 "Usage: compilerBenchmarks.js prepare <checkout>\n" +
11551400 " compilerBenchmarks.js measure <checkout> <out.json> [--instructions]\n" +
11561401 " [--project <checkout>] [--playground]\n" +
1402+ " compilerBenchmarks.js time <base checkout> <head checkout> <out.json>\n" +
1403+ " [--rounds <n>]\n" +
11571404 " compilerBenchmarks.js compare <base.json> <head.json> [--threshold <percent>]\n" +
11581405 " [--summary <file.md>]" ,
11591406 ) ;
0 commit comments