bench: measure the path a request actually takes (#195)

The benchmark timed only `.toSQL()`, which recompiles per call — the right measurement for a query whose shape varies per request, and the wrong one for the shape a request usually has.

    select-all              1438ns -> 51ns    28x
    select-where-eq         3291ns -> 58ns    57x
    select-where-deep-and  10614ns -> 148ns   72x
    insert-values           3356ns -> 63ns    53x
    update-where            3481ns -> 59ns    59x

Also records what a loaded machine does to the perf guard: 9, then 13, then 12 of 48 failing with a different set each run, every failure 0–5% under its floor. At load 2 it passes 48/48.

Author: productdevbook

bench/README.md

@@ -10,7 +10,8 @@ Prisma is intentionally **not** included: Prisma is a code-gen + engine layer ra
 
 ```bash
 pnpm install
-pnpm vitest bench --run bench/compile.bench.ts
+pnpm vitest bench --run bench/compile.bench.ts   # sumak vs kysely vs drizzle
+pnpm vitest bench --run bench/prepared.bench.ts  # the two paths within sumak
 ```
 
 ## Smoke test
@@ -95,6 +96,33 @@ wave (PRs #155, #156, #162, #164, #165, #166): `REGEXP_REPLACE`,
 AST node (extract-field, position-IN keyword form, inlined pattern /
 unit literals) and the competitors fall back to raw template literals.
 
+## The two paths — `bench/prepared.bench.ts`
+
+`compile.bench.ts` above times the whole pipeline, which is what `.toSQL()`
+runs on every call. That is the right measurement for a query whose shape
+genuinely varies per request, and the wrong one for the shape a request
+usually has, where the query was written once and only the values change.
+Nothing measured that until this file existed.
+
+| scenario                | `toSQL()` | `toCompiled()` |     |
+| ----------------------- | --------: | -------------: | --: |
+| `select-all`            |    1438ns |           51ns | 28× |
+| `select-where-eq`       |    3291ns |           58ns | 57× |
+| `select-where-deep-and` |   10614ns |          148ns | 72× |
+| `insert-values`         |    3356ns |           63ns | 53× |
+| `update-where`          |    3481ns |           59ns | 59× |
+
+Everything the pipeline does — plugin transforms, hooks, normalize, optimize,
+print — happens once, at definition. What is left is filling the parameter
+array.
+
+Two things the table does not show. A compiled query's SQL text is fixed, so
+`sumak/drivers/pg` sends it as a named prepared statement and the server keeps
+the plan: 243µs against 303µs for the same query, measured with `PREPARE` /
+`EXECUTE` against pglite. And end to end a single query costs ~330µs either
+way, because the compile this removes is ~1% of it — the split pays on a cold
+start, under CPU pressure, and through that plan reuse.
+
 ## Plugin overhead microbench
 
 `bench/plugin.bench.ts` measures the per-compile cost of each built-in

bench/prepared.bench.ts

@@ -0,0 +1,121 @@
+import { bench, describe } from "vitest"
+
+import { placeholder } from "../src/builder/compiled.ts"
+import { and } from "../src/builder/eb.ts"
+import { pgDialect } from "../src/dialect/pg.ts"
+import { sumak } from "../src/sumak.ts"
+import { tables } from "./src/schema-sumak.ts"
+
+/**
+ * What a request pays, on each of the two paths.
+ *
+ * `compile.bench.ts` times the whole pipeline, which is what `.toSQL()` runs on
+ * every call. That is the right measurement for a query whose shape genuinely
+ * varies per request. It is the wrong one for the shape a request usually has,
+ * where the query was written once and only the values change — and until this
+ * file existed, nothing measured that at all.
+ *
+ * Run with:
+ *   pnpm vitest bench --run bench/prepared.bench.ts
+ */
+
+const db = sumak({ dialect: pgDialect(), tables })
+
+const scenarios = [
+  {
+    name: "select-all",
+    live: () => db.selectFrom("users").selectAll().toSQL(),
+    prepared: db.selectFrom("users").selectAll().toCompiled(),
+    args: {},
+  },
+  {
+    name: "select-where-eq",
+    live: () =>
+      db
+        .selectFrom("users")
+        .select("id", "name")
+        .where(({ id }) => id.eq(1))
+        .toSQL(),
+    prepared: db
+      .selectFrom("users")
+      .select("id", "name")
+      .where(({ id }) => id.eq(placeholder("id") as never))
+      .toCompiled<{ id: number }>(),
+    args: { id: 1 },
+  },
+  {
+    name: "select-where-deep-and",
+    live: () =>
+      db
+        .selectFrom("posts")
+        .selectAll()
+        .where(({ authorId, published, title, body, id }) =>
+          and(authorId.eq(1), published.gt(0), title.neq(""), body.neq(""), id.gt(0)),
+        )
+        .toSQL(),
+    prepared: db
+      .selectFrom("posts")
+      .selectAll()
+      .where(({ authorId, published, title, body, id }) =>
+        and(
+          authorId.eq(placeholder("authorId") as never),
+          published.gt(placeholder("published") as never),
+          title.neq(placeholder("title") as never),
+          body.neq(placeholder("body") as never),
+          id.gt(placeholder("id") as never),
+        ),
+      )
+      .toCompiled<{
+        authorId: number
+        published: number
+        title: string
+        body: string
+        id: number
+      }>(),
+    args: { authorId: 1, published: 0, title: "", body: "", id: 0 },
+  },
+  {
+    name: "insert-values",
+    live: () =>
+      db
+        .insertInto("users")
+        .values({ id: 1, name: "a", email: "a@x.io", createdAt: new Date(0) })
+        .toSQL(),
+    prepared: db
+      .insertInto("users")
+      .values({
+        id: placeholder("id") as never,
+        name: placeholder("name") as never,
+        email: placeholder("email") as never,
+        createdAt: placeholder("createdAt") as never,
+      })
+      .toCompiled<{ id: number; name: string; email: string; createdAt: Date }>(),
+    args: { id: 1, name: "a", email: "a@x.io", createdAt: new Date(0) },
+  },
+  {
+    name: "update-where",
+    live: () =>
+      db
+        .update("users")
+        .set({ name: "a" })
+        .where(({ id }) => id.eq(1))
+        .toSQL(),
+    prepared: db
+      .update("users")
+      .set({ name: placeholder("name") as never })
+      .where(({ id }) => id.eq(placeholder("id") as never))
+      .toCompiled<{ name: string; id: number }>(),
+    args: { name: "a", id: 1 },
+  },
+] as const
+
+for (const sc of scenarios) {
+  describe(sc.name, () => {
+    bench("toSQL — recompiled per call", () => {
+      sc.live()
+    })
+    bench("toCompiled — parameters only", () => {
+      ;(sc.prepared as (args: unknown) => unknown)(sc.args)
+    })
+  })
+}