Merge pull request #16 from ngallodev/feat/output-token-budget

Feat/output token budget

Author: ngallodev

.github/workflows/_lint.yml

@@ -71,9 +71,9 @@ jobs:
       - name: Install LLVM 21 (pinned analyzer toolchain)
         run: |
           wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc
-          echo "deb http://apt.llvm.org/noble/ llvm-toolchain-noble-22 main" | sudo tee /etc/apt/sources.list.d/llvm-22.list
+          echo "deb http://apt.llvm.org/noble/ llvm-toolchain-noble-21 main" | sudo tee /etc/apt/sources.list.d/llvm-21.list
           sudo apt-get update
-          sudo apt-get install -y clang-tidy-22
+          sudo apt-get install -y clang-tidy-21
 
       - name: Memory-analyzer gate
-        run: scripts/ci/lint-mem.sh clang-tidy-22
+        run: scripts/ci/lint-mem.sh clang-tidy-21

.github/workflows/_test.yml

@@ -119,6 +119,32 @@ jobs:
         if: startsWith(matrix.os, 'ubuntu')
         run: sudo apt-get update && sudo apt-get install -y zlib1g-dev ccache
 
+      # Ubuntu 23.10+ ships kernel.apparmor_restrict_unprivileged_userns=1, which
+      # blocks unshare(CLONE_NEWUSER) for unprivileged users. Without this, the
+      # #1830 single-uid-userns smoke test SKIPS on every 24.04 leg, and its only
+      # real-namespace coverage lived on the two ubuntu-22.04 legs -- which are
+      # broad-matrix only (never PR CI) and which GitHub retires on 2027-04-17.
+      # A deliberate relaxation of a security check was therefore guarded by a
+      # test that ran in no PR and was on a clock. Hosted runners have
+      # passwordless sudo, so one sysctl puts it on the CORE matrix instead.
+      # Best-effort: a runner that refuses simply returns to skipping.
+      - name: Permit unprivileged user namespaces (Ubuntu)
+        if: startsWith(matrix.os, 'ubuntu')
+        run: |
+          sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 || \
+            echo "could not lift the userns restriction; namespace tests will skip"
+          # Then SAY whether it worked. The runner prints per-shard aggregates
+          # ("N passed, M skipped") and never individual test names, so a
+          # namespace test that silently returns to skipping is invisible in a
+          # green log -- the same false assurance that let #1830 ship with a
+          # test that never executed. This line is the evidence. Probed exactly
+          # as the test does it: unprivileged, no sudo.
+          if unshare -U -r true 2>/dev/null; then
+            echo "CBM_USERNS_AVAILABLE=yes"
+          else
+            echo "CBM_USERNS_AVAILABLE=no (namespace-dependent tests will skip)"
+          fi
+
       - name: Install ccache (macOS)
         if: startsWith(matrix.os, 'macos')
         run: command -v ccache >/dev/null 2>&1 || brew install ccache
@@ -324,8 +350,14 @@ jobs:
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
 
+      # Pinned: the unversioned `llvm` formula tracks Homebrew stable, so the
+      # leg's compiler changed under us when stable moved 22 -> 23 (a new
+      # -Wunused-but-set-global under -Werror broke the build on whichever
+      # runner drew the 23 bottle, and passed on the ones that drew 22). A
+      # test verdict must be a function of the code, not of the image snapshot;
+      # moving to a newer major is a deliberate one-line bump here.
       - name: Install LSan-capable toolchain
-        run: brew install llvm ccache
+        run: brew install llvm@22 ccache
 
       - name: Compiler cache (content-verified, restore)
         uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
@@ -340,7 +372,7 @@ jobs:
         env:
           ASAN_OPTIONS: "detect_leaks=1:halt_on_error=1"
         run: |
-          LLVM_PREFIX="$(brew --prefix llvm)"
+          LLVM_PREFIX="$(brew --prefix llvm@22)"
           scripts/test.sh CC="$LLVM_PREFIX/bin/clang" CXX="$LLVM_PREFIX/bin/clang++"
 
       - name: Compiler cache (save)

.github/workflows/pages.yml

@@ -41,4 +41,4 @@ jobs:
         with:
           path: ./docs
       - id: deployment
-        uses: actions/deploy-pages@cd2ce8fcbc39b97be8ca5fce6e763baed58fa128 # v5.0.0
+        uses: actions/deploy-pages@368f82528645a54fb793d4d04e342629a3f51346 # v5.0.1

.github/workflows/release.yml

@@ -288,7 +288,7 @@ jobs:
           git tag "$VERSION" "$GITHUB_SHA"
           git push origin "refs/tags/$VERSION"
 
-      - uses: softprops/action-gh-release@3d0d9888cb7fd7b750713d6e236d1fcb99157228 # v2
+      - uses: softprops/action-gh-release@efb35369e0ad2afab669f228072c1b0d510eae64 # v3.0.3
         with:
           tag_name: ${{ inputs.version }}
           draft: true

Makefile.cbm

@@ -100,9 +100,11 @@ KOTLIN_DEDUP_TEST_DEFINE = -DCBM_KOTLIN_DEDUP_TEST_API=1
 CALL_REFERENCE_LOOKUP_TEST_DEFINE = -DCBM_CALL_REFERENCE_LOOKUP_TEST_API=1
 INCREMENTAL_TEST_DEFINE = -DCBM_INCREMENTAL_TEST_API=1
 CFLAGS_TEST_EXTRA ?=
+COVERAGE_MARKER_TEST_DEFINE = -DCBM_COVERAGE_MARKER_TEST_API=1
 CFLAGS_TEST = $(CFLAGS_COMMON) $(EDITOR_TEST_DEFINES) $(SANITIZED_DEFINE) \
 	$(KOTLIN_DEDUP_TEST_DEFINE) $(CALL_REFERENCE_LOOKUP_TEST_DEFINE) \
-	$(INCREMENTAL_TEST_DEFINE) -g -O1 $(SANITIZE) $(CFLAGS_TEST_EXTRA)
+	$(INCREMENTAL_TEST_DEFINE) $(COVERAGE_MARKER_TEST_DEFINE) -g -O1 $(SANITIZE) \
+	$(CFLAGS_TEST_EXTRA)
 CXXFLAGS_TEST = $(CXXFLAGS_COMMON) $(SANITIZED_DEFINE) -g -O1 $(SANITIZE) $(CXX_STDLIB_FLAGS)
 
 # TSan (can't combine with ASan)
@@ -120,6 +122,7 @@ TSAN_SANITIZE = -fsanitize=thread -fno-omit-frame-pointer
 # macro of ours.
 CFLAGS_TSAN = $(CFLAGS_COMMON) $(EDITOR_TEST_DEFINES) $(KOTLIN_DEDUP_TEST_DEFINE) \
 	$(CALL_REFERENCE_LOOKUP_TEST_DEFINE) $(INCREMENTAL_TEST_DEFINE) \
+	$(COVERAGE_MARKER_TEST_DEFINE) \
 	-DCBM_SANITIZED_BUILD=1 -g -O1 $(TSAN_SANITIZE)
 CXXFLAGS_TSAN = $(CXXFLAGS_COMMON) -DCBM_SANITIZED_BUILD=1 -g -O1 \
                 $(TSAN_SANITIZE)
@@ -183,10 +186,35 @@ endif
 # ~259 MB of tree-sitter parse tables was mapped executable at runtime while the
 # amd64 build of the same source mapped them R only. That is a large ROP gadget
 # surface for data that is never executed, and it is invisible to section flags.
+#
+# -z relro + -z now finish the same job on the OTHER writable-but-shouldn't-be
+# region: the GOT. Both are free at runtime here (a static binary resolves
+# everything at link time, so there is no lazy binding left to pay for), and
+# they are added because the SHIPPED artifact measurably lacked the property --
+# not because the linker accepts the flags. Measured on this binary, built the
+# way .github/workflows/_build.yml builds the Linux release (STATIC=1, gcc 13.3
+# / GNU ld 2.42, Ubuntu 24.04 aarch64); PT_GNU_RELRO ends at 0x11DA0000 in both:
+#
+#   without -z now:  .got 0x11d9ec20+0x13c8, .got.plt 0x11d9ffe8+0x40
+#                    -> .got.plt ends at 0x11DA0028, i.e. 40 bytes PAST the
+#                       window: those GOT slots stayed writable for the whole
+#                       process lifetime, in every Linux binary we have shipped
+#   with    -z now:  .got.plt folded into .got (0x11d9ec20+0x13c8, ends
+#                    0x11D9FFE8) -> the whole GOT is inside the window and is
+#                    re-mapped read-only after startup
+#
+# -z relro is already this toolchain's default and changes nothing here; it is
+# named anyway so the property stops depending on one distro's spec file (the
+# musl/portable and glibc-floor images are different toolchains, and a default
+# is not a guarantee). A1c/A1d in scripts/ci/check-binary-composition.sh assert
+# the OUTCOME on the produced binary, because a flag the compiler accepts is
+# not evidence that the artifact gained anything -- the same reason A1 exists
+# next to the .note.GNU-stack annotation.
 ELF_HARDENING_FLAGS :=
 ifeq ($(IS_LINUX),yes)
 ifneq ($(IS_MINGW),yes)
-ELF_HARDENING_FLAGS := -Wl,-z,noexecstack -Wl,-z,separate-code
+ELF_HARDENING_FLAGS := -Wl,-z,noexecstack -Wl,-z,separate-code \
+                       -Wl,-z,relro -Wl,-z,now
 endif
 endif
 
@@ -225,6 +253,7 @@ FOUNDATION_SRCS = \
     src/foundation/compat_fs.c \
     src/foundation/compat_regex.c \
     src/foundation/mem.c \
+    src/foundation/mem_core.c \
     src/foundation/diagnostics.c \
     src/foundation/profile.c \
     src/foundation/dump_verify.c \
@@ -253,6 +282,8 @@ EXTRACTION_SRCS = \
     $(CBM_DIR)/extract_k8s.c \
     $(CBM_DIR)/extract_dbt.c \
     $(CBM_DIR)/helpers.c \
+    $(CBM_DIR)/result_compact.c \
+    $(CBM_DIR)/result_spill.c \
     $(CBM_DIR)/lang_specs.c \
     $(CBM_DIR)/macro_table.c \
     $(CBM_DIR)/iris_export_xml.c \
@@ -304,7 +335,7 @@ STORE_SRCS = src/store/store.c
 # Cypher module (new)
 CYPHER_SRCS = src/cypher/cypher.c
 
-OPERATION_SRCS = src/operations/registry.c src/operations/json_args.c src/operations/reliability_events.c src/operations/execute.c src/operations/result_wire.c src/operations/tool_catalog.c src/operations/index_admission.c src/operations/store_host.c src/operations/session_state.c src/operations/mutation.c src/operations/adr.c src/operations/cross_repo.c src/operations/index.c src/operations/project_arg.c src/operations/index_supervisor.c src/operations/command_runner.c src/operations/read.c src/operations/coverage.c src/operations/snippet.c src/operations/search.c src/operations/trace.c src/operations/trace_ingest.c src/operations/schema.c src/operations/query.c src/operations/architecture.c src/operations/changes.c src/operations/source_search.c src/operations/file_outline.c src/operations/compare.c src/operations/compact_out.c
+OPERATION_SRCS = src/operations/registry.c src/operations/json_args.c src/operations/reliability_events.c src/operations/execute.c src/operations/result_wire.c src/operations/output_budget.c src/operations/tool_catalog.c src/operations/index_admission.c src/operations/store_host.c src/operations/session_state.c src/operations/mutation.c src/operations/adr.c src/operations/cross_repo.c src/operations/index.c src/operations/project_arg.c src/operations/index_supervisor.c src/operations/command_runner.c src/operations/read.c src/operations/coverage.c src/operations/snippet.c src/operations/search.c src/operations/trace.c src/operations/trace_ingest.c src/operations/schema.c src/operations/query.c src/operations/architecture.c src/operations/changes.c src/operations/source_search.c src/operations/file_outline.c src/operations/compare.c src/operations/compact_out.c
 
 # Shared daemon coordination and local transport
 DAEMON_SRCS = \
@@ -684,6 +715,7 @@ TEST_REPRO_SRCS = \
     tests/repro/repro_issue787.c \
     tests/repro/repro_issue842.c \
     tests/repro/repro_issue964.c \
+    tests/repro/repro_issue1692.c \
     tests/repro/repro_invariant_calls.c \
     tests/repro/repro_invariant_graph.c \
     tests/repro/repro_invariant_breadth.c \
@@ -1182,8 +1214,17 @@ lint-no-suppress:
 lint: lint-tidy lint-cppcheck lint-format lint-no-suppress
 	@echo "=== All linters passed ==="
 
+# Memory-core linter: memory is allocated through src/foundation/mem_core.h,
+# not through raw malloc/calloc/realloc/free/strdup. A checked-in baseline
+# (scripts/memory-core-baseline.txt) records each file's raw-site count and the
+# gate goes red the moment any file grows. Files only ever go down; lower the
+# baseline line in the same change that migrates the file.
+lint-memory-core:
+	@echo "=== memory-core linter ==="
+	@python3 scripts/lint-memory-core.py
+
 # CI linters (no clang-tidy — platform-dependent, enforced locally via pre-commit)
-lint-ci: lint-cppcheck lint-format lint-no-suppress
+lint-ci: lint-cppcheck lint-format lint-no-suppress lint-memory-core
 	@echo "=== CI linters passed ==="
 
 # ── Local memory-diagnostic lanes (not PR-CI gates by decision: the diag

README.md

@@ -221,6 +221,7 @@ Design rules:
 
 - stdout is reserved for command results;
 - progress and diagnostics belong on stderr;
+- `cli --progress` forces lifecycle progress when stderr is redirected, `cli --quiet` disables automatic terminal progress and ordinary diagnostics while keeping errors (it cannot be combined with `--progress` or `cli --verbose`), and `cli --verbose` includes routine informational logs;
 - errors produce non-zero exit codes;
 - canonical JSON hides the historical transport envelope;
 - result ordering/pagination remain explicit where the underlying operation supports them.
@@ -247,6 +248,18 @@ The upstream indexing engine is intentionally preserved through the CLI-first mi
 
 The first vertical slice does **not** rewrite the graph schema, parser pipeline, store format, or existing indexes.
 
+### Shared graph artifact
+
+`.codebase-memory/graph.db.zst` is an optional zstd-compressed snapshot of the graph that sits next to the source. `codebase-memory-cli index` writes or refreshes it (the watcher refreshes it with a faster, lower-compression tier), and a first `index` on a clone with no local database imports it before running incremental indexing. A `.codebase-memory/.gitattributes` line with `merge=ours` is created on first export so concurrent edits do not conflict on the binary file.
+
+- **Commit it deliberately.** The artifact is rewritten on every index, including the watcher's fast tier, and git stores each rewrite as a full new blob. Committing every refresh turns a 20 MB file into gigabytes of history. Pick a cadence (a release, a milestone, a nightly job) rather than committing every save.
+- **Git LFS, if it must move on every commit.** Track it from the repo-root `.gitattributes` and leave the auto-created `.codebase-memory/.gitattributes` in place; the nearer file keeps supplying `merge=ours`, and only `filter` comes from the root:
+  ```gitattributes
+  .codebase-memory/graph.db.zst filter=lfs diff=lfs merge=lfs -text
+  ```
+  Track only the `.zst`; `artifact.json` is small and carries the schema version. The attribute applies to future commits only, so a repo that already has the blobs in history needs `git-filter-repo` first. GitHub meters LFS storage and bandwidth, and every teammate needs `git lfs install`; without it the checkout leaves a pointer file, the integrity-checked import refuses it, and they fall back to a full reindex.
+- **Optional.** Add `.codebase-memory/` to `.gitignore` if everyone should reindex from scratch.
+
 ## Configuration
 
 Common commands:
@@ -267,7 +280,7 @@ Important environment variables include:
 | `CBM_ALLOWED_ROOT` | Constrain permissible indexing roots. |
 | `CBM_RUNTIME_DIR` | Override the secure local coordination rendezvous parent. |
 | `CBM_WORKERS` | Override indexing worker count. |
-| `CBM_LOG_LEVEL` | Control runtime logging. |
+| `CBM_LOG_LEVEL` | Control runtime logging. One-shot CLI and hook commands default to `warn`; the detached daemon and supervised index workers default to `info`. |
 
 See [`docs/CONFIGURATION.md`](docs/CONFIGURATION.md) for the full reference.
 

docs/CONFIGURATION.md

@@ -133,7 +133,7 @@ These environment variables affect runtime behavior:
 | `CBM_CACHE_DIR` | `~/.cache/codebase-memory-cli` | Override the cache directory used for indexes, `_config.db`, and UI `config.json`. |
 | `CBM_DIAGNOSTICS` | `false` | Enable periodic `snapshot.json` and retained `trajectory.ndjson` below a fresh owner-private directory in the system temp directory. The daemon records the randomized paths in the `diagnostics.start` discovery record (a single JSON line) in `${CBM_CACHE_DIR}/logs/cbm-daemon.log`; that one record is emitted even when `CBM_LOG_LEVEL` suppresses ordinary logging, so the paths always remain discoverable. |
 | `CBM_DOWNLOAD_URL` | GitHub releases | Override the update download URL. |
-| `CBM_LOG_LEVEL` | `info` | Set the log level to `debug`, `info`, `warn`, `error`, or `none` (or `0`-`4`). One-shot CLI messages use that command's stderr; retained detached-runtime events use `${CBM_CACHE_DIR}/logs/cbm-daemon.log`. |
+| `CBM_LOG_LEVEL` | role-aware | Set the log level to `debug`, `info`, `warn`, `error`, or `none` (or `0`-`4`). Thin CLI/hook frontends default to `warn`; the detached daemon and supervised index workers default to `info`. Physical workers retain INFO liveness records because their private logs drive the supervisor's no-progress timeout. Frontend messages use that command's stderr; detached daemon events use `${CBM_CACHE_DIR}/logs/cbm-daemon.log`. |
 | `CBM_RUNTIME_DIR` | `%LOCALAPPDATA%` (Windows), `/private/tmp` (macOS), `/tmp` (other) | Parent directory for the daemon/CLI rendezvous directory, which CBM creates inside it as `cbm-daemon-<uid>` (`cbm-daemon-<key>` on Windows). When an explicitly configured parent is missing, CBM creates it owner-private before use; existing parents are never modified and must pass the private-directory check. Set it when the default ancestry cannot pass that check — see below. `CBM_CACHE_DIR` does **not** move the rendezvous. |
 | `CBM_WORKERS` | auto-detected | Override the indexing worker count. |
 

graph-ui/src/App.test.tsx

@@ -0,0 +1,59 @@
+/* @vitest-environment jsdom */
+import "@testing-library/jest-dom/vitest";
+import { cleanup, render, screen, waitFor } from "@testing-library/react";
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { App } from "./App";
+import { messages } from "./lib/i18n";
+
+vi.mock("./components/GraphTab", () => ({ GraphTab: () => null }));
+vi.mock("./components/StatsTab", () => ({ StatsTab: () => null }));
+vi.mock("./components/ControlTab", () => ({ ControlTab: () => null }));
+vi.mock("./lib/i18n", async (importOriginal) => {
+  const actual = await importOriginal<typeof import("./lib/i18n")>();
+  return { ...actual, useUiMessages: () => messages.en };
+});
+
+describe("App", () => {
+  afterEach(() => {
+    cleanup();
+    vi.unstubAllGlobals();
+    window.history.replaceState(null, "", "/");
+  });
+
+  it("shows the serving binary version", async () => {
+    vi.stubGlobal("fetch", vi.fn(async () =>
+      new Response(JSON.stringify({ lang: "en", version: "0.10.8" }), {
+        status: 200,
+        headers: { "Content-Type": "application/json" },
+      }),
+    ));
+
+    render(<App />);
+
+    expect(await screen.findByText("v0.10.8")).toBeVisible();
+  });
+
+  it("hides the version when the config has no string version", async () => {
+    const fetchMock = vi.fn(async () =>
+      new Response(JSON.stringify({ lang: "en", version: 108 }), { status: 200 }),
+    );
+    vi.stubGlobal("fetch", fetchMock);
+
+    render(<App />);
+
+    await waitFor(() => expect(fetchMock).toHaveBeenCalledWith("/api/ui-config"));
+    expect(screen.queryByTitle("Server version")).not.toBeInTheDocument();
+  });
+
+  it("hides the version when the config request fails", async () => {
+    const fetchMock = vi.fn(async () => {
+      throw new Error("offline");
+    });
+    vi.stubGlobal("fetch", fetchMock);
+
+    render(<App />);
+
+    await waitFor(() => expect(fetchMock).toHaveBeenCalledWith("/api/ui-config"));
+    expect(screen.queryByTitle("Server version")).not.toBeInTheDocument();
+  });
+});

graph-ui/src/App.tsx

@@ -33,8 +33,24 @@ function routeUrl(tab: TabId, project: string | null): string {
 export function App() {
   const t = useUiMessages();
   const [route, setRoute] = useState<RouteState>(readRoute);
+  const [version, setVersion] = useState<string | null>(null);
   const { tab: activeTab, project: selectedProject } = route;
 
+  useEffect(() => {
+    let cancelled = false;
+    void fetch("/api/ui-config")
+      .then((response) => (response.ok ? response.json() : null))
+      .then((config) => {
+        if (!cancelled && typeof config?.version === "string" && config.version) {
+          setVersion(config.version);
+        }
+      })
+      .catch(() => {});
+    return () => {
+      cancelled = true;
+    };
+  }, []);
+
   /* Normalize the URL on first load so it always carries the current route. */
   useEffect(() => {
     const initial = readRoute();
@@ -73,6 +89,14 @@ export function App() {
             <span className="text-[13px] font-semibold text-foreground/90 tracking-tight">
               Codebase Memory
             </span>
+            {version && (
+              <span
+                className="translate-y-px text-[10px] font-mono text-foreground/30"
+                title="Server version"
+              >
+                {version.startsWith("v") ? version : `v${version}`}
+              </span>
+            )}
           </div>
 
           {/* Tabs inline in header */}