Merge branch 'main' into fix/autoindex-skip-limit-value-1466

Author: rarepops

scripts/extract_nomic_vectors.py

@@ -332,9 +332,18 @@ def write_vectors_h(path: str, token_count: int, dim: int, incbin_path: str):
 
 
 def write_blob_s(path: str, incbin_path: str):
-    """Write assembler .incbin directive."""
-    with open(path, "w") as f:
-        f.write(f"""/* nomic-embed-code vector blob embedded via assembler. */
+    """Write the assembler .incbin wrapper for every target object format.
+
+    Must stay byte-identical to the tracked vendored/nomic/code_vectors_blob.S:
+    tests/test_nomic_blob_generator_contract.sh fails if the two drift. The ELF
+    branch's .note.GNU-stack is load-bearing -- regenerating without it puts an
+    executable stack back into every Linux release binary.
+    """
+    with open(path, "w", encoding="utf-8") as f:
+        f.write(f"""/* nomic-embed-code vector blob embedded via assembler.
+ * Cross-platform: macOS (Mach-O) vs Linux (ELF) vs Windows (COFF). */
+
+#if defined(__APPLE__)
     .section __DATA,__const
     .globl _PRETRAINED_VECTOR_BLOB
     .globl _PRETRAINED_VECTOR_BLOB_LEN
@@ -347,6 +356,46 @@ def write_blob_s(path: str, incbin_path: str):
     .p2align 2
 _PRETRAINED_VECTOR_BLOB_LEN:
     .long _PRETRAINED_VECTOR_BLOB_END - _PRETRAINED_VECTOR_BLOB
+
+#elif defined(_WIN32) || defined(__CYGWIN__) || defined(__MINGW32__)
+    .section .rdata,"dr"
+    .globl PRETRAINED_VECTOR_BLOB
+    .globl PRETRAINED_VECTOR_BLOB_LEN
+    .p2align 4
+PRETRAINED_VECTOR_BLOB:
+    .incbin "{incbin_path}"
+PRETRAINED_VECTOR_BLOB_END:
+
+    .section .rdata,"dr"
+    .p2align 2
+PRETRAINED_VECTOR_BLOB_LEN:
+    .long PRETRAINED_VECTOR_BLOB_END - PRETRAINED_VECTOR_BLOB
+
+#else
+    /* WHY: an ELF object that carries no .note.GNU-stack tells the linker
+     * nothing about its stack requirement, and GNU ld then assumes the WORST
+     * for the whole link — every Linux release binary shipped GNU_STACK RWE
+     * because of this one omission. This is the only assembly source in the
+     * build, so it alone decided that property. The note must stay even though
+     * a blob of constant data obviously never executes: absence is the signal,
+     * not the contents. -Wl,-z,noexecstack in the link flags enforces the
+     * outcome, and scripts/ci/check-binary-composition.sh fails the release if
+     * an executable stack ever comes back. */
+    .section .note.GNU-stack,"",@progbits
+
+    .section .rodata,"a",@progbits
+    .globl PRETRAINED_VECTOR_BLOB
+    .globl PRETRAINED_VECTOR_BLOB_LEN
+    .p2align 4
+PRETRAINED_VECTOR_BLOB:
+    .incbin "{incbin_path}"
+PRETRAINED_VECTOR_BLOB_END:
+
+    .section .rodata,"a",@progbits
+    .p2align 2
+PRETRAINED_VECTOR_BLOB_LEN:
+    .long PRETRAINED_VECTOR_BLOB_END - PRETRAINED_VECTOR_BLOB
+#endif
 """)
     print(f"  {path}: written")
 

scripts/test.sh

@@ -263,6 +263,9 @@ bash "$ROOT/tests/test_runtime_isolation_contract.sh"
 echo "=== Step 0u: shell line-ending contract ==="
 bash "$ROOT/tests/test_shell_line_endings.sh"
 
+echo "=== Step 0v: nomic blob generator contract ==="
+bash "$ROOT/tests/test_nomic_blob_generator_contract.sh"
+
 # Verify compiler supports target arch
 verify_compiler "$CC"
 

src/cli/client_adapter.c

@@ -49,6 +49,39 @@ static void sb_append(adapter_sb_t *sb, const char *s) {
     sb->buf[sb->len] = '\0';
 }
 
+/* Append a single-quoted JavaScript string literal, escaping the characters
+ * that could terminate or corrupt it. Avoids the fixed buffer sizing that
+ * cbm_client_adapter_escape_js imposes, so long registry descriptions fit. */
+static void sb_append_js_string(adapter_sb_t *sb, const char *s) {
+    if (!s) {
+        sb_append(sb, "''");
+        return;
+    }
+    sb_append(sb, "'");
+    for (const char *p = s; *p; p++) {
+        switch (*p) {
+        case '\\':
+            sb_append(sb, "\\\\");
+            break;
+        case '\'':
+            sb_append(sb, "\\'");
+            break;
+        case '\n':
+            sb_append(sb, "\\n");
+            break;
+        case '\r':
+            sb_append(sb, "\\r");
+            break;
+        default: {
+            char ch[2] = {*p, '\0'};
+            sb_append(sb, ch);
+            break;
+        }
+        }
+    }
+    sb_append(sb, "'");
+}
+
 bool cbm_client_adapter_escape_js(const char *in, char *out, size_t out_sz) {
     if (!in || !out || out_sz == 0) {
         return false;
@@ -120,6 +153,10 @@ char *cbm_client_adapter_pi(const char *binary_path) {
 
     adapter_sb_t sb = {0};
     emit_header(&sb, "pi");
+    /* Pin the coding-agent contract so a probe against @mariozechner/pi
+     * (pods CLI) or an old AgentTool arity cannot be mistaken for this file. */
+    sb_append(&sb, "// Target: @earendil-works/pi-coding-agent >= 0.74.0 (verified 0.84.2)\n"
+                   "// ToolDefinition.execute(toolCallId, params, signal, onUpdate, ctx)\n");
     sb_append(&sb, "import { spawn } from 'node:child_process';\n\n");
     sb_append(&sb, "const BIN = '");
     sb_append(&sb, bin);
@@ -131,7 +168,7 @@ char *cbm_client_adapter_pi(const char *binary_path) {
     sb_append(
         &sb, "async function call(tool, args, signal) {\n"
              "  return new Promise((resolve) => {\n"
-             "    const child = spawn(BIN, ['cli', tool, JSON.stringify(args ?? {})], {\n"
+             "    const child = spawn(BIN, ['cli', '--json', tool, JSON.stringify(args ?? {})], {\n"
              "      stdio: ['ignore', 'pipe', 'pipe'],\n"
              "      env: { ...process.env, CBM_LOG_LEVEL: 'error' },\n"
              "    });\n"
@@ -170,11 +207,37 @@ char *cbm_client_adapter_pi(const char *binary_path) {
         if (!name || !name[0]) {
             continue;
         }
-        sb_append(&sb, "  pi.registerTool({ name: '");
-        sb_append(&sb, name);
-        sb_append(&sb, "', run: (args, ctx) => call('");
-        sb_append(&sb, name);
-        sb_append(&sb, "', args, ctx?.signal) });\n");
+        const char *title = cbm_mcp_tool_title(name);
+        const char *description = cbm_mcp_tool_description(name);
+        const char *schema = cbm_mcp_tool_input_schema(name);
+        sb_append(&sb, "  pi.registerTool({\n");
+        sb_append(&sb, "    name: ");
+        sb_append_js_string(&sb, name);
+        sb_append(&sb, ",\n    label: ");
+        sb_append_js_string(&sb, title ? title : name);
+        sb_append(&sb, ",\n    description: ");
+        sb_append_js_string(&sb, description ? description : "");
+        sb_append(&sb, ",\n    parameters: ");
+        /* input_schema is compact JSON, which is a valid JavaScript object
+         * literal; embedding it directly keeps the generated module free of a
+         * JSON.parse indirection and of any escaping drift. */
+        sb_append(&sb, schema ? schema : "{}");
+        /* 0.84.2 calls execute(toolCallId, params, signal, onUpdate, ctx).
+         * The previous (args, ctx) shape bound the call id as the MCP args. */
+        sb_append(&sb, ",\n    execute: async (toolCallId, params, signal, _onUpdate, ctx) => {\n");
+        sb_append(&sb, "      const result = await call(");
+        sb_append_js_string(&sb, name);
+        sb_append(&sb,
+                  ", params, signal ?? ctx?.signal);\n"
+                  "      if (result && typeof result === 'object' && result.error) {\n"
+                  "        throw new Error(String(result.error));\n"
+                  "      }\n"
+                  "      const content = result && Array.isArray(result.content)\n"
+                  "        ? result.content\n"
+                  "        : [{ type: 'text', text: JSON.stringify(result ?? null, null, 2) }];\n"
+                  "      return { content, details: result ?? {} };\n"
+                  "    },\n"
+                  "  });\n");
     }
     sb_append(&sb, "}\n");
 

src/main.c

@@ -44,6 +44,12 @@ enum {
     MAIN_PATH_CAP = 4096,
     MAIN_CONNECT_TIMEOUT_MS = 1000,
     MAIN_STARTUP_TIMEOUT_MS = 10000,
+    /* Backstop for waiting out a held startup transition — see
+     * main_local_transition_acquire. Not a budget for healthy contention: a busy
+     * lock always resolves, either because the live holder finishes or because
+     * the OS finishes reclaiming a dead holder's lock. This only bounds a peer
+     * that never finishes, so a command cannot hang indefinitely. */
+    MAIN_STARTUP_CONTENTION_CEILING_MS = 120000,
     MAIN_MCP_STARTUP_TIMEOUT_MS = 30000,
     MAIN_REQUEST_TIMEOUT_MS = 24 * 60 * 60 * 1000,
     MAIN_HOOK_CONNECT_TIMEOUT_MS = 250,
@@ -1288,14 +1294,47 @@ static bool main_local_cli_feedback_enabled(int argc, char **argv) {
     return cbm_cli_progress_enabled(requested, cli_isatty(2) != 0);
 }
 
+/* Acquire the exclusive startup transition, waiting out whatever currently holds it.
+ *
+ * try_acquire answers one of three things, and they must not be conflated:
+ *    1  acquired
+ *   -1  coordination is unsafe or unverifiable — fail now, waiting cannot help
+ *    0  BUSY: the lock is held
+ *
+ * A busy lock always resolves, by one of two routes:
+ *   - a live peer holds it, and releases when its command finishes;
+ *   - the holder is already dead and the operating system has not finished
+ *     reclaiming the lock yet.
+ *
+ * The second route is Windows-specific and is why this wait needs room. On POSIX
+ * the kernel drops flock the instant the owner dies. Windows byte-range locks do
+ * not work that way: Microsoft documents that after a process terminates holding
+ * one, "the time it takes for the operating system to unlock these locks depends
+ * upon available system resources", and that until then "access to these files
+ * may be denied". A loaded CI runner is precisely where those resources are
+ * scarce, and `tests/windows/test_daemon_stability.py` manufactures the situation
+ * deliberately — it hard-kills daemons with `taskkill /F`, including a crash
+ * recovery section, so the next client meets a lock whose owner no longer exists.
+ *
+ * Waiting was previously capped at MAIN_STARTUP_TIMEOUT_MS (10s), which let a
+ * clock decide a user-visible outcome: a command was refused with "coordination
+ * remained busy" while the only thing wrong was that the OS had not yet swept up
+ * after a killed process. Since both routes above terminate, the correct response
+ * to busy is to keep waiting; the ceiling below is a backstop against a peer that
+ * never finishes, not a budget for healthy contention. Clean exits already
+ * release through main_local_transition_close, so this path is only reached after
+ * an abrupt termination or under genuine concurrency. */
 static int main_local_transition_acquire(const cbm_daemon_ipc_endpoint_t *endpoint, FILE *feedback,
                                          cbm_daemon_ipc_local_transition_t **transition_out) {
-    uint64_t deadline = main_deadline_after(MAIN_STARTUP_TIMEOUT_MS);
+    uint64_t ceiling = main_deadline_after(MAIN_STARTUP_CONTENTION_CEILING_MS);
     bool waiting_reported = false;
     for (;;) {
         int status = cbm_daemon_ipc_local_transition_try_acquire(endpoint, transition_out);
-        if (status != 0 || cbm_now_ms() >= deadline) {
-            return status;
+        if (status != 0) {
+            return status; /* acquired, or a genuine coordination failure */
+        }
+        if (cbm_now_ms() >= ceiling) {
+            return 0; /* still busy after the backstop */
         }
         if (feedback && !waiting_reported) {
             (void)fputs("Waiting for CBM startup coordination...\n", feedback);
@@ -2542,11 +2581,24 @@ int main(int argc, char **argv) {
         int transition_status =
             main_local_transition_acquire(local_endpoint, feedback, &local_transition);
         if (transition_status != 1 || !local_transition) {
-            (void)fprintf(stderr,
-                          "codebase-memory-mcp: CLI startup coordination %s; retry after the "
-                          "active CBM transition exits\n",
-                          transition_status == 0 ? "remained busy"
-                                                 : "could not be verified safely");
+            if (transition_status == 0) {
+                /* The backstop fired. Name both explanations: after this much
+                 * waiting the likely causes are a peer that is genuinely stuck,
+                 * or (on Windows) a lock left behind by a force-killed process
+                 * that the OS has not reclaimed. "Busy" alone sent reporters
+                 * hunting for a CBM session that had already exited. */
+                (void)fprintf(stderr,
+                              "codebase-memory-mcp: CLI startup coordination stayed busy for "
+                              "%d seconds. Either another CBM command is still running, or a "
+                              "previous one was force-killed and the operating system has not "
+                              "released its lock yet. Check for running CBM processes; if there "
+                              "are none, retry shortly.\n",
+                              MAIN_STARTUP_CONTENTION_CEILING_MS / 1000);
+            } else {
+                (void)fprintf(stderr, "codebase-memory-mcp: CLI startup coordination could not "
+                                      "be verified safely; retry after active CBM sessions "
+                                      "exit\n");
+            }
             goto local_cli_cleanup;
         }
         int seal_status = cbm_daemon_ipc_local_transition_seal_legacy(local_transition);

src/mcp/mcp.c

@@ -946,6 +946,30 @@ const char *cbm_mcp_tool_name(int index) {
     return TOOLS[index].name;
 }
 
+const char *cbm_mcp_tool_title(const char *tool_name) {
+    if (!tool_name) {
+        return NULL;
+    }
+    for (int i = 0; i < TOOL_COUNT; i++) {
+        if (strcmp(TOOLS[i].name, tool_name) == 0) {
+            return TOOLS[i].title;
+        }
+    }
+    return NULL;
+}
+
+const char *cbm_mcp_tool_description(const char *tool_name) {
+    if (!tool_name) {
+        return NULL;
+    }
+    for (int i = 0; i < TOOL_COUNT; i++) {
+        if (strcmp(TOOLS[i].name, tool_name) == 0) {
+            return TOOLS[i].description;
+        }
+    }
+    return NULL;
+}
+
 /* Render the top-level --help "Tools:" block from the registry tools/list
  * serves. The list used to be hand-maintained in the help text and drifted
  * when check_index_coverage was added (#1361); deriving it here makes that

src/mcp/mcp.h

@@ -72,6 +72,12 @@ const char *cbm_mcp_tool_input_schema(const char *tool_name);
 int cbm_mcp_tool_count(void);
 const char *cbm_mcp_tool_name(int index);
 
+/* Tool metadata by name (static; do not free; NULL when unknown). Used by the
+ * generated client adapters so they can emit descriptions and parameter
+ * schemas without drifting from tools/list. */
+const char *cbm_mcp_tool_title(const char *tool_name);
+const char *cbm_mcp_tool_description(const char *tool_name);
+
 /* Render the top-level --help "Tools:" block from the registry so the help
  * text cannot drift from tools/list (#1361). Heap-allocated; caller frees. */
 char *cbm_mcp_tools_help_list(void);