String results from host imports, and the measurement that justifies malloc

0017 C says wasmtime takes ownership of what an import callback writes and
frees it, so a string result cannot come from a java.lang.foreign.Arena. That
came from the review; this commit verifies it, and the shape of the failure is
the finding:

  arena pointer, one call    -> "ababab", rc=0
  arena pointer, 2000 calls  -> Abort trap: 6, rc=134

One call succeeds. A test that called once would have licensed the corruption
and reported a green gate, which is why the test loops 500 times.

So the import direction gets libc's malloc -- the only allocator this library
binds, and only here. dev/resources/imps.{wit,wat} is a guest whose import
returns a string and whose export relays it, so both directions of a string
cross in one call.

Two of the assertions I wrote first were nonsense and the suite caught them:
the host function returns n copies of "ab", so expecting "日本語" from it, and
expecting 500 distinct results from 500 identical calls, were both my errors
rather than the code's.

Aggregate results -- list, record, option, variant -- need the same allocator
change threaded through lower-fn and are still refused at instantiate with the
kind named.

101 assertions.

Claude-Session: https://claude.ai/code/session_01XF5Hfq4Ca2N2XYEzWQQuHt

Author: chaploud

dev/resources/imps.wat

@@ -0,0 +1,16 @@
+;; The guest half of dev/resources/imps.wit. A string result flattens to a
+;; return area the callee writes (ptr, len) into, in both directions.
+(module
+  (import "local:imps/host@0.1.0" "mk" (func $mk (param i32) (param i32)))
+  (memory (export "memory") 1)
+  (global $next (mut i32) (i32.const 1024))
+  (func (export "cabi_realloc") (param i32 i32 i32 i32) (result i32)
+    (local $p i32)
+    (local.set $p (global.get $next))
+    (global.set $next (i32.add (local.get $p) (local.get 3)))
+    (local.get $p))
+  (func (export "run") (param $n i32) (result i32)
+    (call $mk (local.get $n) (i32.const 128))
+    (i32.store (i32.const 160) (i32.load (i32.const 128)))
+    (i32.store offset=4 (i32.const 160) (i32.load offset=4 (i32.const 128)))
+    (i32.const 160)))

dev/resources/imps.wit

@@ -0,0 +1,5 @@
+// A host import that returns a string, which is where the two directions stop
+// being mirror images: wasmtime *frees* what the callback writes (`0017` C).
+package local:imps@0.1.0;
+interface host { mk: func(n: u32) -> string; }
+world w { import host; export run: func(n: u32) -> string; }

doc/design/0017-host-imports.md

@@ -76,8 +76,10 @@ returning a pointer into static storage aborts with
 `POINTER_BEING_FREED_WAS_NOT_ALLOCATED` inside
 `wasmtime_component_linker_instance_add_func`'s closure.
 
-`host.clj`'s `lower-fn` allocates **every** payload from a
-`java.lang.foreign.Arena` — string bytes, list/tuple/record element buffers,
+**Confirmed here, and the shape of the failure is the finding.** With an Arena
+pointer, *one call succeeds*; 2000 abort the process (`rc=134`). A test that
+called once would have licensed the corruption. `host.clj`'s `lower-fn`
+allocates **every** payload from a `java.lang.foreign.Arena` — string bytes, list/tuple/record element buffers,
 flags name vectors, and the boxed vals behind `option`/`result`/`variant`.
 Handing wasmtime an Arena pointer to `free()` is heap corruption. The import
 direction needs `malloc` and `wasmtime_component_val_new`, so `lower-fn` needs

src/cljwit/host.clj

@@ -155,6 +155,13 @@
      :err-message   (b "wasmtime_error_message" nil ADDR ADDR)
      :err-delete    (b "wasmtime_error_delete" nil ADDR)
      :err-new       (b "wasmtime_error_new" ADDR ADDR)
+     ;; 0017 C: wasmtime frees what an import callback writes, so a string
+     ;; result cannot come from an Arena. libc's malloc is the allocator it
+     ;; expects, and this is the only place in the library that needs one.
+     :malloc        (let [seg ^MemorySegment (.orElseThrow (.find (.defaultLookup linker) "malloc"))]
+                      (.downcallHandle linker seg
+                                       (FunctionDescriptor/of ADDR (into-array java.lang.foreign.MemoryLayout [I64]))
+                                       (into-array java.lang.foreign.Linker$Option [])))
      :linker-root   (b "wasmtime_component_linker_root" ADDR ADDR)
      :li-add-inst   (b "wasmtime_component_linker_instance_add_instance" ADDR ADDR ADDR I64 ADDR)
      :li-add-func   (b "wasmtime_component_linker_instance_add_func" ADDR ADDR ADDR I64 ADDR ADDR ADDR)}))
@@ -549,15 +556,33 @@
 ;; `malloc` rather than from an Arena (0017 C), which is not built yet. Refusing
 ;; at instantiate with the kind named beats a wrong answer or a corrupt heap.
 (def ^:private IMPORTABLE
-  #{:bool :s8 :u8 :s16 :u16 :s32 :u32 :s64 :u64 :f32 :f64 :char})
+  #{:bool :s8 :u8 :s16 :u16 :s32 :u32 :s64 :u64 :f32 :f64 :char :string})
 
 (defn- import-stub
   "One FFM upcall stub for one host import. Lifts the arguments, calls `f`,
    lowers the result, and converts anything thrown into a wasmtime error --
    a JVM exception unwinding through native frames exits the VM (0017 D)."
   [api ^Arena arena sig f nm dead]
-  (let [lifts  (mapv (fn [[_ t]] (scalar-lift t)) (:params sig))
-        lower  (some-> (:result sig) scalar-lower)
+  (let [lifts  (mapv (fn [[_ t]] (if (= :string t)
+                                   (fn [^MemorySegment seg] (read-str seg UNION))
+                                   (scalar-lift t)))
+                     (:params sig))
+        rt     (:result sig)
+        lower  (cond
+                 (nil? rt) nil
+                 (= :string rt)
+                 (fn [_ ^MemorySegment seg ^String v]
+                   ;; malloc, not the arena: wasmtime takes ownership and frees.
+                   (let [b   (.getBytes v "UTF-8")
+                         n   (max 1 (alength b))
+                         buf ^MemorySegment (.reinterpret
+                                             ^MemorySegment (invoke (:malloc api) (long n))
+                                             (long n))]
+                     (MemorySegment/copy ^bytes b 0 buf I8 0 (alength b))
+                     (.set seg I8 (long 0) (byte (VAL-KIND :string)))
+                     (.set seg I64 (long (+ UNION STR-SIZE)) (long (alength b)))
+                     (.set seg ADDR (long (+ UNION STR-DATA)) buf)))
+                 :else (scalar-lower rt))
         cb (reify ImportCallback
              (call [_ _data _ctx _ty args _n res _nres]
                (try

test/cljwit/host_test.clj

@@ -411,3 +411,31 @@
                   (is (= :instance-dead (:cljwit/error (ex-data ex)))
                       "and the next call says the instance is dead rather than reporting a trap"))))))
         (finally (.delete ^File f))))))
+
+(deftest host-imports-returning-a-string
+  ;; `0017` C: the two directions are not mirror images. wasmtime takes
+  ;; ownership of what an import callback writes and frees it, so a string
+  ;; result must come from `malloc`, not from an `Arena`.
+  ;;
+  ;; Measured, and the shape of the measurement is the point: with an Arena
+  ;; pointer **one call succeeds** and 2000 abort the process (rc=134). A test
+  ;; that called once would have licensed the corruption, which is why this one
+  ;; loops.
+  (if-not lib
+    (println "CLJWIT_WASMTIME_LIB unset — skipping string-import test")
+    (let [f (build-component! "imps")]
+      (try
+        (with-open [e (host/engine)]
+          (with-open [a (host/compile e (io/file f))]
+            (with-open [i (host/instantiate
+                           a {:imports {"local:imps/host@0.1.0#mk"
+                                        (fn [n] (apply str (repeat n "ab")))}})]
+              (testing "a string crosses in both directions within one call"
+                (is (= "ababab" ((i "run") 3)))
+                (is (= "" ((i "run") 0)))
+                (is (= "ab" ((i "run") 1))))
+              (testing "and keeps working, which is what an Arena pointer would not"
+                (is (every? (fn [n] (= (apply str (repeat n "ab")) ((i "run") n)))
+                            (map (fn [k] (inc (mod k 9))) (range 500)))
+                    "500 calls, each result freed by wasmtime")))))
+        (finally (.delete ^File f))))))