cgen: install native fns into the compile path under JOLT_CGEN (jolt-ihdp) (#146)

Wires src/jolt/cgen.janet into the backend's :def emit. With JOLT_CGEN=1 (off by
default, needs direct-linking), a plain defn of a numeric-leaf fn is compiled to
C at def time and the cfunction installed as the var root, so direct-linked
callers embed native code. The fn is not inline-stashed when cgen fires —
callers must call the C fn, not inline the bytecode body. ^:redef/^:dynamic stay
bytecode.

The leaf-first rule falls out: run calls count-point (a user var), so run isn't a
numeric leaf and stays bytecode, calling the native count-point over the cheap
forward crossing. mandelbrot 200: 224ms -> 12.4ms (~18x), result unchanged.

Adds JOLT_CGEN to ctx-shaping-env-vars (rides the disk-cache key) and :cgen? to
resolve-run-mode. Default path (cgen off) is a no-op: cgen-root returns nil and
the normal bytecode emit runs. Gate green (117 files). Test:
test/integration/cgen-pipeline-test.janet.

Co-authored-by: Yogthos <yogthos@gmail.com>

docs/foundational-runtime-lever1-native-codegen.md

@@ -74,6 +74,28 @@ the existing ctx image cache.
 - C API used: `janet_getnumber/getinteger`, `janet_wrap_number`, `janet_fixarity`,
   `janet_getfunction`, `janet_call`, `janet_cfuns`, `JANET_MODULE_ENTRY`.
 
+## Status: wired into the compile path (JOLT_CGEN, opt-in)
+
+`src/jolt/cgen.janet` (IR→C translator) is wired into the backend's `:def` emit
+via `cgen-root`, gated behind **`JOLT_CGEN=1`** (off by default; needs
+direct-linking). When on, a plain defn of a numeric-leaf fn is compiled to C at
+def time and the cfunction installed as the var root — so direct-linked callers
+embed native code. The fn is NOT inline-stashed when cgen fires (callers must
+call the C fn, not inline the bytecode body). `^:redef`/`^:dynamic` defns stay
+bytecode.
+
+The leaf-first rule emerges for free: `run` calls `count-point` (a user var, not
+a native-op), so `run` isn't a numeric leaf and stays bytecode — calling the
+native `count-point` over the cheap forward crossing.
+
+**Measured end-to-end (`jolt -m mandelbrot 200`): 224 ms → 12.4 ms, ~18×**, with
+the correct result — matching the spike's native-C ceiling. The default gate
+(cgen off) is unchanged. Tests: `test/integration/cgen-pipeline-test.janet`.
+
+Known limitation: building *core* with `JOLT_CGEN=1` would try to cgen core
+numeric-leaf fns into the cached ctx image, where embedded cfunctions may not
+serialize — keep cgen for app/user code until image-cache interaction is handled.
+
 ## Open questions for the implementation (next beads)
 
 1. **IR→C for the numeric subset.** Translate jolt IR → C for proven-double

src/jolt/backend.janet

@@ -12,6 +12,7 @@
 (use ./core)
 (use ./evaluator)
 (import ./reader :as r)
+(import ./cgen :as cgen)
 (import ./phm :as phm)
 (import ./phs :as phs)
 (import ./pv :as pv)
@@ -607,6 +608,20 @@
   (def items (map |(emit ctx $) (vview (node :items))))
   (tuple/slice (array/concat @[phs/make-phs] items)))
 
+# Native codegen hook (jolt-ihdp). Under :cgen?, a plain defn of a numeric-leaf
+# fn is compiled to C and the resulting cfunction installed as the var root, so
+# direct-linked callers embed native code. Returns the cfunction or nil (not a
+# candidate / redefable / toolchain absent / cc failed -> normal bytecode path).
+# Skip ^:redef / ^:dynamic: those must stay redefinable bytecode.
+(defn- cgen-root [ctx node]
+  (when (get (ctx :env) :cgen?)
+    (def meta (node :meta))
+    (def redefable (and meta (or (get meta :redef) (get meta :dynamic))))
+    (when (and (not redefable) (cgen/numeric-leaf? (node :init)))
+      (def r (protect (cgen/compile-fn (node :init)
+                                       {:name (string (node :ns) "_" (node :name))})))
+      (and (r 0) (r 1)))))
+
 (set emit
   (fn emit [ctx raw]
     (def node (norm-node raw))
@@ -654,14 +669,23 @@
       :throw ['error (emit ctx (node :expr))]
       :def (let [cell (cell-for ctx (node :ns) (node :name))
                  meta (node :meta)
-                 setter (if (and meta (not (empty? meta))) (var-setter-meta cell meta) (var-setter cell))]
-             (inline-stash! ctx cell node)
-             # Emit the init BEFORE marking the cell unit-defined, so a recursive
-             # self-reference in the init still direct-links; a LATER sibling
-             # reference in this unit then sees it as redefined (jolt-wf4).
-             (let [init-form (emit ctx (node :init))]
-               (when-let [ud (get (ctx :env) :unit-defs)] (put ud cell true))
-               (tuple setter init-form)))
+                 setter (if (and meta (not (empty? meta))) (var-setter-meta cell meta) (var-setter cell))
+                 cfn (cgen-root ctx node)]
+             # cgen path: install the native cfunction as the root and DON'T
+             # inline-stash — callers must call the C fn, not inline the bytecode
+             # body (which would bypass the native version). Otherwise the normal
+             # bytecode path: stash for inlining, emit the init.
+             (if cfn
+               (do (when-let [ud (get (ctx :env) :unit-defs)] (put ud cell true))
+                   (tuple setter cfn))
+               (do (inline-stash! ctx cell node)
+                   # Emit the init BEFORE marking the cell unit-defined, so a
+                   # recursive self-reference in the init still direct-links; a
+                   # LATER sibling reference in this unit then sees it as
+                   # redefined (jolt-wf4).
+                   (let [init-form (emit ctx (node :init))]
+                     (when-let [ud (get (ctx :env) :unit-defs)] (put ud cell true))
+                     (tuple setter init-form)))))
       :let (emit-let ctx node)
       :fn (emit-fn ctx node)
       :invoke (emit-invoke ctx node)

src/jolt/config.janet

@@ -28,7 +28,7 @@
    "JOLT_INTERPRET" "JOLT_INTERPRET_MACROS" "JOLT_DIRECT_LINK"
    "JOLT_NO_DIRECT_LINK" "JOLT_OPTIMIZE" "JOLT_WHOLE_PROGRAM"
    "JOLT_NO_WHOLE_PROGRAM" "JOLT_SHAPE" "JOLT_NO_SHAPE"
-   "JOLT_NO_IR_PASSES" "JOLT_CHECK_HINTS"])
+   "JOLT_NO_IR_PASSES" "JOLT_CHECK_HINTS" "JOLT_CGEN"])
 
 (defn ctx-cache-key
   "Build a disk-cache key from labeled prefix pairs plus the value of every
@@ -69,4 +69,8 @@
    :map-shapes? (and (os/getenv "JOLT_SHAPE") (not (os/getenv "JOLT_NO_SHAPE")))
    :whole-program? (and optimize?
                         (not (os/getenv "JOLT_NO_WHOLE_PROGRAM"))
-                        (or main-entry? (os/getenv "JOLT_WHOLE_PROGRAM")))})
+                        (or main-entry? (os/getenv "JOLT_WHOLE_PROGRAM")))
+   # Native codegen (jolt-ihdp): compile numeric-leaf fns to C at def time and
+   # install the cfunction as the var root, so callers direct-link to native
+   # code. Opt-in (cc latency per fn); needs direct-linking so callers embed it.
+   :cgen? (and dl (= "1" (os/getenv "JOLT_CGEN")))})

test/integration/cgen-pipeline-test.janet

@@ -0,0 +1,47 @@
+# Native codegen pipeline integration (jolt-ihdp): under :cgen?, a defn of a
+# numeric-leaf fn is compiled to C and the cfunction installed as the var root,
+# so direct-linked callers run native code. Pins (1) the root becomes a
+# cfunction, (2) non-leaf / redefable defns stay bytecode, (3) results match.
+# Skips the native legs where the C toolchain (cc + janet.h) is absent.
+(import ../../src/jolt/api :as api)
+(import ../../src/jolt/cgen :as cgen)
+
+(print "Native codegen pipeline (jolt-ihdp)...")
+
+(var failures 0)
+(defn check [label ok] (unless ok (++ failures) (eprintf "  FAIL: %s" label)))
+(defn callable? [x] (or (function? x) (cfunction? x)))
+
+(def ctx (api/init-cached {:compile? true}))
+(put (ctx :env) :direct-linking? true)
+(put (ctx :env) :inline? true)
+(put (ctx :env) :cgen? true)
+(api/eval-string ctx "(ns cgp)")
+
+(if (cgen/toolchain-available?)
+  (do
+    # numeric-leaf fn -> native root
+    (api/eval-string ctx "(defn sq [x] (* x x))")
+    (check "numeric-leaf root is a cfunction" (cfunction? (api/eval-string ctx "sq")))
+    (check "native sq computes correctly" (= 49 (api/eval-string ctx "(sq 7)")))
+
+    # non-leaf fn -> stays bytecode (NOT a cfunction)
+    (api/eval-string ctx "(defn g [x] (str x))")
+    (check "non-leaf root stays bytecode" (not (cfunction? (api/eval-string ctx "g"))))
+    (check "g still works" (= "5" (api/eval-string ctx "(g 5)")))
+
+    # ^:redef stays bytecode even though numeric-leaf
+    (api/eval-string ctx "(defn ^:redef rsq [x] (* x x))")
+    (check "redefable numeric-leaf stays bytecode" (not (cfunction? (api/eval-string ctx "rsq"))))
+
+    # end-to-end: count-point native, run bytecode calling it; grid total matches
+    (api/eval-string ctx "(defn count-point [cr ci cap] (loop [i 0 zr 0.0 zi 0.0] (if (or (>= i cap) (> (+ (* zr zr) (* zi zi)) 4.0)) i (recur (inc i) (+ (- (* zr zr) (* zi zi)) cr) (+ (* 2.0 (* zr zi)) ci)))))")
+    (check "count-point root is native" (cfunction? (api/eval-string ctx "count-point")))
+    (api/eval-string ctx "(defn run [n] (let [cap 200 nd (* 1.0 n)] (loop [y 0 acc 0] (if (< y n) (let [ci (- (/ (* 2.0 y) nd) 1.0) row (loop [x 0 a 0] (if (< x n) (let [cr (- (/ (* 2.0 x) nd) 1.5)] (recur (inc x) (+ a (count-point cr ci cap)))) a))] (recur (inc y) (+ acc row))) acc))))")
+    (check "run stays bytecode (calls a user fn)" (not (cfunction? (api/eval-string ctx "run"))))
+    (check "native count-point gives the right grid total" (= 3288753 (api/eval-string ctx "(run 200)"))))
+  (print "  (toolchain absent — skipping native pipeline legs)"))
+
+(if (= 0 failures)
+  (print "All tests passed.")
+  (do (eprintf "%d cgen-pipeline check(s) failed" failures) (os/exit 1)))