From 393656d8d988e6df979ee18d71922d5652b8493a Mon Sep 17 00:00:00 2001 From: Dmitri Sotnikov Date: Tue, 16 Jun 2026 17:12:48 +0000 Subject: [PATCH] cgen: install native fns into the compile path under JOLT_CGEN (jolt-ihdp) (#146) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wires src/jolt/cgen.janet into the backend's :def emit. With JOLT_CGEN=1 (off by default, needs direct-linking), a plain defn of a numeric-leaf fn is compiled to C at def time and the cfunction installed as the var root, so direct-linked callers embed native code. The fn is not inline-stashed when cgen fires — callers must call the C fn, not inline the bytecode body. ^:redef/^:dynamic stay bytecode. The leaf-first rule falls out: run calls count-point (a user var), so run isn't a numeric leaf and stays bytecode, calling the native count-point over the cheap forward crossing. mandelbrot 200: 224ms -> 12.4ms (~18x), result unchanged. Adds JOLT_CGEN to ctx-shaping-env-vars (rides the disk-cache key) and :cgen? to resolve-run-mode. Default path (cgen off) is a no-op: cgen-root returns nil and the normal bytecode emit runs. Gate green (117 files). Test: test/integration/cgen-pipeline-test.janet. Co-authored-by: Yogthos --- ...ndational-runtime-lever1-native-codegen.md | 22 +++++++++ src/jolt/backend.janet | 40 ++++++++++++---- src/jolt/config.janet | 8 +++- test/integration/cgen-pipeline-test.janet | 47 +++++++++++++++++++ 4 files changed, 107 insertions(+), 10 deletions(-) create mode 100644 test/integration/cgen-pipeline-test.janet diff --git a/docs/foundational-runtime-lever1-native-codegen.md b/docs/foundational-runtime-lever1-native-codegen.md index 11a3030..55e9e01 100644 --- a/docs/foundational-runtime-lever1-native-codegen.md +++ b/docs/foundational-runtime-lever1-native-codegen.md @@ -74,6 +74,28 @@ the existing ctx image cache. - C API used: `janet_getnumber/getinteger`, `janet_wrap_number`, `janet_fixarity`, `janet_getfunction`, `janet_call`, `janet_cfuns`, `JANET_MODULE_ENTRY`. +## Status: wired into the compile path (JOLT_CGEN, opt-in) + +`src/jolt/cgen.janet` (IR→C translator) is wired into the backend's `:def` emit +via `cgen-root`, gated behind **`JOLT_CGEN=1`** (off by default; needs +direct-linking). When on, a plain defn of a numeric-leaf fn is compiled to C at +def time and the cfunction installed as the var root — so direct-linked callers +embed native code. The fn is NOT inline-stashed when cgen fires (callers must +call the C fn, not inline the bytecode body). `^:redef`/`^:dynamic` defns stay +bytecode. + +The leaf-first rule emerges for free: `run` calls `count-point` (a user var, not +a native-op), so `run` isn't a numeric leaf and stays bytecode — calling the +native `count-point` over the cheap forward crossing. + +**Measured end-to-end (`jolt -m mandelbrot 200`): 224 ms → 12.4 ms, ~18×**, with +the correct result — matching the spike's native-C ceiling. The default gate +(cgen off) is unchanged. Tests: `test/integration/cgen-pipeline-test.janet`. + +Known limitation: building *core* with `JOLT_CGEN=1` would try to cgen core +numeric-leaf fns into the cached ctx image, where embedded cfunctions may not +serialize — keep cgen for app/user code until image-cache interaction is handled. + ## Open questions for the implementation (next beads) 1. **IR→C for the numeric subset.** Translate jolt IR → C for proven-double diff --git a/src/jolt/backend.janet b/src/jolt/backend.janet index 6656fc3..c82a5e7 100644 --- a/src/jolt/backend.janet +++ b/src/jolt/backend.janet @@ -12,6 +12,7 @@ (use ./core) (use ./evaluator) (import ./reader :as r) +(import ./cgen :as cgen) (import ./phm :as phm) (import ./phs :as phs) (import ./pv :as pv) @@ -607,6 +608,20 @@ (def items (map |(emit ctx $) (vview (node :items)))) (tuple/slice (array/concat @[phs/make-phs] items))) +# Native codegen hook (jolt-ihdp). Under :cgen?, a plain defn of a numeric-leaf +# fn is compiled to C and the resulting cfunction installed as the var root, so +# direct-linked callers embed native code. Returns the cfunction or nil (not a +# candidate / redefable / toolchain absent / cc failed -> normal bytecode path). +# Skip ^:redef / ^:dynamic: those must stay redefinable bytecode. +(defn- cgen-root [ctx node] + (when (get (ctx :env) :cgen?) + (def meta (node :meta)) + (def redefable (and meta (or (get meta :redef) (get meta :dynamic)))) + (when (and (not redefable) (cgen/numeric-leaf? (node :init))) + (def r (protect (cgen/compile-fn (node :init) + {:name (string (node :ns) "_" (node :name))}))) + (and (r 0) (r 1))))) + (set emit (fn emit [ctx raw] (def node (norm-node raw)) @@ -654,14 +669,23 @@ :throw ['error (emit ctx (node :expr))] :def (let [cell (cell-for ctx (node :ns) (node :name)) meta (node :meta) - setter (if (and meta (not (empty? meta))) (var-setter-meta cell meta) (var-setter cell))] - (inline-stash! ctx cell node) - # Emit the init BEFORE marking the cell unit-defined, so a recursive - # self-reference in the init still direct-links; a LATER sibling - # reference in this unit then sees it as redefined (jolt-wf4). - (let [init-form (emit ctx (node :init))] - (when-let [ud (get (ctx :env) :unit-defs)] (put ud cell true)) - (tuple setter init-form))) + setter (if (and meta (not (empty? meta))) (var-setter-meta cell meta) (var-setter cell)) + cfn (cgen-root ctx node)] + # cgen path: install the native cfunction as the root and DON'T + # inline-stash — callers must call the C fn, not inline the bytecode + # body (which would bypass the native version). Otherwise the normal + # bytecode path: stash for inlining, emit the init. + (if cfn + (do (when-let [ud (get (ctx :env) :unit-defs)] (put ud cell true)) + (tuple setter cfn)) + (do (inline-stash! ctx cell node) + # Emit the init BEFORE marking the cell unit-defined, so a + # recursive self-reference in the init still direct-links; a + # LATER sibling reference in this unit then sees it as + # redefined (jolt-wf4). + (let [init-form (emit ctx (node :init))] + (when-let [ud (get (ctx :env) :unit-defs)] (put ud cell true)) + (tuple setter init-form))))) :let (emit-let ctx node) :fn (emit-fn ctx node) :invoke (emit-invoke ctx node) diff --git a/src/jolt/config.janet b/src/jolt/config.janet index 7444206..ceef368 100644 --- a/src/jolt/config.janet +++ b/src/jolt/config.janet @@ -28,7 +28,7 @@ "JOLT_INTERPRET" "JOLT_INTERPRET_MACROS" "JOLT_DIRECT_LINK" "JOLT_NO_DIRECT_LINK" "JOLT_OPTIMIZE" "JOLT_WHOLE_PROGRAM" "JOLT_NO_WHOLE_PROGRAM" "JOLT_SHAPE" "JOLT_NO_SHAPE" - "JOLT_NO_IR_PASSES" "JOLT_CHECK_HINTS"]) + "JOLT_NO_IR_PASSES" "JOLT_CHECK_HINTS" "JOLT_CGEN"]) (defn ctx-cache-key "Build a disk-cache key from labeled prefix pairs plus the value of every @@ -69,4 +69,8 @@ :map-shapes? (and (os/getenv "JOLT_SHAPE") (not (os/getenv "JOLT_NO_SHAPE"))) :whole-program? (and optimize? (not (os/getenv "JOLT_NO_WHOLE_PROGRAM")) - (or main-entry? (os/getenv "JOLT_WHOLE_PROGRAM")))}) + (or main-entry? (os/getenv "JOLT_WHOLE_PROGRAM"))) + # Native codegen (jolt-ihdp): compile numeric-leaf fns to C at def time and + # install the cfunction as the var root, so callers direct-link to native + # code. Opt-in (cc latency per fn); needs direct-linking so callers embed it. + :cgen? (and dl (= "1" (os/getenv "JOLT_CGEN")))}) diff --git a/test/integration/cgen-pipeline-test.janet b/test/integration/cgen-pipeline-test.janet new file mode 100644 index 0000000..d16bdf1 --- /dev/null +++ b/test/integration/cgen-pipeline-test.janet @@ -0,0 +1,47 @@ +# Native codegen pipeline integration (jolt-ihdp): under :cgen?, a defn of a +# numeric-leaf fn is compiled to C and the cfunction installed as the var root, +# so direct-linked callers run native code. Pins (1) the root becomes a +# cfunction, (2) non-leaf / redefable defns stay bytecode, (3) results match. +# Skips the native legs where the C toolchain (cc + janet.h) is absent. +(import ../../src/jolt/api :as api) +(import ../../src/jolt/cgen :as cgen) + +(print "Native codegen pipeline (jolt-ihdp)...") + +(var failures 0) +(defn check [label ok] (unless ok (++ failures) (eprintf " FAIL: %s" label))) +(defn callable? [x] (or (function? x) (cfunction? x))) + +(def ctx (api/init-cached {:compile? true})) +(put (ctx :env) :direct-linking? true) +(put (ctx :env) :inline? true) +(put (ctx :env) :cgen? true) +(api/eval-string ctx "(ns cgp)") + +(if (cgen/toolchain-available?) + (do + # numeric-leaf fn -> native root + (api/eval-string ctx "(defn sq [x] (* x x))") + (check "numeric-leaf root is a cfunction" (cfunction? (api/eval-string ctx "sq"))) + (check "native sq computes correctly" (= 49 (api/eval-string ctx "(sq 7)"))) + + # non-leaf fn -> stays bytecode (NOT a cfunction) + (api/eval-string ctx "(defn g [x] (str x))") + (check "non-leaf root stays bytecode" (not (cfunction? (api/eval-string ctx "g")))) + (check "g still works" (= "5" (api/eval-string ctx "(g 5)"))) + + # ^:redef stays bytecode even though numeric-leaf + (api/eval-string ctx "(defn ^:redef rsq [x] (* x x))") + (check "redefable numeric-leaf stays bytecode" (not (cfunction? (api/eval-string ctx "rsq")))) + + # end-to-end: count-point native, run bytecode calling it; grid total matches + (api/eval-string ctx "(defn count-point [cr ci cap] (loop [i 0 zr 0.0 zi 0.0] (if (or (>= i cap) (> (+ (* zr zr) (* zi zi)) 4.0)) i (recur (inc i) (+ (- (* zr zr) (* zi zi)) cr) (+ (* 2.0 (* zr zi)) ci)))))") + (check "count-point root is native" (cfunction? (api/eval-string ctx "count-point"))) + (api/eval-string ctx "(defn run [n] (let [cap 200 nd (* 1.0 n)] (loop [y 0 acc 0] (if (< y n) (let [ci (- (/ (* 2.0 y) nd) 1.0) row (loop [x 0 a 0] (if (< x n) (let [cr (- (/ (* 2.0 x) nd) 1.5)] (recur (inc x) (+ a (count-point cr ci cap)))) a))] (recur (inc y) (+ acc row))) acc))))") + (check "run stays bytecode (calls a user fn)" (not (cfunction? (api/eval-string ctx "run")))) + (check "native count-point gives the right grid total" (= 3288753 (api/eval-string ctx "(run 200)")))) + (print " (toolchain absent — skipping native pipeline legs)")) + +(if (= 0 failures) + (print "All tests passed.") + (do (eprintf "%d cgen-pipeline check(s) failed" failures) (os/exit 1)))