From 6cace3db90f04a7d7ee8aad38a6db4357b74d483 Mon Sep 17 00:00:00 2001 From: Dmitri Sotnikov Date: Tue, 16 Jun 2026 20:34:10 +0000 Subject: [PATCH] cgen: single-binary native build via `jolt cgen-build` (jolt-a7ds) (#150) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fuse an app's native-compiled numeric-leaf fns plus its source into one static executable: no sidecar .so, no toolchain on the target. The AOT path (#148) already produced a prebuilt module + manifest; this links them into the jpm-built exe so the app ships as a single file. `jolt cgen-build -m NS -o OUT` stages a build dir (src/jolt-core symlinks into the jolt tree, a generated cg.c of the hot fns, an uberscript bundle of the app, and an entry that bakes the runtime, installs the native fns as var roots, and runs -main), then runs `jpm build` there — declare-native builds cg.a and declare-executable static-links it (jpm's create-executable marshals the module cfns and calls its static entry at startup). Build needs cc + jpm; the result needs neither. Mechanics that bit, codified in cgen_build.janet: stdlib_embed slurps .clj cwd-relative so the build runs in a repo-mirroring dir; jpm hardcodes ./project.janet and sets syspath=modpath; the executable's dofile imports cg and static-links cg.a, neither ordered nor release-built by default, so deps are wired explicitly; cleanup must lstat (the tree symlinks must not be followed); the inner build runs --workers=1 so it doesn't starve siblings in the parallel gate. test/integration/cgen-build-test.janet builds the mandelbrot fixture, runs it from a clean dir with no src/ and no cg.so, and checks the total at native speed. Closes jolt-a7ds. Co-authored-by: Yogthos --- ...ndational-runtime-lever1-native-codegen.md | 37 ++- src/jolt/cgen_build.janet | 211 ++++++++++++++++++ src/jolt/main.janet | 26 +++ test/fixtures/cgen-build/cgapp.clj | 30 +++ test/integration/cgen-build-test.janet | 51 +++++ 5 files changed, 352 insertions(+), 3 deletions(-) create mode 100644 src/jolt/cgen_build.janet create mode 100644 test/fixtures/cgen-build/cgapp.clj create mode 100644 test/integration/cgen-build-test.janet diff --git a/docs/foundational-runtime-lever1-native-codegen.md b/docs/foundational-runtime-lever1-native-codegen.md index 17db927..a037d7e 100644 --- a/docs/foundational-runtime-lever1-native-codegen.md +++ b/docs/foundational-runtime-lever1-native-codegen.md @@ -114,9 +114,40 @@ deploy with `cc` removed from PATH → `count-point` is still native, mandelbrot 3288753 at **12.4 ms** (full 18×). Test: `test/integration/cgen-aot-test.janet`. This removes the runtime-toolchain dependency — the core of the deployment story. -What remains for a literal single binary: fuse the prebuilt `.so` + manifest into -the `jpm`-built executable (declare-native/static-lib link + an uberscript-style -source bundle), so it ships as one file instead of an exe + sidecar `.so`. + +### The literal single binary (`jolt cgen-build`, done) + +`src/jolt/cgen_build.janet` + the `jolt cgen-build -m NS -o OUT` CLI fuse the +native code into the executable, so an app ships as ONE static file — no sidecar +`.so`, no toolchain to run. The driver: + +1. loads the app with `:cgen-collect?` to get its numeric-leaf fns + the source + files loaded (the uberscript-style bundle); +2. emits `cg.c` (one native module of those fns via `cgen/gen-c-module`) + a + positional manifest; +3. stages a build dir: `src`/`jolt-core` symlinks into the jolt tree, `cg.c`, the + app bundle, and an entry that bakes the runtime, installs the native fns as var + roots (`:cgen-prebuilt`), and runs `-main`; +4. runs `jpm build` there — `declare-native` builds `cg.a`, `declare-executable` + static-links it into the final exe (jpm's `create-executable` marshals the + module's cfunctions and calls its static entry at startup). + +Build needs `cc` + `jpm`; the result needs neither. Proven end-to-end: +`test/integration/cgen-build-test.janet` builds the mandelbrot fixture, runs it +from a clean dir with no `src/` and no `cg.so`, and gets the right total at native +speed (the count-point leaf is the linked cfunction). + +Build mechanics that bit (codified in `cgen_build.janet`): `stdlib_embed` slurps +`.clj` relative to cwd, so the build runs in a dir mirroring the repo layout (the +symlinks); jpm hardcodes `./project.janet` and sets `syspath = modpath`; the +executable's dofile imports `cg` and static-links `cg.a`, neither ordered nor +release-built by default, so the deps are wired explicitly; cleanup must `lstat` +(never follow the tree symlinks). The inner build runs `--workers=1` so it doesn't +saturate cores inside the parallel test gate. + +Open follow-ups (filed): widen the cgen grammar (jolt-l1l4) so more of an app's +hot fns qualify; hot-fn auto-detection (jolt-qx70) to drop the manual collect; +DCE on the bundled source (the uberscript path already does it). ## Open questions for the implementation (next beads) diff --git a/src/jolt/cgen_build.janet b/src/jolt/cgen_build.janet new file mode 100644 index 0000000..76a49f0 --- /dev/null +++ b/src/jolt/cgen_build.janet @@ -0,0 +1,211 @@ +# Single-binary native build (jolt-a7ds, epic jolt-5vsp). The AOT path +# (cgen.janet) compiles an app's numeric-leaf fns into one prebuilt .so + a +# manifest, loaded at deploy with no cc. This module fuses that native code into +# the jpm-built executable so an app ships as ONE static file: no sidecar .so, no +# toolchain on the target. +# +# Shape (validated by the spike, docs/foundational-runtime-lever1-native-codegen.md): +# a staging dir with `src`/`jolt-core` symlinks into the jolt source tree, the +# generated cg.c (native module of the app's hot fns), an uberscript-style bundle +# of the app source, and an entry that bakes the jolt runtime, installs the native +# fns as var roots (:cgen-prebuilt), and runs -main. `jpm build` static-links cg.a +# (declare-native builds it) into the final exe. Build needs cc; the result does +# not. See build-app for the driver and `jolt cgen-build` (main.janet) for the CLI. +(import ./api :as api) +(import ./types :as t) +(import ./cgen :as cgen) + +# --- jolt source location ---------------------------------------------------- +# Baking the runtime into the app binary needs the jolt source tree (src/jolt + +# jolt-core). The shipped binary doesn't carry its own source, so locate a +# checkout: JOLT_HOME wins, else walk up from cwd for the marker files. +(defn- jolt-home? [d] + (and (os/stat (string d "/src/jolt/api.janet")) + (os/stat (string d "/jolt-core")))) + +(defn find-jolt-home + "Directory of a jolt source checkout (has src/jolt + jolt-core), or nil. + JOLT_HOME overrides; otherwise walk up from `start` (default cwd)." + [&opt start] + (def env-home (os/getenv "JOLT_HOME")) + (cond + (and env-home (jolt-home? env-home)) env-home + (do + (var d (or start (os/cwd))) + (var found nil) + (while (and (not found) (> (length d) 1)) + (if (jolt-home? d) + (set found d) + (set d (string/slice d 0 (max 0 (last (string/find-all "/" d))))))) + found))) + +# --- generation -------------------------------------------------------------- +(defn gen-c-and-manifest + "collected: [{:ns :name :ir} ...] (numeric-leaf fns from a :cgen-collect? load). + Emit the C source for ONE native module exporting them all, plus a positional + manifest [{:ns :name :sym} ...] mapping each app fn to its C symbol. Mirrors + cgen/aot-build but yields C source (compiled at build) rather than a .so." + [collected] + (def entries @[]) + (def manifest @[]) + (eachp [i e] collected + (def sym (string "f" i)) + (array/push entries {:sym sym :ir (e :ir)}) + (array/push manifest {:ns (e :ns) :name (e :name) :sym sym})) + {:c-src (cgen/gen-c-module entries) :manifest manifest}) + +(defn bundle-source + "Concatenate the app's loaded source files (load order: deps before dependents) + into one closed-world bundle the baked entry can eval. Verbatim — DCE is the + uberscript path's concern, not needed here for correctness." + [files] + (def buf @"") + (each f files + (buffer/push-string buf "; --- " f " ---\n" (slurp f) "\n")) + (string buf)) + +(defn collect-app + "Load main-ns through the full pipeline with :cgen-collect? on (records the + numeric-leaf fns' IR without swapping roots), returning the collected fns and + the source files loaded (for the bundle). source-roots seed :source-paths so + the app + its deps resolve. Throws on a load error." + [main-ns source-roots] + (def ctx (api/init-cached {:compile? true})) # discarded; cached for speed + (each r source-roots + (when (> (length r) 0) (array/push (get (ctx :env) :source-paths) r))) + (put (ctx :env) :direct-linking? true) + (put (ctx :env) :cgen-collect? true) + (put (ctx :env) :loaded-files @[]) + (api/load-string ctx (string "(require '[" main-ns "])") "") + (def seen @{}) + (def files @[]) + (each f (get (ctx :env) :loaded-files) + (unless (get seen f) (put seen f true) (array/push files f))) + {:collected (or (get (ctx :env) :cgen-collected) @[]) :files files}) + +(defn run-main! + "Runtime helper baked into a generated app binary: bind *command-line-args* to + args (a tuple/array of strings) and invoke main-ns/-main." + [ctx main-ns args] + (def core (t/ctx-find-ns ctx "clojure.core")) + (t/ns-intern core "*command-line-args*" (tuple/slice (tuple ;args))) + (api/eval-string ctx (string "(apply " main-ns "/-main *command-line-args*)"))) + +(defn gen-entry-source + "The generated main.janet: imports the jolt runtime + the static cg module, + builds qname->cfunction from the manifest, bakes the ctx with the app compiled + and the native fns installed as roots, and runs -main at startup. Strings are + emitted with %j so embedded source/manifest can't break the quoting." + [main-ns app-source manifest] + (string + "# Generated by `jolt cgen-build` — do not edit.\n" + "(import \"./src/jolt/api\" :as api)\n" + "(import \"./src/jolt/cgen_build\" :as cb)\n" + "(import \"./cg\" :as cg)\n\n" + "(def manifest " (string/format "%j" manifest) ")\n" + "(def app-source " (string/format "%j" app-source) ")\n" + "(def main-ns " (string/format "%j" main-ns) ")\n\n" + "(def cg-env (require \"./cg\"))\n" + "(def prebuilt\n" + " (let [tb @{}]\n" + " (each e manifest\n" + " (when-let [cell (get cg-env (symbol (e :sym)))]\n" + " (put tb (string (e :ns) \"/\" (e :name)) (get cell :value))))\n" + " tb))\n\n" + "(def ctx (api/init {:compile? true}))\n" + "(put (ctx :env) :direct-linking? true)\n" + "(put (ctx :env) :cgen-prebuilt prebuilt)\n" + "(api/load-string ctx app-source \"app.clj\")\n\n" + "(defn main [&]\n" + " (def args (or (dyn :args) @[]))\n" + " (cb/run-main! ctx main-ns (slice args 1)))\n")) + +(defn gen-project-source + "The generated project.janet. The executable's entry imports cg at build + (dofile) and static-links cg.a, so both must build first — jpm infers neither, + and cg.a isn't pulled into a non-release `build` task, so wire deps explicitly." + [exe-name] + (string + "(declare-project :name \"jolt-cgen-app\")\n\n" + "(declare-native :name \"cg\" :source @[\"cg.c\"])\n\n" + "(declare-executable :name " (string/format "%j" exe-name) " :entry \"main.janet\")\n\n" + "(add-dep \"./" exe-name "\" \"./cg.so\")\n" + "(add-dep \"./" exe-name "\" \"./cg.a\")\n")) + +# --- driver ------------------------------------------------------------------ +(defn- mkdir-p [path] + (def parts (filter |(not (empty? $)) (string/split "/" path))) + (var acc (if (string/has-prefix? "/" path) "" ".")) + (each p parts (set acc (string acc "/" p)) (os/mkdir acc))) + +(defn- rm-rf [path] + # lstat, never stat: the staging dir holds symlinks INTO the jolt tree + # (src, jolt-core); following them would delete real source. A symlink — even + # one pointing at a directory — is removed with os/rm, not recursed into. + (when-let [st (os/lstat path)] + (if (= :directory (st :mode)) + (do (each e (os/dir path) (rm-rf (string path "/" e))) (os/rmdir path)) + (os/rm path)))) + +(defn build-app + "Build main-ns into a single static executable at :out. Needs cc + jpm at build + time; the result needs neither. opts: + :main app namespace (string, required) + :out output binary path (required) + :jolt-home jolt source checkout (default: find-jolt-home) + :source-roots extra source roots for the app + deps (default from JOLT_PATH) + :stage staging dir (default: a fresh temp dir; removed unless :keep) + :keep keep the staging dir for inspection + Returns {:out :native-count :stage }." + [opts] + (def main-ns (assert (opts :main) "cgen-build: :main namespace required")) + (def out (assert (opts :out) "cgen-build: :out path required")) + (def home (or (opts :jolt-home) (find-jolt-home))) + (assert home "cgen-build: could not locate jolt source (set JOLT_HOME)") + (def source-roots + (or (opts :source-roots) + (let [jp (os/getenv "JOLT_PATH")] + (if jp (filter |(> (length $) 0) (string/split ":" jp)) @[])))) + (def exe-name "app") + + # 1. Collect the app's numeric-leaf fns + its full source bundle. + (def {:collected collected :files files} (collect-app main-ns source-roots)) + (def {:c-src c-src :manifest manifest} (gen-c-and-manifest collected)) + (def app-source (bundle-source files)) + + # 2. Stage: symlinks into the jolt tree + the generated build inputs. + (def stage + (or (opts :stage) + (string (or (os/getenv "TMPDIR") "/tmp") "/jolt-cgen-build-" + (string/format "%x" (band (hash (string main-ns out)) 0x7fffffff))))) + (rm-rf stage) + (mkdir-p stage) + (os/symlink (string home "/src") (string stage "/src")) + (os/symlink (string home "/jolt-core") (string stage "/jolt-core")) + (spit (string stage "/cg.c") c-src) + (spit (string stage "/main.janet") (gen-entry-source main-ns app-source manifest)) + (spit (string stage "/project.janet") (gen-project-source exe-name)) + + # 3. jpm build in the staging dir (cwd + JANET_BUILDPATH=. so artifacts land + # beside the entry, where ./cg and cg.a resolve). + (def prev-cwd (os/cwd)) + (def prev-bp (os/getenv "JANET_BUILDPATH")) + (defer (do (os/cd prev-cwd) (os/setenv "JANET_BUILDPATH" prev-bp)) + (os/cd stage) + (os/setenv "JANET_BUILDPATH" ".") + # --workers=1: the build shells out to cc; one worker keeps it from saturating + # every core (it runs inside the parallel test gate, where that starves + # siblings into timeouts). Only a few small C files, so the cost is marginal. + (def code (os/execute ["jpm" "--workers=1" "build"] :p)) + (os/cd prev-cwd) + (unless (= 0 code) (error (string "cgen-build: jpm build failed (" code ")")))) + + # 4. Install the binary, then clean up staging. + (def built (string stage "/" exe-name)) + (assert (os/stat built) (string "cgen-build: expected binary at " built)) + (let [od (string/slice out 0 (max 0 (last (or (string/find-all "/" out) [0]))))] + (when (> (length od) 0) (mkdir-p od))) + (spit out (slurp built)) + (os/chmod out 8r755) + (unless (opts :keep) (rm-rf stage)) + {:out out :native-count (length manifest) :stage stage}) diff --git a/src/jolt/main.janet b/src/jolt/main.janet index 83d0fca..9a826c5 100644 --- a/src/jolt/main.janet +++ b/src/jolt/main.janet @@ -12,6 +12,7 @@ (use ./reader) (import ./core :as jcore) (import ./deps :as deps) +(import ./cgen_build :as cgen-build) (def jolt-version "0.1.0") @@ -613,6 +614,28 @@ (print "Wrote " out " (" (length files) " namespace(s)" (if (and dce-ok (> dropped 0)) (string ", " dropped " dead fn(s) dropped") "") ")")) +(defn- run-cgen-build [argv] + # jolt cgen-build -m NS -o OUT [--keep]: fuse the app's hot numeric-leaf fns + # (compiled to native code) + the app source into one static executable. + (var main-ns nil) + (var out nil) + (var keep false) + (var i 0) + (while (< i (length argv)) + (def a (argv i)) + (cond + (or (= a "-m") (= a "--main")) (do (set main-ns (get argv (+ i 1))) (+= i 2)) + (or (= a "-o") (= a "--out")) (do (set out (get argv (+ i 1))) (+= i 2)) + (= a "--keep") (do (set keep true) (++ i)) + (++ i))) + (when (or (nil? main-ns) (nil? out)) + (eprint "Usage: jolt cgen-build -m NS -o OUT [--keep]") + (os/exit 1)) + (try + (let [r (cgen-build/build-app {:main main-ns :out out :keep keep})] + (print "Wrote " (r :out) " (" (r :native-count) " native fn(s) linked)")) + ([err fib] (report-error err fib) (os/exit 1)))) + (defn- print-help [] (print "Jolt — a Clojure interpreter on Janet\n") (print "Usage: jolt [opt] [args]\n") @@ -625,6 +648,8 @@ (print " nrepl-server [addr] Start an nREPL server (addr = [host:]port, default 7888)") (print " (aliases: --nrepl-server, nrepl)") (print " uberscript OUT -m NS Bundle NS + its required namespaces into one .clj") + (print " cgen-build -m NS -o OUT Build NS into a single static binary (hot fns") + (print " compiled to native code, no toolchain to run)") (print " --version, version Print the Jolt version") (print " -h, --help, help Show this help\n") (print "Dependencies (deps.edn, git + :local deps — resolved into JOLT_PATH):") @@ -796,5 +821,6 @@ rest (array/slice argv 2) mi (or (index-of "-m" rest) (index-of "--main" rest))] (run-uberscript out (if mi (get rest (+ mi 1)) nil))) + (= (argv 0) "cgen-build") (run-cgen-build (array/slice argv 1)) (= (argv 0) "-") (run-file "/dev/stdin" (array/slice argv 1)) (run-file (argv 0) (array/slice argv 1)))) diff --git a/test/fixtures/cgen-build/cgapp.clj b/test/fixtures/cgen-build/cgapp.clj new file mode 100644 index 0000000..0cc6a00 --- /dev/null +++ b/test/fixtures/cgen-build/cgapp.clj @@ -0,0 +1,30 @@ +;; Fixture app for the cgen single-binary build test (jolt-a7ds). +;; count-point is a numeric leaf -> compiled to native C and statically linked; +;; run/-main stay bytecode and call into it. -main prints a deterministic total. +(ns cgapp) + +(defn count-point [cr ci cap] + (loop [i 0 zr 0.0 zi 0.0] + (if (or (>= i cap) (> (+ (* zr zr) (* zi zi)) 4.0)) + i + (recur (inc i) + (+ (- (* zr zr) (* zi zi)) cr) + (+ (* 2.0 (* zr zi)) ci))))) + +(defn run [n] + (let [cap 200 + nd (* 1.0 n)] + (loop [y 0 acc 0] + (if (< y n) + (let [ci (- (/ (* 2.0 y) nd) 1.0) + row (loop [x 0 a 0] + (if (< x n) + (let [cr (- (/ (* 2.0 x) nd) 1.5)] + (recur (inc x) (+ a (count-point cr ci cap)))) + a))] + (recur (inc y) (+ acc row))) + acc)))) + +(defn -main [& args] + (let [n (if (seq args) (Integer/parseInt (first args)) 200)] + (println (run n)))) diff --git a/test/integration/cgen-build-test.janet b/test/integration/cgen-build-test.janet new file mode 100644 index 0000000..09e634b --- /dev/null +++ b/test/integration/cgen-build-test.janet @@ -0,0 +1,51 @@ +# Single-binary native build (jolt-a7ds): `jolt cgen-build` fuses an app's hot +# numeric-leaf fns (compiled to C) + the app source into ONE static executable — +# no sidecar .so, no toolchain on the target. This test builds the fixture app to +# a binary, runs it from a CLEAN dir (no src/, no cg.so) to prove it's +# self-contained, and checks the result + native fn count. Skips with no cc. +(import ../../src/jolt/cgen :as cgen) +(import ../../src/jolt/cgen_build :as cb) + +(print "Single-binary native build (jolt-a7ds)...") + +(var failures 0) +(defn check [label ok] (if ok (print " ok: " label) (do (++ failures) (eprintf " FAIL: %s" label)))) + +(def home (os/cwd)) # the repo (gate runs here) +(def fixture-root (string home "/test/fixtures/cgen-build")) + +(if (cgen/toolchain-available?) + (do + (def out (string (or (os/getenv "TMPDIR") "/tmp") "/jolt-cgen-build-test/app")) + (def r (cb/build-app {:main "cgapp" :out out :jolt-home home + :source-roots [fixture-root]})) + (check "build reported the single native leaf (count-point)" (= 1 (r :native-count))) + (check "binary exists at :out" (os/stat out)) + + # Run from a CLEAN dir with NO src/ and NO cg.so beside it — proves the binary + # is self-contained (runtime needs neither the jolt tree nor a sidecar .so). + (def rundir (string (or (os/getenv "TMPDIR") "/tmp") "/jolt-cgen-build-run")) + (when (os/stat rundir) (each e (os/dir rundir) (os/rm (string rundir "/" e))) (os/rmdir rundir)) + (os/mkdir rundir) + (def bin (string rundir "/app")) + (spit bin (slurp out)) + (os/chmod bin 8r755) + + (defn run-app [&opt n] + (def tmp (string rundir "/out.txt")) + (def f (file/open tmp :w)) + (def code (os/execute (if n [bin (string n)] [bin]) :px {:out f} )) + (file/close f) + {:code code :out (string/trim (slurp tmp))}) + + (def res (run-app)) + (check "exits 0" (= 0 (res :code))) + (check "self-contained binary computes the right mandelbrot total (n=200)" + (= "3288753" (res :out))) + (check "passes through command-line args (n=400)" + (= "13162060" ((run-app 400) :out)))) + (print " (toolchain absent — skipping single-binary build)")) + +(if (= 0 failures) + (print "All tests passed.") + (do (eprintf "%d cgen-build check(s) failed" failures) (os/exit 1)))