From ce3c7df24b6eb358a940f04380e2506008f4a06b Mon Sep 17 00:00:00 2001 From: Dmitri Sotnikov Date: Tue, 16 Jun 2026 17:37:35 +0000 Subject: [PATCH] cgen: content-address and cache the compiled .so (jolt-ihdp) (#147) compile-fn now keys the .so on a hash of the generated C + the Janet ABI + the platform, in a persistent cache dir (default jolt-cgen under TMPDIR, override with JOLT_CGEN_CACHE_DIR; JOLT_CGEN_NO_CACHE=1 forces a rebuild). cc runs only on the first build of a given fn; later runs with the same source reuse the cached .so, so the per-startup compile cost is paid once. mandelbrot 100 whole-process wall: cold ~0.71s -> warm ~0.21s (the ~0.5s cc cost). These cache knobs don't shape output, so they stay out of ctx-shaping-env-vars (same as the image-cache knobs). Test asserts the .so is content-addressed and a second compile hits the cache without the source .c. Co-authored-by: Yogthos --- src/jolt/cgen.janet | 53 ++++++++++++++++++++++---------- test/integration/cgen-test.janet | 17 +++++++++- 2 files changed, 53 insertions(+), 17 deletions(-) diff --git a/src/jolt/cgen.janet b/src/jolt/cgen.janet index 7efbd22..6776d0f 100644 --- a/src/jolt/cgen.janet +++ b/src/jolt/cgen.janet @@ -214,30 +214,51 @@ (= 0 (os/execute ["cc" "--version"] :px {:out (file/open "/dev/null" :w) :err (file/open "/dev/null" :w)})))) +(defn- mkdirs [path] + (def parts (filter |(not (empty? $)) (string/split "/" path))) + (var acc (if (string/has-prefix? "/" path) "" ".")) + (each p parts (set acc (string acc "/" p)) (os/mkdir acc))) + +(defn- cache-dir [] + # Persistent across runs (like the ctx image cache, which also defaults to + # TMPDIR). Override with JOLT_CGEN_CACHE_DIR. + (string (or (os/getenv "JOLT_CGEN_CACHE_DIR") (os/getenv "TMPDIR") "/tmp") "/jolt-cgen")) + +(defn- cc-compile [cpath sopath] + # macOS needs -undefined dynamic_lookup so the module's janet_* refs resolve + # against the host at load time; ELF (Linux/BSD) resolves them then anyway. + (def cmd @["cc" "-shared" "-fPIC" "-O2" (string "-I" (header-dir))]) + (when (= :macos (os/which)) (array/push cmd "-undefined") (array/push cmd "dynamic_lookup")) + (array/push cmd cpath "-o" sopath) + (def r (os/execute cmd :p)) + (unless (= 0 r) (error (string "cgen: cc failed (" r ") for " cpath)))) + (defn compile-fn "Compile a numeric-leaf fn IR to a native Janet cfunction. Returns the cfunction, or nil if ir isn't a candidate or the toolchain is unavailable. - opts: :dir (build dir, default \"build/cgen\"), :name (C identifier base). - cc errors propagate (a compiler crash is a cgen bug, not a punt)." + opts: :dir (cache dir, default a persistent jolt-cgen under TMPDIR), :name (C + identifier base). The .so is content-addressed by a hash of the generated C + + the Janet ABI + platform, so cc only runs on the first build of a given fn; + later runs (same source) reuse the cached .so. JOLT_CGEN_NO_CACHE=1 forces a + rebuild. cc errors propagate (a compiler crash is a cgen bug, not a punt)." [ir &opt opts] (default opts {}) (when (and (numeric-leaf? ir) (header-dir)) - (def dir (get opts :dir "build/cgen")) (def c-name (sanitize (get opts :name "jolt_cfn"))) - (os/mkdir dir) (def src (gen-c-fn ir c-name)) - (def cpath (string dir "/" c-name ".c")) - (def sopath (string dir "/" c-name ".so")) - (spit cpath src) - # macOS needs -undefined dynamic_lookup so the module's janet_* refs resolve - # against the host at load time; ELF (Linux/BSD) resolves them then anyway. - (def mac? (= :macos (os/which))) - (def cmd @["cc" "-shared" "-fPIC" "-O2" (string "-I" (header-dir))]) - (when mac? (array/push cmd "-undefined") (array/push cmd "dynamic_lookup")) - (array/push cmd cpath "-o" sopath) - (def r (os/execute cmd :p)) - (unless (= 0 r) (error (string "cgen: cc failed (" r ") for " cpath))) - (def abs (if (= (in sopath 0) (chr "/")) sopath (string (os/cwd) "/" sopath))) + # content address: the C (semantics + symbol name) + ABI + platform fully + # determine the .so, so a matching cache entry is always valid. + (def stamp (string src "|" janet/version "-" janet/build "|" (os/which))) + (def key (string (band (hash stamp) 0x7FFFFFFF) "-" (length src))) + (def dir (get opts :dir (cache-dir))) + (mkdirs dir) + (def base (string dir "/cg-" key)) + (def sopath (string base ".so")) + (def abs (if (string/has-prefix? "/" sopath) sopath (string (os/cwd) "/" sopath))) + (unless (and (not (os/getenv "JOLT_CGEN_NO_CACHE")) (os/stat abs)) + (def cpath (string base ".c")) + (spit cpath src) + (cc-compile cpath sopath)) (def env @{}) (native abs env) ((get env (symbol c-name)) :value))) diff --git a/test/integration/cgen-test.janet b/test/integration/cgen-test.janet index 9cb027e..376a05b 100644 --- a/test/integration/cgen-test.janet +++ b/test/integration/cgen-test.janet @@ -41,6 +41,9 @@ # --- behavioral equivalence (only where the toolchain is present) --- (if (cgen/toolchain-available?) (do + # start from a clean cache dir so the content-addressed-file count is exact + (when (os/stat "build/cgen-test") + (each f (os/dir "build/cgen-test") (os/rm (string "build/cgen-test/" f)))) (api/eval-string ctx count-point-src) (def bc-cp (api/eval-string ctx "count-point")) # the compiled/interpreted fn (def c-cp (cgen/compile-fn (ir-of count-point-src) @@ -63,7 +66,19 @@ (set acc (+ acc a)) (++ y)) acc) (check "C and bytecode agree on the full n=80 grid total" - (= (grid-total c-cp 80) (grid-total bc-cp 80))))) + (= (grid-total c-cp 80) (grid-total bc-cp 80))) + + # caching: the .so is content-addressed, so a second compile of the same + # fn reuses it. Drop the generated .c (keep the .so) then recompile: a + # cache hit loads the existing .so without needing cc/source again. + (def cache-files (filter |(string/has-suffix? ".so" $) (os/dir "build/cgen-test"))) + (check "produced a content-addressed .so" (= 1 (length cache-files))) + (each f (os/dir "build/cgen-test") + (when (string/has-suffix? ".c" f) (os/rm (string "build/cgen-test/" f)))) + (def c-cp2 (cgen/compile-fn (ir-of count-point-src) + {:dir "build/cgen-test" :name "count_point_test"})) + (check "cache hit recompiles without the source .c" + (and (callable? c-cp2) (= (c-cp2 -0.5 0.0 200) 200))))) (print " (toolchain absent — skipping behavioral equivalence)")) (if (= 0 failures)