From dc1f6d975533bdc8f2f068ad2234e3f484d56f7f Mon Sep 17 00:00:00 2001 From: Yogthos Date: Wed, 10 Jun 2026 15:37:32 -0400 Subject: [PATCH] core: compiled macro expansion in every mode (+123 suite passes) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Macros are ordinary compiled fns in Clojure's model; compile mode has had that since the staged bootstrap, but interpret mode — the conformance battery's default — kept interpreted expanders, so every distinct (and ...)/ (cond ...) call form, and every fresh form produced by a recursive expansion, ran an interpreted closure. ensure-macros-compiled! now runs in every mode: interpret-mode init loads the tiers fast-interpreted, then one pass at the end builds the analyzer (which itself stays interpreted there) and compiles all stashed expanders; user defmacros after init compile too. The new :compile-macros? opt (JOLT_INTERPRET_MACROS=1) preserves the fully- interpreted oracle, and joins the ctx-image cache key. Battery: 4700 pass / 90 clean files / 7 timeouts, from 4577 / 87 / 9 — two macro-heavy files stopped timing out and 149 more assertions execute. The compiled-expander delta proper is +67 passes (oracle mode on the same tree measures 4633). Baselines raised 4540->4660, clean 86->88. Interpret init grows 0.12s -> 1.12s for the analyzer build; init-cached amortizes it to ~5ms per process. New macro-expansion-test pins: expanders compiled in interpret mode (core + post-init user defmacros), uncompilable bodies fall back interpreted and still work, compile mode unchanged, oracle opt-out honored. Follow-up filed (jolt-4j3): the same staged-recompile treatment for early overlay DEFNS, which is what still pins keys/vals/empty? to the seed. --- src/jolt/api.janet | 5 +- src/jolt/backend.janet | 16 ++-- src/jolt/types.janet | 9 +++ .../integration/clojure-test-suite-test.janet | 4 +- test/integration/macro-expansion-test.janet | 77 +++++++++++++++++++ 5 files changed, 101 insertions(+), 10 deletions(-) create mode 100644 test/integration/macro-expansion-test.janet diff --git a/src/jolt/api.janet b/src/jolt/api.janet index 1023f56..8d25027 100644 --- a/src/jolt/api.janet +++ b/src/jolt/api.janet @@ -243,13 +243,14 @@ # Opts land in the key via their printed form; an opt that prints unstably # (e.g. a closure in :namespaces) just degrades to a cache miss, never to a # wrong hit. Runtime knobs that shape the ctx outside opts ride along too. - (def key (string/format "%q|%q|%q|%q|%q|%q" + (def key (string/format "%q|%q|%q|%q|%q|%q|%q" (string janet/version "-" janet/build) opts (os/getenv "JOLT_PATH") (os/getenv "JOLT_MUTABLE") (os/getenv "JOLT_AOT_CORE") - (os/getenv "JOLT_FEATURES"))) + (os/getenv "JOLT_FEATURES") + (os/getenv "JOLT_INTERPRET_MACROS"))) (string dir "/jolt-ctx-" (band h 0x7FFFFFFF) "-" len "-" (band (hash key) 0x7FFFFFFF) ".jimg")) (defn init-cached diff --git a/src/jolt/backend.janet b/src/jolt/backend.janet index c4895d1..1768c97 100644 --- a/src/jolt/backend.janet +++ b/src/jolt/backend.janet @@ -510,12 +510,16 @@ n) (defn ensure-macros-compiled! - "Called once the overlay is fully loaded (api/load-core-overlay!): in compile - mode, ensure the analyzer is built, then run the staged macro-recompile pass so - the early (interpreted-during-bootstrap) macro expanders become compiled. No-op - in interpreter mode (no analyzer, macros stay interpreted by design) and cheap to - call again (recompile-macros! skips already-compiled vars)." + "Called once the overlay is fully loaded (api/load-core-overlay!): ensure the + analyzer is built, then run the staged macro-recompile pass so the early + (interpreted-during-bootstrap) macro expanders become compiled. Runs in EVERY + mode — macro expansion is compiled code even when evaluation is interpreted + (in interpret mode the tiers load fast interpreted, then this one pass builds + the analyzer and compiles all stashed expanders; the analyzer itself stays + interpreted there). :compile-macros? false (JOLT_INTERPRET_MACROS=1) skips it, + keeping the fully-interpreted oracle. Cheap to call again (recompile-macros! + skips already-compiled vars)." [ctx] - (when (get (ctx :env) :compile?) + (when (get (ctx :env) :compile-macros?) (ensure-analyzer ctx) (when (analyzer-built? ctx) (recompile-macros! ctx)))) diff --git a/src/jolt/types.janet b/src/jolt/types.janet index 54badeb..6594dcb 100644 --- a/src/jolt/types.janet +++ b/src/jolt/types.janet @@ -408,11 +408,20 @@ # (off unless opted in) and load-core-overlay! flips it on around core. aot-core? (let [o (if opts (get opts :aot-core?) nil)] (if (nil? o) (not (= "0" (os/getenv "JOLT_AOT_CORE"))) o)) + # Macro expanders compile in EVERY mode (macros are ordinary compiled + # fns, as in Clojure) — including interpret mode, where evaluation stays + # interpreted but expansion runs native. :compile-macros? false (or + # JOLT_INTERPRET_MACROS=1) opts back into the fully-interpreted oracle. + compile-macros? (let [o (if opts (get opts :compile-macros?) nil)] + (if (nil? o) + (not (= "1" (os/getenv "JOLT_INTERPRET_MACROS"))) + o)) env @{:namespaces @{} :class->opts @{} :current-ns "user" :compile? compile? :aot-core? aot-core? + :compile-macros? compile-macros? :direct-linking? (if opts (get opts :direct-linking?) nil) # Ordered roots searched (after the stdlib) to resolve a namespace # to a .clj/.cljc file. jolt-core holds the portable Clojure layer diff --git a/test/integration/clojure-test-suite-test.janet b/test/integration/clojure-test-suite-test.janet index ee1ecb7..9577396 100644 --- a/test/integration/clojure-test-suite-test.janet +++ b/test/integration/clojure-test-suite-test.janet @@ -43,9 +43,9 @@ # Raised 4004 -> 4034 / clean 66 -> 67 porting partition-all + repeatedly to the # overlay, which required fixing two leniencies (a char is not callable; take # validates its count) — correct beyond those fns, so the suite rose broadly. -(def baseline-pass 4540) +(def baseline-pass 4660) # A file is "clean" when it ran with zero failures AND zero errors. -(def baseline-clean-files 86) +(def baseline-clean-files 88) # Per-file wall-clock budget (seconds). Normal files finish in well under 1s, so # this normally only fires on genuinely-infinite-sequence hangs. It's an env var # (JOLT_SUITE_TIMEOUT) so CI — whose runners are slower than a dev machine — can diff --git a/test/integration/macro-expansion-test.janet b/test/integration/macro-expansion-test.janet new file mode 100644 index 0000000..2dd21c3 --- /dev/null +++ b/test/integration/macro-expansion-test.janet @@ -0,0 +1,77 @@ +# Compiled macro expansion in EVERY mode (jolt-tzo: the fast macro-expansion +# path that unblocks moving hot fns the 00-syntax expanders depend on). +# +# Macros are ordinary compiled fns in Clojure's model. Compile mode has had +# this since the staged bootstrap (ensure-macros-compiled! recompiles the +# early interpreted expanders once the analyzer is alive); interpret mode — +# the conformance battery's default — used to skip it, so every distinct +# (and ...) / (cond ...) / nested expansion ran an interpreted closure. +# Now interpret-mode init also builds the analyzer once at the end of the +# overlay load and compiles every stashed expander; JOLT_INTERPRET_MACROS=1 +# opts back into the pure interpreted oracle. + +(use ../../src/jolt/api) +(use ../../src/jolt/types) + +(print "compiled macro expansion...") +(os/setenv "JOLT_INTERPRET_MACROS" nil) + +(defn- macro-var [ctx nm] (ns-find (ctx-find-ns ctx "clojure.core") nm)) +(defn- user-var [ctx nm] (ns-find (ctx-find-ns ctx "user") nm)) + +(def probes + ["(= 3 (and 1 2 3))" + "(= 1 (or nil false 1))" + "(= :b (cond false :a :else :b))" + "(= 4 (when-not nil 4))" + "(= 6 (-> 1 inc (* 3)))" + "(= 15 (->> [1 2] (map inc) (reduce +) (* 3)))" + "(= 2 (if-let [x nil] 1 2))" + "(= [0 1] (vec (for [i (range 2)] i)))" + "(= 5 (case 2 1 :one 2 5 :dflt))" + "(= 10 (loop [i 0 a 0] (if (< i 5) (recur (inc i) (+ a 2)) a)))"]) + +(defn- run-probes [ctx label] + (each prog probes + (def got (protect (eval-string ctx prog))) + (assert (and (got 0) (= (got 1) true)) + (string label " probe failed: " prog " => " + (if (got 0) (string/format "%q" (got 1)) (string (got 1))))))) + +# 1. Interpret mode: expanders are COMPILED after init (the new path). +(def ictx (init {})) +(run-probes ictx "interpret") +(each m ["when" "when-not" "and" "or" "cond" "->" "->>" "if-let" "case" "doseq"] + (def v (macro-var ictx m)) + (assert v (string m " var exists")) + (assert (get v :macro-compiled) + (string m " expander is compiled in interpret mode"))) + +# 2. A USER defmacro in an interpret ctx gets a compiled expander too. +(eval-string ictx "(defmacro my-twice [x] `(* 2 ~x))") +(assert (= 10 (eval-string ictx "(my-twice 5)")) "user macro works") +(assert (get (user-var ictx "my-twice") :macro-compiled) + "user macro expander compiled in interpret mode") + +# 3. An expander whose body the analyzer can't compile (here: the `eval` +# special form) falls back to the interpreted closure and still works. +(eval-string ictx "(defmacro evalish [x] (eval `(+ ~x 1)))") +(assert (= 3 (eval-string ictx "(evalish 2)")) "uncompilable expander works") +(assert (not (get (user-var ictx "evalish") :macro-compiled)) + "uncompilable expander stays interpreted") + +# 4. Compile mode unchanged: expanders compiled (pre-existing behavior). +(def cctx (init {:compile? true})) +(run-probes cctx "compile") +(assert (get (macro-var cctx "cond") :macro-compiled) "compile-mode expanders compiled") + +# 5. JOLT_INTERPRET_MACROS=1: the pure interpreted oracle — same semantics, +# expanders NOT compiled. +(os/setenv "JOLT_INTERPRET_MACROS" "1") +(def octx (init {})) +(run-probes octx "oracle") +(assert (not (get (macro-var octx "cond") :macro-compiled)) + "oracle mode keeps interpreted expanders") +(os/setenv "JOLT_INTERPRET_MACROS" nil) + +(print "compiled macro expansion passed!")