Merge pull request #50 from jolt-lang/macro-expansion-path

core: compiled macro expansion in every mode (+123 suite passes)

src/jolt/api.janet

@@ -243,13 +243,14 @@
   # Opts land in the key via their printed form; an opt that prints unstably
   # (e.g. a closure in :namespaces) just degrades to a cache miss, never to a
   # wrong hit. Runtime knobs that shape the ctx outside opts ride along too.
-  (def key (string/format "%q|%q|%q|%q|%q|%q"
+  (def key (string/format "%q|%q|%q|%q|%q|%q|%q"
                           (string janet/version "-" janet/build)
                           opts
                           (os/getenv "JOLT_PATH")
                           (os/getenv "JOLT_MUTABLE")
                           (os/getenv "JOLT_AOT_CORE")
-                          (os/getenv "JOLT_FEATURES")))
+                          (os/getenv "JOLT_FEATURES")
+                          (os/getenv "JOLT_INTERPRET_MACROS")))
   (string dir "/jolt-ctx-" (band h 0x7FFFFFFF) "-" len "-" (band (hash key) 0x7FFFFFFF) ".jimg"))
 
 (defn init-cached

src/jolt/backend.janet

@@ -510,12 +510,16 @@
   n)
 
 (defn ensure-macros-compiled!
-  "Called once the overlay is fully loaded (api/load-core-overlay!): in compile
-  mode, ensure the analyzer is built, then run the staged macro-recompile pass so
-  the early (interpreted-during-bootstrap) macro expanders become compiled. No-op
-  in interpreter mode (no analyzer, macros stay interpreted by design) and cheap to
-  call again (recompile-macros! skips already-compiled vars)."
+  "Called once the overlay is fully loaded (api/load-core-overlay!): ensure the
+  analyzer is built, then run the staged macro-recompile pass so the early
+  (interpreted-during-bootstrap) macro expanders become compiled. Runs in EVERY
+  mode — macro expansion is compiled code even when evaluation is interpreted
+  (in interpret mode the tiers load fast interpreted, then this one pass builds
+  the analyzer and compiles all stashed expanders; the analyzer itself stays
+  interpreted there). :compile-macros? false (JOLT_INTERPRET_MACROS=1) skips it,
+  keeping the fully-interpreted oracle. Cheap to call again (recompile-macros!
+  skips already-compiled vars)."
   [ctx]
-  (when (get (ctx :env) :compile?)
+  (when (get (ctx :env) :compile-macros?)
     (ensure-analyzer ctx)
     (when (analyzer-built? ctx) (recompile-macros! ctx))))

src/jolt/types.janet

@@ -408,11 +408,20 @@
         # (off unless opted in) and load-core-overlay! flips it on around core.
         aot-core? (let [o (if opts (get opts :aot-core?) nil)]
                     (if (nil? o) (not (= "0" (os/getenv "JOLT_AOT_CORE"))) o))
+        # Macro expanders compile in EVERY mode (macros are ordinary compiled
+        # fns, as in Clojure) — including interpret mode, where evaluation stays
+        # interpreted but expansion runs native. :compile-macros? false (or
+        # JOLT_INTERPRET_MACROS=1) opts back into the fully-interpreted oracle.
+        compile-macros? (let [o (if opts (get opts :compile-macros?) nil)]
+                          (if (nil? o)
+                            (not (= "1" (os/getenv "JOLT_INTERPRET_MACROS")))
+                            o))
         env @{:namespaces @{}
               :class->opts @{}
               :current-ns "user"
               :compile? compile?
               :aot-core? aot-core?
+              :compile-macros? compile-macros?
               :direct-linking? (if opts (get opts :direct-linking?) nil)
               # Ordered roots searched (after the stdlib) to resolve a namespace
               # to a .clj/.cljc file. jolt-core holds the portable Clojure layer

test/integration/clojure-test-suite-test.janet

@@ -43,9 +43,9 @@
 # Raised 4004 -> 4034 / clean 66 -> 67 porting partition-all + repeatedly to the
 # overlay, which required fixing two leniencies (a char is not callable; take
 # validates its count) — correct beyond those fns, so the suite rose broadly.
-(def baseline-pass 4540)
+(def baseline-pass 4660)
 # A file is "clean" when it ran with zero failures AND zero errors.
-(def baseline-clean-files 86)
+(def baseline-clean-files 88)
 # Per-file wall-clock budget (seconds). Normal files finish in well under 1s, so
 # this normally only fires on genuinely-infinite-sequence hangs. It's an env var
 # (JOLT_SUITE_TIMEOUT) so CI — whose runners are slower than a dev machine — can

test/integration/macro-expansion-test.janet

@@ -0,0 +1,77 @@
+# Compiled macro expansion in EVERY mode (jolt-tzo: the fast macro-expansion
+# path that unblocks moving hot fns the 00-syntax expanders depend on).
+#
+# Macros are ordinary compiled fns in Clojure's model. Compile mode has had
+# this since the staged bootstrap (ensure-macros-compiled! recompiles the
+# early interpreted expanders once the analyzer is alive); interpret mode —
+# the conformance battery's default — used to skip it, so every distinct
+# (and ...) / (cond ...) / nested expansion ran an interpreted closure.
+# Now interpret-mode init also builds the analyzer once at the end of the
+# overlay load and compiles every stashed expander; JOLT_INTERPRET_MACROS=1
+# opts back into the pure interpreted oracle.
+
+(use ../../src/jolt/api)
+(use ../../src/jolt/types)
+
+(print "compiled macro expansion...")
+(os/setenv "JOLT_INTERPRET_MACROS" nil)
+
+(defn- macro-var [ctx nm] (ns-find (ctx-find-ns ctx "clojure.core") nm))
+(defn- user-var [ctx nm] (ns-find (ctx-find-ns ctx "user") nm))
+
+(def probes
+  ["(= 3 (and 1 2 3))"
+   "(= 1 (or nil false 1))"
+   "(= :b (cond false :a :else :b))"
+   "(= 4 (when-not nil 4))"
+   "(= 6 (-> 1 inc (* 3)))"
+   "(= 15 (->> [1 2] (map inc) (reduce +) (* 3)))"
+   "(= 2 (if-let [x nil] 1 2))"
+   "(= [0 1] (vec (for [i (range 2)] i)))"
+   "(= 5 (case 2 1 :one 2 5 :dflt))"
+   "(= 10 (loop [i 0 a 0] (if (< i 5) (recur (inc i) (+ a 2)) a)))"])
+
+(defn- run-probes [ctx label]
+  (each prog probes
+    (def got (protect (eval-string ctx prog)))
+    (assert (and (got 0) (= (got 1) true))
+            (string label " probe failed: " prog " => "
+                    (if (got 0) (string/format "%q" (got 1)) (string (got 1)))))))
+
+# 1. Interpret mode: expanders are COMPILED after init (the new path).
+(def ictx (init {}))
+(run-probes ictx "interpret")
+(each m ["when" "when-not" "and" "or" "cond" "->" "->>" "if-let" "case" "doseq"]
+  (def v (macro-var ictx m))
+  (assert v (string m " var exists"))
+  (assert (get v :macro-compiled)
+          (string m " expander is compiled in interpret mode")))
+
+# 2. A USER defmacro in an interpret ctx gets a compiled expander too.
+(eval-string ictx "(defmacro my-twice [x] `(* 2 ~x))")
+(assert (= 10 (eval-string ictx "(my-twice 5)")) "user macro works")
+(assert (get (user-var ictx "my-twice") :macro-compiled)
+        "user macro expander compiled in interpret mode")
+
+# 3. An expander whose body the analyzer can't compile (here: the `eval`
+#    special form) falls back to the interpreted closure and still works.
+(eval-string ictx "(defmacro evalish [x] (eval `(+ ~x 1)))")
+(assert (= 3 (eval-string ictx "(evalish 2)")) "uncompilable expander works")
+(assert (not (get (user-var ictx "evalish") :macro-compiled))
+        "uncompilable expander stays interpreted")
+
+# 4. Compile mode unchanged: expanders compiled (pre-existing behavior).
+(def cctx (init {:compile? true}))
+(run-probes cctx "compile")
+(assert (get (macro-var cctx "cond") :macro-compiled) "compile-mode expanders compiled")
+
+# 5. JOLT_INTERPRET_MACROS=1: the pure interpreted oracle — same semantics,
+#    expanders NOT compiled.
+(os/setenv "JOLT_INTERPRET_MACROS" "1")
+(def octx (init {}))
+(run-probes octx "oracle")
+(assert (not (get (macro-var octx "cond") :macro-compiled))
+        "oracle mode keeps interpreted expanders")
+(os/setenv "JOLT_INTERPRET_MACROS" nil)
+
+(print "compiled macro expansion passed!")