From a3cc035782d924c8f4f94780c5674d32834bca35 Mon Sep 17 00:00:00 2001 From: Yogthos Date: Sat, 6 Jun 2026 17:13:54 -0400 Subject: [PATCH] compiler: direct-linking (call-site/unit, Clojure model) + :aot-core? flag MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Calls compile direct (target value embedded) iff the compiling unit has direct-linking on, the target isn't ^:redef/^:dynamic, and it's an already-defined fn; otherwise indirect (live var deref, redefinable). Direct emission is guarded to function values only — embedding a non-function would make Janet evaluate it as code. :aot-core? (init opt, default true; JOLT_AOT_CORE=0 disables) compiles the core tiers with direct-linking on so core->core calls are direct. User/REPL code compiles indirect by default, so redefining functions in the REPL works with no annotation, exactly like Clojure. :aot-core? false makes core indirect too — the whole language becomes redefinable. ^:redef / ^:dynamic on a target force it indirect even inside a direct-linked unit. Also fixes a pre-existing gap this surfaced: compiled def dropped ALL var metadata (so ^:dynamic was silently lost under compilation, and ^:redef couldn't work). def metadata now flows analyzer -> IR -> back end (new form-sym-meta host fn, def-node :meta, var-setter-meta) and is applied to the cell. Matrix test in test/integration/direct-linking-test.janet. Conformance 218/218 in all three modes under both :aot-core? true and false; battery holds 3916; binary rebuilt. --- jolt-core/jolt/analyzer.clj | 4 +- jolt-core/jolt/ir.clj | 6 ++- src/jolt/api.janet | 10 +++- src/jolt/backend.janet | 36 ++++++++++++- src/jolt/host_iface.janet | 5 ++ src/jolt/types.janet | 9 ++++ test/integration/direct-linking-test.janet | 63 ++++++++++++++++++++++ 7 files changed, 127 insertions(+), 6 deletions(-) create mode 100644 test/integration/direct-linking-test.janet diff --git a/jolt-core/jolt/analyzer.clj b/jolt-core/jolt/analyzer.clj index d3a8463..ba187a3 100644 --- a/jolt-core/jolt/analyzer.clj +++ b/jolt-core/jolt/analyzer.clj @@ -22,7 +22,7 @@ form-literal? form-elements form-vec-items form-map-pairs form-special? compile-ns form-macro? form-expand-1 resolve-global - host-intern!]])) + form-sym-meta host-intern!]])) (declare analyze) @@ -140,7 +140,7 @@ nm (form-sym-name name-sym) cur (compile-ns ctx)] (host-intern! ctx cur nm) - (def-node cur nm (analyze ctx (nth items 2) env))) + (def-node cur nm (analyze ctx (nth items 2) env) (form-sym-meta name-sym))) "let*" (let [bvec (vec (form-vec-items (nth items 1))) r (analyze-bindings ctx bvec env)] (let-node (first r) (analyze-seq ctx (drop 2 items) (second r)))) diff --git a/jolt-core/jolt/ir.clj b/jolt-core/jolt/ir.clj index 0179755..0624810 100644 --- a/jolt-core/jolt/ir.clj +++ b/jolt-core/jolt/ir.clj @@ -29,7 +29,11 @@ (defn invoke [f args] {:op :invoke :fn f :args args}) -(defn def-node [ns name init] {:op :def :ns ns :name name :init init}) +;; meta is the var metadata (e.g. {:dynamic true} / {:redef true}) the back end +;; applies to the cell; nil when the def name carried none. +(defn def-node + ([ns name init] {:op :def :ns ns :name name :init init :meta nil}) + ([ns name init meta] {:op :def :ns ns :name name :init init :meta meta})) (defn let-node [bindings body] {:op :let :bindings bindings :body body}) diff --git a/src/jolt/api.janet b/src/jolt/api.janet index a4eb883..bb6bc91 100644 --- a/src/jolt/api.janet +++ b/src/jolt/api.janet @@ -51,14 +51,22 @@ "Load the Clojure portion of clojure.core in dependency-ordered tiers. See core-tiers and jolt-core/clojure/core/." [ctx] - (def compile? (get (ctx :env) :compile?)) + (def env (ctx :env)) + (def compile? (get env :compile?)) + # Core compiles with direct-linking on when :aot-core? (so core->core calls + # are direct). The flag is restored to the user-code default afterward, so + # user/REPL code stays indirect and fully redefinable. + (def user-dl (get env :direct-linking?)) + (def core-dl (get env :aot-core?)) (def saved (ctx-current-ns ctx)) (ctx-set-current-ns ctx "clojure.core") (each tier core-tiers (when-let [src (get stdlib-embed/sources (tier :ns))] + (put env :direct-linking? core-dl) (if (and compile? (tier :kernel)) (backend/bootstrap-load-source ctx "clojure.core" src) (eval-overlay-source ctx src)))) + (put env :direct-linking? user-dl) (ctx-set-current-ns ctx saved)) (defn init diff --git a/src/jolt/backend.janet b/src/jolt/backend.janet index 325a2cf..a43c1b5 100644 --- a/src/jolt/backend.janet +++ b/src/jolt/backend.janet @@ -24,9 +24,35 @@ (or (get cell :jolt/setter) (let [s (fn [v] (bind-root cell v) cell)] (put cell :jolt/setter s) s))) +# Setter that also applies def metadata to the var (so ^:dynamic / ^:redef / +# ^:private survive compilation, matching the interpreter's def). Not memoized: +# the meta is specific to this def site. +(defn- var-setter-meta [cell meta] + (fn [v] + (bind-root cell v) + (put cell :meta (merge (or (cell :meta) {}) meta)) + (when (get meta :dynamic) (put cell :dynamic true)) + cell)) + (defn- cell-for [ctx ns-name nm] (ns-intern (ctx-find-ns ctx ns-name) nm)) +# Direct-linking decision (call-site/unit property, Clojure-style). A var +# reference compiles to its embedded value (direct) iff: +# - the compiling unit has direct-linking on (env :direct-linking?), +# - the target opts in (NOT ^:redef / ^:dynamic — those force indirect), +# - the target is already defined AND its root is a Janet function. +# The function? guard is essential: embedding a non-function value (a jolt +# collection/symbol) into the emitted form would make Janet evaluate it AS code. +# So we direct-link exactly the call-optimization case; everything else stays +# indirect (live var deref → redefinable). Default user/REPL units: flag off, +# so all user calls are indirect and redefinable with no annotation. +(defn- direct-var? [ctx cell] + (and (get (ctx :env) :direct-linking?) + (not (cell :dynamic)) + (not (let [m (cell :meta)] (and m (get m :redef)))) + (function? (cell :root)))) + # Fresh Janet symbol for back-end-introduced bindings (arity dispatch). NOT # Janet's `gensym` — `(use ./core)` shadows it with Jolt's, which returns a jolt # symbol struct (invalid in a Janet param position). @@ -186,14 +212,20 @@ :const (node :val) :local (symbol (node :name)) :host (symbol (node :name)) - :var (tuple (var-getter (cell-for ctx (node :ns) (node :name)))) + :var (let [cell (cell-for ctx (node :ns) (node :name))] + (if (direct-var? ctx cell) + (cell :root) # direct link: embed the fn value + (tuple (var-getter cell)))) # indirect: live, redefinable :if ['if (emit ctx (node :test)) (emit ctx (node :then)) (emit ctx (node :else))] :do (emit-seq ctx node) :loop (emit-loop ctx node) :recur (emit-recur ctx node) :try (emit-try ctx node) :throw ['error (emit ctx (node :expr))] - :def (tuple (var-setter (cell-for ctx (node :ns) (node :name))) (emit ctx (node :init))) + :def (let [cell (cell-for ctx (node :ns) (node :name)) + meta (node :meta)] + (tuple (if (and meta (not (empty? meta))) (var-setter-meta cell meta) (var-setter cell)) + (emit ctx (node :init)))) :let (emit-let ctx node) :fn (emit-fn ctx node) :invoke (emit-invoke ctx node) diff --git a/src/jolt/host_iface.janet b/src/jolt/host_iface.janet index 5b7278e..4e43638 100644 --- a/src/jolt/host_iface.janet +++ b/src/jolt/host_iface.janet @@ -29,6 +29,10 @@ (defn h-sym? [form] (and (struct? form) (= :symbol (form :jolt/type)))) (defn h-sym-name [form] (form :name)) (defn h-sym-ns [form] (form :ns)) +# Reader metadata on a symbol (e.g. ^:dynamic / ^:redef / ^:private on a def +# name). Returns the meta map or nil. Lets the analyzer carry def metadata that +# the back end applies to the var — without it, compiled defs drop all var meta. +(defn h-sym-meta [form] (form :meta)) (defn h-list? [form] (array? form)) # a call / list (reader: array) (defn h-vector? [form] (tuple? form)) # a vector literal (reader: tuple) @@ -133,6 +137,7 @@ # intercepting them as the value-level predicates. (def- exports {"form-sym?" h-sym? "form-sym-name" h-sym-name "form-sym-ns" h-sym-ns + "form-sym-meta" h-sym-meta "form-list?" h-list? "form-vec?" h-vector? "form-map?" h-map? "form-set?" h-set? "form-char?" h-char? "form-literal?" h-literal? "form-elements" h-elements "form-vec-items" h-vector-items diff --git a/src/jolt/types.janet b/src/jolt/types.janet index 9580267..b6d9823 100644 --- a/src/jolt/types.janet +++ b/src/jolt/types.janet @@ -375,10 +375,19 @@ [&opt opts] (default opts nil) (let [compile? (if opts (get opts :compile?) false) + # Direct-linking (call-site/unit property, like Clojure). :aot-core? + # (default true; JOLT_AOT_CORE=0 disables) compiles the core tiers + + # compiler with direct-linking on. :direct-linking? is the per-unit flag + # the back end reads while emitting; it defaults to the user-code setting + # (off unless opted in) and load-core-overlay! flips it on around core. + aot-core? (let [o (if opts (get opts :aot-core?) nil)] + (if (nil? o) (not (= "0" (os/getenv "JOLT_AOT_CORE"))) o)) env @{:namespaces @{} :class->opts @{} :current-ns "user" :compile? compile? + :aot-core? aot-core? + :direct-linking? (if opts (get opts :direct-linking?) nil) # Ordered roots searched (after the stdlib) to resolve a namespace # to a .clj/.cljc file. jolt-core holds the portable Clojure layer # (analyzer/IR/core); deps.edn resolution appends dep src dirs. diff --git a/test/integration/direct-linking-test.janet b/test/integration/direct-linking-test.janet new file mode 100644 index 0000000..0bb1fe5 --- /dev/null +++ b/test/integration/direct-linking-test.janet @@ -0,0 +1,63 @@ +# Direct-linking / redefinition matrix (jolt-d9j, jolt-g86). +# +# Direct-linking is a per-compilation-UNIT property (Clojure model). A call +# compiles direct iff the unit has direct-linking on AND the target is not +# ^:redef/^:dynamic AND the target is an already-defined fn. Otherwise indirect +# (live var deref → redefinable). This pins the user-visible semantics: +# - default user/REPL unit (direct-linking off): redefine anything, callers see it +# - direct-linked unit: callers don't see a later redef (unless target ^:redef) +# - :aot-core? gates whether the core tiers compile direct-linked + +(use ../../src/jolt/api) + +(var failures 0) +(defn- check [label got want] + (unless (= got want) + (++ failures) + (printf "FAIL [%s] got %q want %q" label got want))) + +# 1. Default unit (direct-linking OFF): redefinition reaches compiled callers. +(let [ctx (init {:compile? true})] + (eval-string ctx "(defn add [a b] (+ a b))") + (eval-string ctx "(defn caller [] (add 1 2))") + (check "default before redef" (eval-string ctx "(caller)") 3) + (eval-string ctx "(defn add [a b] (* a b))") + (check "default sees redef (indirect)" (eval-string ctx "(caller)") 2)) + +# 2. Direct-linked unit: compiled caller keeps the original target after a redef. +(let [ctx (init {:compile? true :direct-linking? true})] + (eval-string ctx "(defn add [a b] (+ a b))") + (eval-string ctx "(defn caller [] (add 1 2))") + (check "direct before redef" (eval-string ctx "(caller)") 3) + (eval-string ctx "(defn add [a b] (* a b))") + (check "direct ignores redef (sealed)" (eval-string ctx "(caller)") 3) + # the var itself is still redefined; only the direct-linked call is frozen + (check "direct var still updated" (eval-string ctx "(add 3 4)") 12)) + +# 3. ^:redef opts a var OUT of direct-linking even in a direct-linked unit. +(let [ctx (init {:compile? true :direct-linking? true})] + (eval-string ctx "(defn ^:redef add [a b] (+ a b))") + (eval-string ctx "(defn caller [] (add 1 2))") + (check "redef-tagged before" (eval-string ctx "(caller)") 3) + (eval-string ctx "(defn ^:redef add [a b] (* a b))") + (check "redef-tagged sees redef" (eval-string ctx "(caller)") 2)) + +# 4. :aot-core? true (default): redefining a core fn (in clojure.core) is still +# seen by USER code, because user calls are indirect regardless of core being +# direct-linked internally. +(let [ctx (init {:compile? true})] + (eval-string ctx "(defn uses-last [] (last [1 2 3]))") + (check "core call before" (eval-string ctx "(uses-last)") 3) + (eval-string ctx "(in-ns (quote clojure.core))") + (eval-string ctx "(def last (fn [coll] :patched))") + (eval-string ctx "(in-ns (quote user))") + (check "user sees core redef (indirect)" (eval-string ctx "(uses-last)") :patched)) + +# 5. :aot-core? false: core compiles indirect too, so even core-internal callers +# see a redef — the whole language is redefinable. +(let [ctx (init {:compile? true :aot-core? false})] + (check "aot-core off still correct" (eval-string ctx "(last [1 2 3])") 3)) + +(if (pos? failures) + (do (printf "direct-linking: %d failure(s)" failures) (os/exit 1)) + (print "direct-linking: all matrix cases passed"))