From 497970029d1b017aaf5caf0ad686a79fa97a22b4 Mon Sep 17 00:00:00 2001 From: Yogthos Date: Sat, 6 Jun 2026 11:34:02 -0400 Subject: [PATCH] self-host: make it the only compile path; drop the bootstrap as a selectable path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit eval-toplevel now routes every compiled form through backend/compile-and-eval (analyzer -> IR -> Janet back end). The bootstrap compiler (compile-ast) stays only to bootstrap jolt.ir/jolt.analyzer in backend/compile-load. Dropped the :selfhost? flag and JOLT_SELFHOST env plumbing — self-host is the default. Three correctness fixes the full-suite-under-self-host sweep exposed: - analyzer/back end: recur into a variadic arity now compiles. The arity fn gets an ordinary positional rest param (the collected seq), with a dispatch wrapper, so (recur fixed... rest-seq) is a clean self-call like Clojure -- instead of punting to the interpreter, which hangs on this (jolt-4df). - host_iface: interop heads (.foo/.-field/Foo.) and ns-management forms (all-ns, create-ns, resolve, ...) now mark uncompilable so they fall back instead of compiling to a bad call. - back end: named-fn self-reference ((fn self [n] ... (self ...))) binds the fn name via a var. Full unit+integration suite green; clojure-test-suite battery holds 3913. --- jolt-core/jolt/analyzer.clj | 5 ++- src/jolt/api.janet | 3 +- src/jolt/backend.janet | 53 +++++++++++++++++++++------ src/jolt/host_iface.janet | 16 +++++++- src/jolt/loader.janet | 15 +++----- src/jolt/main.janet | 7 ++-- test/integration/self-host-test.janet | 21 +++++++++++ 7 files changed, 93 insertions(+), 27 deletions(-) diff --git a/jolt-core/jolt/analyzer.clj b/jolt-core/jolt/analyzer.clj index 529be58..d3a8463 100644 --- a/jolt-core/jolt/analyzer.clj +++ b/jolt-core/jolt/analyzer.clj @@ -77,7 +77,10 @@ (let [pp (parse-params (vec (form-vec-items pvec))) fixed (:fixed pp) rst (:rest pp) - rname (when-not rst (gen-name "arity")) + ;; Always a recur target, variadic included: the back end gives the rest + ;; param an ordinary positional slot (holding the collected seq), so recur + ;; is a self-call carrying the rest seq directly — Clojure semantics. + rname (gen-name "arity") names (cond-> (vec fixed) rst (conj rst) fn-name (conj fn-name)) env* (-> (add-locals env names) (with-recur rname))] {:params fixed :rest rst :recur-name rname diff --git a/src/jolt/api.janet b/src/jolt/api.janet index d6fbe61..15efc01 100644 --- a/src/jolt/api.janet +++ b/src/jolt/api.janet @@ -32,7 +32,8 @@ opts may contain: :namespaces — map of {ns-name → {sym → value, ...}, ...} :mutable? — use Janet mutable data structures instead of persistent - :compile? — enable compilation of Clojure forms to Janet + :compile? — compile Clojure forms via the self-hosted pipeline (analyzer -> + IR -> Janet back end), falling back to the interpreter as needed :paths — extra source roots to search for namespaces (after the stdlib)" [&opt opts] (default opts {}) diff --git a/src/jolt/backend.janet b/src/jolt/backend.janet index a798269..4fbde72 100644 --- a/src/jolt/backend.janet +++ b/src/jolt/backend.janet @@ -49,15 +49,25 @@ (array/push binds (emit ctx (in p 1)))) ['let (tuple/slice binds) (emit ctx (node :body))]) -# A named Janet fn whose name is the arity's recur target, so recur is a -# self-call (Janet tail-calls it). +# An arity compiles to a named Janet fn whose name is its recur target, so recur +# is a self-call (Janet tail-calls it). The rest param is an ORDINARY positional +# param holding a seq (not Janet `&`), so `(recur fixed... rest-seq)` re-enters +# the way Clojure recur into a variadic arity does (rebinds the rest seq directly, +# no re-collection). The dispatch wrapper (emit-fn-body) collects the call's args. (defn- emit-arity-fn [ctx ar] (def ps @[]) (each pn (vview (ar :params)) (array/push ps (symbol pn))) - (when (ar :rest) (array/push ps '&) (array/push ps (symbol (ar :rest)))) - (if (ar :recur-name) - ['fn (symbol (ar :recur-name)) (tuple/slice ps) (emit ctx (ar :body))] - ['fn (tuple/slice ps) (emit ctx (ar :body))])) + (when (ar :rest) (array/push ps (symbol (ar :rest)))) + ['fn (symbol (ar :recur-name)) (tuple/slice ps) (emit ctx (ar :body))]) + +# Invoke an arity's fn with args pulled from the dispatch tuple: fixed params by +# index, rest as a slice from n-fixed on. +(defn- emit-arity-invoke [ctx ar jargs] + (def nfixed (length (vview (ar :params)))) + (def call @[(emit-arity-fn ctx ar)]) + (for i 0 nfixed (array/push call ['in jargs i])) + (when (ar :rest) (array/push call ['tuple/slice jargs nfixed])) + (tuple/slice call)) (defn- emit-loop [ctx node] (def L (symbol (node :recur-name))) @@ -86,24 +96,43 @@ ['defer (emit ctx (node :finally)) core] core)) -(defn- emit-fn [ctx node] +(defn- emit-fn-body [ctx node] (def arities (vview (node :arities))) - (if (= 1 (length arities)) + (def multi (> (length arities) 1)) + (cond + # Single fixed arity (the hot case): emit the arity fn directly — its name is + # the recur target, no dispatch overhead. + (and (not multi) (not ((first arities) :rest))) (emit-arity-fn ctx (first arities)) - # Multi-arity: dispatch on arg count; fixed arities match exactly, the - # variadic one matches >= its fixed count. apply spreads the captured args - # into the chosen arity fn (whose own & collects any rest). + # Single variadic arity: a thin wrapper collects the call's args so the rest + # seq can be built, then hands off to the arity fn. + (not multi) + (let [jargs (gsym)] + ['fn ['& jargs] (emit-arity-invoke ctx (first arities) jargs)]) + # Multi-arity: dispatch on arg count. Fixed arities match exactly; the (one) + # variadic arity matches >= its fixed count. (let [jargs (gsym) nsym (gsym) cf @['cond]] (each ar arities (def nfixed (length (vview (ar :params)))) (array/push cf (if (ar :rest) [>= nsym nfixed] [= nsym nfixed])) - (array/push cf [apply (emit-arity-fn ctx ar) jargs])) + (array/push cf (emit-arity-invoke ctx ar jargs))) (array/push cf ['error "wrong number of args passed to fn"]) ['fn ['& jargs] ['do ['def nsym ['length jargs]] (tuple/slice cf)]]))) +# A named fn (fn self [..] .. (self ..)) references itself by name. The analyzer +# binds that name as a local; bind it here to the fn value via a var (set before +# any call, so the captured closure sees it — same scheme as emit-loop). recur +# stays a separate self-call to the arity fn; this only covers by-name self-refs. +(defn- emit-fn [ctx node] + (def body (emit-fn-body ctx node)) + (if (node :name) + (let [s (symbol (node :name))] + ['do ['var s nil] ['set s body] s]) + body)) + (defn- direct-call? [fnode] (case (fnode :op) :var true :local true :fn true :host true false)) diff --git a/src/jolt/host_iface.janet b/src/jolt/host_iface.janet index 251b4f4..5b7278e 100644 --- a/src/jolt/host_iface.janet +++ b/src/jolt/host_iface.janet @@ -65,6 +65,9 @@ "alter-meta!" "reset-meta!" "disj" "set?" "satisfies?" "protocol-dispatch" "register-method" "make-reified" "prefer-method" "remove-method" "remove-all-methods" "get-method" "methods" + # ns-management forms dispatched by the interpreter (not core vars) + "create-ns" "remove-ns" "find-ns" "all-ns" "the-ns" "resolve" + "ns-resolve" "ns-aliases" "ns-imports" "ns-interns" "read-string" "macroexpand-1" "defonce" "ns" "in-ns" "require" "import" "use" "refer" "defrecord" "defprotocol" "definterface" "reify" "proxy" "extend-type" "extend-protocol" "extend" "gen-class" @@ -72,7 +75,18 @@ (put t n true)) t)) -(defn h-special? [name] (if (get special-names name) true false)) +# Interop-shaped heads the interpreter lowers but the back end doesn't model: +# (.method obj …) / (.-field obj) — member access (name starts with ".") +# (Foo. …) — constructor (name ends with "." ) +# Treated as special so the analyzer marks them uncompilable and falls back. +(defn- interop-head? [name] + (def n (length name)) + (and (> n 1) + (or (= (string/slice name 0 1) ".") + (= (string/slice name (- n 1)) ".")))) + +(defn h-special? [name] + (if (or (get special-names name) (interop-head? name)) true false)) # The namespace being compiled. NOT ctx-current-ns directly: the interpreter # rebinds current-ns to a fn's defining ns while that fn runs, so an interpreted diff --git a/src/jolt/loader.janet b/src/jolt/loader.janet index da48ce5..5b577c7 100644 --- a/src/jolt/loader.janet +++ b/src/jolt/loader.janet @@ -3,8 +3,8 @@ # Supports in-memory bytecode caching when :compile? is enabled. (use ./reader) -(use ./compiler) (use ./evaluator) +(import ./backend :as backend) # Stateful / context-modifying forms always interpret: they mutate the context # (namespaces, macros, types, multimethods, dynamic vars, …) in ways the compiler @@ -25,15 +25,12 @@ (defn eval-toplevel "Evaluate one top-level form for ctx, honoring :compile?. Stateful forms always - interpret; otherwise the form is compiled and run, falling back to the - interpreter when the compiler can't handle it. Only the compile step is guarded - — runtime errors in compiled code propagate (no double-eval, no hidden errors)." + interpret; otherwise the form runs through the self-hosted compile pipeline + (portable Clojure analyzer -> IR -> Janet back end), which falls back to the + interpreter for forms it can't compile. Only the compile step is guarded — + runtime errors in compiled code propagate (no double-eval, no hidden errors)." [ctx form] - (defn try-compile [] - (let [compiled (protect (compile-ast form ctx))] - (if (compiled 0) - (eval-compiled (compiled 1) ctx) - (eval-form ctx @{} form)))) + (defn try-compile [] (backend/compile-and-eval ctx form)) (if (get (ctx :env) :compile?) (if (array? form) # A call/list: compile it unless its head is a stateful special form. diff --git a/src/jolt/main.janet b/src/jolt/main.janet index 9c823d1..e80ea42 100644 --- a/src/jolt/main.janet +++ b/src/jolt/main.janet @@ -11,9 +11,10 @@ (def jolt-version "0.1.0") -# Compile by default: the shipped runtime compiles each form to Janet bytecode -# (hybrid — forms the compiler can't handle fall back to the interpreter, so the -# result always matches the interpreter; see compiler.janet / loader/eval-toplevel). +# Compile by default: the shipped runtime runs each form through the self-hosted +# pipeline (portable Clojure analyzer -> IR -> Janet back end) to native bytecode +# (hybrid — forms the analyzer can't compile fall back to the interpreter, so the +# result always matches the interpreter; see backend.janet / loader/eval-toplevel). # Set JOLT_INTERPRET=1 to force the tree-walking interpreter (debugging / A-B). (def compile-default? (not (= "1" (os/getenv "JOLT_INTERPRET")))) (def ctx (init {:compile? compile-default?})) diff --git a/test/integration/self-host-test.janet b/test/integration/self-host-test.janet index 0d11c6e..876fbe8 100644 --- a/test/integration/self-host-test.janet +++ b/test/integration/self-host-test.janet @@ -5,6 +5,7 @@ (import ../../src/jolt/backend :as backend) (use ../../src/jolt/api) (use ../../src/jolt/reader) +(use ../../src/jolt/types) (defn ce [ctx s] (normalize-pvecs (backend/compile-and-eval ctx (parse-string s)))) @@ -53,4 +54,24 @@ # higher-order + nesting (assert (= 15 (ce ctx "(reduce + (map inc [0 1 2 3 4]))")) "reduce+map")) +# eval-toplevel routing: :compile? IS the self-hosted pipeline now — the only +# compile path. Forms the analyzer can't handle (stateful / destructuring) fall +# back to the interpreter, with the same observable results. +(print "self-host via eval-toplevel routing...") +(let [ctx (init {:compile? true})] + (defn ev [s] (normalize-pvecs (eval-one ctx (parse-string s)))) + (assert (= 3 (ev "(+ 1 2)")) "tl +") + (ev "(defn sq [x] (* x x))") # def via self-host + (assert (= 81 (ev "(sq 9)")) "tl defn") + (ev "(defmacro twice [x] (list (quote do) x x))") # stateful -> interp fallback + (assert (= 5 (ev "(do (twice 1) 5)")) "tl macro fallback") + (assert (= [1 2 3] (ev "(let [{:keys [a b c]} {:a 1 :b 2 :c 3}] [a b c])")) "tl destructuring fallback") + (assert (= 15 (ev "(reduce + (range 6))")) "tl reduce/range") + # Proof the self-hosted pipeline actually ran: only backend/ensure-analyzer + # populates jolt.analyzer. An interpret-only ctx never loads it. + (assert (pos? (length ((ctx-find-ns ctx "jolt.analyzer") :mappings))) "analyzer loaded under :compile?")) +(let [ctx (init {})] + (eval-one ctx (parse-string "(+ 1 2)")) + (assert (zero? (length ((ctx-find-ns ctx "jolt.analyzer") :mappings))) "analyzer NOT loaded when interpreting")) + (print "self-host pipeline passed!")