From f3f2c63ee7f72dc429452581a2b7504b794a97c9 Mon Sep 17 00:00:00 2001 From: Yogthos Date: Fri, 5 Jun 2026 23:51:03 -0400 Subject: [PATCH] bake the .clj stdlib into the binary MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Clojure stdlib namespaces (clojure.string/set/walk/edn/zip, jolt.http/ interop/shell/nrepl) were loaded cwd-relative, so the built binary couldn't load them outside the repo. Now stdlib_embed slurps them all at build time into a {ns -> source} map frozen in the image, and the loader falls back to it when a namespace isn't found on disk. The artifact is self-contained — clojure.string works from any directory. Drops the one-off nREPL source embed (jolt.nrepl is covered by this now). The baked-in stdlib is excluded from uberscript bundles since it's part of the runtime. clojure.core is unchanged (Janet, auto-referred). Test: embedded-stdlib-test requires from the embedded copy with FS roots cleared. --- doc/building-and-deps.md | 10 ++++++- src/jolt/api.janet | 5 ++++ src/jolt/evaluator.janet | 30 +++++++++++-------- src/jolt/main.janet | 16 ++-------- src/jolt/stdlib_embed.janet | 33 +++++++++++++++++++++ test/integration/embedded-stdlib-test.janet | 33 +++++++++++++++++++++ 6 files changed, 99 insertions(+), 28 deletions(-) create mode 100644 src/jolt/stdlib_embed.janet create mode 100644 test/integration/embedded-stdlib-test.janet diff --git a/doc/building-and-deps.md b/doc/building-and-deps.md index 44a34e8..fa6a499 100644 --- a/doc/building-and-deps.md +++ b/doc/building-and-deps.md @@ -13,7 +13,11 @@ jpm build This produces two executables under `build/`: -- **`jolt`** — the runtime: REPL, file/expr runner, nREPL server. +- **`jolt`** — the runtime: REPL, file/expr runner, nREPL server. The whole `.clj` + standard library (`clojure.string`/`set`/`walk`/`edn`/`zip`, `jolt.http`/ + `interop`/`shell`/`nrepl`) is baked into this binary at build time, so it loads + from any directory — the build artifact is self-contained. (`clojure.core` is + built into the runtime in Janet and auto-referred, so it's always available.) - **`jolt-deps`** — a separate tool that resolves a `deps.edn` (see below). It sits beside the runtime the way `jpm` sits beside `janet`; the runtime itself knows nothing about deps.edn. @@ -37,6 +41,10 @@ come from: at runtime; - the `:paths` option to `init` when embedding Jolt as a library. +If a namespace isn't found on any root, the loader falls back to the stdlib baked +into the binary — that's how `clojure.string` and friends resolve when you run +the binary outside the source tree. + So you can point Jolt at a directory of Clojure source with no deps machinery at all: diff --git a/src/jolt/api.janet b/src/jolt/api.janet index 215ec6b..8f568ae 100644 --- a/src/jolt/api.janet +++ b/src/jolt/api.janet @@ -10,6 +10,7 @@ (use ./compiler) (use ./loader) (use ./async) +(import ./stdlib_embed :as stdlib-embed) (defn normalize-pvecs "Deep-convert any sequential (pvec/tuple/array) to a Janet tuple. Test helper @@ -35,6 +36,10 @@ [&opt opts] (default opts {}) (let [ctx (make-ctx opts)] + # The .clj stdlib (clojure.string, jolt.http, …) baked into the image at build + # time, so it loads from any directory; the loader falls back to this when a + # namespace isn't found on disk. (See stdlib-embed.) + (put (ctx :env) :embedded-sources stdlib-embed/sources) # Extra source roots: opts :paths, then JOLT_PATH (colon-separated). These are # searched after the stdlib so (require ...) finds deps.edn-resolved libs. (let [roots (get (ctx :env) :source-paths)] diff --git a/src/jolt/evaluator.janet b/src/jolt/evaluator.janet index 0b54ce5..c4e3011 100644 --- a/src/jolt/evaluator.janet +++ b/src/jolt/evaluator.janet @@ -204,33 +204,37 @@ (when (os/stat p) (set found p)))))) found)) -(defn- load-ns-file - "Parse and evaluate every form in a .clj file in the given context." - [ctx path] - (var s (slurp path)) +(defn- load-ns-source + "Parse and evaluate every form of a namespace's source in the given context." + [ctx src] + (var s src) (while (> (length (string/trim s)) 0) (def [f r] (parse-next s)) (set s r) (when (not (nil? f)) (eval-form ctx @{} f)))) (defn- maybe-require-ns - "If namespace ns-name isn't populated yet and source for it exists on the - context's source roots, load it. Restores the current namespace afterwards (a - library's own `ns` form, or our manual switch for ns-form-less stdlib files, - changes it). No-op for already-loaded namespaces." + "If namespace ns-name isn't populated yet, load its source — from a file on the + context's source roots, else from the stdlib baked into the image. Restores the + current namespace afterwards (a library's own `ns` form, or our manual switch + for ns-form-less stdlib files, changes it). No-op for already-loaded namespaces." [ctx ns-name] (let [ns (ctx-find-ns ctx ns-name)] (when (and (= 0 (length (ns :mappings))) (not= ns-name "clojure.core")) - (let [path (find-ns-file ctx ns-name)] - (when path + (let [path (find-ns-file ctx ns-name) + embedded (get (get (ctx :env) :embedded-sources @{}) ns-name) + stdlib? (not (nil? embedded))] + (when (or path embedded) (let [saved (ctx-current-ns ctx)] # Stdlib files have no `ns` form, so switch into the target ns first # (their defs intern there); a library's own `ns` form overrides this. (ctx-set-current-ns ctx ns-name) - (load-ns-file ctx path) + (if path (load-ns-source ctx (slurp path)) (load-ns-source ctx embedded)) # Record load order for tooling (uberscript): a dependency finishes - # loading before the file that required it, so this is topological. - (when-let [lf (get (ctx :env) :loaded-files)] (array/push lf path)) + # loading before its requirer, so this is topological. Skip the + # baked-in stdlib — it's part of the runtime, not something to bundle. + (when (and path (not stdlib?)) + (when-let [lf (get (ctx :env) :loaded-files)] (array/push lf path))) (ctx-set-current-ns ctx saved))))))) (defn- eval-require diff --git a/src/jolt/main.janet b/src/jolt/main.janet index 8f72372..a9a1f5d 100644 --- a/src/jolt/main.janet +++ b/src/jolt/main.janet @@ -9,11 +9,6 @@ (use ./config) (use ./reader) -# Embed the Clojure nREPL source at build time (cwd is the repo during `jpm -# build`), so `jolt nrepl` is self-contained and works from any directory — the -# stdlib .clj files otherwise load cwd-relative. -(def nrepl-clj-source (try (slurp "src/jolt/jolt/nrepl.clj") ([_] nil))) - (def jolt-version "0.1.0") (def ctx (init)) @@ -276,15 +271,8 @@ ([err fib] (report-error err fib) (os/exit 1)))) (defn- ensure-nrepl-loaded [] - # Prefer a normal require (running from the repo); otherwise load the source - # embedded at build time into the jolt.nrepl namespace. - (eval-string ctx "(require '[jolt.nrepl])") - (let [ns (ctx-find-ns ctx "jolt.nrepl")] - (when (and (or (nil? ns) (= 0 (length (ns :mappings)))) nrepl-clj-source) - (let [saved (ctx-current-ns ctx)] - (ctx-set-current-ns ctx "jolt.nrepl") - (load-string ctx nrepl-clj-source) - (ctx-set-current-ns ctx saved))))) + # jolt.nrepl is part of the baked-in stdlib, so require finds it anywhere. + (eval-string ctx "(require '[jolt.nrepl])")) (defn- run-nrepl [argv] # addr is [host:]port; bare number is a port. Default 127.0.0.1:7888. diff --git a/src/jolt/stdlib_embed.janet b/src/jolt/stdlib_embed.janet new file mode 100644 index 0000000..e63c239 --- /dev/null +++ b/src/jolt/stdlib_embed.janet @@ -0,0 +1,33 @@ +# Bakes the Clojure stdlib (.clj/.cljc under src/jolt/clojure and src/jolt/jolt) +# into the image at build time, so the runtime can load clojure.string, jolt.http, +# jolt.nrepl, etc. from any directory — not just when run from the repo. +# +# `sources` is built at module-load time. During `jpm build` that's the build +# (cwd = repo), so the map is captured into the image and frozen; in the shipped +# binary the files are never read from disk. Running from source rebuilds it. + +(defn- relpath->ns [rel] + # string/replace-all takes the string LAST, so thread with ->> + (->> rel (string/replace-all "/" ".") (string/replace-all "_" "-"))) + +(defn- strip-ext [name] + (cond + (string/has-suffix? ".cljc" name) (string/slice name 0 (- (length name) 5)) + (string/has-suffix? ".clj" name) (string/slice name 0 (- (length name) 4)) + name)) + +(defn- collect [root prefix acc] + (when (os/stat root) + (each e (os/dir root) + (def p (string root "/" e)) + (cond + (= :directory (os/stat p :mode)) (collect p (string prefix e "/") acc) + (or (string/has-suffix? ".clj" e) (string/has-suffix? ".cljc" e)) + (put acc (relpath->ns (string prefix (strip-ext e))) (slurp p))))) + acc) + +(def sources + (let [acc @{}] + (collect "src/jolt/clojure" "clojure/" acc) + (collect "src/jolt/jolt" "jolt/" acc) + acc)) diff --git a/test/integration/embedded-stdlib-test.janet b/test/integration/embedded-stdlib-test.janet new file mode 100644 index 0000000..d11540f --- /dev/null +++ b/test/integration/embedded-stdlib-test.janet @@ -0,0 +1,33 @@ +# The .clj stdlib (clojure.string, clojure.set, jolt.interop, …) is baked into the +# image at build time, so it loads even when the files aren't on disk. We simulate +# the shipped-binary-elsewhere case by clearing the filesystem source roots, so a +# require can only be satisfied by the embedded copy. + +(use ../../src/jolt/api) +(use ../../src/jolt/types) + +(def ctx (init)) +(ctx-set-current-ns ctx "user") +(put (ctx :env) :source-paths @[]) # no FS roots — embedded fallback only + +(var fails 0) +(defn check [label expr expected] + (let [r (protect (eval-string ctx expr)) + got (if (r 0) (normalize-pvecs (r 1)) (string "ERR " (r 1)))] + (if (= got expected) (print " ok " label) + (do (++ fails) (printf " FAIL %s: want %q, got %q" label expected got))))) + +(assert (> (length (get (ctx :env) :embedded-sources)) 0) "embedded-sources should be populated") + +(check "clojure.string from embedded" + "(do (require (quote [clojure.string :as s])) (s/upper-case \"hi\"))" "HI") +(check "clojure.set from embedded" + "(do (require (quote [clojure.set :as set])) (vec (set/union #{1} #{2})))" [2 1]) +(check "clojure.walk from embedded" + "(do (require (quote [clojure.walk :as w])) (w/keywordize-keys {\"a\" 1}))" {:a 1}) +(check "jolt.interop from embedded" + "(do (require (quote [jolt.interop :as j])) (j/janet-type 1))" :number) + +(if (> fails 0) + (error (string "embedded-stdlib-test: " fails " failing check(s)")) + (print "\nAll embedded-stdlib tests passed!"))