Merge pull request #6 from jolt-lang/bake-stdlib

Bake the .clj stdlib into the binary

doc/building-and-deps.md

@@ -13,7 +13,11 @@ jpm build
 
 This produces two executables under `build/`:
 
-- **`jolt`** — the runtime: REPL, file/expr runner, nREPL server.
+- **`jolt`** — the runtime: REPL, file/expr runner, nREPL server. The whole `.clj`
+  standard library (`clojure.string`/`set`/`walk`/`edn`/`zip`, `jolt.http`/
+  `interop`/`shell`/`nrepl`) is baked into this binary at build time, so it loads
+  from any directory — the build artifact is self-contained. (`clojure.core` is
+  built into the runtime in Janet and auto-referred, so it's always available.)
 - **`jolt-deps`** — a separate tool that resolves a `deps.edn` (see below). It
   sits beside the runtime the way `jpm` sits beside `janet`; the runtime itself
   knows nothing about deps.edn.
@@ -37,6 +41,10 @@ come from:
   at runtime;
 - the `:paths` option to `init` when embedding Jolt as a library.
 
+If a namespace isn't found on any root, the loader falls back to the stdlib baked
+into the binary — that's how `clojure.string` and friends resolve when you run
+the binary outside the source tree.
+
 So you can point Jolt at a directory of Clojure source with no deps machinery at
 all:
 

src/jolt/api.janet

@@ -10,6 +10,7 @@
 (use ./compiler)
 (use ./loader)
 (use ./async)
+(import ./stdlib_embed :as stdlib-embed)
 
 (defn normalize-pvecs
   "Deep-convert any sequential (pvec/tuple/array) to a Janet tuple. Test helper
@@ -35,6 +36,10 @@
   [&opt opts]
   (default opts {})
   (let [ctx (make-ctx opts)]
+    # The .clj stdlib (clojure.string, jolt.http, …) baked into the image at build
+    # time, so it loads from any directory; the loader falls back to this when a
+    # namespace isn't found on disk. (See stdlib-embed.)
+    (put (ctx :env) :embedded-sources stdlib-embed/sources)
     # Extra source roots: opts :paths, then JOLT_PATH (colon-separated). These are
     # searched after the stdlib so (require ...) finds deps.edn-resolved libs.
     (let [roots (get (ctx :env) :source-paths)]

src/jolt/evaluator.janet

@@ -204,33 +204,37 @@
             (when (os/stat p) (set found p))))))
     found))
 
-(defn- load-ns-file
-  "Parse and evaluate every form in a .clj file in the given context."
-  [ctx path]
-  (var s (slurp path))
+(defn- load-ns-source
+  "Parse and evaluate every form of a namespace's source in the given context."
+  [ctx src]
+  (var s src)
   (while (> (length (string/trim s)) 0)
     (def [f r] (parse-next s))
     (set s r)
     (when (not (nil? f)) (eval-form ctx @{} f))))
 
 (defn- maybe-require-ns
-  "If namespace ns-name isn't populated yet and source for it exists on the
-  context's source roots, load it. Restores the current namespace afterwards (a
-  library's own `ns` form, or our manual switch for ns-form-less stdlib files,
-  changes it). No-op for already-loaded namespaces."
+  "If namespace ns-name isn't populated yet, load its source — from a file on the
+  context's source roots, else from the stdlib baked into the image. Restores the
+  current namespace afterwards (a library's own `ns` form, or our manual switch
+  for ns-form-less stdlib files, changes it). No-op for already-loaded namespaces."
   [ctx ns-name]
   (let [ns (ctx-find-ns ctx ns-name)]
     (when (and (= 0 (length (ns :mappings))) (not= ns-name "clojure.core"))
-      (let [path (find-ns-file ctx ns-name)]
-        (when path
+      (let [path (find-ns-file ctx ns-name)
+            embedded (get (get (ctx :env) :embedded-sources @{}) ns-name)
+            stdlib? (not (nil? embedded))]
+        (when (or path embedded)
           (let [saved (ctx-current-ns ctx)]
             # Stdlib files have no `ns` form, so switch into the target ns first
             # (their defs intern there); a library's own `ns` form overrides this.
             (ctx-set-current-ns ctx ns-name)
-            (load-ns-file ctx path)
+            (if path (load-ns-source ctx (slurp path)) (load-ns-source ctx embedded))
             # Record load order for tooling (uberscript): a dependency finishes
-            # loading before the file that required it, so this is topological.
-            (when-let [lf (get (ctx :env) :loaded-files)] (array/push lf path))
+            # loading before its requirer, so this is topological. Skip the
+            # baked-in stdlib — it's part of the runtime, not something to bundle.
+            (when (and path (not stdlib?))
+              (when-let [lf (get (ctx :env) :loaded-files)] (array/push lf path)))
             (ctx-set-current-ns ctx saved)))))))
 
 (defn- eval-require

src/jolt/main.janet

@@ -9,11 +9,6 @@
 (use ./config)
 (use ./reader)
 
-# Embed the Clojure nREPL source at build time (cwd is the repo during `jpm
-# build`), so `jolt nrepl` is self-contained and works from any directory — the
-# stdlib .clj files otherwise load cwd-relative.
-(def nrepl-clj-source (try (slurp "src/jolt/jolt/nrepl.clj") ([_] nil)))
-
 (def jolt-version "0.1.0")
 
 (def ctx (init))
@@ -276,15 +271,8 @@
     ([err fib] (report-error err fib) (os/exit 1))))
 
 (defn- ensure-nrepl-loaded []
-  # Prefer a normal require (running from the repo); otherwise load the source
-  # embedded at build time into the jolt.nrepl namespace.
-  (eval-string ctx "(require '[jolt.nrepl])")
-  (let [ns (ctx-find-ns ctx "jolt.nrepl")]
-    (when (and (or (nil? ns) (= 0 (length (ns :mappings)))) nrepl-clj-source)
-      (let [saved (ctx-current-ns ctx)]
-        (ctx-set-current-ns ctx "jolt.nrepl")
-        (load-string ctx nrepl-clj-source)
-        (ctx-set-current-ns ctx saved)))))
+  # jolt.nrepl is part of the baked-in stdlib, so require finds it anywhere.
+  (eval-string ctx "(require '[jolt.nrepl])"))
 
 (defn- run-nrepl [argv]
   # addr is [host:]port; bare number is a port. Default 127.0.0.1:7888.

src/jolt/stdlib_embed.janet

@@ -0,0 +1,33 @@
+# Bakes the Clojure stdlib (.clj/.cljc under src/jolt/clojure and src/jolt/jolt)
+# into the image at build time, so the runtime can load clojure.string, jolt.http,
+# jolt.nrepl, etc. from any directory — not just when run from the repo.
+#
+# `sources` is built at module-load time. During `jpm build` that's the build
+# (cwd = repo), so the map is captured into the image and frozen; in the shipped
+# binary the files are never read from disk. Running from source rebuilds it.
+
+(defn- relpath->ns [rel]
+  # string/replace-all takes the string LAST, so thread with ->>
+  (->> rel (string/replace-all "/" ".") (string/replace-all "_" "-")))
+
+(defn- strip-ext [name]
+  (cond
+    (string/has-suffix? ".cljc" name) (string/slice name 0 (- (length name) 5))
+    (string/has-suffix? ".clj" name)  (string/slice name 0 (- (length name) 4))
+    name))
+
+(defn- collect [root prefix acc]
+  (when (os/stat root)
+    (each e (os/dir root)
+      (def p (string root "/" e))
+      (cond
+        (= :directory (os/stat p :mode)) (collect p (string prefix e "/") acc)
+        (or (string/has-suffix? ".clj" e) (string/has-suffix? ".cljc" e))
+          (put acc (relpath->ns (string prefix (strip-ext e))) (slurp p)))))
+  acc)
+
+(def sources
+  (let [acc @{}]
+    (collect "src/jolt/clojure" "clojure/" acc)
+    (collect "src/jolt/jolt" "jolt/" acc)
+    acc))

test/integration/embedded-stdlib-test.janet

@@ -0,0 +1,33 @@
+# The .clj stdlib (clojure.string, clojure.set, jolt.interop, …) is baked into the
+# image at build time, so it loads even when the files aren't on disk. We simulate
+# the shipped-binary-elsewhere case by clearing the filesystem source roots, so a
+# require can only be satisfied by the embedded copy.
+
+(use ../../src/jolt/api)
+(use ../../src/jolt/types)
+
+(def ctx (init))
+(ctx-set-current-ns ctx "user")
+(put (ctx :env) :source-paths @[])   # no FS roots — embedded fallback only
+
+(var fails 0)
+(defn check [label expr expected]
+  (let [r (protect (eval-string ctx expr))
+        got (if (r 0) (normalize-pvecs (r 1)) (string "ERR " (r 1)))]
+    (if (= got expected) (print "  ok   " label)
+      (do (++ fails) (printf "  FAIL %s: want %q, got %q" label expected got)))))
+
+(assert (> (length (get (ctx :env) :embedded-sources)) 0) "embedded-sources should be populated")
+
+(check "clojure.string from embedded"
+  "(do (require (quote [clojure.string :as s])) (s/upper-case \"hi\"))" "HI")
+(check "clojure.set from embedded"
+  "(do (require (quote [clojure.set :as set])) (vec (set/union #{1} #{2})))" [2 1])
+(check "clojure.walk from embedded"
+  "(do (require (quote [clojure.walk :as w])) (w/keywordize-keys {\"a\" 1}))" {:a 1})
+(check "jolt.interop from embedded"
+  "(do (require (quote [jolt.interop :as j])) (j/janet-type 1))" :number)
+
+(if (> fails 0)
+  (error (string "embedded-stdlib-test: " fails " failing check(s)"))
+  (print "\nAll embedded-stdlib tests passed!"))