jolt build: bundle native libs + resources into standalone binaries

A built binary dropped its deps.edn :jolt/native declarations and its
resource roots, so an FFI+resources app (ring-app) failed at runtime:
sockets/sqlite gave 'no entry for socket' and io/resource returned nil.
The buildsmoke fixture is pure compute, so neither path was exercised.

The launcher now loads required + :process native libs before the app's
top-level forms (a library's defcfn resolves its foreign-procedure symbols
at top-level eval during startup, so the libs must be loaded first);
optional libs load in the scheme-start launcher, where a missing lib is
caught rather than aborting the heap build.

deps.edn :jolt/build {:embed [dirs]} bakes those dirs' files into the heap
(register-embedded-resource! at heap build), so io/resource serves them with
no files on disk. Non-embedded resources resolve at runtime against JOLT_PWD,
and io/file reads (e.g. config.edn) stay external.

build-binary now takes the encoded natives, embed dirs, and project paths
from cmd-build; deps/resolve-project surfaces them. Buildsmoke fixture grows
an embedded resource + a :process native to cover both paths.

docs/tools-deps.md

@@ -56,11 +56,41 @@ roots, and de-sugars the argv into a run:
 - `-A:alias` → add the alias's paths/deps, then run the rest;
 - `repl` → a line REPL;
 - `path` → print the resolved roots;
+- `build -m NS [-o OUT] [--opt|--dev]` → AOT-compile the app into a standalone binary;
 - `<task>` → run a `deps.edn` `:tasks` entry.
 
 The resolver lives in the overlay alongside the runtime, but the runtime's only
 dependency interface is the list of source roots it's handed.
 
+## Native libraries
+
+A library that binds C declares the shared objects it needs under `:jolt/native`,
+so `jolt.main` loads them before the namespace is required and its `foreign-fn`
+bindings resolve. Each entry is a map — `{:name "sqlite3" :darwin
+["libsqlite3.0.dylib" …] :linux ["libsqlite3.so.0" …]}` — with optional
+`:optional true` (absence is fine, a feature-gated dep) and `:process true` (use
+the running process's own symbols, e.g. libc sockets, no external file). A
+project inherits its dependencies' `:jolt/native`.
+
+## Standalone binaries
+
+`joltc build -m NS -o OUT` compiles the app and every library into one
+executable (the runtime + compiler are baked in). It loads the resolved
+`:jolt/native` libs at startup, so an FFI app — sockets, SQLite — runs with no
+jolt or Chez on the path.
+
+`:jolt/build {:embed ["resources" …]}` bakes those directories' files into the
+binary; `io/resource` serves them from the image with no files on disk. Resources
+not embedded resolve at runtime against `JOLT_PWD` (or the cwd), so the
+ship-the-binary-with-its-`resources/`-dir model also works. Files read through
+`io/file` (e.g. a `config.edn` a config library loads) stay external by design —
+edit them without rebuilding.
+
+A standalone build needs Chez's kernel dev files (`libkernel.a`, `scheme.h`) and
+a C compiler; `JOLT_CHEZ_CSV` overrides the auto-detected `csv<ver>/<machine>`
+dir. `--opt` turns on the inference/flatten/scalar-replace passes; the default
+`release` mode is const-fold only.
+
 ## Limitations
 
 - Pure `clj`/`cljc` only — JVM interop, host classes, and unimplemented

host/chez/build-smoke.sh

@@ -36,9 +36,12 @@ if ! JOLT_PWD="$app" bin/joltc build -m app.core -o "$out" >/dev/null 2>&1; then
 fi
 [ -x "$out" ] || { echo "  FAIL: no executable produced"; exit 1; }
 
-# Run from a neutral cwd with args; check the three output lines.
+# Run from a neutral cwd with args. The first line is an embedded resource
+# (deps.edn :jolt/build :embed), proving io/resource resolves from the binary with
+# no resources/ dir on disk; the rest exercise a macro, cross-ns, and args.
 got="$(cd / && "$out" alpha bb ccc 2>&1)"
-want='HELLO FROM A BUILT BINARY!
+want='embedded resource ok
+HELLO FROM A BUILT BINARY!
 HELLO FROM A BUILT BINARY!
 args: [alpha bb ccc]
 sum: 10'

host/chez/build.ss

@@ -146,12 +146,72 @@
 ;; The loop itself is emit-image's ei-emit-ns* (optimize? #t, guard? #f).
 (define (bld-emit-ns ns-name src) (ei-emit-ns* ns-name src #t #f))
 
+;; --- bundling: native libs + resources --------------------------------------
+;; A jolt seq of jolt strings -> a Scheme list of Scheme strings.
+(define (bld-strs x) (map jolt-str-render-one (seq->list x)))
+
+;; Emit native-library loads. `natives` is the encoded jolt seq jolt.main/
+;; encode-natives produced: each entry is ["process"] | ["req" cand…] | ["opt" cand…].
+;; `which` selects 'required (process + req) or 'optional. Required + process loads
+;; are emitted before the app forms (the app's defcfn foreign-procedures resolve
+;; their symbols at top-level eval during startup, so the libs must be loaded
+;; first); a load-shared-object failure there is fatal — correct for a required
+;; lib. Optional loads run in the scheme-start launcher, where guard catches a
+;; missing lib (an optional lib's namespace is only present when the app requires
+;; it, so its foreign-procedures aren't among the baked top-level forms).
+(define (bld-emit-natives out natives which)
+  (for-each
+    (lambda (entry)
+      (let* ((parts (bld-strs entry)) (kind (car parts)) (cands (cdr parts))
+             (cand-lits (fold-left (lambda (s c) (string-append s (ei-str-lit c) " ")) "" cands)))
+        (cond
+          ((and (eq? which 'required) (string=? kind "process"))
+           (put-string out "(jolt-build-load-native '() #f #t)\n"))
+          ((and (eq? which 'required) (string=? kind "req"))
+           (put-string out (string-append "(jolt-build-load-native (list " cand-lits ") #f #f)\n")))
+          ((and (eq? which 'optional) (string=? kind "opt"))
+           (put-string out (string-append "(jolt-build-load-native (list " cand-lits ") #t #f)\n"))))))
+    (seq->list natives)))
+
+;; Walk an embed root recursively; return (resource-name . abspath) pairs, where
+;; resource-name is the "/"-joined path under the root (what io/resource is asked for).
+(define (bld-walk-files root rel acc)
+  (let ((dir (if (string=? rel "") root (string-append root "/" rel))))
+    (fold-left
+      (lambda (acc name)
+        (let* ((relpath (if (string=? rel "") name (string-append rel "/" name)))
+               (full (string-append root "/" relpath)))
+          (if (file-directory? full)
+              (bld-walk-files root relpath acc)
+              (cons (cons relpath full) acc))))
+      acc
+      (directory-list dir))))
+
+;; Emit register-embedded-resource! per file under each embed dir. Emitted BEFORE
+;; the app forms so the (read-file-string ABSPATH) runs at heap build — the file's
+;; contents bake into the boot image and io/resource serves them with no file on
+;; disk. ABSPATH only has to exist at build time.
+(define (bld-emit-embeds out embed-dirs)
+  (for-each
+    (lambda (root)
+      (when (file-directory? root)
+        (for-each
+          (lambda (rp)
+            (put-string out (string-append
+                              "(register-embedded-resource! " (ei-str-lit (car rp))
+                              " (read-file-string " (ei-str-lit (cdr rp)) "))\n")))
+          (bld-walk-files root "" '()))))
+    (bld-strs embed-dirs)))
+
 ;; --- the build --------------------------------------------------------------
 ;; entry-ns: the app's main namespace (a string). out-path: the binary to write.
 ;; mode: "dev" | "release" | "optimized". Every form runs through jolt.passes/
 ;; run-passes (const-fold always; inline + type inference when optimized turns on
 ;; direct-linking). Deps + source roots are already applied by the caller.
-(define (build-binary entry-ns out-path mode)
+;; natives: encoded :jolt/native libs to load at startup. embed-dirs: dirs whose
+;; files bake into the binary (single-file). ext-roots: project-relative io/resource
+;; roots resolved at runtime against JOLT_PWD (ship-alongside resources).
+(define (build-binary entry-ns out-path mode natives embed-dirs ext-roots)
   (bld-check-toolchain)
   ;; 1. record app namespaces in dependency order as they finish loading.
   (let ((app-order '()))
@@ -180,16 +240,40 @@
         ;; 3. flat source = runtime + app + launcher.
         (let ((out (open-output-file flat-ss 'replace)))
           (bld-emit-runtime out)
+          ;; Load native libs, bake embedded resources, and point source roots at
+          ;; the build-time app roots — all BEFORE the app forms. The app's
+          ;; top-level forms run at binary startup (Sbuild_heap), and they include
+          ;; foreign-procedure evals (a library's defcfn) and (slurp (io/resource …))
+          ;; reads. So the libraries must be loaded and resources resolvable by the
+          ;; time those forms run, not later in the scheme-start launcher.
+          (put-string out "\n;; === native libraries (required) ===\n")
+          (bld-emit-natives out natives 'required)
+          (put-string out "\n;; === embedded resources ===\n")
+          (bld-emit-embeds out embed-dirs)
+          (put-string out (string-append
+                            "(set-source-roots! (list "
+                            (fold-left (lambda (s r) (string-append s (ei-str-lit r) " ")) ""
+                                       (get-source-roots))
+                            "))\n"))
           (put-string out "\n;; === app ===\n")
           (for-each (lambda (s) (put-string out s) (put-string out "\n")) app-strs)
           ;; The launcher runs as Chez's scheme-start (so argv reaches -main —
           ;; top-level boot forms run during heap build, before args are set), and
-          ;; suppresses the interactive greeting.
+          ;; suppresses the interactive greeting. It resets source roots to the
+          ;; app's resource dirs resolved against JOLT_PWD (or cwd) so a runtime
+          ;; io/resource that wasn't embedded still resolves next to the binary.
           (put-string out "\n;; === launcher ===\n")
+          (put-string out "(suppress-greeting #t)\n")
+          (put-string out "(scheme-start\n  (lambda args\n")
+          (bld-emit-natives out natives 'optional)
+          (put-string out (string-append
+                            "    (let ((base (or (getenv \"JOLT_PWD\") \".\")))\n"
+                            "      (set-source-roots!\n"
+                            "        (append (map (lambda (r) (string-append base \"/\" r)) (list "
+                            (fold-left (lambda (s r) (string-append s (ei-str-lit r) " ")) "" (bld-strs ext-roots))
+                            "))\n"
+                            "                (list \"jolt-core\" \"stdlib\"))))\n"))
           (put-string out (string-append
-                            "(suppress-greeting #t)\n"
-                            "(scheme-start\n"
-                            "  (lambda args\n"
                             "    (let ((mainv (var-deref " (ei-str-lit entry-ns) " \"-main\")))\n"
                             "      (apply jolt-invoke mainv args))\n"
                             "    (exit 0)))\n"))
@@ -233,8 +317,9 @@
         (display (string-append "jolt build: wrote " out-path "\n"))))))
 
 (def-var! "jolt.host" "build-binary"
-  (lambda (entry out mode)
+  (lambda (entry out mode natives embed-dirs ext-roots)
     (build-binary (jolt-str-render-one entry)
                   (jolt-str-render-one out)
-                  (jolt-str-render-one mode))
+                  (jolt-str-render-one mode)
+                  natives embed-dirs ext-roots)
     jolt-nil))

host/chez/ffi.ss

@@ -134,6 +134,24 @@
     (when co (unlock-object co) (hashtable-delete! ffi-callable-table a))
     jolt-nil))
 
+;; --- native libraries for a standalone binary -------------------------------
+;; `jolt build` bakes a project's deps.edn :jolt/native declarations into the
+;; launcher, which loads them at startup (load-shared-object isn't part of the
+;; saved heap, so it must run in the built process, not at heap build). process?
+;; loads the running binary's own symbols (libc sockets); otherwise try each
+;; platform candidate in turn and fail unless the spec is optional.
+(define (jolt-build-load-native cands optional? process?)
+  (if process?
+      (begin (load-shared-object #f) #t)
+      (let loop ((cs cands))
+        (cond
+          ((null? cs)
+           (unless optional?
+             (error 'jolt-build "required native library not found" cands))
+           #f)
+          ((guard (e (#t #f)) (load-shared-object (car cs)) #t) #t)
+          (else (loop (cdr cs)))))))
+
 ;; --- expose under jolt.ffi ---------------------------------------------------
 (def-var! "jolt.ffi" "free-callable" ffi-free-callable)
 (def-var! "jolt.ffi" "load-library" ffi-load-library)

host/chez/io.ss

@@ -19,6 +19,16 @@
 ;; path string of any value: a jfile -> its path, else its str rendering.
 (define (file-path-of x) (if (jfile? x) (jfile-path x) (jolt-str-render-one x)))
 
+;; Resources baked into a standalone binary by `jolt build` (deps.edn
+;; :jolt/build :embed). The build emits a register-embedded-resource! per file at
+;; heap-build time, so the contents live in the boot image — io/resource serves
+;; them with no file on disk. An embedded hit reads through slurp/reader exactly
+;; like a jfile would.
+(define embedded-resources (make-hashtable equal-hash equal?))
+(define (register-embedded-resource! name content)
+  (hashtable-set! embedded-resources name content))
+(define-record-type embedded-res (fields name content) (nongenerative jolt-embres-v1))
+
 ;; A user-facing relative path resolves against JOLT_PWD — the user's cwd before
 ;; the launcher cd'd to the jolt repo root — matching the JVM, where io/file is
 ;; cwd-relative. (io/resource builds jfiles from the source roots directly, so it
@@ -191,6 +201,7 @@
 (define (jolt-slurp src . opts)
   (cond
     ((jfile? src) (read-file-string (jfile-path src)))
+    ((embedded-res? src) (embedded-res-content src))
     ((reader-jhost? src) (drain-reader src))
     ;; bytes (a bytevector or a jolt byte-array): decode with :encoding (UTF-8
     ;; default). clj-http-lite slurps response-body byte arrays.
@@ -286,6 +297,7 @@
   (cond
     ((reader-jhost? x) x)
     ((jfile? x) (host-new "StringReader" (read-file-string (jfile-path x))))
+    ((embedded-res? x) (host-new "StringReader" (embedded-res-content x)))
     ((and (jhost? x) (string=? (jhost-tag x) "url"))
      (host-new "StringReader" (read-file-string (url-strip-scheme (url-spec x)))))
     ((string? x) (host-new "StringReader" (read-file-string x)))
@@ -317,11 +329,13 @@
 ;; (slurp/reader-able) for the first match, else nil. get-source-roots is the
 ;; loader's accessor (loader.ss), resolved at call time — the runtime CLI loads it.
 (define (jolt-io-resource name)
-  (let ((nm (jolt-str-render-one name)))
-    (let loop ((roots (get-source-roots)))
-      (cond ((null? roots) jolt-nil)
-            ((file-exists? (string-append (car roots) "/" nm)) (make-jfile (string-append (car roots) "/" nm)))
-            (else (loop (cdr roots)))))))
+  (let* ((nm (jolt-str-render-one name))
+         (emb (hashtable-ref embedded-resources nm #f)))
+    (if emb (make-embedded-res nm emb)
+        (let loop ((roots (get-source-roots)))
+          (cond ((null? roots) jolt-nil)
+                ((file-exists? (string-append (car roots) "/" nm)) (make-jfile (string-append (car roots) "/" nm)))
+                (else (loop (cdr roots))))))))
 (def-var! "clojure.java.io" "resource" jolt-io-resource)
 ;; as-url honors a library-registered URL class (e.g. jolt-lang/http-client's full
 ;; java.net.URL shim) so io/as-url and (URL. spec) agree; else the file-only jhost.

jolt-core/jolt/deps.clj

@@ -127,6 +127,14 @@
          {dep-roots :roots dep-natives :natives} (resolve-deps all-deps project-dir)]
      {:roots (vec (distinct (concat project-roots dep-roots)))
       :main-opts main-opts
+      ;; the project's own paths (relative to project-dir) and absolute resource
+      ;; roots, plus its :jolt/build options — `jolt build` uses these to bundle
+      ;; resources into / alongside a standalone binary.
+      :project-dir project-dir
+      :project-paths (vec project-paths)
+      :project-roots (vec project-roots)
+      :build (:jolt/build edn)
+      :embed-dirs (mapv #(abspath project-dir %) (:embed (:jolt/build edn)))
       :tasks (:tasks edn)
       :natives (vec (distinct (concat (:jolt/native edn) dep-natives)))
       ;; nREPL middleware a library contributes (jolt.nrepl composes them over its

jolt-core/jolt/main.clj

@@ -8,6 +8,12 @@
 
 (defn- project-dir [] (or (jolt.host/getenv "JOLT_PWD") "."))
 
+(defn- current-platform []
+  (let [os (str/lower-case (or (System/getProperty "os.name") ""))]
+    (cond (str/includes? os "mac") :darwin
+          (str/includes? os "win") :windows
+          :else :linux)))
+
 ;; Load a library's declared native shared objects (deps.edn :jolt/native) before
 ;; its Clojure is required, so its foreign-fn bindings resolve. Each entry is a
 ;; map: {:name "sqlite3" :darwin ["libsqlite3.0.dylib" ...] :linux ["libsqlite3.so.0" ...]}
@@ -15,10 +21,7 @@
 ;; (use the running process's symbols, e.g. libc sockets — no external file).
 (defn- load-natives! [natives]
   (when (seq natives)
-    (let [os (str/lower-case (or (System/getProperty "os.name") ""))
-          plat (cond (str/includes? os "mac") :darwin
-                     (str/includes? os "win") :windows
-                     :else :linux)]
+    (let [plat (current-platform)]
       (doseq [spec natives]
         (if (:process spec)
           (jolt.ffi/load-library)
@@ -116,23 +119,41 @@
 ;; standalone executable. Resolves deps + roots like `run`, then hands the entry
 ;; namespace to the host build driver (jolt.host/build-binary, defined by
 ;; build.ss). Default mode is release; --opt selects optimized, --dev unoptimized.
+;; Encode a deps.edn :jolt/native spec for the build launcher, resolving the
+;; current platform's candidate list now (the binary runs on this OS). Each entry
+;; becomes a vector the launcher (build.ss) reads: ["process"] for the running
+;; binary's own symbols, else ["req"|"opt" cand…] to try in turn.
+(defn- encode-natives [natives]
+  (let [plat (current-platform)]
+    (vec (for [spec natives]
+           (if (:process spec)
+             ["process"]
+             (let [c (get spec plat)
+                   cands (if (string? c) [c] (vec c))]
+               (into [(if (:optional spec) "opt" "req")] cands)))))))
+
 (defn- cmd-build [more]
-  (apply-project! (deps/resolve-project (project-dir)))
-  (let [opts (loop [a more, entry nil, out nil]
-               (cond
-                 (empty? a)          {:entry entry :out out}
-                 (= "-m" (first a))  (recur (drop 2 a) (second a) out)
-                 (= "-o" (first a))  (recur (drop 2 a) entry (second a))
-                 (str/starts-with? (first a) "-") (recur (rest a) entry out)
-                 :else               (recur (rest a) (or entry (first a)) out)))
-        entry (:entry opts)
-        mode  (cond (some #{"--opt"} more) "optimized"
-                    (some #{"--dev"} more) "dev"
-                    :else                  "release")]
-    (when (nil? entry)
-      (throw (ex-info "build needs an entry: -m NS" {})))
-    (let [out (or (:out opts) (first (str/split entry #"\.")))]
-      (jolt.host/build-binary entry out mode))))
+  (let [{:keys [project-paths embed-dirs] :as resolved}
+        (deps/resolve-project (project-dir))]
+    (apply-project! resolved)
+    (let [opts (loop [a more, entry nil, out nil]
+                 (cond
+                   (empty? a)          {:entry entry :out out}
+                   (= "-m" (first a))  (recur (drop 2 a) (second a) out)
+                   (= "-o" (first a))  (recur (drop 2 a) entry (second a))
+                   (str/starts-with? (first a) "-") (recur (rest a) entry out)
+                   :else               (recur (rest a) (or entry (first a)) out)))
+          entry (:entry opts)
+          mode  (cond (some #{"--opt"} more) "optimized"
+                      (some #{"--dev"} more) "dev"
+                      :else                  "release")]
+      (when (nil? entry)
+        (throw (ex-info "build needs an entry: -m NS" {})))
+      (let [out (or (:out opts) (first (str/split entry #"\.")))
+            natives (encode-natives (:natives resolved))]
+        ;; embed-dirs (absolute) are walked + baked into the binary by the driver;
+        ;; project-paths (relative) become runtime io/resource roots (ship-alongside).
+        (jolt.host/build-binary entry out mode natives embed-dirs project-paths)))))
 
 (defn- nrepl [more]
   ;; resolve the project (deps on the roots, native libs loaded), then start the

test/chez/build-app/deps.edn

@@ -1 +1,8 @@
-{:paths ["src"]}
+{:paths ["src" "resources"]
+
+ ;; exercise the build's native-lib loading: a :process spec loads the running
+ ;; binary's own symbols (libc) at startup — no external file, always succeeds.
+ :jolt/native [{:name "libc" :process true}]
+
+ ;; bake resources/ into the binary so io/resource resolves with no file on disk.
+ :jolt/build {:embed ["resources"]}}

test/chez/build-app/resources/greeting.txt

@@ -0,0 +1 @@
+embedded resource ok

test/chez/build-app/src/app/core.clj

@@ -1,7 +1,11 @@
 (ns app.core
-  (:require [app.util :as util]))
+  (:require [app.util :as util]
+            [clojure.java.io :as io]))
 
 (defn -main [& args]
+  ;; the resource is baked into the binary (deps.edn :jolt/build :embed), so this
+  ;; resolves with no resources/ dir on disk, run from any cwd.
+  (println (slurp (io/resource "greeting.txt")))
   (util/twice (println (util/shout "hello from a built binary")))
   (println "args:" (vec args))
   (println "sum:" (reduce + (map count args))))