From 1d345bfd0fe2d325fb4cc678d904879a47731055 Mon Sep 17 00:00:00 2001 From: Yogthos Date: Tue, 23 Jun 2026 13:19:33 -0400 Subject: [PATCH] jolt build: bundle native libs + resources into standalone binaries A built binary dropped its deps.edn :jolt/native declarations and its resource roots, so an FFI+resources app (ring-app) failed at runtime: sockets/sqlite gave 'no entry for socket' and io/resource returned nil. The buildsmoke fixture is pure compute, so neither path was exercised. The launcher now loads required + :process native libs before the app's top-level forms (a library's defcfn resolves its foreign-procedure symbols at top-level eval during startup, so the libs must be loaded first); optional libs load in the scheme-start launcher, where a missing lib is caught rather than aborting the heap build. deps.edn :jolt/build {:embed [dirs]} bakes those dirs' files into the heap (register-embedded-resource! at heap build), so io/resource serves them with no files on disk. Non-embedded resources resolve at runtime against JOLT_PWD, and io/file reads (e.g. config.edn) stay external. build-binary now takes the encoded natives, embed dirs, and project paths from cmd-build; deps/resolve-project surfaces them. Buildsmoke fixture grows an embedded resource + a :process native to cover both paths. --- docs/tools-deps.md | 30 +++++++ host/chez/build-smoke.sh | 7 +- host/chez/build.ss | 99 ++++++++++++++++++++-- host/chez/ffi.ss | 18 ++++ host/chez/io.ss | 24 ++++-- jolt-core/jolt/deps.clj | 8 ++ jolt-core/jolt/main.clj | 61 ++++++++----- test/chez/build-app/deps.edn | 9 +- test/chez/build-app/resources/greeting.txt | 1 + test/chez/build-app/src/app/core.clj | 6 +- 10 files changed, 227 insertions(+), 36 deletions(-) create mode 100644 test/chez/build-app/resources/greeting.txt diff --git a/docs/tools-deps.md b/docs/tools-deps.md index 84461ca..e0dab76 100644 --- a/docs/tools-deps.md +++ b/docs/tools-deps.md @@ -56,11 +56,41 @@ roots, and de-sugars the argv into a run: - `-A:alias` → add the alias's paths/deps, then run the rest; - `repl` → a line REPL; - `path` → print the resolved roots; +- `build -m NS [-o OUT] [--opt|--dev]` → AOT-compile the app into a standalone binary; - `` → run a `deps.edn` `:tasks` entry. The resolver lives in the overlay alongside the runtime, but the runtime's only dependency interface is the list of source roots it's handed. +## Native libraries + +A library that binds C declares the shared objects it needs under `:jolt/native`, +so `jolt.main` loads them before the namespace is required and its `foreign-fn` +bindings resolve. Each entry is a map — `{:name "sqlite3" :darwin +["libsqlite3.0.dylib" …] :linux ["libsqlite3.so.0" …]}` — with optional +`:optional true` (absence is fine, a feature-gated dep) and `:process true` (use +the running process's own symbols, e.g. libc sockets, no external file). A +project inherits its dependencies' `:jolt/native`. + +## Standalone binaries + +`joltc build -m NS -o OUT` compiles the app and every library into one +executable (the runtime + compiler are baked in). It loads the resolved +`:jolt/native` libs at startup, so an FFI app — sockets, SQLite — runs with no +jolt or Chez on the path. + +`:jolt/build {:embed ["resources" …]}` bakes those directories' files into the +binary; `io/resource` serves them from the image with no files on disk. Resources +not embedded resolve at runtime against `JOLT_PWD` (or the cwd), so the +ship-the-binary-with-its-`resources/`-dir model also works. Files read through +`io/file` (e.g. a `config.edn` a config library loads) stay external by design — +edit them without rebuilding. + +A standalone build needs Chez's kernel dev files (`libkernel.a`, `scheme.h`) and +a C compiler; `JOLT_CHEZ_CSV` overrides the auto-detected `csv/` +dir. `--opt` turns on the inference/flatten/scalar-replace passes; the default +`release` mode is const-fold only. + ## Limitations - Pure `clj`/`cljc` only — JVM interop, host classes, and unimplemented diff --git a/host/chez/build-smoke.sh b/host/chez/build-smoke.sh index 883b853..9187211 100755 --- a/host/chez/build-smoke.sh +++ b/host/chez/build-smoke.sh @@ -36,9 +36,12 @@ if ! JOLT_PWD="$app" bin/joltc build -m app.core -o "$out" >/dev/null 2>&1; then fi [ -x "$out" ] || { echo " FAIL: no executable produced"; exit 1; } -# Run from a neutral cwd with args; check the three output lines. +# Run from a neutral cwd with args. The first line is an embedded resource +# (deps.edn :jolt/build :embed), proving io/resource resolves from the binary with +# no resources/ dir on disk; the rest exercise a macro, cross-ns, and args. got="$(cd / && "$out" alpha bb ccc 2>&1)" -want='HELLO FROM A BUILT BINARY! +want='embedded resource ok +HELLO FROM A BUILT BINARY! HELLO FROM A BUILT BINARY! args: [alpha bb ccc] sum: 10' diff --git a/host/chez/build.ss b/host/chez/build.ss index 3d52646..755647a 100644 --- a/host/chez/build.ss +++ b/host/chez/build.ss @@ -146,12 +146,72 @@ ;; The loop itself is emit-image's ei-emit-ns* (optimize? #t, guard? #f). (define (bld-emit-ns ns-name src) (ei-emit-ns* ns-name src #t #f)) +;; --- bundling: native libs + resources -------------------------------------- +;; A jolt seq of jolt strings -> a Scheme list of Scheme strings. +(define (bld-strs x) (map jolt-str-render-one (seq->list x))) + +;; Emit native-library loads. `natives` is the encoded jolt seq jolt.main/ +;; encode-natives produced: each entry is ["process"] | ["req" cand…] | ["opt" cand…]. +;; `which` selects 'required (process + req) or 'optional. Required + process loads +;; are emitted before the app forms (the app's defcfn foreign-procedures resolve +;; their symbols at top-level eval during startup, so the libs must be loaded +;; first); a load-shared-object failure there is fatal — correct for a required +;; lib. Optional loads run in the scheme-start launcher, where guard catches a +;; missing lib (an optional lib's namespace is only present when the app requires +;; it, so its foreign-procedures aren't among the baked top-level forms). +(define (bld-emit-natives out natives which) + (for-each + (lambda (entry) + (let* ((parts (bld-strs entry)) (kind (car parts)) (cands (cdr parts)) + (cand-lits (fold-left (lambda (s c) (string-append s (ei-str-lit c) " ")) "" cands))) + (cond + ((and (eq? which 'required) (string=? kind "process")) + (put-string out "(jolt-build-load-native '() #f #t)\n")) + ((and (eq? which 'required) (string=? kind "req")) + (put-string out (string-append "(jolt-build-load-native (list " cand-lits ") #f #f)\n"))) + ((and (eq? which 'optional) (string=? kind "opt")) + (put-string out (string-append "(jolt-build-load-native (list " cand-lits ") #t #f)\n")))))) + (seq->list natives))) + +;; Walk an embed root recursively; return (resource-name . abspath) pairs, where +;; resource-name is the "/"-joined path under the root (what io/resource is asked for). +(define (bld-walk-files root rel acc) + (let ((dir (if (string=? rel "") root (string-append root "/" rel)))) + (fold-left + (lambda (acc name) + (let* ((relpath (if (string=? rel "") name (string-append rel "/" name))) + (full (string-append root "/" relpath))) + (if (file-directory? full) + (bld-walk-files root relpath acc) + (cons (cons relpath full) acc)))) + acc + (directory-list dir)))) + +;; Emit register-embedded-resource! per file under each embed dir. Emitted BEFORE +;; the app forms so the (read-file-string ABSPATH) runs at heap build — the file's +;; contents bake into the boot image and io/resource serves them with no file on +;; disk. ABSPATH only has to exist at build time. +(define (bld-emit-embeds out embed-dirs) + (for-each + (lambda (root) + (when (file-directory? root) + (for-each + (lambda (rp) + (put-string out (string-append + "(register-embedded-resource! " (ei-str-lit (car rp)) + " (read-file-string " (ei-str-lit (cdr rp)) "))\n"))) + (bld-walk-files root "" '())))) + (bld-strs embed-dirs))) + ;; --- the build -------------------------------------------------------------- ;; entry-ns: the app's main namespace (a string). out-path: the binary to write. ;; mode: "dev" | "release" | "optimized". Every form runs through jolt.passes/ ;; run-passes (const-fold always; inline + type inference when optimized turns on ;; direct-linking). Deps + source roots are already applied by the caller. -(define (build-binary entry-ns out-path mode) +;; natives: encoded :jolt/native libs to load at startup. embed-dirs: dirs whose +;; files bake into the binary (single-file). ext-roots: project-relative io/resource +;; roots resolved at runtime against JOLT_PWD (ship-alongside resources). +(define (build-binary entry-ns out-path mode natives embed-dirs ext-roots) (bld-check-toolchain) ;; 1. record app namespaces in dependency order as they finish loading. (let ((app-order '())) @@ -180,16 +240,40 @@ ;; 3. flat source = runtime + app + launcher. (let ((out (open-output-file flat-ss 'replace))) (bld-emit-runtime out) + ;; Load native libs, bake embedded resources, and point source roots at + ;; the build-time app roots — all BEFORE the app forms. The app's + ;; top-level forms run at binary startup (Sbuild_heap), and they include + ;; foreign-procedure evals (a library's defcfn) and (slurp (io/resource …)) + ;; reads. So the libraries must be loaded and resources resolvable by the + ;; time those forms run, not later in the scheme-start launcher. + (put-string out "\n;; === native libraries (required) ===\n") + (bld-emit-natives out natives 'required) + (put-string out "\n;; === embedded resources ===\n") + (bld-emit-embeds out embed-dirs) + (put-string out (string-append + "(set-source-roots! (list " + (fold-left (lambda (s r) (string-append s (ei-str-lit r) " ")) "" + (get-source-roots)) + "))\n")) (put-string out "\n;; === app ===\n") (for-each (lambda (s) (put-string out s) (put-string out "\n")) app-strs) ;; The launcher runs as Chez's scheme-start (so argv reaches -main — ;; top-level boot forms run during heap build, before args are set), and - ;; suppresses the interactive greeting. + ;; suppresses the interactive greeting. It resets source roots to the + ;; app's resource dirs resolved against JOLT_PWD (or cwd) so a runtime + ;; io/resource that wasn't embedded still resolves next to the binary. (put-string out "\n;; === launcher ===\n") + (put-string out "(suppress-greeting #t)\n") + (put-string out "(scheme-start\n (lambda args\n") + (bld-emit-natives out natives 'optional) + (put-string out (string-append + " (let ((base (or (getenv \"JOLT_PWD\") \".\")))\n" + " (set-source-roots!\n" + " (append (map (lambda (r) (string-append base \"/\" r)) (list " + (fold-left (lambda (s r) (string-append s (ei-str-lit r) " ")) "" (bld-strs ext-roots)) + "))\n" + " (list \"jolt-core\" \"stdlib\"))))\n")) (put-string out (string-append - "(suppress-greeting #t)\n" - "(scheme-start\n" - " (lambda args\n" " (let ((mainv (var-deref " (ei-str-lit entry-ns) " \"-main\")))\n" " (apply jolt-invoke mainv args))\n" " (exit 0)))\n")) @@ -233,8 +317,9 @@ (display (string-append "jolt build: wrote " out-path "\n")))))) (def-var! "jolt.host" "build-binary" - (lambda (entry out mode) + (lambda (entry out mode natives embed-dirs ext-roots) (build-binary (jolt-str-render-one entry) (jolt-str-render-one out) - (jolt-str-render-one mode)) + (jolt-str-render-one mode) + natives embed-dirs ext-roots) jolt-nil)) diff --git a/host/chez/ffi.ss b/host/chez/ffi.ss index c595b40..ca665e5 100644 --- a/host/chez/ffi.ss +++ b/host/chez/ffi.ss @@ -134,6 +134,24 @@ (when co (unlock-object co) (hashtable-delete! ffi-callable-table a)) jolt-nil)) +;; --- native libraries for a standalone binary ------------------------------- +;; `jolt build` bakes a project's deps.edn :jolt/native declarations into the +;; launcher, which loads them at startup (load-shared-object isn't part of the +;; saved heap, so it must run in the built process, not at heap build). process? +;; loads the running binary's own symbols (libc sockets); otherwise try each +;; platform candidate in turn and fail unless the spec is optional. +(define (jolt-build-load-native cands optional? process?) + (if process? + (begin (load-shared-object #f) #t) + (let loop ((cs cands)) + (cond + ((null? cs) + (unless optional? + (error 'jolt-build "required native library not found" cands)) + #f) + ((guard (e (#t #f)) (load-shared-object (car cs)) #t) #t) + (else (loop (cdr cs))))))) + ;; --- expose under jolt.ffi --------------------------------------------------- (def-var! "jolt.ffi" "free-callable" ffi-free-callable) (def-var! "jolt.ffi" "load-library" ffi-load-library) diff --git a/host/chez/io.ss b/host/chez/io.ss index 04ce13b..e8a84cf 100644 --- a/host/chez/io.ss +++ b/host/chez/io.ss @@ -19,6 +19,16 @@ ;; path string of any value: a jfile -> its path, else its str rendering. (define (file-path-of x) (if (jfile? x) (jfile-path x) (jolt-str-render-one x))) +;; Resources baked into a standalone binary by `jolt build` (deps.edn +;; :jolt/build :embed). The build emits a register-embedded-resource! per file at +;; heap-build time, so the contents live in the boot image — io/resource serves +;; them with no file on disk. An embedded hit reads through slurp/reader exactly +;; like a jfile would. +(define embedded-resources (make-hashtable equal-hash equal?)) +(define (register-embedded-resource! name content) + (hashtable-set! embedded-resources name content)) +(define-record-type embedded-res (fields name content) (nongenerative jolt-embres-v1)) + ;; A user-facing relative path resolves against JOLT_PWD — the user's cwd before ;; the launcher cd'd to the jolt repo root — matching the JVM, where io/file is ;; cwd-relative. (io/resource builds jfiles from the source roots directly, so it @@ -191,6 +201,7 @@ (define (jolt-slurp src . opts) (cond ((jfile? src) (read-file-string (jfile-path src))) + ((embedded-res? src) (embedded-res-content src)) ((reader-jhost? src) (drain-reader src)) ;; bytes (a bytevector or a jolt byte-array): decode with :encoding (UTF-8 ;; default). clj-http-lite slurps response-body byte arrays. @@ -286,6 +297,7 @@ (cond ((reader-jhost? x) x) ((jfile? x) (host-new "StringReader" (read-file-string (jfile-path x)))) + ((embedded-res? x) (host-new "StringReader" (embedded-res-content x))) ((and (jhost? x) (string=? (jhost-tag x) "url")) (host-new "StringReader" (read-file-string (url-strip-scheme (url-spec x))))) ((string? x) (host-new "StringReader" (read-file-string x))) @@ -317,11 +329,13 @@ ;; (slurp/reader-able) for the first match, else nil. get-source-roots is the ;; loader's accessor (loader.ss), resolved at call time — the runtime CLI loads it. (define (jolt-io-resource name) - (let ((nm (jolt-str-render-one name))) - (let loop ((roots (get-source-roots))) - (cond ((null? roots) jolt-nil) - ((file-exists? (string-append (car roots) "/" nm)) (make-jfile (string-append (car roots) "/" nm))) - (else (loop (cdr roots))))))) + (let* ((nm (jolt-str-render-one name)) + (emb (hashtable-ref embedded-resources nm #f))) + (if emb (make-embedded-res nm emb) + (let loop ((roots (get-source-roots))) + (cond ((null? roots) jolt-nil) + ((file-exists? (string-append (car roots) "/" nm)) (make-jfile (string-append (car roots) "/" nm))) + (else (loop (cdr roots)))))))) (def-var! "clojure.java.io" "resource" jolt-io-resource) ;; as-url honors a library-registered URL class (e.g. jolt-lang/http-client's full ;; java.net.URL shim) so io/as-url and (URL. spec) agree; else the file-only jhost. diff --git a/jolt-core/jolt/deps.clj b/jolt-core/jolt/deps.clj index e3dd000..343974e 100644 --- a/jolt-core/jolt/deps.clj +++ b/jolt-core/jolt/deps.clj @@ -127,6 +127,14 @@ {dep-roots :roots dep-natives :natives} (resolve-deps all-deps project-dir)] {:roots (vec (distinct (concat project-roots dep-roots))) :main-opts main-opts + ;; the project's own paths (relative to project-dir) and absolute resource + ;; roots, plus its :jolt/build options — `jolt build` uses these to bundle + ;; resources into / alongside a standalone binary. + :project-dir project-dir + :project-paths (vec project-paths) + :project-roots (vec project-roots) + :build (:jolt/build edn) + :embed-dirs (mapv #(abspath project-dir %) (:embed (:jolt/build edn))) :tasks (:tasks edn) :natives (vec (distinct (concat (:jolt/native edn) dep-natives))) ;; nREPL middleware a library contributes (jolt.nrepl composes them over its diff --git a/jolt-core/jolt/main.clj b/jolt-core/jolt/main.clj index 40e674a..7ab0aa3 100644 --- a/jolt-core/jolt/main.clj +++ b/jolt-core/jolt/main.clj @@ -8,6 +8,12 @@ (defn- project-dir [] (or (jolt.host/getenv "JOLT_PWD") ".")) +(defn- current-platform [] + (let [os (str/lower-case (or (System/getProperty "os.name") ""))] + (cond (str/includes? os "mac") :darwin + (str/includes? os "win") :windows + :else :linux))) + ;; Load a library's declared native shared objects (deps.edn :jolt/native) before ;; its Clojure is required, so its foreign-fn bindings resolve. Each entry is a ;; map: {:name "sqlite3" :darwin ["libsqlite3.0.dylib" ...] :linux ["libsqlite3.so.0" ...]} @@ -15,10 +21,7 @@ ;; (use the running process's symbols, e.g. libc sockets — no external file). (defn- load-natives! [natives] (when (seq natives) - (let [os (str/lower-case (or (System/getProperty "os.name") "")) - plat (cond (str/includes? os "mac") :darwin - (str/includes? os "win") :windows - :else :linux)] + (let [plat (current-platform)] (doseq [spec natives] (if (:process spec) (jolt.ffi/load-library) @@ -116,23 +119,41 @@ ;; standalone executable. Resolves deps + roots like `run`, then hands the entry ;; namespace to the host build driver (jolt.host/build-binary, defined by ;; build.ss). Default mode is release; --opt selects optimized, --dev unoptimized. +;; Encode a deps.edn :jolt/native spec for the build launcher, resolving the +;; current platform's candidate list now (the binary runs on this OS). Each entry +;; becomes a vector the launcher (build.ss) reads: ["process"] for the running +;; binary's own symbols, else ["req"|"opt" cand…] to try in turn. +(defn- encode-natives [natives] + (let [plat (current-platform)] + (vec (for [spec natives] + (if (:process spec) + ["process"] + (let [c (get spec plat) + cands (if (string? c) [c] (vec c))] + (into [(if (:optional spec) "opt" "req")] cands))))))) + (defn- cmd-build [more] - (apply-project! (deps/resolve-project (project-dir))) - (let [opts (loop [a more, entry nil, out nil] - (cond - (empty? a) {:entry entry :out out} - (= "-m" (first a)) (recur (drop 2 a) (second a) out) - (= "-o" (first a)) (recur (drop 2 a) entry (second a)) - (str/starts-with? (first a) "-") (recur (rest a) entry out) - :else (recur (rest a) (or entry (first a)) out))) - entry (:entry opts) - mode (cond (some #{"--opt"} more) "optimized" - (some #{"--dev"} more) "dev" - :else "release")] - (when (nil? entry) - (throw (ex-info "build needs an entry: -m NS" {}))) - (let [out (or (:out opts) (first (str/split entry #"\.")))] - (jolt.host/build-binary entry out mode)))) + (let [{:keys [project-paths embed-dirs] :as resolved} + (deps/resolve-project (project-dir))] + (apply-project! resolved) + (let [opts (loop [a more, entry nil, out nil] + (cond + (empty? a) {:entry entry :out out} + (= "-m" (first a)) (recur (drop 2 a) (second a) out) + (= "-o" (first a)) (recur (drop 2 a) entry (second a)) + (str/starts-with? (first a) "-") (recur (rest a) entry out) + :else (recur (rest a) (or entry (first a)) out))) + entry (:entry opts) + mode (cond (some #{"--opt"} more) "optimized" + (some #{"--dev"} more) "dev" + :else "release")] + (when (nil? entry) + (throw (ex-info "build needs an entry: -m NS" {}))) + (let [out (or (:out opts) (first (str/split entry #"\."))) + natives (encode-natives (:natives resolved))] + ;; embed-dirs (absolute) are walked + baked into the binary by the driver; + ;; project-paths (relative) become runtime io/resource roots (ship-alongside). + (jolt.host/build-binary entry out mode natives embed-dirs project-paths))))) (defn- nrepl [more] ;; resolve the project (deps on the roots, native libs loaded), then start the diff --git a/test/chez/build-app/deps.edn b/test/chez/build-app/deps.edn index ccd9a31..fff0290 100644 --- a/test/chez/build-app/deps.edn +++ b/test/chez/build-app/deps.edn @@ -1 +1,8 @@ -{:paths ["src"]} +{:paths ["src" "resources"] + + ;; exercise the build's native-lib loading: a :process spec loads the running + ;; binary's own symbols (libc) at startup — no external file, always succeeds. + :jolt/native [{:name "libc" :process true}] + + ;; bake resources/ into the binary so io/resource resolves with no file on disk. + :jolt/build {:embed ["resources"]}} diff --git a/test/chez/build-app/resources/greeting.txt b/test/chez/build-app/resources/greeting.txt new file mode 100644 index 0000000..835112d --- /dev/null +++ b/test/chez/build-app/resources/greeting.txt @@ -0,0 +1 @@ +embedded resource ok \ No newline at end of file diff --git a/test/chez/build-app/src/app/core.clj b/test/chez/build-app/src/app/core.clj index 4fd95a0..fd50bba 100644 --- a/test/chez/build-app/src/app/core.clj +++ b/test/chez/build-app/src/app/core.clj @@ -1,7 +1,11 @@ (ns app.core - (:require [app.util :as util])) + (:require [app.util :as util] + [clojure.java.io :as io])) (defn -main [& args] + ;; the resource is baked into the binary (deps.edn :jolt/build :embed), so this + ;; resolves with no resources/ dir on disk, run from any cwd. + (println (slurp (io/resource "greeting.txt"))) (util/twice (println (util/shout "hello from a built binary"))) (println "args:" (vec args)) (println "sum:" (reduce + (map count args))))