jolt build: bundle native libs + resources into standalone binaries
A built binary dropped its deps.edn :jolt/native declarations and its
resource roots, so an FFI+resources app (ring-app) failed at runtime:
sockets/sqlite gave 'no entry for socket' and io/resource returned nil.
The buildsmoke fixture is pure compute, so neither path was exercised.
The launcher now loads required + :process native libs before the app's
top-level forms (a library's defcfn resolves its foreign-procedure symbols
at top-level eval during startup, so the libs must be loaded first);
optional libs load in the scheme-start launcher, where a missing lib is
caught rather than aborting the heap build.
deps.edn :jolt/build {:embed [dirs]} bakes those dirs' files into the heap
(register-embedded-resource! at heap build), so io/resource serves them with
no files on disk. Non-embedded resources resolve at runtime against JOLT_PWD,
and io/file reads (e.g. config.edn) stay external.
build-binary now takes the encoded natives, embed dirs, and project paths
from cmd-build; deps/resolve-project surfaces them. Buildsmoke fixture grows
an embedded resource + a :process native to cover both paths.
This commit is contained in:
Yogthos
parent
22c08d30f2
commit
1d345bfd0f
10 changed files with 227 additions and 36 deletions
|
|
@ -36,9 +36,12 @@ if ! JOLT_PWD="$app" bin/joltc build -m app.core -o "$out" >/dev/null 2>&1; then
|
|||
fi
|
||||
[ -x "$out" ] || { echo " FAIL: no executable produced"; exit 1; }
|
||||
|
||||
# Run from a neutral cwd with args; check the three output lines.
|
||||
# Run from a neutral cwd with args. The first line is an embedded resource
|
||||
# (deps.edn :jolt/build :embed), proving io/resource resolves from the binary with
|
||||
# no resources/ dir on disk; the rest exercise a macro, cross-ns, and args.
|
||||
got="$(cd / && "$out" alpha bb ccc 2>&1)"
|
||||
want='HELLO FROM A BUILT BINARY!
|
||||
want='embedded resource ok
|
||||
HELLO FROM A BUILT BINARY!
|
||||
HELLO FROM A BUILT BINARY!
|
||||
args: [alpha bb ccc]
|
||||
sum: 10'
|
||||
|
|
|
|||
|
|
@ -146,12 +146,72 @@
|
|||
;; The loop itself is emit-image's ei-emit-ns* (optimize? #t, guard? #f).
|
||||
(define (bld-emit-ns ns-name src) (ei-emit-ns* ns-name src #t #f))
|
||||
|
||||
;; --- bundling: native libs + resources --------------------------------------
|
||||
;; A jolt seq of jolt strings -> a Scheme list of Scheme strings.
|
||||
(define (bld-strs x) (map jolt-str-render-one (seq->list x)))
|
||||
|
||||
;; Emit native-library loads. `natives` is the encoded jolt seq jolt.main/
|
||||
;; encode-natives produced: each entry is ["process"] | ["req" cand…] | ["opt" cand…].
|
||||
;; `which` selects 'required (process + req) or 'optional. Required + process loads
|
||||
;; are emitted before the app forms (the app's defcfn foreign-procedures resolve
|
||||
;; their symbols at top-level eval during startup, so the libs must be loaded
|
||||
;; first); a load-shared-object failure there is fatal — correct for a required
|
||||
;; lib. Optional loads run in the scheme-start launcher, where guard catches a
|
||||
;; missing lib (an optional lib's namespace is only present when the app requires
|
||||
;; it, so its foreign-procedures aren't among the baked top-level forms).
|
||||
(define (bld-emit-natives out natives which)
|
||||
(for-each
|
||||
(lambda (entry)
|
||||
(let* ((parts (bld-strs entry)) (kind (car parts)) (cands (cdr parts))
|
||||
(cand-lits (fold-left (lambda (s c) (string-append s (ei-str-lit c) " ")) "" cands)))
|
||||
(cond
|
||||
((and (eq? which 'required) (string=? kind "process"))
|
||||
(put-string out "(jolt-build-load-native '() #f #t)\n"))
|
||||
((and (eq? which 'required) (string=? kind "req"))
|
||||
(put-string out (string-append "(jolt-build-load-native (list " cand-lits ") #f #f)\n")))
|
||||
((and (eq? which 'optional) (string=? kind "opt"))
|
||||
(put-string out (string-append "(jolt-build-load-native (list " cand-lits ") #t #f)\n"))))))
|
||||
(seq->list natives)))
|
||||
|
||||
;; Walk an embed root recursively; return (resource-name . abspath) pairs, where
|
||||
;; resource-name is the "/"-joined path under the root (what io/resource is asked for).
|
||||
(define (bld-walk-files root rel acc)
|
||||
(let ((dir (if (string=? rel "") root (string-append root "/" rel))))
|
||||
(fold-left
|
||||
(lambda (acc name)
|
||||
(let* ((relpath (if (string=? rel "") name (string-append rel "/" name)))
|
||||
(full (string-append root "/" relpath)))
|
||||
(if (file-directory? full)
|
||||
(bld-walk-files root relpath acc)
|
||||
(cons (cons relpath full) acc))))
|
||||
acc
|
||||
(directory-list dir))))
|
||||
|
||||
;; Emit register-embedded-resource! per file under each embed dir. Emitted BEFORE
|
||||
;; the app forms so the (read-file-string ABSPATH) runs at heap build — the file's
|
||||
;; contents bake into the boot image and io/resource serves them with no file on
|
||||
;; disk. ABSPATH only has to exist at build time.
|
||||
(define (bld-emit-embeds out embed-dirs)
|
||||
(for-each
|
||||
(lambda (root)
|
||||
(when (file-directory? root)
|
||||
(for-each
|
||||
(lambda (rp)
|
||||
(put-string out (string-append
|
||||
"(register-embedded-resource! " (ei-str-lit (car rp))
|
||||
" (read-file-string " (ei-str-lit (cdr rp)) "))\n")))
|
||||
(bld-walk-files root "" '()))))
|
||||
(bld-strs embed-dirs)))
|
||||
|
||||
;; --- the build --------------------------------------------------------------
|
||||
;; entry-ns: the app's main namespace (a string). out-path: the binary to write.
|
||||
;; mode: "dev" | "release" | "optimized". Every form runs through jolt.passes/
|
||||
;; run-passes (const-fold always; inline + type inference when optimized turns on
|
||||
;; direct-linking). Deps + source roots are already applied by the caller.
|
||||
(define (build-binary entry-ns out-path mode)
|
||||
;; natives: encoded :jolt/native libs to load at startup. embed-dirs: dirs whose
|
||||
;; files bake into the binary (single-file). ext-roots: project-relative io/resource
|
||||
;; roots resolved at runtime against JOLT_PWD (ship-alongside resources).
|
||||
(define (build-binary entry-ns out-path mode natives embed-dirs ext-roots)
|
||||
(bld-check-toolchain)
|
||||
;; 1. record app namespaces in dependency order as they finish loading.
|
||||
(let ((app-order '()))
|
||||
|
|
@ -180,16 +240,40 @@
|
|||
;; 3. flat source = runtime + app + launcher.
|
||||
(let ((out (open-output-file flat-ss 'replace)))
|
||||
(bld-emit-runtime out)
|
||||
;; Load native libs, bake embedded resources, and point source roots at
|
||||
;; the build-time app roots — all BEFORE the app forms. The app's
|
||||
;; top-level forms run at binary startup (Sbuild_heap), and they include
|
||||
;; foreign-procedure evals (a library's defcfn) and (slurp (io/resource …))
|
||||
;; reads. So the libraries must be loaded and resources resolvable by the
|
||||
;; time those forms run, not later in the scheme-start launcher.
|
||||
(put-string out "\n;; === native libraries (required) ===\n")
|
||||
(bld-emit-natives out natives 'required)
|
||||
(put-string out "\n;; === embedded resources ===\n")
|
||||
(bld-emit-embeds out embed-dirs)
|
||||
(put-string out (string-append
|
||||
"(set-source-roots! (list "
|
||||
(fold-left (lambda (s r) (string-append s (ei-str-lit r) " ")) ""
|
||||
(get-source-roots))
|
||||
"))\n"))
|
||||
(put-string out "\n;; === app ===\n")
|
||||
(for-each (lambda (s) (put-string out s) (put-string out "\n")) app-strs)
|
||||
;; The launcher runs as Chez's scheme-start (so argv reaches -main —
|
||||
;; top-level boot forms run during heap build, before args are set), and
|
||||
;; suppresses the interactive greeting.
|
||||
;; suppresses the interactive greeting. It resets source roots to the
|
||||
;; app's resource dirs resolved against JOLT_PWD (or cwd) so a runtime
|
||||
;; io/resource that wasn't embedded still resolves next to the binary.
|
||||
(put-string out "\n;; === launcher ===\n")
|
||||
(put-string out "(suppress-greeting #t)\n")
|
||||
(put-string out "(scheme-start\n (lambda args\n")
|
||||
(bld-emit-natives out natives 'optional)
|
||||
(put-string out (string-append
|
||||
" (let ((base (or (getenv \"JOLT_PWD\") \".\")))\n"
|
||||
" (set-source-roots!\n"
|
||||
" (append (map (lambda (r) (string-append base \"/\" r)) (list "
|
||||
(fold-left (lambda (s r) (string-append s (ei-str-lit r) " ")) "" (bld-strs ext-roots))
|
||||
"))\n"
|
||||
" (list \"jolt-core\" \"stdlib\"))))\n"))
|
||||
(put-string out (string-append
|
||||
"(suppress-greeting #t)\n"
|
||||
"(scheme-start\n"
|
||||
" (lambda args\n"
|
||||
" (let ((mainv (var-deref " (ei-str-lit entry-ns) " \"-main\")))\n"
|
||||
" (apply jolt-invoke mainv args))\n"
|
||||
" (exit 0)))\n"))
|
||||
|
|
@ -233,8 +317,9 @@
|
|||
(display (string-append "jolt build: wrote " out-path "\n"))))))
|
||||
|
||||
(def-var! "jolt.host" "build-binary"
|
||||
(lambda (entry out mode)
|
||||
(lambda (entry out mode natives embed-dirs ext-roots)
|
||||
(build-binary (jolt-str-render-one entry)
|
||||
(jolt-str-render-one out)
|
||||
(jolt-str-render-one mode))
|
||||
(jolt-str-render-one mode)
|
||||
natives embed-dirs ext-roots)
|
||||
jolt-nil))
|
||||
|
|
|
|||
|
|
@ -134,6 +134,24 @@
|
|||
(when co (unlock-object co) (hashtable-delete! ffi-callable-table a))
|
||||
jolt-nil))
|
||||
|
||||
;; --- native libraries for a standalone binary -------------------------------
|
||||
;; `jolt build` bakes a project's deps.edn :jolt/native declarations into the
|
||||
;; launcher, which loads them at startup (load-shared-object isn't part of the
|
||||
;; saved heap, so it must run in the built process, not at heap build). process?
|
||||
;; loads the running binary's own symbols (libc sockets); otherwise try each
|
||||
;; platform candidate in turn and fail unless the spec is optional.
|
||||
(define (jolt-build-load-native cands optional? process?)
|
||||
(if process?
|
||||
(begin (load-shared-object #f) #t)
|
||||
(let loop ((cs cands))
|
||||
(cond
|
||||
((null? cs)
|
||||
(unless optional?
|
||||
(error 'jolt-build "required native library not found" cands))
|
||||
#f)
|
||||
((guard (e (#t #f)) (load-shared-object (car cs)) #t) #t)
|
||||
(else (loop (cdr cs)))))))
|
||||
|
||||
;; --- expose under jolt.ffi ---------------------------------------------------
|
||||
(def-var! "jolt.ffi" "free-callable" ffi-free-callable)
|
||||
(def-var! "jolt.ffi" "load-library" ffi-load-library)
|
||||
|
|
|
|||
|
|
@ -19,6 +19,16 @@
|
|||
;; path string of any value: a jfile -> its path, else its str rendering.
|
||||
(define (file-path-of x) (if (jfile? x) (jfile-path x) (jolt-str-render-one x)))
|
||||
|
||||
;; Resources baked into a standalone binary by `jolt build` (deps.edn
|
||||
;; :jolt/build :embed). The build emits a register-embedded-resource! per file at
|
||||
;; heap-build time, so the contents live in the boot image — io/resource serves
|
||||
;; them with no file on disk. An embedded hit reads through slurp/reader exactly
|
||||
;; like a jfile would.
|
||||
(define embedded-resources (make-hashtable equal-hash equal?))
|
||||
(define (register-embedded-resource! name content)
|
||||
(hashtable-set! embedded-resources name content))
|
||||
(define-record-type embedded-res (fields name content) (nongenerative jolt-embres-v1))
|
||||
|
||||
;; A user-facing relative path resolves against JOLT_PWD — the user's cwd before
|
||||
;; the launcher cd'd to the jolt repo root — matching the JVM, where io/file is
|
||||
;; cwd-relative. (io/resource builds jfiles from the source roots directly, so it
|
||||
|
|
@ -191,6 +201,7 @@
|
|||
(define (jolt-slurp src . opts)
|
||||
(cond
|
||||
((jfile? src) (read-file-string (jfile-path src)))
|
||||
((embedded-res? src) (embedded-res-content src))
|
||||
((reader-jhost? src) (drain-reader src))
|
||||
;; bytes (a bytevector or a jolt byte-array): decode with :encoding (UTF-8
|
||||
;; default). clj-http-lite slurps response-body byte arrays.
|
||||
|
|
@ -286,6 +297,7 @@
|
|||
(cond
|
||||
((reader-jhost? x) x)
|
||||
((jfile? x) (host-new "StringReader" (read-file-string (jfile-path x))))
|
||||
((embedded-res? x) (host-new "StringReader" (embedded-res-content x)))
|
||||
((and (jhost? x) (string=? (jhost-tag x) "url"))
|
||||
(host-new "StringReader" (read-file-string (url-strip-scheme (url-spec x)))))
|
||||
((string? x) (host-new "StringReader" (read-file-string x)))
|
||||
|
|
@ -317,11 +329,13 @@
|
|||
;; (slurp/reader-able) for the first match, else nil. get-source-roots is the
|
||||
;; loader's accessor (loader.ss), resolved at call time — the runtime CLI loads it.
|
||||
(define (jolt-io-resource name)
|
||||
(let ((nm (jolt-str-render-one name)))
|
||||
(let loop ((roots (get-source-roots)))
|
||||
(cond ((null? roots) jolt-nil)
|
||||
((file-exists? (string-append (car roots) "/" nm)) (make-jfile (string-append (car roots) "/" nm)))
|
||||
(else (loop (cdr roots)))))))
|
||||
(let* ((nm (jolt-str-render-one name))
|
||||
(emb (hashtable-ref embedded-resources nm #f)))
|
||||
(if emb (make-embedded-res nm emb)
|
||||
(let loop ((roots (get-source-roots)))
|
||||
(cond ((null? roots) jolt-nil)
|
||||
((file-exists? (string-append (car roots) "/" nm)) (make-jfile (string-append (car roots) "/" nm)))
|
||||
(else (loop (cdr roots))))))))
|
||||
(def-var! "clojure.java.io" "resource" jolt-io-resource)
|
||||
;; as-url honors a library-registered URL class (e.g. jolt-lang/http-client's full
|
||||
;; java.net.URL shim) so io/as-url and (URL. spec) agree; else the file-only jhost.
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue