jolt build: compile an app to a standalone binary (Phase 4 stages 1-2)
Restores the standalone-binary capability the Janet host had. `bin/joltc build -m NS -o OUT` AOT-compiles an app into a single self-contained executable — the whole runtime, clojure.core, stdlib and compiler embedded, no Chez install or jolt source needed at runtime. Pipeline (host/chez/build.ss, host primitive jolt.host/build-binary driven by jolt.main's build command): resolve deps, load the entry namespace recording the app namespaces in dependency order, re-emit each to Scheme, textually inline the cli.ss runtime load sequence into one flat source + the app + a launcher, then compile-file -> make-boot-file -> embed the boot as C bytes -> cc-link against libkernel.a. Two non-obvious bits: the compile pass runs in a fresh Chez, not the loaded runtime (regex.ss shadows top-level `error`, which otherwise bakes a broken reference into the boot); and the launcher installs scheme-start rather than running -main at top level, since boot top-level forms execute during heap build before argv is set, so args only reach -main through scheme-start. Loader: a require of an in-memory namespace with no source file now no-ops, so AOT'd app namespaces satisfy require in a built binary. Mode flags (--opt/--dev, default release) are plumbed; the optimization passes they gate come in a later stage. RFC 0007 has the design. Gated by `make buildsmoke`.
This commit is contained in:
Yogthos
parent
33eff7c7d8
commit
43778eafd7
10 changed files with 554 additions and 12 deletions
253
host/chez/build.ss
Normal file
253
host/chez/build.ss
Normal file
|
|
@ -0,0 +1,253 @@
|
|||
;; build.ss — `jolt build`: AOT-compile an app into a standalone executable.
|
||||
;;
|
||||
;; Loaded on demand by cli.ss when the command is `build`. Defines the host
|
||||
;; primitive jolt.host/build-binary, which jolt.main's build command calls after
|
||||
;; resolving the project's deps + source roots.
|
||||
;;
|
||||
;; The pipeline (Phase 4 stage 2):
|
||||
;; 1. load the entry namespace — registers its macros/vars and follows requires,
|
||||
;; recording the app namespaces in dependency order (loader's ns-loaded-hook).
|
||||
;; 2. re-emit each app namespace to Scheme (the emit-image cross-compile path),
|
||||
;; now that its macros are registered.
|
||||
;; 3. textually inline the cli.ss runtime load sequence into one flat source,
|
||||
;; append the app emission + a launcher that calls the entry's -main.
|
||||
;; 4. compile-file -> make-boot-file -> embed the boot as C bytes -> cc-link
|
||||
;; against libkernel.a into a single self-contained binary.
|
||||
;;
|
||||
;; emit-image.ss supplies the cross-compiler (ei-* helpers); it's loaded here so a
|
||||
;; normal run never pays for it.
|
||||
|
||||
(load "host/chez/emit-image.ss")
|
||||
|
||||
;; --- shell helpers ----------------------------------------------------------
|
||||
;; Run a command, return its stdout as one trimmed string ("" on no output).
|
||||
(define (bld-sh-capture cmd)
|
||||
(let* ((p (process cmd)) (in (car p)))
|
||||
(let loop ((acc '()))
|
||||
(let ((l (get-line in)))
|
||||
(if (eof-object? l)
|
||||
(begin (close-port in)
|
||||
(let ((s (apply string-append (reverse acc))))
|
||||
;; trim a trailing newline-equivalent (we joined without them)
|
||||
s))
|
||||
(loop (cons l acc)))))))
|
||||
|
||||
(define (bld-system cmd)
|
||||
(let ((rc (system cmd)))
|
||||
(unless (zero? rc)
|
||||
(error 'jolt-build (string-append "command failed (" (number->string rc) "): " cmd)))))
|
||||
|
||||
(define (bld-contains? s sub)
|
||||
(let ((ns (string-length s)) (nsub (string-length sub)))
|
||||
(let loop ((i 0))
|
||||
(cond ((> (+ i nsub) ns) #f)
|
||||
((string=? (substring s i (+ i nsub)) sub) #t)
|
||||
(else (loop (+ i 1)))))))
|
||||
|
||||
;; --- toolchain discovery ----------------------------------------------------
|
||||
(define bld-machine (symbol->string (machine-type)))
|
||||
(define bld-osx? (bld-contains? bld-machine "osx"))
|
||||
|
||||
;; The Chez executable, for the isolated compile pass (see build-binary step 4).
|
||||
(define bld-chez
|
||||
(let ((p (bld-sh-capture "command -v chez || command -v scheme || command -v petite")))
|
||||
(if (> (string-length p) 0) p "chez")))
|
||||
|
||||
;; Chez version off (scheme-version) "Chez Scheme Version X.Y.Z" — last token.
|
||||
(define bld-version
|
||||
(let* ((s (scheme-version)) (n (string-length s)))
|
||||
(let loop ((i n))
|
||||
(if (or (= i 0) (char=? (string-ref s (- i 1)) #\space))
|
||||
(substring s i n)
|
||||
(loop (- i 1))))))
|
||||
|
||||
;; The csv<ver>/<machine> dir holding scheme.h, libkernel.a, *.boot. Derived from
|
||||
;; the chez executable's location; JOLT_CHEZ_CSV overrides.
|
||||
(define bld-csv-dir
|
||||
(let ((env (getenv "JOLT_CHEZ_CSV")))
|
||||
(or (and env (> (string-length env) 0) env)
|
||||
(let* ((bindir (bld-sh-capture "dirname \"$(command -v chez || command -v scheme || command -v petite)\""))
|
||||
(cand (string-append bindir "/../lib/csv" bld-version "/" bld-machine)))
|
||||
cand))))
|
||||
|
||||
(define (bld-check-toolchain)
|
||||
(for-each
|
||||
(lambda (f)
|
||||
(let ((p (string-append bld-csv-dir "/" f)))
|
||||
(unless (file-exists? p)
|
||||
(error 'jolt-build (string-append "Chez build file missing: " p
|
||||
"\nSet JOLT_CHEZ_CSV to the csv<ver>/<machine> dir.")))))
|
||||
'("scheme.h" "libkernel.a" "petite.boot" "scheme.boot")))
|
||||
|
||||
;; Link flags. macOS Homebrew layout for the kernel's lz4/zlib/ncurses deps.
|
||||
(define (bld-link-libs)
|
||||
(if bld-osx?
|
||||
(let ((lz4 (bld-sh-capture "brew --prefix lz4 2>/dev/null")))
|
||||
(string-append
|
||||
(if (> (string-length lz4) 0) (string-append "-L" lz4 "/lib ") "")
|
||||
"-llz4 -lz -lncurses -framework Foundation -liconv -lm"))
|
||||
;; Best-effort Linux/other; untested here.
|
||||
"-llz4 -lz -lncurses -ldl -lpthread -lm"))
|
||||
|
||||
;; --- runtime manifest (mirrors host/chez/cli.ss's load order) ---------------
|
||||
(define bld-runtime-manifest
|
||||
(list
|
||||
"(load \"host/chez/rt.ss\")"
|
||||
"(set-chez-ns! \"clojure.core\")"
|
||||
"(load \"host/chez/seed/prelude.ss\")"
|
||||
"(load \"host/chez/post-prelude.ss\")"
|
||||
"(set-chez-ns! \"user\")"
|
||||
"(load \"host/chez/host-contract.ss\")"
|
||||
"(load \"host/chez/seed/image.ss\")"
|
||||
"(load \"host/chez/compile-eval.ss\")"
|
||||
"(load \"host/chez/png.ss\")"
|
||||
"(load \"host/chez/loader.ss\")"
|
||||
"(load \"host/chez/ffi.ss\")"
|
||||
"(set-source-roots! (list \"jolt-core\" \"stdlib\"))"))
|
||||
|
||||
;; A single-line top-level `(load "PATH")` -> PATH, else #f.
|
||||
(define (bld-load-path line)
|
||||
(let ((s (let trim ((i 0))
|
||||
(if (and (< i (string-length line))
|
||||
(memv (string-ref line i) '(#\space #\tab)))
|
||||
(trim (+ i 1))
|
||||
(substring line i (string-length line))))))
|
||||
(and (>= (string-length s) 7)
|
||||
(string=? (substring s 0 6) "(load ")
|
||||
(let* ((q1 (let scan ((i 6)) (if (char=? (string-ref s i) #\") i (scan (+ i 1)))))
|
||||
(q2 (let scan ((i (+ q1 1))) (if (char=? (string-ref s i) #\") i (scan (+ i 1))))))
|
||||
(substring s (+ q1 1) q2)))))
|
||||
|
||||
(define (bld-file-lines path)
|
||||
(call-with-input-file path
|
||||
(lambda (p)
|
||||
(let loop ((acc '()))
|
||||
(let ((l (get-line p)))
|
||||
(if (eof-object? l) (reverse acc) (loop (cons l acc))))))))
|
||||
|
||||
;; Emit one line to OUT, recursively inlining a `(load ...)` of a repo file.
|
||||
(define (bld-inline-line line out depth)
|
||||
(when (> depth 50) (error 'jolt-build "load nesting too deep"))
|
||||
(let ((p (bld-load-path line)))
|
||||
(if p
|
||||
(for-each (lambda (l) (bld-inline-line l out (+ depth 1))) (bld-file-lines p))
|
||||
(begin (put-string out line) (put-string out "\n")))))
|
||||
|
||||
(define (bld-emit-runtime out)
|
||||
(for-each (lambda (l) (bld-inline-line l out 0)) bld-runtime-manifest))
|
||||
|
||||
;; --- app emission -----------------------------------------------------------
|
||||
;; Re-emit one app namespace to a list of Scheme strings. Like emit-image's
|
||||
;; ei-emit-ns but WITHOUT the silent (guard ...) wrapper — a form that fails to
|
||||
;; emit must fail the build, not vanish.
|
||||
(define (bld-emit-ns ns-name src)
|
||||
(let loop ((forms (ei-read-all src)) (acc '()))
|
||||
(if (null? forms)
|
||||
(reverse acc)
|
||||
(let ((f (car forms)))
|
||||
(ce-scan-requires! f ns-name)
|
||||
(cond
|
||||
((ei-ns-form? f) (loop (cdr forms) acc))
|
||||
((ce-macro-form? f)
|
||||
(let-values (((nm fn-form) (ce-defmacro->fn f)))
|
||||
(let ((scm (let ((ctx (make-analyze-ctx ns-name)))
|
||||
(jolt-ce-emit (jolt-ce-analyze ctx fn-form)))))
|
||||
(loop (cdr forms)
|
||||
(cons (string-append
|
||||
"(def-var! " (ei-str-lit ns-name) " " (ei-str-lit nm) "\n "
|
||||
scm ")\n(mark-macro! "
|
||||
(ei-str-lit ns-name) " " (ei-str-lit nm) ")")
|
||||
acc)))))
|
||||
(else
|
||||
(let* ((ctx (make-analyze-ctx ns-name))
|
||||
(scm (jolt-ce-emit (jolt-ce-analyze ctx f))))
|
||||
(loop (cdr forms) (cons scm acc)))))))))
|
||||
|
||||
;; --- the build --------------------------------------------------------------
|
||||
;; entry-ns: the app's main namespace (a string). out-path: the binary to write.
|
||||
;; mode: "dev" | "release" | "optimized" (recorded; optimization passes wired in a
|
||||
;; later stage). Deps + source roots are already applied by the caller.
|
||||
(define (build-binary entry-ns out-path mode)
|
||||
(bld-check-toolchain)
|
||||
;; 1. record app namespaces in dependency order as they finish loading.
|
||||
(let ((app-order '()))
|
||||
(set-ns-loaded-hook!
|
||||
(lambda (name file) (set! app-order (cons (cons name file) app-order))))
|
||||
(load-namespace entry-ns)
|
||||
(set-ns-loaded-hook! (lambda (name file) #f))
|
||||
(let ((ordered (reverse app-order))) ; deps first, entry last
|
||||
(when (null? ordered)
|
||||
(error 'jolt-build (string-append "no source namespace loaded for " entry-ns
|
||||
" — is it on the source roots?")))
|
||||
;; 2. emit each app namespace.
|
||||
(let* ((app-strs (apply append
|
||||
(map (lambda (nf) (bld-emit-ns (car nf) (read-file-string (cdr nf))))
|
||||
ordered)))
|
||||
(builddir (string-append out-path ".build"))
|
||||
(flat-ss (string-append builddir "/flat.ss"))
|
||||
(flat-so (string-append builddir "/flat.so"))
|
||||
(boot (string-append builddir "/jolt.boot"))
|
||||
(boot-h (string-append builddir "/boot_data.h"))
|
||||
(main-c (string-append builddir "/main.c")))
|
||||
(bld-system (string-append "mkdir -p '" builddir "'"))
|
||||
;; 3. flat source = runtime + app + launcher.
|
||||
(let ((out (open-output-file flat-ss 'replace)))
|
||||
(bld-emit-runtime out)
|
||||
(put-string out "\n;; === app ===\n")
|
||||
(for-each (lambda (s) (put-string out s) (put-string out "\n")) app-strs)
|
||||
;; The launcher runs as Chez's scheme-start (so argv reaches -main —
|
||||
;; top-level boot forms run during heap build, before args are set), and
|
||||
;; suppresses the interactive greeting.
|
||||
(put-string out "\n;; === launcher ===\n")
|
||||
(put-string out (string-append
|
||||
"(suppress-greeting #t)\n"
|
||||
"(scheme-start\n"
|
||||
" (lambda args\n"
|
||||
" (let ((mainv (var-deref " (ei-str-lit entry-ns) " \"-main\")))\n"
|
||||
" (apply jolt-invoke mainv args))\n"
|
||||
" (exit 0)))\n"))
|
||||
(close-port out))
|
||||
;; 4. compile -> boot -> embed -> link.
|
||||
;; compile-file/make-boot-file run in a FRESH Chez, not this process: the
|
||||
;; loaded runtime shadows `error` (regex.ss, for irregex), which would
|
||||
;; otherwise bake a broken `error` reference into the boot.
|
||||
(display (string-append "jolt build: compiling " entry-ns " (" mode " mode)\n"))
|
||||
(let ((cs (string-append builddir "/compile.ss")))
|
||||
(let ((p (open-output-file cs 'replace)))
|
||||
(put-string p
|
||||
(string-append
|
||||
"(import (chezscheme))\n"
|
||||
"(compile-file " (ei-str-lit flat-ss) " " (ei-str-lit flat-so) ")\n"
|
||||
"(make-boot-file " (ei-str-lit boot) " '()\n "
|
||||
(ei-str-lit (string-append bld-csv-dir "/petite.boot")) "\n "
|
||||
(ei-str-lit (string-append bld-csv-dir "/scheme.boot")) "\n "
|
||||
(ei-str-lit flat-so) ")\n"))
|
||||
(close-port p))
|
||||
(bld-system (string-append bld-chez " --script '" cs "'")))
|
||||
(bld-system (string-append "xxd -i '" boot "' > '" boot-h "'"))
|
||||
;; The xxd symbol is derived from the path; normalize to jolt_boot.
|
||||
(bld-system (string-append
|
||||
"sed -i.bak -E 's/unsigned char [A-Za-z0-9_]+\\[\\]/unsigned char jolt_boot[]/; "
|
||||
"s/unsigned int [A-Za-z0-9_]+_len/unsigned int jolt_boot_len/' '" boot-h "'"))
|
||||
(let ((mc (open-output-file main-c 'replace)))
|
||||
(put-string mc
|
||||
(string-append
|
||||
"#include \"scheme.h\"\n#include \"boot_data.h\"\n"
|
||||
"int main(int argc, char *argv[]) {\n"
|
||||
" Sscheme_init(0);\n"
|
||||
" Sregister_boot_file_bytes(\"jolt\", jolt_boot, jolt_boot_len);\n"
|
||||
" Sbuild_heap(0, 0);\n"
|
||||
" int status = Sscheme_start(argc, (const char **)argv);\n"
|
||||
" Sscheme_deinit();\n return status;\n}\n"))
|
||||
(close-port mc))
|
||||
(bld-system (string-append
|
||||
"cc -O2 -I'" bld-csv-dir "' '" main-c "' '" bld-csv-dir "/libkernel.a' "
|
||||
"-o '" out-path "' " (bld-link-libs)))
|
||||
(display (string-append "jolt build: wrote " out-path "\n"))))))
|
||||
|
||||
(def-var! "jolt.host" "build-binary"
|
||||
(lambda (entry out mode)
|
||||
(build-binary (jolt-str-render-one entry)
|
||||
(jolt-str-render-one out)
|
||||
(jolt-str-render-one mode))
|
||||
jolt-nil))
|
||||
Loading…
Add table
Add a link
Reference in a new issue