Build joltc as a self-contained binary (make joltc-release / joltc-debug)
host/chez/build-joltc.ss builds joltc into target/<profile>/joltc: it emits a flat source of the full runtime + compiler image + inlined build.ss + every jolt-core/stdlib file as a baked string literal + a cli.ss-style launcher, then (in a fresh Chez, so the inlined runtime's redefinition of error doesn't strand early references and runtime eval still sees the runtime's top-level procedures) compiles it and cc-links it with the Chez petite/scheme boots and the launcher stub embedded as C arrays. The launcher reads those arrays via FFI on (jolt-materialize-bundles!) and registers them so build-self-contained can spill them. joltc itself is cc-linked (clean signature for Homebrew); only the apps it later builds use the appended-stub path. build.ss: skip the csv toolchain check on the self-contained path and create the build dir with a subprocess-free bld-mkdir-p, so a from the distributed binary shells out to nothing. release = optimize-level 3 + no inspector info + compressed; debug = optimize-level 0 + inspector + procedure source + debug-on-exception. joltc-selfbuild-smoke.sh (make joltcsmoke) builds joltc and, with an empty environment (no chez/cc/PATH), drives it through the build-app fixture, asserting the produced binary's output. .gitignore ignores target/.
This commit is contained in:
Yogthos
parent
0420cd4d79
commit
242eeac5c6
5 changed files with 320 additions and 14 deletions
206
host/chez/build-joltc.ss
Normal file
206
host/chez/build-joltc.ss
Normal file
|
|
@ -0,0 +1,206 @@
|
|||
;; build-joltc.ss — build joltc itself as a self-contained native binary (jolt-eaj).
|
||||
;;
|
||||
;; chez --script host/chez/build-joltc.ss <profile> <out-path>
|
||||
;; profile: "release" | "debug" out-path: e.g. target/release/joltc
|
||||
;;
|
||||
;; Runs on a dev/CI machine that HAS Chez + cc. Produces a binary that needs
|
||||
;; NEITHER: it bakes the full runtime + compiler image + all jolt-core/stdlib
|
||||
;; source + the Chez petite/scheme boots + a prebuilt launcher stub into one
|
||||
;; cc-linked executable, so the resulting joltc can run AND `build` jolt apps on
|
||||
;; its own. joltc itself is cc-linked (not appended) so its signature stays clean
|
||||
;; for Homebrew/codesign, like dirge's binaries; only the apps it later builds use
|
||||
;; the appended-stub path (host/chez/build.ss build-self-contained).
|
||||
;;
|
||||
;; Pipeline:
|
||||
;; 0. cc-compile host/chez/stub/launcher.c against the Chez kernel.
|
||||
;; 1. emit flat.ss = runtime + compiler image (cli.ss load order) + inlined
|
||||
;; build.ss + every jolt-core/stdlib file as a baked string literal + the
|
||||
;; joltc launcher.
|
||||
;; 2. in-process compile-file + make-boot-file (profile Chez settings), error
|
||||
;; restored around the call (the runtime shadows it; regex.ss/%chez-error).
|
||||
;; 3. xxd the joltc boot + petite/scheme boots + stub into C arrays, generate
|
||||
;; main.c, cc-link -> out-path. The launcher reads the petite/scheme/stub
|
||||
;; arrays via FFI on `build` (jolt-materialize-bundles!).
|
||||
|
||||
(import (chezscheme))
|
||||
|
||||
(load "host/chez/rt.ss")
|
||||
(set-chez-ns! "clojure.core")
|
||||
(load "host/chez/seed/prelude.ss")
|
||||
(load "host/chez/post-prelude.ss")
|
||||
(set-chez-ns! "user")
|
||||
(load "host/chez/host-contract.ss")
|
||||
(load "host/chez/seed/image.ss")
|
||||
(load "host/chez/compile-eval.ss")
|
||||
(load "host/chez/png.ss")
|
||||
(load "host/chez/loader.ss")
|
||||
(load "host/chez/java/ffi.ss")
|
||||
(set-source-roots! (list "jolt-core" "stdlib"))
|
||||
(load "host/chez/build.ss") ; bld-* helpers, ei-* (emit-image), dce
|
||||
|
||||
(define jb-args (cdr (command-line)))
|
||||
(define jb-profile (if (pair? jb-args) (car jb-args) "release"))
|
||||
(define jb-out (if (and (pair? jb-args) (pair? (cdr jb-args))) (cadr jb-args)
|
||||
(string-append "target/" jb-profile "/joltc")))
|
||||
(define jb-release? (string=? jb-profile "release"))
|
||||
(unless (or jb-release? (string=? jb-profile "debug"))
|
||||
(error 'build-joltc "profile must be \"release\" or \"debug\"" jb-profile))
|
||||
|
||||
(define jb-build (string-append jb-out ".build"))
|
||||
(bld-check-toolchain)
|
||||
(bld-system (string-append "mkdir -p '" (path-parent jb-out) "' '" jb-build "'"))
|
||||
|
||||
;; --- 0. compile the launcher stub -------------------------------------------
|
||||
(define jb-stub (string-append jb-build "/launcher"))
|
||||
(display "build-joltc: compiling launcher stub\n")
|
||||
(bld-system (string-append
|
||||
"cc -O2 -I'" bld-csv-dir "' 'host/chez/stub/launcher.c' '"
|
||||
bld-csv-dir "/libkernel.a' -o '" jb-stub "' " (bld-link-libs)))
|
||||
|
||||
;; --- 1. emit flat.ss --------------------------------------------------------
|
||||
(define jb-flat-ss (string-append jb-build "/flat.ss"))
|
||||
(define (str-suffix? s suf)
|
||||
(let ((n (string-length s)) (m (string-length suf)))
|
||||
(and (>= n m) (string=? (substring s (- n m) n) suf))))
|
||||
|
||||
;; Bake every jolt-core/stdlib source file as an in-heap string literal keyed by
|
||||
;; its root-relative path ("jolt/main.clj", "clojure/string.clj") — exactly what
|
||||
;; resolve-on-roots probes. Literals (not read-file-string at startup) because
|
||||
;; flat.ss top-level forms run at every startup, with no source on disk.
|
||||
(define (jb-emit-source-embeds out)
|
||||
(for-each
|
||||
(lambda (root)
|
||||
(for-each
|
||||
(lambda (rp)
|
||||
(let ((rel (car rp)) (abs (cdr rp)))
|
||||
(when (or (str-suffix? rel ".clj") (str-suffix? rel ".cljc"))
|
||||
(put-string out (string-append
|
||||
"(register-embedded-resource! " (ei-str-lit rel) " "
|
||||
(ei-str-lit (read-file-string abs)) ")\n")))))
|
||||
(bld-walk-files root "" '())))
|
||||
(list "jolt-core" "stdlib")))
|
||||
|
||||
;; The launcher (Chez scheme-start): replicates host/chez/cli.ss but reads argv
|
||||
;; from the scheme-start lambda and has no repo root to cd into (all source is
|
||||
;; embedded; JOLT_PWD defaults to cwd via io/jolt.main). build.ss is already
|
||||
;; inlined, so `build` dispatches straight to jolt.host/build-binary after the
|
||||
;; bundled boots/stub are materialized from the binary's own C arrays.
|
||||
(define (jb-emit-launcher out)
|
||||
(put-string out "
|
||||
;; Materialize the bundled Chez boots + launcher stub (cc-linked into this binary
|
||||
;; as C arrays) into the embedded-bytes store, so build-self-contained can spill
|
||||
;; them. Done lazily on `build` only.
|
||||
(define (jolt-materialize-bundles!)
|
||||
(load-shared-object #f)
|
||||
(let ((memcpy (foreign-procedure \"memcpy\" (u8* uptr uptr) void*)))
|
||||
(for-each
|
||||
(lambda (spec)
|
||||
(let* ((len (foreign-ref 'unsigned-int (foreign-entry (caddr spec)) 0))
|
||||
(bv (make-bytevector len)))
|
||||
(memcpy bv (foreign-entry (cadr spec)) len)
|
||||
(register-embedded-bytes! (car spec) bv)))
|
||||
'((\"csv/petite.boot\" \"jolt_petite_boot\" \"jolt_petite_boot_len\")
|
||||
(\"csv/scheme.boot\" \"jolt_scheme_boot\" \"jolt_scheme_boot_len\")
|
||||
(\"stub/launcher\" \"jolt_stub\" \"jolt_stub_len\")))))
|
||||
|
||||
(suppress-greeting #t)
|
||||
(scheme-start
|
||||
(lambda args
|
||||
(set-source-roots! (list \"jolt-core\" \"stdlib\"))
|
||||
(guard (v (#t (jolt-report-throwable v (current-error-port))
|
||||
(let ((bt (jolt-backtrace-string v)))
|
||||
(when bt (display \" trace:\\n\" (current-error-port))
|
||||
(display bt (current-error-port))))
|
||||
(exit 1)))
|
||||
(cond
|
||||
((and (= (length args) 2) (string=? (car args) \"-e\"))
|
||||
(let ((result (jolt-final-str
|
||||
(jolt-compile-eval (string-append \"(do \" (cadr args) \")\") \"user\"))))
|
||||
(unless (string=? result \"\") (display result) (newline))))
|
||||
(else
|
||||
(when (and (pair? args) (string=? (car args) \"build\"))
|
||||
(jolt-materialize-bundles!))
|
||||
(load-namespace \"jolt.main\")
|
||||
(apply jolt-invoke (var-deref \"jolt.main\" \"-main\") args))))
|
||||
(exit 0)))
|
||||
"))
|
||||
|
||||
(display "build-joltc: emitting flat source\n")
|
||||
(let ((out (open-output-file jb-flat-ss 'replace)))
|
||||
;; full runtime + compiler image: keep the compiler (joltc evals at runtime).
|
||||
(bld-emit-runtime out #f #f)
|
||||
(put-string out "\n;; === build driver (inlined for self-contained `jolt build`) ===\n")
|
||||
(bld-inline-line "(load \"host/chez/build.ss\")" out 0)
|
||||
(put-string out "\n;; === embedded jolt-core + stdlib source ===\n")
|
||||
(jb-emit-source-embeds out)
|
||||
(put-string out "\n;; === joltc launcher ===\n")
|
||||
(jb-emit-launcher out)
|
||||
(close-port out))
|
||||
|
||||
;; --- 2. compile + boot in a FRESH Chez (profile Chez settings) --------------
|
||||
;; joltc is a compiler/REPL: it evals jolt-compiled Scheme at runtime, which must
|
||||
;; resolve the runtime's top-level procedures (var-deref, jolt-inc, …) through the
|
||||
;; boot's interaction-environment. compile-file's top-level defines are visible
|
||||
;; there only when compiled in the REAL interaction-environment, and `error` (and
|
||||
;; other primitives the inlined runtime references before redefining) bind to the
|
||||
;; kernel primitive only when compiled against a clean chezscheme env. A fresh
|
||||
;; Chez process gives both at once — exactly the legacy build-with-cc pass. The
|
||||
;; in-process compile in build.ss/build-self-contained is for the distributed
|
||||
;; joltc building (non-eval) apps, where no Chez is available.
|
||||
(define jb-flat-so (string-append jb-build "/flat.so"))
|
||||
(define jb-boot (string-append jb-build "/joltc.boot"))
|
||||
(define jb-bool (lambda (b) (if b "#t" "#f")))
|
||||
(display (string-append "build-joltc: compiling (" jb-profile " profile)\n"))
|
||||
(let ((cs (string-append jb-build "/compile.ss")))
|
||||
(let ((p (open-output-file cs 'replace)))
|
||||
(put-string p
|
||||
(string-append
|
||||
"(import (chezscheme))\n"
|
||||
"(optimize-level " (if jb-release? "3" "0") ")\n"
|
||||
"(generate-inspector-information " (jb-bool (not jb-release?)) ")\n"
|
||||
"(generate-procedure-source-information " (jb-bool (not jb-release?)) ")\n"
|
||||
"(debug-on-exception " (jb-bool (not jb-release?)) ")\n"
|
||||
"(fasl-compressed " (jb-bool jb-release?) ")\n"
|
||||
"(compile-file " (ei-str-lit jb-flat-ss) " " (ei-str-lit jb-flat-so) ")\n"
|
||||
"(make-boot-file " (ei-str-lit jb-boot) " '()\n "
|
||||
(ei-str-lit (string-append bld-csv-dir "/petite.boot")) "\n "
|
||||
(ei-str-lit (string-append bld-csv-dir "/scheme.boot")) "\n "
|
||||
(ei-str-lit jb-flat-so) ")\n"))
|
||||
(close-port p))
|
||||
(bld-system (string-append bld-chez " --script '" cs "'")))
|
||||
|
||||
;; --- 3. embed boots/stub as C arrays + cc-link ------------------------------
|
||||
;; xxd a file into header H and rename its symbol to NAME / NAME_len.
|
||||
(define (jb-c-array file h name)
|
||||
(bld-system (string-append "xxd -i '" file "' > '" h "'"))
|
||||
(bld-system (string-append
|
||||
"sed -i.bak -E 's/unsigned char [A-Za-z0-9_]+\\[\\]/unsigned char " name "[]/; "
|
||||
"s/unsigned int [A-Za-z0-9_]+_len/unsigned int " name "_len/' '" h "'")))
|
||||
|
||||
(display "build-joltc: embedding boots + stub, linking\n")
|
||||
(jb-c-array jb-boot (string-append jb-build "/boot_data.h") "jolt_boot")
|
||||
(jb-c-array (string-append bld-csv-dir "/petite.boot") (string-append jb-build "/petite_data.h") "jolt_petite_boot")
|
||||
(jb-c-array (string-append bld-csv-dir "/scheme.boot") (string-append jb-build "/scheme_data.h") "jolt_scheme_boot")
|
||||
(jb-c-array jb-stub (string-append jb-build "/stub_data.h") "jolt_stub")
|
||||
|
||||
(define jb-main-c (string-append jb-build "/main.c"))
|
||||
(let ((mc (open-output-file jb-main-c 'replace)))
|
||||
(put-string mc
|
||||
(string-append
|
||||
"#include \"scheme.h\"\n"
|
||||
"#include \"boot_data.h\"\n"
|
||||
"#include \"petite_data.h\"\n"
|
||||
"#include \"scheme_data.h\"\n"
|
||||
"#include \"stub_data.h\"\n"
|
||||
"int main(int argc, char *argv[]) {\n"
|
||||
" Sscheme_init(0);\n"
|
||||
" Sregister_boot_file_bytes(\"jolt\", jolt_boot, jolt_boot_len);\n"
|
||||
" Sbuild_heap(0, 0);\n"
|
||||
" int status = Sscheme_start(argc, (const char **)argv);\n"
|
||||
" Sscheme_deinit();\n return status;\n}\n"))
|
||||
(close-port mc))
|
||||
|
||||
(bld-system (string-append
|
||||
"cc -O2 -I'" bld-csv-dir "' -I'" jb-build "' '" jb-main-c "' '"
|
||||
bld-csv-dir "/libkernel.a' -o '" jb-out "' " (bld-link-libs)))
|
||||
(display (string-append "build-joltc: wrote " jb-out "\n"))
|
||||
|
|
@ -41,6 +41,12 @@
|
|||
(unless (zero? rc)
|
||||
(error 'jolt-build (string-append "command failed (" (number->string rc) "): " cmd)))))
|
||||
|
||||
;; mkdir -p without a subprocess (the self-contained build shells out to nothing).
|
||||
(define (bld-mkdir-p dir)
|
||||
(unless (or (string=? dir "") (string=? dir "/") (string=? dir ".") (file-exists? dir))
|
||||
(bld-mkdir-p (path-parent dir))
|
||||
(guard (e (#t #f)) (mkdir dir))))
|
||||
|
||||
(define (bld-contains? s sub)
|
||||
(let ((ns (string-length s)) (nsub (string-length sub)))
|
||||
(let loop ((i 0))
|
||||
|
|
@ -329,7 +335,9 @@
|
|||
;; no runtime redefinition). Off by default in every mode — release stays
|
||||
;; dynamically linked.
|
||||
(define (build-binary entry-ns out-path mode natives embed-dirs ext-roots direct-link? tree-shake?)
|
||||
(bld-check-toolchain)
|
||||
;; The self-contained path (jolt-embedded-bytes "stub/launcher") needs no csv
|
||||
;; kernel files, no Chez, no cc — only the legacy cc path does.
|
||||
(unless (jolt-embedded-bytes "stub/launcher") (bld-check-toolchain))
|
||||
;; 1. record app namespaces in dependency order as they finish loading.
|
||||
(let ((app-order '()))
|
||||
(set-ns-loaded-hook!
|
||||
|
|
@ -397,7 +405,7 @@
|
|||
(boot (string-append builddir "/jolt.boot"))
|
||||
(boot-h (string-append builddir "/boot_data.h"))
|
||||
(main-c (string-append builddir "/main.c")))
|
||||
(bld-system (string-append "mkdir -p '" builddir "'"))
|
||||
(bld-mkdir-p builddir)
|
||||
;; 3. flat source = runtime + app + launcher.
|
||||
(let ((out (open-output-file flat-ss 'replace)))
|
||||
(bld-emit-runtime out drop-compiler? core-strs)
|
||||
|
|
@ -456,23 +464,21 @@
|
|||
(build-with-cc entry-ns out-path mode builddir flat-ss flat-so boot boot-h main-c)))))))
|
||||
|
||||
;; --- self-contained link (in-process compile + append the boot to the stub) ---
|
||||
;; The system `error` is restored around compile-file/make-boot-file: the loaded
|
||||
;; jolt runtime shadows it (regex.ss, %chez-error holds the kernel original).
|
||||
;; flat.ss opens with (import (chezscheme)) so the boot binds the real error
|
||||
;; regardless, but the save/restore keeps any compile-time diagnostic honest.
|
||||
;; compile-file runs with a FRESH scheme-environment as the interaction
|
||||
;; environment: the loaded jolt runtime redefines `error` (regex.ss), and flat.ss
|
||||
;; inlines that same runtime, so an early reference to `error` (before its define
|
||||
;; runs) must bind to the kernel primitive — a clean env gives exactly that, the
|
||||
;; same as the legacy path compiling in a fresh Chez process. Without it the boot
|
||||
;; dies at startup with "variable error is not bound".
|
||||
(define (build-self-contained entry-ns out-path mode builddir flat-ss flat-so boot)
|
||||
(let ((petite (string-append builddir "/petite.boot"))
|
||||
(scheme (string-append builddir "/scheme.boot")))
|
||||
(jolt-spill-embedded! "csv/petite.boot" petite)
|
||||
(jolt-spill-embedded! "csv/scheme.boot" scheme)
|
||||
(display (string-append "jolt build: compiling " entry-ns " (" mode " mode, self-contained)\n"))
|
||||
(let ((saved-err error))
|
||||
(dynamic-wind
|
||||
(lambda () (set! error %chez-error))
|
||||
(lambda ()
|
||||
(compile-file flat-ss flat-so)
|
||||
(make-boot-file boot '() petite scheme flat-so))
|
||||
(lambda () (set! error saved-err))))
|
||||
(parameterize ((interaction-environment (copy-environment (scheme-environment))))
|
||||
(compile-file flat-ss flat-so)
|
||||
(make-boot-file boot '() petite scheme flat-so))
|
||||
;; link: stub bytes ++ boot ++ frame, then make it executable.
|
||||
(jolt-spill-embedded! "stub/launcher" out-path)
|
||||
(jolt-append-payload! out-path (read-file-bytes boot))
|
||||
|
|
|
|||
76
host/chez/joltc-selfbuild-smoke.sh
Normal file
76
host/chez/joltc-selfbuild-smoke.sh
Normal file
|
|
@ -0,0 +1,76 @@
|
|||
#!/bin/sh
|
||||
# joltc self-build smoke (jolt-eaj): build joltc as a self-contained binary, then
|
||||
# use THAT binary to compile a jolt app with Chez and cc removed from the
|
||||
# environment — the whole point of the feature. The produced app must then run
|
||||
# and match the same expected output as build-smoke.sh.
|
||||
root="$(CDPATH= cd -- "$(dirname -- "$0")/../.." && pwd)"
|
||||
cd "$root"
|
||||
|
||||
# Preflight: building joltc itself needs the Chez kernel dev files (libkernel.a +
|
||||
# scheme.h) and a C compiler, same as build-smoke.sh. A distro chezscheme package
|
||||
# ships neither, so skip there (CI included).
|
||||
csv="$JOLT_CHEZ_CSV"
|
||||
if [ -z "$csv" ]; then
|
||||
chez_bin="$(command -v chez || command -v scheme || command -v petite || true)"
|
||||
if [ -n "$chez_bin" ]; then
|
||||
base="$(cd "$(dirname "$chez_bin")/.." 2>/dev/null && pwd)"
|
||||
for d in "$base"/lib/csv*/*/; do
|
||||
[ -f "${d}libkernel.a" ] && csv="${d%/}" && break
|
||||
done
|
||||
fi
|
||||
fi
|
||||
if ! command -v cc >/dev/null 2>&1 || [ -z "$csv" ] || [ ! -f "$csv/scheme.h" ] || [ ! -f "$csv/libkernel.a" ]; then
|
||||
echo "joltc self-build smoke: skipped (Chez kernel dev files or C compiler not available)"
|
||||
exit 0
|
||||
fi
|
||||
export JOLT_CHEZ_CSV="$csv"
|
||||
|
||||
# 1. Build joltc (debug profile — faster; the self-contained app-build mechanism
|
||||
# is identical to release, only Chez compile settings differ).
|
||||
joltc="$root/target/debug/joltc"
|
||||
echo "joltc self-build smoke: building $joltc"
|
||||
if ! chez --script host/chez/build-joltc.ss debug "$joltc" >/dev/null 2>&1; then
|
||||
echo " FAIL: build-joltc.ss exited non-zero"
|
||||
exit 1
|
||||
fi
|
||||
[ -x "$joltc" ] || { echo " FAIL: no joltc executable produced"; exit 1; }
|
||||
|
||||
# 2. The distributed joltc must run with no Chez install: a basic eval.
|
||||
got_e="$(env -i HOME="$HOME" "$joltc" -e '(reduce + (range 10))' 2>&1)"
|
||||
if [ "$got_e" != "45" ]; then
|
||||
echo " FAIL: joltc -e under empty env gave '$got_e', want 45"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# 3. Build an app through the distributed joltc with an EMPTY environment — no
|
||||
# PATH at all, so no chez, no cc, no shell tools are reachable. This is the core
|
||||
# guarantee: joltc compiles apps entirely on its own.
|
||||
app="$(mktemp -d)/build-app"
|
||||
cp -r "$root/test/chez/build-app" "$app"
|
||||
out="$app/app"
|
||||
echo "joltc self-build smoke: compiling app.core via the binary (no chez/cc on PATH)"
|
||||
if ! env -i HOME="$HOME" JOLT_PWD="$app" "$joltc" build -m app.core -o "$out" >/dev/null 2>&1; then
|
||||
echo " FAIL: self-contained jolt build exited non-zero"
|
||||
rm -rf "$(dirname "$app")"
|
||||
exit 1
|
||||
fi
|
||||
[ -x "$out" ] || { echo " FAIL: no app executable produced"; rm -rf "$(dirname "$app")"; exit 1; }
|
||||
|
||||
# 4. The produced app runs from a neutral cwd and matches build-smoke's output.
|
||||
got="$(cd / && "$out" alpha bb ccc 2>&1)"
|
||||
want='embedded resource ok
|
||||
HELLO FROM A BUILT BINARY!
|
||||
HELLO FROM A BUILT BINARY!
|
||||
args: [alpha bb ccc]
|
||||
sum: 10
|
||||
greet-default: greet:default
|
||||
greet-loud: greet:loud
|
||||
greet-soft: greet:soft'
|
||||
rm -rf "$(dirname "$app")"
|
||||
if [ "$got" != "$want" ]; then
|
||||
echo " FAIL: produced app output mismatch"
|
||||
echo "--- want ---"; echo "$want"
|
||||
echo "--- got ----"; echo "$got"
|
||||
exit 1
|
||||
fi
|
||||
echo "joltc self-build smoke: passed (joltc runs + builds a working app with no external toolchain)"
|
||||
Loading…
Add table
Add a link
Reference in a new issue