From 0420cd4d7999de5e64fe7c691e1ac3b969366cc4 Mon Sep 17 00:00:00 2001 From: Yogthos Date: Mon, 29 Jun 2026 20:48:44 -0400 Subject: [PATCH 1/3] Self-contained build foundation: embedded-bytes helpers, launcher stub, in-process app link Adds the pieces a toolchain-free joltc needs to compile apps with no external Chez or cc: - host/chez/java/io.ss: register-embedded-bytes!/jolt-embedded-bytes, read-file-bytes, jolt-spill-embedded!, jolt-append-payload! (frames an app boot onto the stub as [stub][boot][len:le64]["JOLTBOOT"]), and jolt-chmod-755 via load-shared-object #f (no subprocess). - host/chez/stub/launcher.c: a native stub that locates its own executable, reads the trailing frame, and hands the appended boot to the Chez kernel. - host/chez/loader.ss: resolve-on-roots consults the embedded source store before disk; ldr-read-source reads baked source. Dev (empty store) is unaffected. - host/chez/build.ss: build-binary step 4 splits into build-self-contained (in-process compile-file/make-boot-file with the system error restored, then append the boot to a copy of the embedded stub) and build-with-cc (the existing dev path). The self-contained path is taken only when the stub is embedded. The legacy cc path is unchanged behaviorally; make buildsmoke still passes. --- host/chez/build.ss | 119 +++++++++++++++++++++++++------------- host/chez/java/io.ss | 59 +++++++++++++++++++ host/chez/loader.ss | 33 ++++++++--- host/chez/stub/launcher.c | 103 +++++++++++++++++++++++++++++++++ 4 files changed, 266 insertions(+), 48 deletions(-) create mode 100644 host/chez/stub/launcher.c diff --git a/host/chez/build.ss b/host/chez/build.ss index 5d28735..200c8ad 100644 --- a/host/chez/build.ss +++ b/host/chez/build.ss @@ -190,7 +190,7 @@ (for-each (lambda (nf) (set-chez-ns! (car nf)) - (let ((src (read-file-string (cdr nf)))) + (let ((src (ldr-read-source (cdr nf)))) (parameterize ((rdr-source-file (cdr nf))) (for-each (lambda (f) @@ -370,7 +370,7 @@ ;; ns-prelude forms (always kept, no fqn/refs) set the ;; ns + register aliases before this ns's forms; dce ;; keeps original order. - (let ((src (read-file-string (cdr nf)))) + (let ((src (ldr-read-source (cdr nf)))) (parameterize ((rdr-source-file (cdr nf))) (append (map (lambda (s) (dce-rec #t #f '() s)) @@ -381,7 +381,7 @@ (values #f (apply append (map (lambda (nf) - (let ((src (read-file-string (cdr nf)))) + (let ((src (ldr-read-source (cdr nf)))) (parameterize ((rdr-source-file (cdr nf))) (append (bld-ns-prelude (car nf) src) (bld-emit-ns (car nf) src))))) @@ -442,43 +442,82 @@ " (apply jolt-invoke mainv args)))\n" " (exit 0)))\n")) (close-port out)) - ;; 4. compile -> boot -> embed -> link. - ;; compile-file/make-boot-file run in a FRESH Chez, not this process: the - ;; loaded runtime shadows `error` (regex.ss, for irregex), which would - ;; otherwise bake a broken `error` reference into the boot. - (display (string-append "jolt build: compiling " entry-ns " (" mode " mode)\n")) - (let ((cs (string-append builddir "/compile.ss"))) - (let ((p (open-output-file cs 'replace))) - (put-string p - (string-append - "(import (chezscheme))\n" - "(compile-file " (ei-str-lit flat-ss) " " (ei-str-lit flat-so) ")\n" - "(make-boot-file " (ei-str-lit boot) " '()\n " - (ei-str-lit (string-append bld-csv-dir "/petite.boot")) "\n " - (ei-str-lit (string-append bld-csv-dir "/scheme.boot")) "\n " - (ei-str-lit flat-so) ")\n")) - (close-port p)) - (bld-system (string-append bld-chez " --script '" cs "'"))) - (bld-system (string-append "xxd -i '" boot "' > '" boot-h "'")) - ;; The xxd symbol is derived from the path; normalize to jolt_boot. - (bld-system (string-append - "sed -i.bak -E 's/unsigned char [A-Za-z0-9_]+\\[\\]/unsigned char jolt_boot[]/; " - "s/unsigned int [A-Za-z0-9_]+_len/unsigned int jolt_boot_len/' '" boot-h "'")) - (let ((mc (open-output-file main-c 'replace))) - (put-string mc - (string-append - "#include \"scheme.h\"\n#include \"boot_data.h\"\n" - "int main(int argc, char *argv[]) {\n" - " Sscheme_init(0);\n" - " Sregister_boot_file_bytes(\"jolt\", jolt_boot, jolt_boot_len);\n" - " Sbuild_heap(0, 0);\n" - " int status = Sscheme_start(argc, (const char **)argv);\n" - " Sscheme_deinit();\n return status;\n}\n")) - (close-port mc)) - (bld-system (string-append - "cc -O2 -I'" bld-csv-dir "' '" main-c "' '" bld-csv-dir "/libkernel.a' " - "-o '" out-path "' " (bld-link-libs))) - (display (string-append "jolt build: wrote " out-path "\n"))))))) + ;; 4. compile -> boot -> link. Two paths, chosen by whether this process + ;; carries the bundled Chez boots + launcher stub: + ;; - SELF-CONTAINED (the distributed joltc, jolt-eaj): compile-file + + ;; make-boot-file run IN PROCESS (the compiler is resident — joltc is + ;; built from scheme.boot), then the boot is appended to a copy of the + ;; embedded stub. No external Chez, no cc. + ;; - LEGACY (dev bin/joltc): spawn a fresh Chez for compile-file/ + ;; make-boot-file, then xxd the boot into a C array and cc-link against + ;; libkernel.a. Kept so `make buildsmoke` still exercises the cc path. + (if (jolt-embedded-bytes "stub/launcher") + (build-self-contained entry-ns out-path mode builddir flat-ss flat-so boot) + (build-with-cc entry-ns out-path mode builddir flat-ss flat-so boot boot-h main-c))))))) + +;; --- self-contained link (in-process compile + append the boot to the stub) --- +;; The system `error` is restored around compile-file/make-boot-file: the loaded +;; jolt runtime shadows it (regex.ss, %chez-error holds the kernel original). +;; flat.ss opens with (import (chezscheme)) so the boot binds the real error +;; regardless, but the save/restore keeps any compile-time diagnostic honest. +(define (build-self-contained entry-ns out-path mode builddir flat-ss flat-so boot) + (let ((petite (string-append builddir "/petite.boot")) + (scheme (string-append builddir "/scheme.boot"))) + (jolt-spill-embedded! "csv/petite.boot" petite) + (jolt-spill-embedded! "csv/scheme.boot" scheme) + (display (string-append "jolt build: compiling " entry-ns " (" mode " mode, self-contained)\n")) + (let ((saved-err error)) + (dynamic-wind + (lambda () (set! error %chez-error)) + (lambda () + (compile-file flat-ss flat-so) + (make-boot-file boot '() petite scheme flat-so)) + (lambda () (set! error saved-err)))) + ;; link: stub bytes ++ boot ++ frame, then make it executable. + (jolt-spill-embedded! "stub/launcher" out-path) + (jolt-append-payload! out-path (read-file-bytes boot)) + (jolt-chmod-755 out-path) + (display (string-append "jolt build: wrote " out-path "\n")) + (when bld-osx? + (display (string-append + "jolt build: note — on macOS this binary is unsigned; to share it,\n" + " `xattr -d com.apple.quarantine " out-path "` on the target, or sign it.\n"))))) + +;; --- legacy cc link (dev bin/joltc): fresh Chez compile + xxd + cc ------------ +(define (build-with-cc entry-ns out-path mode builddir flat-ss flat-so boot boot-h main-c) + (display (string-append "jolt build: compiling " entry-ns " (" mode " mode)\n")) + (let ((cs (string-append builddir "/compile.ss"))) + (let ((p (open-output-file cs 'replace))) + (put-string p + (string-append + "(import (chezscheme))\n" + "(compile-file " (ei-str-lit flat-ss) " " (ei-str-lit flat-so) ")\n" + "(make-boot-file " (ei-str-lit boot) " '()\n " + (ei-str-lit (string-append bld-csv-dir "/petite.boot")) "\n " + (ei-str-lit (string-append bld-csv-dir "/scheme.boot")) "\n " + (ei-str-lit flat-so) ")\n")) + (close-port p)) + (bld-system (string-append bld-chez " --script '" cs "'"))) + (bld-system (string-append "xxd -i '" boot "' > '" boot-h "'")) + ;; The xxd symbol is derived from the path; normalize to jolt_boot. + (bld-system (string-append + "sed -i.bak -E 's/unsigned char [A-Za-z0-9_]+\\[\\]/unsigned char jolt_boot[]/; " + "s/unsigned int [A-Za-z0-9_]+_len/unsigned int jolt_boot_len/' '" boot-h "'")) + (let ((mc (open-output-file main-c 'replace))) + (put-string mc + (string-append + "#include \"scheme.h\"\n#include \"boot_data.h\"\n" + "int main(int argc, char *argv[]) {\n" + " Sscheme_init(0);\n" + " Sregister_boot_file_bytes(\"jolt\", jolt_boot, jolt_boot_len);\n" + " Sbuild_heap(0, 0);\n" + " int status = Sscheme_start(argc, (const char **)argv);\n" + " Sscheme_deinit();\n return status;\n}\n")) + (close-port mc)) + (bld-system (string-append + "cc -O2 -I'" bld-csv-dir "' '" main-c "' '" bld-csv-dir "/libkernel.a' " + "-o '" out-path "' " (bld-link-libs))) + (display (string-append "jolt build: wrote " out-path "\n"))) (def-var! "jolt.host" "build-binary" (lambda (entry out mode natives embed-dirs ext-roots direct-link? tree-shake?) diff --git a/host/chez/java/io.ss b/host/chez/java/io.ss index aa3ac33..ce7243e 100644 --- a/host/chez/java/io.ss +++ b/host/chez/java/io.ss @@ -29,6 +29,65 @@ (hashtable-set! embedded-resources name content)) (define-record-type embedded-res (fields name content) (nongenerative jolt-embres-v1)) +;; --- self-contained build artifacts (jolt-eaj) ------------------------------ +;; A toolchain-free `jolt build` (the distributed joltc) carries the Chez +;; petite/scheme boots and a prebuilt launcher stub baked into its own boot image. +;; They live in the same table as embedded-resources, but keyed under bytevector +;; values (register-embedded-bytes!) rather than strings; resolve-on-roots / +;; io/resource only ever ask for the string-keyed source entries, so the two +;; coexist. The build driver reads them at heap-build time from files that exist +;; only on the dev machine. +(define (register-embedded-bytes! name bv) (hashtable-set! embedded-resources name bv)) +(define (jolt-embedded-bytes name) + (let ((v (hashtable-ref embedded-resources name #f))) + (and (bytevector? v) v))) + +;; Read a whole file as a bytevector ("" -> empty). Used to slurp boot/stub files. +(define (read-file-bytes path) + (let ((p (open-file-input-port path))) + (let ((bv (get-bytevector-all p))) + (close-port p) + (if (eof-object? bv) (bytevector) bv)))) + +;; Write an embedded bytevector resource out to a path. make-boot-file needs the +;; petite/scheme boots as files, so they are spilled to scratch before the call. +(define (jolt-spill-embedded! name path) + (let ((bv (jolt-embedded-bytes name))) + (unless bv (error 'jolt-spill-embedded! "no embedded bytes for" name)) + (let ((p (open-file-output-port path (file-options no-fail) (buffer-mode block)))) + (put-bytevector p bv) + (close-port p)))) + +;; Frame an app boot onto a file that already holds the stub bytes. Layout: +;; [stub][boot][boot-length:le64]["JOLTBOOT"]. The stub (host/chez/stub/launcher.c) +;; reads the trailing 16 bytes — the 8-byte magic, then the preceding 8-byte LE +;; length — to locate and register the boot, so a boot that itself contains the +;; magic bytes can't be mistaken for the frame. +(define jolt-payload-magic (string->utf8 "JOLTBOOT")) +(define (jolt-append-payload! path boot-bv) + (let ((head (read-file-bytes path))) ; the stub bytes already written + (let ((p (open-file-output-port path (file-options no-fail) (buffer-mode block))) + (lb (make-bytevector 8 0))) + (bytevector-u64-set! lb 0 (bytevector-length boot-bv) (endianness little)) + (put-bytevector p head) + (put-bytevector p boot-bv) + (put-bytevector p lb) + (put-bytevector p jolt-payload-magic) + (close-port p)))) + +;; chmod 0755 via libc, so the produced binary is executable. load-shared-object +;; with #f pulls the running process's own symbols (chmod is in libc, linked into +;; every Chez binary) — no external toolchain. Falls back to /bin/sh chmod if the +;; symbol can't be resolved. +(define jolt-chmod-755 + (let ((c (guard (e (#t #f)) + (load-shared-object #f) + (foreign-procedure "chmod" (string int) int)))) + (lambda (path) + (if c + (c path #o755) + (system (string-append "chmod 755 '" path "'")))))) + ;; A user-facing relative path resolves against JOLT_PWD — the user's cwd before ;; the launcher cd'd to the jolt repo root — matching the JVM, where io/file is ;; cwd-relative. (io/resource builds jfiles from the source roots directly, so it diff --git a/host/chez/loader.ss b/host/chez/loader.ss index c5d6ee7..ad57dfd 100644 --- a/host/chez/loader.ss +++ b/host/chez/loader.ss @@ -122,14 +122,31 @@ (else (loop (cdr cs) (cons (car cs) seg) segs))))) ;; First existing /rel.clj or /rel.cljc on the search roots, else #f. +;; A self-contained joltc binary embeds jolt-core + stdlib source keyed by their +;; root-relative path ("clojure/string.clj"); those are checked first, so a +;; `require` resolves with no source on disk. The dev bin/joltc has an empty +;; source store, so the two hashtable probes miss and it falls straight to disk. (define (resolve-on-roots rel) - (let loop ((roots source-roots)) - (if (null? roots) #f - (let ((clj (string-append (car roots) "/" rel ".clj")) - (cljc (string-append (car roots) "/" rel ".cljc"))) - (cond ((file-exists? clj) clj) - ((file-exists? cljc) cljc) - (else (loop (cdr roots)))))))) + (let ((eclj (string-append rel ".clj")) (ecljc (string-append rel ".cljc"))) + (cond + ((string? (hashtable-ref embedded-resources eclj #f)) eclj) + ((string? (hashtable-ref embedded-resources ecljc #f)) ecljc) + (else + (let loop ((roots source-roots)) + (if (null? roots) #f + (let ((clj (string-append (car roots) "/" rel ".clj")) + (cljc (string-append (car roots) "/" rel ".cljc"))) + (cond ((file-exists? clj) clj) + ((file-exists? cljc) cljc) + (else (loop (cdr roots))))))))))) + +;; Read a namespace source. An embedded key (resolve-on-roots above, or the +;; build driver's app-order entries) reads its baked string; everything else is +;; a real path read off disk. Bytevector entries (the bundled boots/stub) are not +;; source, so a string? guard skips them. +(define (ldr-read-source path) + (let ((emb (hashtable-ref embedded-resources path #f))) + (if (string? emb) emb (read-file-string path)))) (define (find-ns-file name) (resolve-on-roots (ns-name->rel name))) @@ -176,7 +193,7 @@ ;; more forms", which would silently drop the entire rest of the file; here we ;; skip the no-op form and continue to true end-of-string. (define (load-jolt-file path) - (let* ((src (read-file-string path)) (end (string-length src))) + (let* ((src (ldr-read-source path)) (end (string-length src))) ;; parameterize (not a bare set!) so a require nested in this file's ns form ;; restores path when control returns to the rest of this file. (parameterize ((rdr-source-file path)) ; list forms read here carry :file = path diff --git a/host/chez/stub/launcher.c b/host/chez/stub/launcher.c new file mode 100644 index 0000000..0cc7cee --- /dev/null +++ b/host/chez/stub/launcher.c @@ -0,0 +1,103 @@ +/* launcher.c — the native stub for self-contained jolt binaries (jolt-eaj). + * + * A toolchain-free `jolt build` (and joltc itself) produces an executable by + * appending a Chez boot image to a copy of this prebuilt stub, framed as: + * + * [stub bytes][boot bytes][boot-length : little-endian u64]["JOLTBOOT"] + * + * (see host/chez/java/io.ss jolt-append-payload!). At startup the stub locates + * its own executable, reads the trailing 16-byte frame to find the boot, and + * hands the boot to the Chez kernel — no external boot file, no Chez install. + * + * Built once at joltc-build time against the Chez kernel (libkernel.a + scheme.h) + * by host/chez/build-joltc.ss; the resulting binary is embedded into joltc and + * copied per app build. Inherently per-platform (the boot targets the host + * machine-type), like a native compiler. + */ +#include "scheme.h" +#include +#include +#include +#include + +#if defined(__APPLE__) +#include +static int self_path(char *buf, uint32_t size) { + /* _NSGetExecutablePath fills buf and reports the needed size on overflow. */ + return _NSGetExecutablePath(buf, &size); +} +#else +#include +static int self_path(char *buf, uint32_t size) { + ssize_t n = readlink("/proc/self/exe", buf, (size_t)size - 1); + if (n < 0) return -1; + buf[n] = '\0'; + return 0; +} +#endif + +#define JOLT_MAGIC "JOLTBOOT" +#define JOLT_MAGIC_LEN 8 +#define JOLT_TRAILER_LEN 16 /* u64 length + 8-byte magic */ + +int main(int argc, char *argv[]) { + char path[4096]; + if (self_path(path, (uint32_t)sizeof(path)) != 0) { + fprintf(stderr, "jolt: cannot resolve own executable path\n"); + return 1; + } + + FILE *f = fopen(path, "rb"); + if (!f) { fprintf(stderr, "jolt: cannot open self for reading\n"); return 1; } + + if (fseek(f, 0, SEEK_END) != 0) { fclose(f); return 1; } + long fsize = ftell(f); + if (fsize < JOLT_TRAILER_LEN) { + fprintf(stderr, "jolt: no boot payload (run was not produced by jolt build)\n"); + fclose(f); + return 1; + } + + unsigned char trailer[JOLT_TRAILER_LEN]; + if (fseek(f, fsize - JOLT_TRAILER_LEN, SEEK_SET) != 0 || + fread(trailer, 1, JOLT_TRAILER_LEN, f) != JOLT_TRAILER_LEN) { + fclose(f); + return 1; + } + if (memcmp(trailer + 8, JOLT_MAGIC, JOLT_MAGIC_LEN) != 0) { + fprintf(stderr, "jolt: boot payload not found\n"); + fclose(f); + return 1; + } + + uint64_t boot_len = 0; + for (int i = 0; i < 8; i++) + boot_len |= ((uint64_t)trailer[i]) << (8 * i); + + long boot_off = fsize - JOLT_TRAILER_LEN - (long)boot_len; + if (boot_off < 0) { + fprintf(stderr, "jolt: corrupt boot payload\n"); + fclose(f); + return 1; + } + + /* The kernel keeps the boot bytes for the life of the process (demand-loaded), + * so this buffer is freed only after Sscheme_deinit. */ + void *boot = malloc((size_t)boot_len); + if (!boot) { fclose(f); return 1; } + if (fseek(f, boot_off, SEEK_SET) != 0 || + fread(boot, 1, (size_t)boot_len, f) != (size_t)boot_len) { + free(boot); + fclose(f); + return 1; + } + fclose(f); + + Sscheme_init(0); + Sregister_boot_file_bytes("jolt", boot, (iptr)boot_len); + Sbuild_heap(0, 0); + int status = Sscheme_start(argc, (const char **)argv); + Sscheme_deinit(); + free(boot); + return status; +} From 242eeac5c6fe732ceae174cb80df2e477e7bc10d Mon Sep 17 00:00:00 2001 From: Yogthos Date: Mon, 29 Jun 2026 21:04:23 -0400 Subject: [PATCH 2/3] Build joltc as a self-contained binary (make joltc-release / joltc-debug) host/chez/build-joltc.ss builds joltc into target//joltc: it emits a flat source of the full runtime + compiler image + inlined build.ss + every jolt-core/stdlib file as a baked string literal + a cli.ss-style launcher, then (in a fresh Chez, so the inlined runtime's redefinition of error doesn't strand early references and runtime eval still sees the runtime's top-level procedures) compiles it and cc-links it with the Chez petite/scheme boots and the launcher stub embedded as C arrays. The launcher reads those arrays via FFI on (jolt-materialize-bundles!) and registers them so build-self-contained can spill them. joltc itself is cc-linked (clean signature for Homebrew); only the apps it later builds use the appended-stub path. build.ss: skip the csv toolchain check on the self-contained path and create the build dir with a subprocess-free bld-mkdir-p, so a from the distributed binary shells out to nothing. release = optimize-level 3 + no inspector info + compressed; debug = optimize-level 0 + inspector + procedure source + debug-on-exception. joltc-selfbuild-smoke.sh (make joltcsmoke) builds joltc and, with an empty environment (no chez/cc/PATH), drives it through the build-app fixture, asserting the produced binary's output. .gitignore ignores target/. --- .gitignore | 1 + Makefile | 19 ++- host/chez/build-joltc.ss | 206 +++++++++++++++++++++++++++++ host/chez/build.ss | 32 +++-- host/chez/joltc-selfbuild-smoke.sh | 76 +++++++++++ 5 files changed, 320 insertions(+), 14 deletions(-) create mode 100644 host/chez/build-joltc.ss create mode 100644 host/chez/joltc-selfbuild-smoke.sh diff --git a/.gitignore b/.gitignore index b62bfd2..ab386cf 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ AGENTS.md .DS_Store CLAUDE.md build/ +target/ .clj-kondo/ .dirge/ .claude/ diff --git a/Makefile b/Makefile index bdef687..1c8d6f0 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ # build step. `make test` is the full gate. `make remint` rebuilds the seed after a # source change. -.PHONY: test ci values corpus unit smoke buildsmoke selfhost sci certify ffi transient infer wp devirt fieldread numwp fieldnum protoret narrow directlink numeric inline shakesmoke remint +.PHONY: test ci values corpus unit smoke buildsmoke selfhost sci certify ffi transient infer wp devirt fieldread numwp fieldnum protoret narrow directlink numeric inline shakesmoke remint joltc joltc-release joltc-debug joltcsmoke # Full gate (dev machine). Includes the self-host byte-fixpoint, which only holds # on the same Chez that minted the seed. @@ -42,6 +42,23 @@ smoke: buildsmoke: @sh host/chez/build-smoke.sh +# Build joltc as a self-contained native binary into target//joltc. The +# binary bundles the runtime, compiler, jolt-core + stdlib source, the Chez boots, +# and a launcher stub, so it runs AND compiles jolt apps with no Chez or cc on the +# machine. Built on a dev/CI host that HAS Chez + cc. release = optimize-level 3, +# no inspector info, compressed; debug = optimize-level 0 + inspector + debug info. +joltc-release: + @chez --script host/chez/build-joltc.ss release target/release/joltc +joltc-debug: + @chez --script host/chez/build-joltc.ss debug target/debug/joltc +# Re-mint the seed first so the embedded compiler image is current, then both builds. +joltc: selfhost joltc-release joltc-debug + @echo "OK: target/release/joltc and target/debug/joltc built" + +# Self-build smoke: the distributed joltc compiles an app with Chez + cc removed. +joltcsmoke: + @sh host/chez/joltc-selfbuild-smoke.sh + # SCI conformance: load borkdude/sci's source through joltc (floor-gated). sci: @chez --script host/chez/run-sci.ss diff --git a/host/chez/build-joltc.ss b/host/chez/build-joltc.ss new file mode 100644 index 0000000..b8c5cca --- /dev/null +++ b/host/chez/build-joltc.ss @@ -0,0 +1,206 @@ +;; build-joltc.ss — build joltc itself as a self-contained native binary (jolt-eaj). +;; +;; chez --script host/chez/build-joltc.ss +;; profile: "release" | "debug" out-path: e.g. target/release/joltc +;; +;; Runs on a dev/CI machine that HAS Chez + cc. Produces a binary that needs +;; NEITHER: it bakes the full runtime + compiler image + all jolt-core/stdlib +;; source + the Chez petite/scheme boots + a prebuilt launcher stub into one +;; cc-linked executable, so the resulting joltc can run AND `build` jolt apps on +;; its own. joltc itself is cc-linked (not appended) so its signature stays clean +;; for Homebrew/codesign, like dirge's binaries; only the apps it later builds use +;; the appended-stub path (host/chez/build.ss build-self-contained). +;; +;; Pipeline: +;; 0. cc-compile host/chez/stub/launcher.c against the Chez kernel. +;; 1. emit flat.ss = runtime + compiler image (cli.ss load order) + inlined +;; build.ss + every jolt-core/stdlib file as a baked string literal + the +;; joltc launcher. +;; 2. in-process compile-file + make-boot-file (profile Chez settings), error +;; restored around the call (the runtime shadows it; regex.ss/%chez-error). +;; 3. xxd the joltc boot + petite/scheme boots + stub into C arrays, generate +;; main.c, cc-link -> out-path. The launcher reads the petite/scheme/stub +;; arrays via FFI on `build` (jolt-materialize-bundles!). + +(import (chezscheme)) + +(load "host/chez/rt.ss") +(set-chez-ns! "clojure.core") +(load "host/chez/seed/prelude.ss") +(load "host/chez/post-prelude.ss") +(set-chez-ns! "user") +(load "host/chez/host-contract.ss") +(load "host/chez/seed/image.ss") +(load "host/chez/compile-eval.ss") +(load "host/chez/png.ss") +(load "host/chez/loader.ss") +(load "host/chez/java/ffi.ss") +(set-source-roots! (list "jolt-core" "stdlib")) +(load "host/chez/build.ss") ; bld-* helpers, ei-* (emit-image), dce + +(define jb-args (cdr (command-line))) +(define jb-profile (if (pair? jb-args) (car jb-args) "release")) +(define jb-out (if (and (pair? jb-args) (pair? (cdr jb-args))) (cadr jb-args) + (string-append "target/" jb-profile "/joltc"))) +(define jb-release? (string=? jb-profile "release")) +(unless (or jb-release? (string=? jb-profile "debug")) + (error 'build-joltc "profile must be \"release\" or \"debug\"" jb-profile)) + +(define jb-build (string-append jb-out ".build")) +(bld-check-toolchain) +(bld-system (string-append "mkdir -p '" (path-parent jb-out) "' '" jb-build "'")) + +;; --- 0. compile the launcher stub ------------------------------------------- +(define jb-stub (string-append jb-build "/launcher")) +(display "build-joltc: compiling launcher stub\n") +(bld-system (string-append + "cc -O2 -I'" bld-csv-dir "' 'host/chez/stub/launcher.c' '" + bld-csv-dir "/libkernel.a' -o '" jb-stub "' " (bld-link-libs))) + +;; --- 1. emit flat.ss -------------------------------------------------------- +(define jb-flat-ss (string-append jb-build "/flat.ss")) +(define (str-suffix? s suf) + (let ((n (string-length s)) (m (string-length suf))) + (and (>= n m) (string=? (substring s (- n m) n) suf)))) + +;; Bake every jolt-core/stdlib source file as an in-heap string literal keyed by +;; its root-relative path ("jolt/main.clj", "clojure/string.clj") — exactly what +;; resolve-on-roots probes. Literals (not read-file-string at startup) because +;; flat.ss top-level forms run at every startup, with no source on disk. +(define (jb-emit-source-embeds out) + (for-each + (lambda (root) + (for-each + (lambda (rp) + (let ((rel (car rp)) (abs (cdr rp))) + (when (or (str-suffix? rel ".clj") (str-suffix? rel ".cljc")) + (put-string out (string-append + "(register-embedded-resource! " (ei-str-lit rel) " " + (ei-str-lit (read-file-string abs)) ")\n"))))) + (bld-walk-files root "" '()))) + (list "jolt-core" "stdlib"))) + +;; The launcher (Chez scheme-start): replicates host/chez/cli.ss but reads argv +;; from the scheme-start lambda and has no repo root to cd into (all source is +;; embedded; JOLT_PWD defaults to cwd via io/jolt.main). build.ss is already +;; inlined, so `build` dispatches straight to jolt.host/build-binary after the +;; bundled boots/stub are materialized from the binary's own C arrays. +(define (jb-emit-launcher out) + (put-string out " +;; Materialize the bundled Chez boots + launcher stub (cc-linked into this binary +;; as C arrays) into the embedded-bytes store, so build-self-contained can spill +;; them. Done lazily on `build` only. +(define (jolt-materialize-bundles!) + (load-shared-object #f) + (let ((memcpy (foreign-procedure \"memcpy\" (u8* uptr uptr) void*))) + (for-each + (lambda (spec) + (let* ((len (foreign-ref 'unsigned-int (foreign-entry (caddr spec)) 0)) + (bv (make-bytevector len))) + (memcpy bv (foreign-entry (cadr spec)) len) + (register-embedded-bytes! (car spec) bv))) + '((\"csv/petite.boot\" \"jolt_petite_boot\" \"jolt_petite_boot_len\") + (\"csv/scheme.boot\" \"jolt_scheme_boot\" \"jolt_scheme_boot_len\") + (\"stub/launcher\" \"jolt_stub\" \"jolt_stub_len\"))))) + +(suppress-greeting #t) +(scheme-start + (lambda args + (set-source-roots! (list \"jolt-core\" \"stdlib\")) + (guard (v (#t (jolt-report-throwable v (current-error-port)) + (let ((bt (jolt-backtrace-string v))) + (when bt (display \" trace:\\n\" (current-error-port)) + (display bt (current-error-port)))) + (exit 1))) + (cond + ((and (= (length args) 2) (string=? (car args) \"-e\")) + (let ((result (jolt-final-str + (jolt-compile-eval (string-append \"(do \" (cadr args) \")\") \"user\")))) + (unless (string=? result \"\") (display result) (newline)))) + (else + (when (and (pair? args) (string=? (car args) \"build\")) + (jolt-materialize-bundles!)) + (load-namespace \"jolt.main\") + (apply jolt-invoke (var-deref \"jolt.main\" \"-main\") args)))) + (exit 0))) +")) + +(display "build-joltc: emitting flat source\n") +(let ((out (open-output-file jb-flat-ss 'replace))) + ;; full runtime + compiler image: keep the compiler (joltc evals at runtime). + (bld-emit-runtime out #f #f) + (put-string out "\n;; === build driver (inlined for self-contained `jolt build`) ===\n") + (bld-inline-line "(load \"host/chez/build.ss\")" out 0) + (put-string out "\n;; === embedded jolt-core + stdlib source ===\n") + (jb-emit-source-embeds out) + (put-string out "\n;; === joltc launcher ===\n") + (jb-emit-launcher out) + (close-port out)) + +;; --- 2. compile + boot in a FRESH Chez (profile Chez settings) -------------- +;; joltc is a compiler/REPL: it evals jolt-compiled Scheme at runtime, which must +;; resolve the runtime's top-level procedures (var-deref, jolt-inc, …) through the +;; boot's interaction-environment. compile-file's top-level defines are visible +;; there only when compiled in the REAL interaction-environment, and `error` (and +;; other primitives the inlined runtime references before redefining) bind to the +;; kernel primitive only when compiled against a clean chezscheme env. A fresh +;; Chez process gives both at once — exactly the legacy build-with-cc pass. The +;; in-process compile in build.ss/build-self-contained is for the distributed +;; joltc building (non-eval) apps, where no Chez is available. +(define jb-flat-so (string-append jb-build "/flat.so")) +(define jb-boot (string-append jb-build "/joltc.boot")) +(define jb-bool (lambda (b) (if b "#t" "#f"))) +(display (string-append "build-joltc: compiling (" jb-profile " profile)\n")) +(let ((cs (string-append jb-build "/compile.ss"))) + (let ((p (open-output-file cs 'replace))) + (put-string p + (string-append + "(import (chezscheme))\n" + "(optimize-level " (if jb-release? "3" "0") ")\n" + "(generate-inspector-information " (jb-bool (not jb-release?)) ")\n" + "(generate-procedure-source-information " (jb-bool (not jb-release?)) ")\n" + "(debug-on-exception " (jb-bool (not jb-release?)) ")\n" + "(fasl-compressed " (jb-bool jb-release?) ")\n" + "(compile-file " (ei-str-lit jb-flat-ss) " " (ei-str-lit jb-flat-so) ")\n" + "(make-boot-file " (ei-str-lit jb-boot) " '()\n " + (ei-str-lit (string-append bld-csv-dir "/petite.boot")) "\n " + (ei-str-lit (string-append bld-csv-dir "/scheme.boot")) "\n " + (ei-str-lit jb-flat-so) ")\n")) + (close-port p)) + (bld-system (string-append bld-chez " --script '" cs "'"))) + +;; --- 3. embed boots/stub as C arrays + cc-link ------------------------------ +;; xxd a file into header H and rename its symbol to NAME / NAME_len. +(define (jb-c-array file h name) + (bld-system (string-append "xxd -i '" file "' > '" h "'")) + (bld-system (string-append + "sed -i.bak -E 's/unsigned char [A-Za-z0-9_]+\\[\\]/unsigned char " name "[]/; " + "s/unsigned int [A-Za-z0-9_]+_len/unsigned int " name "_len/' '" h "'"))) + +(display "build-joltc: embedding boots + stub, linking\n") +(jb-c-array jb-boot (string-append jb-build "/boot_data.h") "jolt_boot") +(jb-c-array (string-append bld-csv-dir "/petite.boot") (string-append jb-build "/petite_data.h") "jolt_petite_boot") +(jb-c-array (string-append bld-csv-dir "/scheme.boot") (string-append jb-build "/scheme_data.h") "jolt_scheme_boot") +(jb-c-array jb-stub (string-append jb-build "/stub_data.h") "jolt_stub") + +(define jb-main-c (string-append jb-build "/main.c")) +(let ((mc (open-output-file jb-main-c 'replace))) + (put-string mc + (string-append + "#include \"scheme.h\"\n" + "#include \"boot_data.h\"\n" + "#include \"petite_data.h\"\n" + "#include \"scheme_data.h\"\n" + "#include \"stub_data.h\"\n" + "int main(int argc, char *argv[]) {\n" + " Sscheme_init(0);\n" + " Sregister_boot_file_bytes(\"jolt\", jolt_boot, jolt_boot_len);\n" + " Sbuild_heap(0, 0);\n" + " int status = Sscheme_start(argc, (const char **)argv);\n" + " Sscheme_deinit();\n return status;\n}\n")) + (close-port mc)) + +(bld-system (string-append + "cc -O2 -I'" bld-csv-dir "' -I'" jb-build "' '" jb-main-c "' '" + bld-csv-dir "/libkernel.a' -o '" jb-out "' " (bld-link-libs))) +(display (string-append "build-joltc: wrote " jb-out "\n")) diff --git a/host/chez/build.ss b/host/chez/build.ss index 200c8ad..8b1b290 100644 --- a/host/chez/build.ss +++ b/host/chez/build.ss @@ -41,6 +41,12 @@ (unless (zero? rc) (error 'jolt-build (string-append "command failed (" (number->string rc) "): " cmd))))) +;; mkdir -p without a subprocess (the self-contained build shells out to nothing). +(define (bld-mkdir-p dir) + (unless (or (string=? dir "") (string=? dir "/") (string=? dir ".") (file-exists? dir)) + (bld-mkdir-p (path-parent dir)) + (guard (e (#t #f)) (mkdir dir)))) + (define (bld-contains? s sub) (let ((ns (string-length s)) (nsub (string-length sub))) (let loop ((i 0)) @@ -329,7 +335,9 @@ ;; no runtime redefinition). Off by default in every mode — release stays ;; dynamically linked. (define (build-binary entry-ns out-path mode natives embed-dirs ext-roots direct-link? tree-shake?) - (bld-check-toolchain) + ;; The self-contained path (jolt-embedded-bytes "stub/launcher") needs no csv + ;; kernel files, no Chez, no cc — only the legacy cc path does. + (unless (jolt-embedded-bytes "stub/launcher") (bld-check-toolchain)) ;; 1. record app namespaces in dependency order as they finish loading. (let ((app-order '())) (set-ns-loaded-hook! @@ -397,7 +405,7 @@ (boot (string-append builddir "/jolt.boot")) (boot-h (string-append builddir "/boot_data.h")) (main-c (string-append builddir "/main.c"))) - (bld-system (string-append "mkdir -p '" builddir "'")) + (bld-mkdir-p builddir) ;; 3. flat source = runtime + app + launcher. (let ((out (open-output-file flat-ss 'replace))) (bld-emit-runtime out drop-compiler? core-strs) @@ -456,23 +464,21 @@ (build-with-cc entry-ns out-path mode builddir flat-ss flat-so boot boot-h main-c))))))) ;; --- self-contained link (in-process compile + append the boot to the stub) --- -;; The system `error` is restored around compile-file/make-boot-file: the loaded -;; jolt runtime shadows it (regex.ss, %chez-error holds the kernel original). -;; flat.ss opens with (import (chezscheme)) so the boot binds the real error -;; regardless, but the save/restore keeps any compile-time diagnostic honest. +;; compile-file runs with a FRESH scheme-environment as the interaction +;; environment: the loaded jolt runtime redefines `error` (regex.ss), and flat.ss +;; inlines that same runtime, so an early reference to `error` (before its define +;; runs) must bind to the kernel primitive — a clean env gives exactly that, the +;; same as the legacy path compiling in a fresh Chez process. Without it the boot +;; dies at startup with "variable error is not bound". (define (build-self-contained entry-ns out-path mode builddir flat-ss flat-so boot) (let ((petite (string-append builddir "/petite.boot")) (scheme (string-append builddir "/scheme.boot"))) (jolt-spill-embedded! "csv/petite.boot" petite) (jolt-spill-embedded! "csv/scheme.boot" scheme) (display (string-append "jolt build: compiling " entry-ns " (" mode " mode, self-contained)\n")) - (let ((saved-err error)) - (dynamic-wind - (lambda () (set! error %chez-error)) - (lambda () - (compile-file flat-ss flat-so) - (make-boot-file boot '() petite scheme flat-so)) - (lambda () (set! error saved-err)))) + (parameterize ((interaction-environment (copy-environment (scheme-environment)))) + (compile-file flat-ss flat-so) + (make-boot-file boot '() petite scheme flat-so)) ;; link: stub bytes ++ boot ++ frame, then make it executable. (jolt-spill-embedded! "stub/launcher" out-path) (jolt-append-payload! out-path (read-file-bytes boot)) diff --git a/host/chez/joltc-selfbuild-smoke.sh b/host/chez/joltc-selfbuild-smoke.sh new file mode 100644 index 0000000..7681e10 --- /dev/null +++ b/host/chez/joltc-selfbuild-smoke.sh @@ -0,0 +1,76 @@ +#!/bin/sh +# joltc self-build smoke (jolt-eaj): build joltc as a self-contained binary, then +# use THAT binary to compile a jolt app with Chez and cc removed from the +# environment — the whole point of the feature. The produced app must then run +# and match the same expected output as build-smoke.sh. +root="$(CDPATH= cd -- "$(dirname -- "$0")/../.." && pwd)" +cd "$root" + +# Preflight: building joltc itself needs the Chez kernel dev files (libkernel.a + +# scheme.h) and a C compiler, same as build-smoke.sh. A distro chezscheme package +# ships neither, so skip there (CI included). +csv="$JOLT_CHEZ_CSV" +if [ -z "$csv" ]; then + chez_bin="$(command -v chez || command -v scheme || command -v petite || true)" + if [ -n "$chez_bin" ]; then + base="$(cd "$(dirname "$chez_bin")/.." 2>/dev/null && pwd)" + for d in "$base"/lib/csv*/*/; do + [ -f "${d}libkernel.a" ] && csv="${d%/}" && break + done + fi +fi +if ! command -v cc >/dev/null 2>&1 || [ -z "$csv" ] || [ ! -f "$csv/scheme.h" ] || [ ! -f "$csv/libkernel.a" ]; then + echo "joltc self-build smoke: skipped (Chez kernel dev files or C compiler not available)" + exit 0 +fi +export JOLT_CHEZ_CSV="$csv" + +# 1. Build joltc (debug profile — faster; the self-contained app-build mechanism +# is identical to release, only Chez compile settings differ). +joltc="$root/target/debug/joltc" +echo "joltc self-build smoke: building $joltc" +if ! chez --script host/chez/build-joltc.ss debug "$joltc" >/dev/null 2>&1; then + echo " FAIL: build-joltc.ss exited non-zero" + exit 1 +fi +[ -x "$joltc" ] || { echo " FAIL: no joltc executable produced"; exit 1; } + +# 2. The distributed joltc must run with no Chez install: a basic eval. +got_e="$(env -i HOME="$HOME" "$joltc" -e '(reduce + (range 10))' 2>&1)" +if [ "$got_e" != "45" ]; then + echo " FAIL: joltc -e under empty env gave '$got_e', want 45" + exit 1 +fi + +# 3. Build an app through the distributed joltc with an EMPTY environment — no +# PATH at all, so no chez, no cc, no shell tools are reachable. This is the core +# guarantee: joltc compiles apps entirely on its own. +app="$(mktemp -d)/build-app" +cp -r "$root/test/chez/build-app" "$app" +out="$app/app" +echo "joltc self-build smoke: compiling app.core via the binary (no chez/cc on PATH)" +if ! env -i HOME="$HOME" JOLT_PWD="$app" "$joltc" build -m app.core -o "$out" >/dev/null 2>&1; then + echo " FAIL: self-contained jolt build exited non-zero" + rm -rf "$(dirname "$app")" + exit 1 +fi +[ -x "$out" ] || { echo " FAIL: no app executable produced"; rm -rf "$(dirname "$app")"; exit 1; } + +# 4. The produced app runs from a neutral cwd and matches build-smoke's output. +got="$(cd / && "$out" alpha bb ccc 2>&1)" +want='embedded resource ok +HELLO FROM A BUILT BINARY! +HELLO FROM A BUILT BINARY! +args: [alpha bb ccc] +sum: 10 +greet-default: greet:default +greet-loud: greet:loud +greet-soft: greet:soft' +rm -rf "$(dirname "$app")" +if [ "$got" != "$want" ]; then + echo " FAIL: produced app output mismatch" + echo "--- want ---"; echo "$want" + echo "--- got ----"; echo "$got" + exit 1 +fi +echo "joltc self-build smoke: passed (joltc runs + builds a working app with no external toolchain)" From df4653e57f04954fea319ae11afe4579363b2eee Mon Sep 17 00:00:00 2001 From: Yogthos Date: Mon, 29 Jun 2026 21:16:26 -0400 Subject: [PATCH 3/3] Add release workflow: build joltc binaries on a v* tag MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On a pushed v* tag, build the self-contained joltc (make joltc-release) for x86_64-linux, x86_64-macos, and aarch64-macos, package each as a tar.gz plus a SHA256, and attach them to the GitHub Release. Linux builds Chez from source like tests.yml (the apt package lacks the kernel dev files build-joltc cc-links against); macOS uses Homebrew chezscheme, which ships chez and the csv kernel files. No notarization, matching dirge — macOS tarball users de-quarantine once or install via a Homebrew tap. The Homebrew tap update job is a separate follow-on; this covers building and publishing the release assets. --- .github/workflows/release.yml | 113 ++++++++++++++++++++++++++++++++++ 1 file changed, 113 insertions(+) create mode 100644 .github/workflows/release.yml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..09f615a --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,113 @@ +name: release + +# Build the self-contained joltc binary for each platform and attach it to the +# GitHub Release when a v* tag is pushed. The binary bundles the runtime, +# compiler, jolt-core + stdlib source, the Chez boots, and a launcher stub, so it +# runs AND compiles jolt apps with no Chez or cc on the user's machine (jolt-eaj). +# +# No Apple notarization, mirroring dirge: macOS users who download the tarball +# clear Gatekeeper quarantine once (`xattr -d com.apple.quarantine joltc`), or +# install via a Homebrew tap that de-quarantines on install. +on: + push: + tags: + - 'v*' + +permissions: + contents: write # create/update the GitHub Release and upload assets + +jobs: + build: + name: build ${{ matrix.target }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + include: + - os: ubuntu-latest + target: x86_64-linux + - os: macos-13 + target: x86_64-macos + - os: macos-14 + target: aarch64-macos + steps: + - uses: actions/checkout@v5 + with: + submodules: recursive # vendor/irregex, used by the Chez regex shim + + # --- Linux: build Chez from source. The apt chezscheme ships petite+scheme + # only, with no kernel dev files (libkernel.a, scheme.h), which build-joltc + # needs to cc-link. Same setup as .github/workflows/tests.yml. --- + - name: Install build dependencies (Linux) + if: runner.os == 'Linux' + run: | + sudo apt-get update + sudo apt-get install -y build-essential git liblz4-dev zlib1g-dev libncurses-dev uuid-dev + + - name: Cache Chez Scheme (Linux) + if: runner.os == 'Linux' + id: cache-chez + uses: actions/cache@v4 + with: + path: /opt/chez + key: chez-${{ runner.os }}-v10.4.1-x11off + + - name: Build Chez Scheme from source (Linux) + if: runner.os == 'Linux' && steps.cache-chez.outputs.cache-hit != 'true' + run: | + git clone --depth 1 --branch v10.4.1 https://github.com/cisco/ChezScheme.git /tmp/chez-src + cd /tmp/chez-src + ./configure --installprefix=/opt/chez --threads --disable-x11 + make -j"$(nproc)" + sudo make install + sudo chown -R "$USER" /opt/chez + + - name: Put chez on PATH (Linux) + if: runner.os == 'Linux' + run: | + # Installed as `scheme`; the build invokes `chez`. A wrapper that execs + # scheme keeps argv0 so Chez finds its boot files, and sits next to + # scheme so build.ss derives the csv dir (libkernel.a/scheme.h) from it. + printf '#!/bin/sh\nexec /opt/chez/bin/scheme "$@"\n' > /opt/chez/bin/chez + chmod +x /opt/chez/bin/chez + echo '/opt/chez/bin' >> "$GITHUB_PATH" + + # --- macOS: Homebrew chezscheme ships `chez` plus the csv kernel dev files + # (libkernel.a, scheme.h, *.boot), which is all build-joltc needs. --- + - name: Install Chez Scheme (macOS) + if: runner.os == 'macOS' + run: brew install chezscheme lz4 + + - name: Show Chez version + run: chez --version + + # build-joltc compiles in a fresh Chez and cc-links; the checked-in seed is + # the compiler image, so no selfhost re-mint (that byte-fixpoint is a + # dev-machine check — see jolt-8479). `make joltc-release`, not `make joltc`. + - name: Build joltc (release) + run: make joltc-release + + # Sanity: the built binary runs (no Chez needed) and self-reports a value. + - name: Smoke the binary + run: | + out="$(./target/release/joltc -e '(reduce + (range 10))')" + test "$out" = "45" || { echo "joltc -e gave '$out', want 45"; exit 1; } + + - name: Package + run: | + ver="${GITHUB_REF_NAME}" + name="joltc-${ver}-${{ matrix.target }}" + mkdir -p "dist/${name}" + cp target/release/joltc "dist/${name}/joltc" + cp README.md LICENSE "dist/${name}/" + tar -C dist -czf "dist/${name}.tar.gz" "${name}" + ( cd dist && shasum -a 256 "${name}.tar.gz" > "${name}.tar.gz.sha256" ) + ls -la dist + + - name: Upload to the GitHub Release + uses: softprops/action-gh-release@v2 + with: + files: | + dist/*.tar.gz + dist/*.tar.gz.sha256 + fail_on_unmatched_files: true