diff --git a/.gitignore b/.gitignore index b62bfd2..ab386cf 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ AGENTS.md .DS_Store CLAUDE.md build/ +target/ .clj-kondo/ .dirge/ .claude/ diff --git a/Makefile b/Makefile index bdef687..1c8d6f0 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ # build step. `make test` is the full gate. `make remint` rebuilds the seed after a # source change. -.PHONY: test ci values corpus unit smoke buildsmoke selfhost sci certify ffi transient infer wp devirt fieldread numwp fieldnum protoret narrow directlink numeric inline shakesmoke remint +.PHONY: test ci values corpus unit smoke buildsmoke selfhost sci certify ffi transient infer wp devirt fieldread numwp fieldnum protoret narrow directlink numeric inline shakesmoke remint joltc joltc-release joltc-debug joltcsmoke # Full gate (dev machine). Includes the self-host byte-fixpoint, which only holds # on the same Chez that minted the seed. @@ -42,6 +42,23 @@ smoke: buildsmoke: @sh host/chez/build-smoke.sh +# Build joltc as a self-contained native binary into target//joltc. The +# binary bundles the runtime, compiler, jolt-core + stdlib source, the Chez boots, +# and a launcher stub, so it runs AND compiles jolt apps with no Chez or cc on the +# machine. Built on a dev/CI host that HAS Chez + cc. release = optimize-level 3, +# no inspector info, compressed; debug = optimize-level 0 + inspector + debug info. +joltc-release: + @chez --script host/chez/build-joltc.ss release target/release/joltc +joltc-debug: + @chez --script host/chez/build-joltc.ss debug target/debug/joltc +# Re-mint the seed first so the embedded compiler image is current, then both builds. +joltc: selfhost joltc-release joltc-debug + @echo "OK: target/release/joltc and target/debug/joltc built" + +# Self-build smoke: the distributed joltc compiles an app with Chez + cc removed. +joltcsmoke: + @sh host/chez/joltc-selfbuild-smoke.sh + # SCI conformance: load borkdude/sci's source through joltc (floor-gated). sci: @chez --script host/chez/run-sci.ss diff --git a/host/chez/build-joltc.ss b/host/chez/build-joltc.ss new file mode 100644 index 0000000..b8c5cca --- /dev/null +++ b/host/chez/build-joltc.ss @@ -0,0 +1,206 @@ +;; build-joltc.ss — build joltc itself as a self-contained native binary (jolt-eaj). +;; +;; chez --script host/chez/build-joltc.ss +;; profile: "release" | "debug" out-path: e.g. target/release/joltc +;; +;; Runs on a dev/CI machine that HAS Chez + cc. Produces a binary that needs +;; NEITHER: it bakes the full runtime + compiler image + all jolt-core/stdlib +;; source + the Chez petite/scheme boots + a prebuilt launcher stub into one +;; cc-linked executable, so the resulting joltc can run AND `build` jolt apps on +;; its own. joltc itself is cc-linked (not appended) so its signature stays clean +;; for Homebrew/codesign, like dirge's binaries; only the apps it later builds use +;; the appended-stub path (host/chez/build.ss build-self-contained). +;; +;; Pipeline: +;; 0. cc-compile host/chez/stub/launcher.c against the Chez kernel. +;; 1. emit flat.ss = runtime + compiler image (cli.ss load order) + inlined +;; build.ss + every jolt-core/stdlib file as a baked string literal + the +;; joltc launcher. +;; 2. in-process compile-file + make-boot-file (profile Chez settings), error +;; restored around the call (the runtime shadows it; regex.ss/%chez-error). +;; 3. xxd the joltc boot + petite/scheme boots + stub into C arrays, generate +;; main.c, cc-link -> out-path. The launcher reads the petite/scheme/stub +;; arrays via FFI on `build` (jolt-materialize-bundles!). + +(import (chezscheme)) + +(load "host/chez/rt.ss") +(set-chez-ns! "clojure.core") +(load "host/chez/seed/prelude.ss") +(load "host/chez/post-prelude.ss") +(set-chez-ns! "user") +(load "host/chez/host-contract.ss") +(load "host/chez/seed/image.ss") +(load "host/chez/compile-eval.ss") +(load "host/chez/png.ss") +(load "host/chez/loader.ss") +(load "host/chez/java/ffi.ss") +(set-source-roots! (list "jolt-core" "stdlib")) +(load "host/chez/build.ss") ; bld-* helpers, ei-* (emit-image), dce + +(define jb-args (cdr (command-line))) +(define jb-profile (if (pair? jb-args) (car jb-args) "release")) +(define jb-out (if (and (pair? jb-args) (pair? (cdr jb-args))) (cadr jb-args) + (string-append "target/" jb-profile "/joltc"))) +(define jb-release? (string=? jb-profile "release")) +(unless (or jb-release? (string=? jb-profile "debug")) + (error 'build-joltc "profile must be \"release\" or \"debug\"" jb-profile)) + +(define jb-build (string-append jb-out ".build")) +(bld-check-toolchain) +(bld-system (string-append "mkdir -p '" (path-parent jb-out) "' '" jb-build "'")) + +;; --- 0. compile the launcher stub ------------------------------------------- +(define jb-stub (string-append jb-build "/launcher")) +(display "build-joltc: compiling launcher stub\n") +(bld-system (string-append + "cc -O2 -I'" bld-csv-dir "' 'host/chez/stub/launcher.c' '" + bld-csv-dir "/libkernel.a' -o '" jb-stub "' " (bld-link-libs))) + +;; --- 1. emit flat.ss -------------------------------------------------------- +(define jb-flat-ss (string-append jb-build "/flat.ss")) +(define (str-suffix? s suf) + (let ((n (string-length s)) (m (string-length suf))) + (and (>= n m) (string=? (substring s (- n m) n) suf)))) + +;; Bake every jolt-core/stdlib source file as an in-heap string literal keyed by +;; its root-relative path ("jolt/main.clj", "clojure/string.clj") — exactly what +;; resolve-on-roots probes. Literals (not read-file-string at startup) because +;; flat.ss top-level forms run at every startup, with no source on disk. +(define (jb-emit-source-embeds out) + (for-each + (lambda (root) + (for-each + (lambda (rp) + (let ((rel (car rp)) (abs (cdr rp))) + (when (or (str-suffix? rel ".clj") (str-suffix? rel ".cljc")) + (put-string out (string-append + "(register-embedded-resource! " (ei-str-lit rel) " " + (ei-str-lit (read-file-string abs)) ")\n"))))) + (bld-walk-files root "" '()))) + (list "jolt-core" "stdlib"))) + +;; The launcher (Chez scheme-start): replicates host/chez/cli.ss but reads argv +;; from the scheme-start lambda and has no repo root to cd into (all source is +;; embedded; JOLT_PWD defaults to cwd via io/jolt.main). build.ss is already +;; inlined, so `build` dispatches straight to jolt.host/build-binary after the +;; bundled boots/stub are materialized from the binary's own C arrays. +(define (jb-emit-launcher out) + (put-string out " +;; Materialize the bundled Chez boots + launcher stub (cc-linked into this binary +;; as C arrays) into the embedded-bytes store, so build-self-contained can spill +;; them. Done lazily on `build` only. +(define (jolt-materialize-bundles!) + (load-shared-object #f) + (let ((memcpy (foreign-procedure \"memcpy\" (u8* uptr uptr) void*))) + (for-each + (lambda (spec) + (let* ((len (foreign-ref 'unsigned-int (foreign-entry (caddr spec)) 0)) + (bv (make-bytevector len))) + (memcpy bv (foreign-entry (cadr spec)) len) + (register-embedded-bytes! (car spec) bv))) + '((\"csv/petite.boot\" \"jolt_petite_boot\" \"jolt_petite_boot_len\") + (\"csv/scheme.boot\" \"jolt_scheme_boot\" \"jolt_scheme_boot_len\") + (\"stub/launcher\" \"jolt_stub\" \"jolt_stub_len\"))))) + +(suppress-greeting #t) +(scheme-start + (lambda args + (set-source-roots! (list \"jolt-core\" \"stdlib\")) + (guard (v (#t (jolt-report-throwable v (current-error-port)) + (let ((bt (jolt-backtrace-string v))) + (when bt (display \" trace:\\n\" (current-error-port)) + (display bt (current-error-port)))) + (exit 1))) + (cond + ((and (= (length args) 2) (string=? (car args) \"-e\")) + (let ((result (jolt-final-str + (jolt-compile-eval (string-append \"(do \" (cadr args) \")\") \"user\")))) + (unless (string=? result \"\") (display result) (newline)))) + (else + (when (and (pair? args) (string=? (car args) \"build\")) + (jolt-materialize-bundles!)) + (load-namespace \"jolt.main\") + (apply jolt-invoke (var-deref \"jolt.main\" \"-main\") args)))) + (exit 0))) +")) + +(display "build-joltc: emitting flat source\n") +(let ((out (open-output-file jb-flat-ss 'replace))) + ;; full runtime + compiler image: keep the compiler (joltc evals at runtime). + (bld-emit-runtime out #f #f) + (put-string out "\n;; === build driver (inlined for self-contained `jolt build`) ===\n") + (bld-inline-line "(load \"host/chez/build.ss\")" out 0) + (put-string out "\n;; === embedded jolt-core + stdlib source ===\n") + (jb-emit-source-embeds out) + (put-string out "\n;; === joltc launcher ===\n") + (jb-emit-launcher out) + (close-port out)) + +;; --- 2. compile + boot in a FRESH Chez (profile Chez settings) -------------- +;; joltc is a compiler/REPL: it evals jolt-compiled Scheme at runtime, which must +;; resolve the runtime's top-level procedures (var-deref, jolt-inc, …) through the +;; boot's interaction-environment. compile-file's top-level defines are visible +;; there only when compiled in the REAL interaction-environment, and `error` (and +;; other primitives the inlined runtime references before redefining) bind to the +;; kernel primitive only when compiled against a clean chezscheme env. A fresh +;; Chez process gives both at once — exactly the legacy build-with-cc pass. The +;; in-process compile in build.ss/build-self-contained is for the distributed +;; joltc building (non-eval) apps, where no Chez is available. +(define jb-flat-so (string-append jb-build "/flat.so")) +(define jb-boot (string-append jb-build "/joltc.boot")) +(define jb-bool (lambda (b) (if b "#t" "#f"))) +(display (string-append "build-joltc: compiling (" jb-profile " profile)\n")) +(let ((cs (string-append jb-build "/compile.ss"))) + (let ((p (open-output-file cs 'replace))) + (put-string p + (string-append + "(import (chezscheme))\n" + "(optimize-level " (if jb-release? "3" "0") ")\n" + "(generate-inspector-information " (jb-bool (not jb-release?)) ")\n" + "(generate-procedure-source-information " (jb-bool (not jb-release?)) ")\n" + "(debug-on-exception " (jb-bool (not jb-release?)) ")\n" + "(fasl-compressed " (jb-bool jb-release?) ")\n" + "(compile-file " (ei-str-lit jb-flat-ss) " " (ei-str-lit jb-flat-so) ")\n" + "(make-boot-file " (ei-str-lit jb-boot) " '()\n " + (ei-str-lit (string-append bld-csv-dir "/petite.boot")) "\n " + (ei-str-lit (string-append bld-csv-dir "/scheme.boot")) "\n " + (ei-str-lit jb-flat-so) ")\n")) + (close-port p)) + (bld-system (string-append bld-chez " --script '" cs "'"))) + +;; --- 3. embed boots/stub as C arrays + cc-link ------------------------------ +;; xxd a file into header H and rename its symbol to NAME / NAME_len. +(define (jb-c-array file h name) + (bld-system (string-append "xxd -i '" file "' > '" h "'")) + (bld-system (string-append + "sed -i.bak -E 's/unsigned char [A-Za-z0-9_]+\\[\\]/unsigned char " name "[]/; " + "s/unsigned int [A-Za-z0-9_]+_len/unsigned int " name "_len/' '" h "'"))) + +(display "build-joltc: embedding boots + stub, linking\n") +(jb-c-array jb-boot (string-append jb-build "/boot_data.h") "jolt_boot") +(jb-c-array (string-append bld-csv-dir "/petite.boot") (string-append jb-build "/petite_data.h") "jolt_petite_boot") +(jb-c-array (string-append bld-csv-dir "/scheme.boot") (string-append jb-build "/scheme_data.h") "jolt_scheme_boot") +(jb-c-array jb-stub (string-append jb-build "/stub_data.h") "jolt_stub") + +(define jb-main-c (string-append jb-build "/main.c")) +(let ((mc (open-output-file jb-main-c 'replace))) + (put-string mc + (string-append + "#include \"scheme.h\"\n" + "#include \"boot_data.h\"\n" + "#include \"petite_data.h\"\n" + "#include \"scheme_data.h\"\n" + "#include \"stub_data.h\"\n" + "int main(int argc, char *argv[]) {\n" + " Sscheme_init(0);\n" + " Sregister_boot_file_bytes(\"jolt\", jolt_boot, jolt_boot_len);\n" + " Sbuild_heap(0, 0);\n" + " int status = Sscheme_start(argc, (const char **)argv);\n" + " Sscheme_deinit();\n return status;\n}\n")) + (close-port mc)) + +(bld-system (string-append + "cc -O2 -I'" bld-csv-dir "' -I'" jb-build "' '" jb-main-c "' '" + bld-csv-dir "/libkernel.a' -o '" jb-out "' " (bld-link-libs))) +(display (string-append "build-joltc: wrote " jb-out "\n")) diff --git a/host/chez/build.ss b/host/chez/build.ss index 200c8ad..8b1b290 100644 --- a/host/chez/build.ss +++ b/host/chez/build.ss @@ -41,6 +41,12 @@ (unless (zero? rc) (error 'jolt-build (string-append "command failed (" (number->string rc) "): " cmd))))) +;; mkdir -p without a subprocess (the self-contained build shells out to nothing). +(define (bld-mkdir-p dir) + (unless (or (string=? dir "") (string=? dir "/") (string=? dir ".") (file-exists? dir)) + (bld-mkdir-p (path-parent dir)) + (guard (e (#t #f)) (mkdir dir)))) + (define (bld-contains? s sub) (let ((ns (string-length s)) (nsub (string-length sub))) (let loop ((i 0)) @@ -329,7 +335,9 @@ ;; no runtime redefinition). Off by default in every mode — release stays ;; dynamically linked. (define (build-binary entry-ns out-path mode natives embed-dirs ext-roots direct-link? tree-shake?) - (bld-check-toolchain) + ;; The self-contained path (jolt-embedded-bytes "stub/launcher") needs no csv + ;; kernel files, no Chez, no cc — only the legacy cc path does. + (unless (jolt-embedded-bytes "stub/launcher") (bld-check-toolchain)) ;; 1. record app namespaces in dependency order as they finish loading. (let ((app-order '())) (set-ns-loaded-hook! @@ -397,7 +405,7 @@ (boot (string-append builddir "/jolt.boot")) (boot-h (string-append builddir "/boot_data.h")) (main-c (string-append builddir "/main.c"))) - (bld-system (string-append "mkdir -p '" builddir "'")) + (bld-mkdir-p builddir) ;; 3. flat source = runtime + app + launcher. (let ((out (open-output-file flat-ss 'replace))) (bld-emit-runtime out drop-compiler? core-strs) @@ -456,23 +464,21 @@ (build-with-cc entry-ns out-path mode builddir flat-ss flat-so boot boot-h main-c))))))) ;; --- self-contained link (in-process compile + append the boot to the stub) --- -;; The system `error` is restored around compile-file/make-boot-file: the loaded -;; jolt runtime shadows it (regex.ss, %chez-error holds the kernel original). -;; flat.ss opens with (import (chezscheme)) so the boot binds the real error -;; regardless, but the save/restore keeps any compile-time diagnostic honest. +;; compile-file runs with a FRESH scheme-environment as the interaction +;; environment: the loaded jolt runtime redefines `error` (regex.ss), and flat.ss +;; inlines that same runtime, so an early reference to `error` (before its define +;; runs) must bind to the kernel primitive — a clean env gives exactly that, the +;; same as the legacy path compiling in a fresh Chez process. Without it the boot +;; dies at startup with "variable error is not bound". (define (build-self-contained entry-ns out-path mode builddir flat-ss flat-so boot) (let ((petite (string-append builddir "/petite.boot")) (scheme (string-append builddir "/scheme.boot"))) (jolt-spill-embedded! "csv/petite.boot" petite) (jolt-spill-embedded! "csv/scheme.boot" scheme) (display (string-append "jolt build: compiling " entry-ns " (" mode " mode, self-contained)\n")) - (let ((saved-err error)) - (dynamic-wind - (lambda () (set! error %chez-error)) - (lambda () - (compile-file flat-ss flat-so) - (make-boot-file boot '() petite scheme flat-so)) - (lambda () (set! error saved-err)))) + (parameterize ((interaction-environment (copy-environment (scheme-environment)))) + (compile-file flat-ss flat-so) + (make-boot-file boot '() petite scheme flat-so)) ;; link: stub bytes ++ boot ++ frame, then make it executable. (jolt-spill-embedded! "stub/launcher" out-path) (jolt-append-payload! out-path (read-file-bytes boot)) diff --git a/host/chez/joltc-selfbuild-smoke.sh b/host/chez/joltc-selfbuild-smoke.sh new file mode 100644 index 0000000..7681e10 --- /dev/null +++ b/host/chez/joltc-selfbuild-smoke.sh @@ -0,0 +1,76 @@ +#!/bin/sh +# joltc self-build smoke (jolt-eaj): build joltc as a self-contained binary, then +# use THAT binary to compile a jolt app with Chez and cc removed from the +# environment — the whole point of the feature. The produced app must then run +# and match the same expected output as build-smoke.sh. +root="$(CDPATH= cd -- "$(dirname -- "$0")/../.." && pwd)" +cd "$root" + +# Preflight: building joltc itself needs the Chez kernel dev files (libkernel.a + +# scheme.h) and a C compiler, same as build-smoke.sh. A distro chezscheme package +# ships neither, so skip there (CI included). +csv="$JOLT_CHEZ_CSV" +if [ -z "$csv" ]; then + chez_bin="$(command -v chez || command -v scheme || command -v petite || true)" + if [ -n "$chez_bin" ]; then + base="$(cd "$(dirname "$chez_bin")/.." 2>/dev/null && pwd)" + for d in "$base"/lib/csv*/*/; do + [ -f "${d}libkernel.a" ] && csv="${d%/}" && break + done + fi +fi +if ! command -v cc >/dev/null 2>&1 || [ -z "$csv" ] || [ ! -f "$csv/scheme.h" ] || [ ! -f "$csv/libkernel.a" ]; then + echo "joltc self-build smoke: skipped (Chez kernel dev files or C compiler not available)" + exit 0 +fi +export JOLT_CHEZ_CSV="$csv" + +# 1. Build joltc (debug profile — faster; the self-contained app-build mechanism +# is identical to release, only Chez compile settings differ). +joltc="$root/target/debug/joltc" +echo "joltc self-build smoke: building $joltc" +if ! chez --script host/chez/build-joltc.ss debug "$joltc" >/dev/null 2>&1; then + echo " FAIL: build-joltc.ss exited non-zero" + exit 1 +fi +[ -x "$joltc" ] || { echo " FAIL: no joltc executable produced"; exit 1; } + +# 2. The distributed joltc must run with no Chez install: a basic eval. +got_e="$(env -i HOME="$HOME" "$joltc" -e '(reduce + (range 10))' 2>&1)" +if [ "$got_e" != "45" ]; then + echo " FAIL: joltc -e under empty env gave '$got_e', want 45" + exit 1 +fi + +# 3. Build an app through the distributed joltc with an EMPTY environment — no +# PATH at all, so no chez, no cc, no shell tools are reachable. This is the core +# guarantee: joltc compiles apps entirely on its own. +app="$(mktemp -d)/build-app" +cp -r "$root/test/chez/build-app" "$app" +out="$app/app" +echo "joltc self-build smoke: compiling app.core via the binary (no chez/cc on PATH)" +if ! env -i HOME="$HOME" JOLT_PWD="$app" "$joltc" build -m app.core -o "$out" >/dev/null 2>&1; then + echo " FAIL: self-contained jolt build exited non-zero" + rm -rf "$(dirname "$app")" + exit 1 +fi +[ -x "$out" ] || { echo " FAIL: no app executable produced"; rm -rf "$(dirname "$app")"; exit 1; } + +# 4. The produced app runs from a neutral cwd and matches build-smoke's output. +got="$(cd / && "$out" alpha bb ccc 2>&1)" +want='embedded resource ok +HELLO FROM A BUILT BINARY! +HELLO FROM A BUILT BINARY! +args: [alpha bb ccc] +sum: 10 +greet-default: greet:default +greet-loud: greet:loud +greet-soft: greet:soft' +rm -rf "$(dirname "$app")" +if [ "$got" != "$want" ]; then + echo " FAIL: produced app output mismatch" + echo "--- want ---"; echo "$want" + echo "--- got ----"; echo "$got" + exit 1 +fi +echo "joltc self-build smoke: passed (joltc runs + builds a working app with no external toolchain)"