From f0293fb4ee7e1a8760434cf2533ed9f665a5003a Mon Sep 17 00:00:00 2001 From: Dmitri Sotnikov Date: Tue, 16 Jun 2026 14:23:02 +0000 Subject: [PATCH] Parallelize the test gate; cache cold-init tests, drop the benchmark from it (#139) run-tests.janet runs the same file set as `jpm test` across a pool of worker processes (one `janet FILE` each, ev-based). The full gate goes from ~790s serial to ~98s here (8x), and more on CI where the heavy files don't thrash on swap. CI and the docs point at it; `jpm test` still works serially. Three things dominated the wall: - Nine integration tests cold-built a compile ctx (~8s each); switch them to api/init-cached so they share the prebuilt image. The cache key already fingerprints the ctx-shaping env vars, so the direct-link ones share one DL image and the rest share the plain one. - core-bench's main ran on every gate (~35s of benchmark loops that assert nothing); gate it behind JOLT_BENCH=1. - cli-test spawned `janet src/jolt/main.janet` ~20 times at ~8s cold each (340s under parallel load, and it was the whole wall); prefer build/jolt (~20ms baked ctx) when present, fall back to from-source for an unbuilt tree. type-check-test stays on cold init: a snapshot-loaded ctx loses the success checker's op/msg detail (jolt-vley). jolt-pria tracks caching from-source startup generally, which would let cli-test drop the build/jolt preference. Co-authored-by: Yogthos --- .github/workflows/tests.yml | 5 +- CLAUDE.md | 5 +- run-tests.janet | 100 ++++++++++++++++++ test/bench/core-bench.janet | 10 +- test/integration/cli-test.janet | 13 ++- test/integration/inline-sra-test.janet | 2 +- test/integration/predicate-fold-test.janet | 2 +- .../scalar-replace-record-test.janet | 2 +- test/integration/sorted-rbtree-test.janet | 2 +- test/integration/struct-hint-test.janet | 2 +- test/integration/type-infer-phase1-test.janet | 2 +- test/integration/type-infer-phase2-test.janet | 2 +- test/integration/type-infer-phase3-test.janet | 2 +- test/integration/type-infer-test.janet | 2 +- 14 files changed, 134 insertions(+), 17 deletions(-) create mode 100644 run-tests.janet diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index fd95e2c..05fd2bb 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -54,4 +54,7 @@ jobs: run: jpm build - name: Run tests - run: jpm test + # Parallel runner (same file set as `jpm test`, across CPU workers). + # The executable is built in the previous step, so binary-spawning tests + # don't skip. + run: janet run-tests.janet diff --git a/CLAUDE.md b/CLAUDE.md index 18db596..aa36b0b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -55,10 +55,11 @@ bd close # Complete work ```bash jpm build # build/jolt (one binary; ctx baked at build time) -jpm test # FULL gate — recursive over test/ (spec, unit, integration, bench) +jpm build; janet run-tests.janet # FULL gate, PARALLEL (across CPU workers) — build first so binary-spawning tests don't skip; JOLT_TEST_JOBS overrides worker count, -v shows all output +jpm test # FULL gate, serial (same file set; slower — recursive over test/) janet test/spec/.janet # one spec file janet test/integration/conformance-test.janet # 3-mode conformance (interpret/compile/self-host) -janet test/bench/core-bench.janet # bench — compare back-to-back vs main, never absolute +JOLT_BENCH=1 janet test/bench/core-bench.janet # bench (opt-in; skipped in the gate) — back-to-back vs main, never absolute ``` **Run the gate with a REAL exit code.** `jpm test | grep ...` reports grep's diff --git a/run-tests.janet b/run-tests.janet new file mode 100644 index 0000000..1f460cd --- /dev/null +++ b/run-tests.janet @@ -0,0 +1,100 @@ +#!/usr/bin/env janet +# Parallel test runner — the gate, faster. +# +# `jpm test` runs every file under test/ serially; this runs the same set across +# a pool of worker processes (each test file is an independent `janet FILE`). +# Build the executable FIRST (`jpm build`) so the binary-spawning tests don't +# skip: jpm build && janet run-tests.janet +# +# Worker count is the detected CPU count; override with JOLT_TEST_JOBS. Per-file +# stdout+stderr is captured to a temp file and only printed for failures (and, +# with -v / JOLT_TEST_VERBOSE, for everything). Exits non-zero if any file fails +# and prints the literal "All tests passed." on success (CI greps for it). + +(defn- detect-jobs [] + (or (when-let [j (os/getenv "JOLT_TEST_JOBS")] (scan-number j)) + (try + (let [p (os/spawn ["sh" "-c" "nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null"] + :p {:out :pipe}) + out (:read (p :out) :all)] + (os/proc-wait p) + (scan-number (string/trim (or out "")))) + ([_] nil)) + 4)) + +(defn- find-tests [dir acc] + (each name (sorted (os/dir dir)) + (def full (string dir "/" name)) + (case (os/stat full :mode) + # support/ holds shared libraries (harness, shims), not standalone tests — + # jpm doesn't run them and neither do we. + :directory (unless (= name "support") (find-tests full acc)) + :file (when (string/has-suffix? ".janet" name) (array/push acc full)))) + acc) + +(def verbose? (or (has-value? (dyn :args) "-v") (os/getenv "JOLT_TEST_VERBOSE"))) +(def janet-bin (dyn :executable)) +(def tmpdir (string (or (os/getenv "TMPDIR") "/tmp") "/jolt-test-" (os/getpid))) +(os/mkdir tmpdir) + +(def files (find-tests "test" @[])) +(def total (length files)) +(def jobs (max 1 (min 16 (detect-jobs)))) + +(def results @[]) +(var done 0) + +(defn- run-one [file] + (def t0 (os/clock)) + (def of (string tmpdir "/" (string/replace-all "/" "_" file) ".out")) + # let the shell own redirection — avoids pipe-buffer deadlock on chatty tests. + (def cmd (string/format "exec '%s' '%s' > '%s' 2>&1" janet-bin file of)) + (def rc (os/proc-wait (os/spawn ["sh" "-c" cmd] :p))) + (def dt (- (os/clock) t0)) + (array/push results {:file file :rc rc :out of :dt dt}) + (++ done) + (printf "[%d/%d] %s %6.2fs %s%s" done total (if (zero? rc) "ok " "FAIL") + dt file (if (zero? rc) "" (string/format " (rc=%d)" rc))) + (flush)) + +# A :stop sentinel per worker signals end-of-queue — ev/chan-close discards +# buffered items (Janet 1.41), so we can't just close and drain. +(def q (ev/chan (+ total jobs))) +(each f files (ev/give q f)) +(loop [_ :range [0 jobs]] (ev/give q :stop)) + +(defn- worker [&] + (forever + (def f (ev/take q)) + (if (= f :stop) (break) (run-one f)))) + +(printf "running %d test files across %d workers...\n" total jobs) +(flush) +(def wall0 (os/clock)) +(def super (ev/chan)) +(loop [_ :range [0 jobs]] (ev/go worker nil super)) +(loop [_ :range [0 jobs]] (ev/take super)) +(def wall (- (os/clock) wall0)) + +(def failed (filter |(not (zero? ($ :rc))) results)) +(def slow (->> results (sorted-by |(- ($ :dt))) (take 5))) + +(print "\n--- slowest ---") +(each r slow (printf " %6.2fs %s" (r :dt) (r :file))) + +(when verbose? + (each r results + (printf "\n===== %s (rc=%d, %.2fs) =====" (r :file) (r :rc) (r :dt)) + (print (slurp (r :out))))) + +(unless (empty? failed) + (print "\n--- FAILURES ---") + (each r failed + (printf "\n===== %s (rc=%d) =====" (r :file) (r :rc)) + (print (slurp (r :out))))) + +(printf "\n%d files, %d failed, %.1fs wall (%d workers)" + total (length failed) wall jobs) +(if (empty? failed) + (do (print "\nAll tests passed.") (os/exit 0)) + (do (printf "\n%d test files FAILED." (length failed)) (os/exit 1))) diff --git a/test/bench/core-bench.janet b/test/bench/core-bench.janet index d995b10..cc95a60 100644 --- a/test/bench/core-bench.janet +++ b/test/bench/core-bench.janet @@ -3,7 +3,8 @@ # Times representative core operations end-to-end (compile path) so a phase that # moves fns from native Janet to the self-hosted Clojure overlay can be checked # for regressions. Same programs before/after a phase -> relative delta is the -# migration's perf impact. Run: janet test/bench/core-bench.janet +# migration's perf impact. Run: JOLT_BENCH=1 janet test/bench/core-bench.janet +# (skipped under `jpm test` — it asserts nothing; see main). # # Each program carries its own internal iteration so the measured work dominates # parse/compile overhead. Reports the min of N runs (least noisy). @@ -32,6 +33,13 @@ best) (defn main [&] + # `jpm test` recurses test/ and would run this every gate, but it's a manual + # perf tool that asserts nothing (just reports timings) — so skip it unless + # opted in with JOLT_BENCH=1. Keeps ~35s of unasserted benchmark work out of + # the correctness gate (same pattern as suite-worker's no-arg no-op). + (unless (os/getenv "JOLT_BENCH") + (print "core-bench: SKIP (set JOLT_BENCH=1 to run)") + (os/exit 0)) (def ctx (api/init {:compile? true})) (print "bench (compile mode), min of " runs " runs, ms:") (var total 0) diff --git a/test/integration/cli-test.janet b/test/integration/cli-test.janet index 184d1b9..e4404f1 100644 --- a/test/integration/cli-test.janet +++ b/test/integration/cli-test.janet @@ -1,14 +1,19 @@ -# Smoke-test the command-line flags by running main.janet from source (no build -# needed, so it can't go stale). +# Smoke-test the command-line flags. Prefer the built binary (baked ctx, ~20ms +# startup); from-source is ~8s cold per invocation, and with ~20 invocations this +# file dominated the (parallel) gate's wall clock. Falls back to running +# main.janet from source when there's no binary, so an unbuilt tree still works — +# the gate (`jpm build && janet run-tests.janet`) builds first, so it's fresh. +(def- jolt-cmd + (if (os/stat "build/jolt") ["build/jolt"] ["janet" "src/jolt/main.janet"])) (defn- run [& args] - (def p (os/spawn ["janet" "src/jolt/main.janet" ;args] :p {:out :pipe :err :pipe})) + (def p (os/spawn [;jolt-cmd ;args] :p {:out :pipe :err :pipe})) (def out (:read (p :out) :all)) (os/proc-wait p) (string (or out ""))) (defn- run-err [& args] - (def p (os/spawn ["janet" "src/jolt/main.janet" ;args] :p {:out :pipe :err :pipe})) + (def p (os/spawn [;jolt-cmd ;args] :p {:out :pipe :err :pipe})) (def err (:read (p :err) :all)) (os/proc-wait p) (string (or err ""))) diff --git a/test/integration/inline-sra-test.janet b/test/integration/inline-sra-test.janet index 1cb24e5..b713dcd 100644 --- a/test/integration/inline-sra-test.janet +++ b/test/integration/inline-sra-test.janet @@ -10,7 +10,7 @@ (print "Inline + scalar replacement (jolt-87f)...") # A ctx with inlining ON (independent of the build-time JOLT_DIRECT_LINK). -(def ctx (api/init {:compile? true})) +(def ctx (api/init-cached {:compile? true})) (put (ctx :env) :direct-linking? true) (put (ctx :env) :inline? true) (api/eval-string ctx "(ns rt)") diff --git a/test/integration/predicate-fold-test.janet b/test/integration/predicate-fold-test.janet index d0bf618..61afb2f 100644 --- a/test/integration/predicate-fold-test.janet +++ b/test/integration/predicate-fold-test.janet @@ -13,7 +13,7 @@ (print "Predicate folding (jolt-wcw)...") (os/setenv "JOLT_DIRECT_LINK" "1") -(def ctx (api/init {:compile? true})) +(def ctx (api/init-cached {:compile? true})) (api/eval-string ctx "(ns pf)") (defn code [src] diff --git a/test/integration/scalar-replace-record-test.janet b/test/integration/scalar-replace-record-test.janet index b84b556..977e543 100644 --- a/test/integration/scalar-replace-record-test.janet +++ b/test/integration/scalar-replace-record-test.janet @@ -14,7 +14,7 @@ (print "Scalar-replace of records (jolt-15jq)...") (os/setenv "JOLT_DIRECT_LINK" "1") -(def ctx (api/init {:compile? true})) +(def ctx (api/init-cached {:compile? true})) (api/eval-string ctx "(ns srr)") (api/eval-string ctx "(defrecord V3 [r g b])") diff --git a/test/integration/sorted-rbtree-test.janet b/test/integration/sorted-rbtree-test.janet index da81715..6c5094f 100644 --- a/test/integration/sorted-rbtree-test.janet +++ b/test/integration/sorted-rbtree-test.janet @@ -8,7 +8,7 @@ (print "sorted-map/set red-black tree (jolt-0hbr)...") (os/setenv "JOLT_DIRECT_LINK" "1") -(def ctx (api/init {:compile? true})) +(def ctx (api/init-cached {:compile? true})) (defn ev [s] (api/eval-string ctx s)) (var fails 0) diff --git a/test/integration/struct-hint-test.janet b/test/integration/struct-hint-test.janet index f91ffa0..3fba4f2 100644 --- a/test/integration/struct-hint-test.janet +++ b/test/integration/struct-hint-test.janet @@ -13,7 +13,7 @@ (print "Type hints (jolt-94n)...") (os/setenv "JOLT_DIRECT_LINK" "1") # inline on, so hint-through-inline is exercised -(def ctx (api/init {:compile? true})) +(def ctx (api/init-cached {:compile? true})) (api/eval-string ctx "(ns sh)") (api/eval-string ctx "(defrecord Vec3r [r g b])") (each s ["(defn v3 [r g b] {:r r :g g :b b})" diff --git a/test/integration/type-infer-phase1-test.janet b/test/integration/type-infer-phase1-test.janet index e60fbc1..5d8fac7 100644 --- a/test/integration/type-infer-phase1-test.janet +++ b/test/integration/type-infer-phase1-test.janet @@ -12,7 +12,7 @@ (print "Type inference Phase 1 (jolt-767)...") (os/setenv "JOLT_DIRECT_LINK" "1") -(def ctx (api/init {:compile? true})) +(def ctx (api/init-cached {:compile? true})) (api/eval-string ctx "(ns p1)") # closed-world unit. mk is small (inlined away). rd is RECURSIVE, so it survives # inlining and is called via its var — exactly the shape (big/recursive fn with diff --git a/test/integration/type-infer-phase2-test.janet b/test/integration/type-infer-phase2-test.janet index 94c8aeb..8ac347a 100644 --- a/test/integration/type-infer-phase2-test.janet +++ b/test/integration/type-infer-phase2-test.janet @@ -8,7 +8,7 @@ (import ../../src/jolt/reader :as reader) (print "Type inference Phase 2 (vector ops)...") (os/setenv "JOLT_DIRECT_LINK" "1") -(def ctx (api/init {:compile? true})) +(def ctx (api/init-cached {:compile? true})) (api/eval-string ctx "(ns p2)") (def reinfer (types/var-get (types/ns-find (types/ctx-find-ns ctx "jolt.passes") "reinfer-def"))) (defn estr [src ptmap] diff --git a/test/integration/type-infer-phase3-test.janet b/test/integration/type-infer-phase3-test.janet index e9dc1c5..2160b04 100644 --- a/test/integration/type-infer-phase3-test.janet +++ b/test/integration/type-infer-phase3-test.janet @@ -10,7 +10,7 @@ (print "Type inference Phase 3 (jolt-d6u)...") (os/setenv "JOLT_DIRECT_LINK" "1") -(def ctx (api/init {:compile? true})) +(def ctx (api/init-cached {:compile? true})) (api/eval-string ctx "(ns p3)") (def pns (types/ctx-find-ns ctx "jolt.passes")) (def reinfer (types/var-get (types/ns-find pns "reinfer-def"))) diff --git a/test/integration/type-infer-test.janet b/test/integration/type-infer-test.janet index 016500a..e010870 100644 --- a/test/integration/type-infer-test.janet +++ b/test/integration/type-infer-test.janet @@ -15,7 +15,7 @@ (print "Type inference Phase 0 (jolt-6sr)...") (os/setenv "JOLT_DIRECT_LINK" "1") -(def ctx (api/init {:compile? true})) +(def ctx (api/init-cached {:compile? true})) (api/eval-string ctx "(ns ti)") (defn guards [src]