Parallelize the test gate; cache cold-init tests, drop the benchmark from it (#139)

run-tests.janet runs the same file set as `jpm test` across a pool of worker
processes (one `janet FILE` each, ev-based). The full gate goes from ~790s
serial to ~98s here (8x), and more on CI where the heavy files don't thrash on
swap. CI and the docs point at it; `jpm test` still works serially.

Three things dominated the wall:

- Nine integration tests cold-built a compile ctx (~8s each); switch them to
  api/init-cached so they share the prebuilt image. The cache key already
  fingerprints the ctx-shaping env vars, so the direct-link ones share one DL
  image and the rest share the plain one.
- core-bench's main ran on every gate (~35s of benchmark loops that assert
  nothing); gate it behind JOLT_BENCH=1.
- cli-test spawned `janet src/jolt/main.janet` ~20 times at ~8s cold each
  (340s under parallel load, and it was the whole wall); prefer build/jolt
  (~20ms baked ctx) when present, fall back to from-source for an unbuilt tree.

type-check-test stays on cold init: a snapshot-loaded ctx loses the success
checker's op/msg detail (jolt-vley). jolt-pria tracks caching from-source
startup generally, which would let cli-test drop the build/jolt preference.

Co-authored-by: Yogthos <yogthos@gmail.com>

.github/workflows/tests.yml

@@ -54,4 +54,7 @@ jobs:
         run: jpm build
 
       - name: Run tests
-        run: jpm test
+        # Parallel runner (same file set as `jpm test`, across CPU workers).
+        # The executable is built in the previous step, so binary-spawning tests
+        # don't skip.
+        run: janet run-tests.janet

CLAUDE.md

@@ -55,10 +55,11 @@ bd close <id>         # Complete work
 
 ```bash
 jpm build              # build/jolt (one binary; ctx baked at build time)
-jpm test               # FULL gate — recursive over test/ (spec, unit, integration, bench)
+jpm build; janet run-tests.janet   # FULL gate, PARALLEL (across CPU workers) — build first so binary-spawning tests don't skip; JOLT_TEST_JOBS overrides worker count, -v shows all output
+jpm test               # FULL gate, serial (same file set; slower — recursive over test/)
 janet test/spec/<f>.janet            # one spec file
 janet test/integration/conformance-test.janet   # 3-mode conformance (interpret/compile/self-host)
-janet test/bench/core-bench.janet    # bench — compare back-to-back vs main, never absolute
+JOLT_BENCH=1 janet test/bench/core-bench.janet   # bench (opt-in; skipped in the gate) — back-to-back vs main, never absolute
 ```
 
 **Run the gate with a REAL exit code.** `jpm test | grep ...` reports grep's

run-tests.janet

@@ -0,0 +1,100 @@
+#!/usr/bin/env janet
+# Parallel test runner — the gate, faster.
+#
+# `jpm test` runs every file under test/ serially; this runs the same set across
+# a pool of worker processes (each test file is an independent `janet FILE`).
+# Build the executable FIRST (`jpm build`) so the binary-spawning tests don't
+# skip:  jpm build && janet run-tests.janet
+#
+# Worker count is the detected CPU count; override with JOLT_TEST_JOBS. Per-file
+# stdout+stderr is captured to a temp file and only printed for failures (and,
+# with -v / JOLT_TEST_VERBOSE, for everything). Exits non-zero if any file fails
+# and prints the literal "All tests passed." on success (CI greps for it).
+
+(defn- detect-jobs []
+  (or (when-let [j (os/getenv "JOLT_TEST_JOBS")] (scan-number j))
+      (try
+        (let [p (os/spawn ["sh" "-c" "nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null"]
+                          :p {:out :pipe})
+              out (:read (p :out) :all)]
+          (os/proc-wait p)
+          (scan-number (string/trim (or out ""))))
+        ([_] nil))
+      4))
+
+(defn- find-tests [dir acc]
+  (each name (sorted (os/dir dir))
+    (def full (string dir "/" name))
+    (case (os/stat full :mode)
+      # support/ holds shared libraries (harness, shims), not standalone tests —
+      # jpm doesn't run them and neither do we.
+      :directory (unless (= name "support") (find-tests full acc))
+      :file (when (string/has-suffix? ".janet" name) (array/push acc full))))
+  acc)
+
+(def verbose? (or (has-value? (dyn :args) "-v") (os/getenv "JOLT_TEST_VERBOSE")))
+(def janet-bin (dyn :executable))
+(def tmpdir (string (or (os/getenv "TMPDIR") "/tmp") "/jolt-test-" (os/getpid)))
+(os/mkdir tmpdir)
+
+(def files (find-tests "test" @[]))
+(def total (length files))
+(def jobs (max 1 (min 16 (detect-jobs))))
+
+(def results @[])
+(var done 0)
+
+(defn- run-one [file]
+  (def t0 (os/clock))
+  (def of (string tmpdir "/" (string/replace-all "/" "_" file) ".out"))
+  # let the shell own redirection — avoids pipe-buffer deadlock on chatty tests.
+  (def cmd (string/format "exec '%s' '%s' > '%s' 2>&1" janet-bin file of))
+  (def rc (os/proc-wait (os/spawn ["sh" "-c" cmd] :p)))
+  (def dt (- (os/clock) t0))
+  (array/push results {:file file :rc rc :out of :dt dt})
+  (++ done)
+  (printf "[%d/%d] %s %6.2fs  %s%s" done total (if (zero? rc) "ok  " "FAIL")
+          dt file (if (zero? rc) "" (string/format " (rc=%d)" rc)))
+  (flush))
+
+# A :stop sentinel per worker signals end-of-queue — ev/chan-close discards
+# buffered items (Janet 1.41), so we can't just close and drain.
+(def q (ev/chan (+ total jobs)))
+(each f files (ev/give q f))
+(loop [_ :range [0 jobs]] (ev/give q :stop))
+
+(defn- worker [&]
+  (forever
+    (def f (ev/take q))
+    (if (= f :stop) (break) (run-one f))))
+
+(printf "running %d test files across %d workers...\n" total jobs)
+(flush)
+(def wall0 (os/clock))
+(def super (ev/chan))
+(loop [_ :range [0 jobs]] (ev/go worker nil super))
+(loop [_ :range [0 jobs]] (ev/take super))
+(def wall (- (os/clock) wall0))
+
+(def failed (filter |(not (zero? ($ :rc))) results))
+(def slow (->> results (sorted-by |(- ($ :dt))) (take 5)))
+
+(print "\n--- slowest ---")
+(each r slow (printf "  %6.2fs  %s" (r :dt) (r :file)))
+
+(when verbose?
+  (each r results
+    (printf "\n===== %s (rc=%d, %.2fs) =====" (r :file) (r :rc) (r :dt))
+    (print (slurp (r :out)))))
+
+(unless (empty? failed)
+  (print "\n--- FAILURES ---")
+  (each r failed
+    (printf "\n===== %s (rc=%d) =====" (r :file) (r :rc))
+    (print (slurp (r :out)))))
+
+(printf "\n%d files, %d failed, %.1fs wall (%d workers)"
+        total (length failed) wall jobs)
+(if (empty? failed)
+  (do (print "\nAll tests passed.") (os/exit 0))
+  (do (printf "\n%d test files FAILED." (length failed)) (os/exit 1)))

test/bench/core-bench.janet

@@ -3,7 +3,8 @@
 # Times representative core operations end-to-end (compile path) so a phase that
 # moves fns from native Janet to the self-hosted Clojure overlay can be checked
 # for regressions. Same programs before/after a phase -> relative delta is the
-# migration's perf impact. Run: janet test/bench/core-bench.janet
+# migration's perf impact. Run: JOLT_BENCH=1 janet test/bench/core-bench.janet
+# (skipped under `jpm test` — it asserts nothing; see main).
 #
 # Each program carries its own internal iteration so the measured work dominates
 # parse/compile overhead. Reports the min of N runs (least noisy).
@@ -32,6 +33,13 @@
   best)
 
 (defn main [&]
+  # `jpm test` recurses test/ and would run this every gate, but it's a manual
+  # perf tool that asserts nothing (just reports timings) — so skip it unless
+  # opted in with JOLT_BENCH=1. Keeps ~35s of unasserted benchmark work out of
+  # the correctness gate (same pattern as suite-worker's no-arg no-op).
+  (unless (os/getenv "JOLT_BENCH")
+    (print "core-bench: SKIP (set JOLT_BENCH=1 to run)")
+    (os/exit 0))
   (def ctx (api/init {:compile? true}))
   (print "bench (compile mode), min of " runs " runs, ms:")
   (var total 0)

test/integration/cli-test.janet

@@ -1,14 +1,19 @@
-# Smoke-test the command-line flags by running main.janet from source (no build
-# needed, so it can't go stale).
+# Smoke-test the command-line flags. Prefer the built binary (baked ctx, ~20ms
+# startup); from-source is ~8s cold per invocation, and with ~20 invocations this
+# file dominated the (parallel) gate's wall clock. Falls back to running
+# main.janet from source when there's no binary, so an unbuilt tree still works —
+# the gate (`jpm build && janet run-tests.janet`) builds first, so it's fresh.
+(def- jolt-cmd
+  (if (os/stat "build/jolt") ["build/jolt"] ["janet" "src/jolt/main.janet"]))
 
 (defn- run [& args]
-  (def p (os/spawn ["janet" "src/jolt/main.janet" ;args] :p {:out :pipe :err :pipe}))
+  (def p (os/spawn [;jolt-cmd ;args] :p {:out :pipe :err :pipe}))
   (def out (:read (p :out) :all))
   (os/proc-wait p)
   (string (or out "")))
 
 (defn- run-err [& args]
-  (def p (os/spawn ["janet" "src/jolt/main.janet" ;args] :p {:out :pipe :err :pipe}))
+  (def p (os/spawn [;jolt-cmd ;args] :p {:out :pipe :err :pipe}))
   (def err (:read (p :err) :all))
   (os/proc-wait p)
   (string (or err "")))

test/integration/inline-sra-test.janet

@@ -10,7 +10,7 @@
 (print "Inline + scalar replacement (jolt-87f)...")
 
 # A ctx with inlining ON (independent of the build-time JOLT_DIRECT_LINK).
-(def ctx (api/init {:compile? true}))
+(def ctx (api/init-cached {:compile? true}))
 (put (ctx :env) :direct-linking? true)
 (put (ctx :env) :inline? true)
 (api/eval-string ctx "(ns rt)")

test/integration/predicate-fold-test.janet

@@ -13,7 +13,7 @@
 (print "Predicate folding (jolt-wcw)...")
 
 (os/setenv "JOLT_DIRECT_LINK" "1")
-(def ctx (api/init {:compile? true}))
+(def ctx (api/init-cached {:compile? true}))
 (api/eval-string ctx "(ns pf)")
 
 (defn code [src]

test/integration/scalar-replace-record-test.janet

@@ -14,7 +14,7 @@
 (print "Scalar-replace of records (jolt-15jq)...")
 
 (os/setenv "JOLT_DIRECT_LINK" "1")
-(def ctx (api/init {:compile? true}))
+(def ctx (api/init-cached {:compile? true}))
 (api/eval-string ctx "(ns srr)")
 (api/eval-string ctx "(defrecord V3 [r g b])")
 

test/integration/sorted-rbtree-test.janet

@@ -8,7 +8,7 @@
 (print "sorted-map/set red-black tree (jolt-0hbr)...")
 
 (os/setenv "JOLT_DIRECT_LINK" "1")
-(def ctx (api/init {:compile? true}))
+(def ctx (api/init-cached {:compile? true}))
 (defn ev [s] (api/eval-string ctx s))
 
 (var fails 0)

test/integration/struct-hint-test.janet

@@ -13,7 +13,7 @@
 (print "Type hints (jolt-94n)...")
 
 (os/setenv "JOLT_DIRECT_LINK" "1")  # inline on, so hint-through-inline is exercised
-(def ctx (api/init {:compile? true}))
+(def ctx (api/init-cached {:compile? true}))
 (api/eval-string ctx "(ns sh)")
 (api/eval-string ctx "(defrecord Vec3r [r g b])")
 (each s ["(defn v3 [r g b] {:r r :g g :b b})"

test/integration/type-infer-phase1-test.janet

@@ -12,7 +12,7 @@
 (print "Type inference Phase 1 (jolt-767)...")
 
 (os/setenv "JOLT_DIRECT_LINK" "1")
-(def ctx (api/init {:compile? true}))
+(def ctx (api/init-cached {:compile? true}))
 (api/eval-string ctx "(ns p1)")
 # closed-world unit. mk is small (inlined away). rd is RECURSIVE, so it survives
 # inlining and is called via its var — exactly the shape (big/recursive fn with

test/integration/type-infer-phase2-test.janet

@@ -8,7 +8,7 @@
 (import ../../src/jolt/reader :as reader)
 (print "Type inference Phase 2 (vector ops)...")
 (os/setenv "JOLT_DIRECT_LINK" "1")
-(def ctx (api/init {:compile? true}))
+(def ctx (api/init-cached {:compile? true}))
 (api/eval-string ctx "(ns p2)")
 (def reinfer (types/var-get (types/ns-find (types/ctx-find-ns ctx "jolt.passes") "reinfer-def")))
 (defn estr [src ptmap]

test/integration/type-infer-phase3-test.janet

@@ -10,7 +10,7 @@
 (print "Type inference Phase 3 (jolt-d6u)...")
 
 (os/setenv "JOLT_DIRECT_LINK" "1")
-(def ctx (api/init {:compile? true}))
+(def ctx (api/init-cached {:compile? true}))
 (api/eval-string ctx "(ns p3)")
 (def pns (types/ctx-find-ns ctx "jolt.passes"))
 (def reinfer (types/var-get (types/ns-find pns "reinfer-def")))

test/integration/type-infer-test.janet

@@ -15,7 +15,7 @@
 (print "Type inference Phase 0 (jolt-6sr)...")
 
 (os/setenv "JOLT_DIRECT_LINK" "1")
-(def ctx (api/init {:compile? true}))
+(def ctx (api/init-cached {:compile? true}))
 (api/eval-string ctx "(ns ti)")
 
 (defn guards [src]