simon/jolt
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

test: port clojure-test-suite as baseline-guarded integration battery

Browse source 
Run the external cross-dialect clojure-test-suite (lread/clojure-test-suite,
240 per-fn .cljc files in clojure.test format) against Jolt:

- test/support/clojure_test.clj: minimal clojure.test + portability shims
  (deftest/is/testing/are + when-var-exists/thrown?/big-int?/lazy-seq?) — just
  enough surface to load the suite and tally pass/fail/error. Pre-loaded so the
  suite's (require [clojure.test ...]) finds it already populated.
- test/integration/suite-worker.janet: one-shot worker that loads the shim,
  the suite's number-range helper ns, and a single .cljc file, then prints
  'pass fail error'.
- test/integration/clojure-test-suite-test.janet: spawns a worker per file
  under an ev/with-deadline wall-clock budget (so infinite-seq tests that hang
  Jolt's eager evaluator are auto-contained, not a manual skip-list) and asserts
  pass/clean-file counts stay at/above a baseline. References ~/src/clojure-test-suite
  if present; skips cleanly when absent, like the jank battery.

Current: 210 files run, 7 timed out, 2233 assertions -> 1683 pass / 350 fail /
200 error, 23 clean files. Remaining fails are genuine divergences (float/ratio/
bigint, lenient transients where Clojure throws), tracked separately.

Fixes two real evaluator bugs the suite surfaced:
- :refer now preserves a referred macro's :macro flag (was interned as a plain
  value, degrading referred macros to functions).
- resolve-var now resolves ns aliases (like resolve-sym), so aliased macros
  (e.g. p/thrown? via :as p) dispatch as macros instead of being called as fns.
This commit is contained in:
Yogthos 2026-06-05 09:04:38 -04:00
parent f38d402445
commit 0ca678a159
6 changed files with 280 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

107
test/integration/clojure-test-suite-test.janet Normal file
View file

@ -0,0 +1,107 @@
# clojure-test-suite conformance: runs the external, cross-dialect
# clojure-test-suite (https://github.com/lread/clojure-test-suite, EPL) against
# Jolt and asserts the number of passing per-function test files stays at/above
# a baseline. Like the jank battery, this does NOT vendor the suite — it
# references ~/src/clojure-test-suite if present and SKIPS cleanly when absent.
#
# Each suite file is a `clojure.test` namespace (one per clojure.core/string
# function). A minimal clojure.test + portability shim (test/support/clojure_test.clj)
# lets Jolt load them; `when-var-exists` auto-skips fns Jolt doesn't implement.
#
# Files are run in a one-shot worker subprocess (test/integration/suite-worker.janet)
# under a wall-clock deadline. Some suite tests build infinite sequences
# (cycle/range/transducers-over-infinite) that Jolt's eager evaluator can't
# truncate and so HANG rather than fail; the deadline contains them — a timed-out
# file is reported as :timeout and contributes nothing, no manual skip-list needed.
(def suite-dir (string (os/getenv "HOME") "/src/clojure-test-suite/test/clojure"))
# Baseline: assertions Jolt currently passes across the suite. Raise as Jolt
# improves so a regression (previously-passing assertion breaking) is caught.
(def baseline-pass 1650)
# A file is "clean" when it ran with zero failures AND zero errors.
(def baseline-clean-files 22)
# Per-file wall-clock budget (seconds). Normal files finish in well under 1s;
# this only fires on infinite-sequence hangs.
(def per-file-timeout 6)
(defn- walk [dir acc]
(each e (os/dir dir)
(def p (string dir "/" e))
(case ((os/stat p) :mode)
:directory (walk p acc)
:file (when (and (string/has-suffix? ".cljc" p)
(not (string/has-suffix? "portability.cljc" p)))
(array/push acc p))))
acc)
# Run one file in a worker subprocess; return its "pass fail error" stdout, or
# nil if it exceeded the deadline (hang) or crashed.
(defn- run-file [path]
(def proc (os/spawn ["janet" "test/integration/suite-worker.janet" path] :p {:out :pipe}))
(def out (proc :out))
(var data nil)
(def ok
(try
(ev/with-deadline per-file-timeout
(set data (ev/read out 0x10000)) # workers print a single short line
(os/proc-wait proc)
true)
([err] false)))
(when (not ok)
(protect (os/proc-kill proc true))
(protect (ev/with-deadline 2 (os/proc-wait proc))))
(protect (:close out))
(if (and ok data) (string data) nil))
(defn- parse-counts [s]
# s is "pass fail error"
(def parts (string/split " " (string/trim s)))
(if (= 3 (length parts))
[(scan-number (parts 0)) (scan-number (parts 1)) (scan-number (parts 2))]
nil))
(if (not (os/stat suite-dir))
(print "clojure-test-suite: ~/src/clojure-test-suite not present — skipped")
(do
(def progress? (os/getenv "SUITE_PROGRESS"))
(def files (sort (walk suite-dir @[])))
(var total-pass 0)
(var total-fail 0)
(var total-error 0)
(var clean-files 0)
(var ran-files 0)
(var timeouts 0)
(def worst @[])
(each path files
(def rel (string/slice path (+ 1 (length suite-dir))))
(when progress? (eprintf " %s" rel) (eflush))
(def out (run-file path))
(def counts (and out (parse-counts out)))
(cond
(nil? out) (do (++ timeouts) (when progress? (eprint " TIMEOUT")))
(nil? counts) (when progress? (eprint " (no counts)"))
(let [[pn fn* en] counts]
(++ ran-files)
(+= total-pass pn)
(+= total-fail fn*)
(+= total-error en)
(when (and (= 0 fn*) (= 0 en) (> pn 0)) (++ clean-files))
(when (> (+ fn* en) 0) (array/push worst [(+ fn* en) rel pn fn* en])))))
(def total (+ total-pass total-fail total-error))
(printf "\nclojure-test-suite: %d files ran (%d timed out), %d assertions — %d pass / %d fail / %d error"
ran-files timeouts total total-pass total-fail total-error)
(printf "\n clean files (0 fail/error, >0 pass): %d" clean-files)
(sort-by (fn [x] (- (x 0))) worst)
(when (> (length worst) 0)
(print " top files by fail+error:")
(each w (slice worst 0 (min 15 (length worst)))
(printf " %-40s pass=%d fail=%d err=%d" (w 1) (w 2) (w 3) (w 4))))
(assert (>= total-pass baseline-pass)
(string/format "regression: total-pass %d < baseline %d" total-pass baseline-pass))
(assert (>= clean-files baseline-clean-files)
(string/format "regression: clean-files %d < baseline %d" clean-files baseline-clean-files))
(printf "\nclojure-test-suite: OK (>= %d pass, >= %d clean files)\n" baseline-pass baseline-clean-files)))

40
test/integration/suite-worker.janet Normal file
View file

@ -0,0 +1,40 @@
# One-file worker for the clojure-test-suite battery. Loads the clojure.test
# shim, evaluates a single suite .cljc file, runs its deftests, and prints
# "pass fail error" to stdout. Used by the discovery pass to find files that
# hang under Jolt's eager evaluation (run under an external timeout).
(use ../../src/jolt/api)
(use ../../src/jolt/reader)
(use ../../src/jolt/evaluator)
(defn- parse-forms [src]
(var s src) (def fs @[]) (var go true)
(while (and go (> (length (string/trim s)) 0))
(def r (protect (parse-next s)))
(if (not (r 0)) (set go false)
(let [p (r 1)] (set s (p 1)) (when (not (nil? (p 0))) (array/push fs (p 0))))))
fs)
# A helper, not a standalone test: it needs a .cljc path argument. When `jpm
# test` runs it with no args, no-op cleanly so it doesn't count as a failure.
(def path (get (dyn :args) 1))
(when path
(def ctx (init))
(each f (parse-forms (slurp "test/support/clojure_test.clj")) (eval-form ctx @{} f))
# Pre-load the suite's own clojure.core-test.number-range helper ns if present
# (35 files require it for r/max-int, r/max-double, … — its :default branches are
# plain numeric literals Jolt can read). Its `ns` form sets the namespace; the
# test file's own `ns` form switches back afterwards.
(let [dir (string/slice path 0 (- (length path) (length (last (string/split "/" path)))))
nr (string dir "number_range.cljc")]
(when (os/stat nr)
(each f (parse-forms (slurp nr)) (protect (eval-form ctx @{} f)))))
(eval-string ctx "(clojure.test/reset-report!)")
(each form (parse-forms (slurp path)) (protect (eval-form ctx @{} form)))
(protect (eval-string ctx "(clojure.test/run-registered)"))
(def p (eval-string ctx "(clojure.test/n-pass)"))
(def f (eval-string ctx "(clojure.test/n-fail)"))
(def e (eval-string ctx "(clojure.test/n-error)"))
(printf "%d %d %d" (if (number? p) p 0) (if (number? f) f 0) (if (number? e) e 0)))

116
test/support/clojure_test.clj Normal file
View file

@ -0,0 +1,116 @@
;; Minimal `clojure.test` + `clojure.core-test.portability` shims.
;;
;; These exist solely so Jolt can load the external clojure-test-suite
;; (https://github.com/lread/clojure-test-suite, EPL) — 240 per-function
;; `.cljc` files written against `clojure.test`. They are NOT a full
;; clojure.test: just enough surface (deftest/is/testing/are + the suite's
;; portability helpers) to run the suite and tally pass/fail/error.
;;
;; Loaded by test/integration/clojure-test-suite-test.janet, which pre-loads
;; this file so the suite's `(require [clojure.test ...])` finds it already
;; populated (Jolt's require is a no-op for already-loaded namespaces).
(ns clojure.test)
;; --- result accumulator (plain atoms; no dynamic vars needed) -------------
(def jolt-report (atom {:pass 0 :fail 0 :error 0 :fails []}))
(def jolt-ctx (atom [])) ;; stack of `testing` context strings
(def registry (atom [])) ;; deftest fns defined since last reset
(defn reset-report! []
(reset! jolt-report {:pass 0 :fail 0 :error 0 :fails []})
(reset! jolt-ctx [])
(reset! registry []))
(defn- ctx-str [] (apply str (interpose " " @jolt-ctx)))
(defn inc-pass! [] (swap! jolt-report update :pass inc))
(defn fail! [form]
(swap! jolt-report
(fn [r] (-> r (update :fail inc)
(update :fails conj (str (ctx-str) " FAIL: " form))))))
(defn err! [form]
(swap! jolt-report
(fn [r] (-> r (update :error inc)
(update :fails conj (str (ctx-str) " ERROR: " form))))))
(defn n-pass [] (:pass @jolt-report))
(defn n-fail [] (:fail @jolt-report))
(defn n-error [] (:error @jolt-report))
(defn failures [] (:fails @jolt-report))
;; --- assertion macros ------------------------------------------------------
;; `(is form)` / `(is form msg)` — the optional msg is absorbed and ignored.
(defmacro is [form & _]
`(try
(if ~form
(clojure.test/inc-pass!)
(clojure.test/fail! (pr-str '~form)))
(catch :default e#
(clojure.test/err! (str (pr-str '~form) " threw")))))
(defmacro testing [s & body]
`(do
(swap! clojure.test/jolt-ctx conj ~s)
(try
(do ~@body)
(finally (swap! clojure.test/jolt-ctx pop)))))
(defmacro deftest [name & body]
`(do
(def ~name (fn [] ~@body))
(swap! clojure.test/registry conj ~name)
~name))
;; `(are [bindings] expr & data)` — substitute each row of `data` into the
;; template and assert via `is`. Expands to a `do` of let+is forms.
(defmacro are [argv expr & data]
(let [n (count argv)
rows (partition n data)]
`(do ~@(map (fn [row]
`(let [~@(interleave argv row)]
(clojure.test/is ~expr)))
rows))))
;; Run every deftest registered since the last reset, isolating crashes.
(defn run-registered []
(doseq [t @registry]
(try (t) (catch :default e (clojure.test/err! "deftest crashed"))))
nil)
;; clojure.test entry points the suite may call — no-ops; the Janet runner
;; drives execution via run-registered instead.
(defn run-tests [& _] nil)
(defn run-test [& _] nil)
(defn test-var [& _] nil)
;; --- clojure.core-test.portability ----------------------------------------
(ns clojure.core-test.portability)
;; Gate a test on whether its target var exists in this dialect. Jolt only
;; implements a subset of clojure.core, so unimplemented fns get skipped
;; cleanly rather than erroring.
(defmacro when-var-exists [var-sym & body]
(if (resolve var-sym)
`(do ~@body)
`(println "SKIP -" '~var-sym)))
;; `(thrown? body)` — true iff evaluating body throws. The suite always uses
;; the single-arg (no exception-class) form via this portability helper.
(defmacro thrown? [& body]
`(try (do ~@body false) (catch :default e# true)))
(defn big-int? [n]
(and (integer? n) (not (int? n))))
(defn lazy-seq? [x]
;; Jolt has no public LazySeq type test; approximate with seq?-of-non-vector.
(and (seq? x) (not (vector? x))))
(defn sleep [ms] nil)