0ca678a159
Run the external cross-dialect clojure-test-suite (lread/clojure-test-suite, 240 per-fn .cljc files in clojure.test format) against Jolt: - test/support/clojure_test.clj: minimal clojure.test + portability shims (deftest/is/testing/are + when-var-exists/thrown?/big-int?/lazy-seq?) — just enough surface to load the suite and tally pass/fail/error. Pre-loaded so the suite's (require [clojure.test ...]) finds it already populated. - test/integration/suite-worker.janet: one-shot worker that loads the shim, the suite's number-range helper ns, and a single .cljc file, then prints 'pass fail error'. - test/integration/clojure-test-suite-test.janet: spawns a worker per file under an ev/with-deadline wall-clock budget (so infinite-seq tests that hang Jolt's eager evaluator are auto-contained, not a manual skip-list) and asserts pass/clean-file counts stay at/above a baseline. References ~/src/clojure-test-suite if present; skips cleanly when absent, like the jank battery. Current: 210 files run, 7 timed out, 2233 assertions -> 1683 pass / 350 fail / 200 error, 23 clean files. Remaining fails are genuine divergences (float/ratio/ bigint, lenient transients where Clojure throws), tracked separately. Fixes two real evaluator bugs the suite surfaced: - :refer now preserves a referred macro's :macro flag (was interned as a plain value, degrading referred macros to functions). - resolve-var now resolves ns aliases (like resolve-sym), so aliased macros (e.g. p/thrown? via :as p) dispatch as macros instead of being called as fns.
107 lines
4.7 KiB
Text
107 lines
4.7 KiB
Text
# clojure-test-suite conformance: runs the external, cross-dialect
|
|
# clojure-test-suite (https://github.com/lread/clojure-test-suite, EPL) against
|
|
# Jolt and asserts the number of passing per-function test files stays at/above
|
|
# a baseline. Like the jank battery, this does NOT vendor the suite — it
|
|
# references ~/src/clojure-test-suite if present and SKIPS cleanly when absent.
|
|
#
|
|
# Each suite file is a `clojure.test` namespace (one per clojure.core/string
|
|
# function). A minimal clojure.test + portability shim (test/support/clojure_test.clj)
|
|
# lets Jolt load them; `when-var-exists` auto-skips fns Jolt doesn't implement.
|
|
#
|
|
# Files are run in a one-shot worker subprocess (test/integration/suite-worker.janet)
|
|
# under a wall-clock deadline. Some suite tests build infinite sequences
|
|
# (cycle/range/transducers-over-infinite) that Jolt's eager evaluator can't
|
|
# truncate and so HANG rather than fail; the deadline contains them — a timed-out
|
|
# file is reported as :timeout and contributes nothing, no manual skip-list needed.
|
|
|
|
(def suite-dir (string (os/getenv "HOME") "/src/clojure-test-suite/test/clojure"))
|
|
|
|
# Baseline: assertions Jolt currently passes across the suite. Raise as Jolt
|
|
# improves so a regression (previously-passing assertion breaking) is caught.
|
|
(def baseline-pass 1650)
|
|
# A file is "clean" when it ran with zero failures AND zero errors.
|
|
(def baseline-clean-files 22)
|
|
# Per-file wall-clock budget (seconds). Normal files finish in well under 1s;
|
|
# this only fires on infinite-sequence hangs.
|
|
(def per-file-timeout 6)
|
|
|
|
(defn- walk [dir acc]
|
|
(each e (os/dir dir)
|
|
(def p (string dir "/" e))
|
|
(case ((os/stat p) :mode)
|
|
:directory (walk p acc)
|
|
:file (when (and (string/has-suffix? ".cljc" p)
|
|
(not (string/has-suffix? "portability.cljc" p)))
|
|
(array/push acc p))))
|
|
acc)
|
|
|
|
# Run one file in a worker subprocess; return its "pass fail error" stdout, or
|
|
# nil if it exceeded the deadline (hang) or crashed.
|
|
(defn- run-file [path]
|
|
(def proc (os/spawn ["janet" "test/integration/suite-worker.janet" path] :p {:out :pipe}))
|
|
(def out (proc :out))
|
|
(var data nil)
|
|
(def ok
|
|
(try
|
|
(ev/with-deadline per-file-timeout
|
|
(set data (ev/read out 0x10000)) # workers print a single short line
|
|
(os/proc-wait proc)
|
|
true)
|
|
([err] false)))
|
|
(when (not ok)
|
|
(protect (os/proc-kill proc true))
|
|
(protect (ev/with-deadline 2 (os/proc-wait proc))))
|
|
(protect (:close out))
|
|
(if (and ok data) (string data) nil))
|
|
|
|
(defn- parse-counts [s]
|
|
# s is "pass fail error"
|
|
(def parts (string/split " " (string/trim s)))
|
|
(if (= 3 (length parts))
|
|
[(scan-number (parts 0)) (scan-number (parts 1)) (scan-number (parts 2))]
|
|
nil))
|
|
|
|
(if (not (os/stat suite-dir))
|
|
(print "clojure-test-suite: ~/src/clojure-test-suite not present — skipped")
|
|
(do
|
|
(def progress? (os/getenv "SUITE_PROGRESS"))
|
|
(def files (sort (walk suite-dir @[])))
|
|
(var total-pass 0)
|
|
(var total-fail 0)
|
|
(var total-error 0)
|
|
(var clean-files 0)
|
|
(var ran-files 0)
|
|
(var timeouts 0)
|
|
(def worst @[])
|
|
|
|
(each path files
|
|
(def rel (string/slice path (+ 1 (length suite-dir))))
|
|
(when progress? (eprintf " %s" rel) (eflush))
|
|
(def out (run-file path))
|
|
(def counts (and out (parse-counts out)))
|
|
(cond
|
|
(nil? out) (do (++ timeouts) (when progress? (eprint " TIMEOUT")))
|
|
(nil? counts) (when progress? (eprint " (no counts)"))
|
|
(let [[pn fn* en] counts]
|
|
(++ ran-files)
|
|
(+= total-pass pn)
|
|
(+= total-fail fn*)
|
|
(+= total-error en)
|
|
(when (and (= 0 fn*) (= 0 en) (> pn 0)) (++ clean-files))
|
|
(when (> (+ fn* en) 0) (array/push worst [(+ fn* en) rel pn fn* en])))))
|
|
|
|
(def total (+ total-pass total-fail total-error))
|
|
(printf "\nclojure-test-suite: %d files ran (%d timed out), %d assertions — %d pass / %d fail / %d error"
|
|
ran-files timeouts total total-pass total-fail total-error)
|
|
(printf "\n clean files (0 fail/error, >0 pass): %d" clean-files)
|
|
(sort-by (fn [x] (- (x 0))) worst)
|
|
(when (> (length worst) 0)
|
|
(print " top files by fail+error:")
|
|
(each w (slice worst 0 (min 15 (length worst)))
|
|
(printf " %-40s pass=%d fail=%d err=%d" (w 1) (w 2) (w 3) (w 4))))
|
|
|
|
(assert (>= total-pass baseline-pass)
|
|
(string/format "regression: total-pass %d < baseline %d" total-pass baseline-pass))
|
|
(assert (>= clean-files baseline-clean-files)
|
|
(string/format "regression: clean-files %d < baseline %d" clean-files baseline-clean-files))
|
|
(printf "\nclojure-test-suite: OK (>= %d pass, >= %d clean files)\n" baseline-pass baseline-clean-files)))
|