From edfd67a3220427df34589a8dafa287aa159cc6e5 Mon Sep 17 00:00:00 2001 From: Yogthos Date: Thu, 2 Jul 2026 03:46:57 -0400 Subject: [PATCH] Vendor clojure-test-suite as a standing gate (make cts) jank-lang/clojure-test-suite (per-core-fn clojure.test suites shared across Clojure dialects) joins the default gate as vendor/clojure-test-suite, run by host/chez/cts.sh: one joltc process per test namespace (a hang or crash is contained by a per-process timeout), through the test/chez/cts-app project and its cts-run runner, parallel workers. Gating is exact per namespace against test/chez/cts-known-failures.txt, like certify's allowlist: a namespace doing worse than the baseline fails, and one doing better also fails as stale until the baseline is updated in the same change. JOLT_CTS_WRITE_BASELINE=1 regenerates it; JOLT_CTS_NS runs a subset verbosely. Current standing: 243 namespaces, 5302 assertions pass, 340 fail + 236 error across 88 namespaces pinned in the baseline (dominant clusters: BigDecimal arithmetic operands, derive/ancestors hierarchy, transients, special-symbol?, clojure.string case fns, the accepted narrow-int and seq-type-model divergences). Two consecutive full runs produce identical counts. Wired into make ci; skips cleanly when the submodule isn't checked out. --- Makefile | 10 ++- host/chez/cts.sh | 120 ++++++++++++++++++++++++++++++ test/chez/README.md | 10 +++ test/chez/cts-app/deps.edn | 2 + test/chez/cts-app/src/cts_run.clj | 18 +++++ test/chez/cts-known-failures.txt | 91 ++++++++++++++++++++++ 6 files changed, 249 insertions(+), 2 deletions(-) create mode 100755 host/chez/cts.sh create mode 100644 test/chez/cts-app/deps.edn create mode 100644 test/chez/cts-app/src/cts_run.clj create mode 100644 test/chez/cts-known-failures.txt diff --git a/Makefile b/Makefile index 95b4260..84a569a 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ # build step. `make test` is the full gate. `make remint` rebuilds the seed after a # source change. -.PHONY: test ci values corpus unit smoke buildsmoke staticnativesmoke selfhost sci certify ffi transient infer wp devirt fieldread numwp fieldnum protoret narrow directlink numeric inline shakesmoke remint joltc joltc-release joltc-debug joltcsmoke +.PHONY: test ci values corpus unit smoke buildsmoke staticnativesmoke selfhost sci cts certify ffi transient infer wp devirt fieldread numwp fieldnum protoret narrow directlink numeric inline shakesmoke remint joltc joltc-release joltc-debug joltcsmoke # Full gate (dev machine). Includes the self-host byte-fixpoint, which only holds # on the same Chez that minted the seed. @@ -15,7 +15,7 @@ test: selfhost ci # lockfile) — it RUNS correctly on any Chez, but `selfhost` rebuilds it and a # different Chez version may emit byte-different (gensym/order) output, so the # byte-fixpoint is a dev-machine check, not a CI one (jolt-8479). -ci: values corpus unit smoke buildsmoke staticnativesmoke sci ffi transient infer wp devirt fieldread numwp fieldnum protoret narrow directlink numeric inline certify +ci: values corpus unit smoke buildsmoke staticnativesmoke sci cts ffi transient infer wp devirt fieldread numwp fieldnum protoret narrow directlink numeric inline certify @echo "OK: CI gates passed" # Self-host fixpoint: bootstrap.ss rebuild == checked-in seed. @@ -68,6 +68,12 @@ joltcsmoke: sci: @chez --script host/chez/run-sci.ss +# clojure-test-suite conformance: run the vendored jank-lang/clojure-test-suite +# per-namespace under joltc, gated on the per-namespace baseline +# (test/chez/cts-known-failures.txt). +cts: + @bash host/chez/cts.sh + # FFI: bind native functions (typed foreign-procedure), memory, and that a # :blocking call is collect-safe (a parked thread doesn't pin the collector). ffi: diff --git a/host/chez/cts.sh b/host/chez/cts.sh new file mode 100755 index 0000000..0c2f1cb --- /dev/null +++ b/host/chez/cts.sh @@ -0,0 +1,120 @@ +#!/bin/bash +# clojure-test-suite gate: run the vendored jank-lang/clojure-test-suite +# (vendor/clojure-test-suite) against joltc, one process per test namespace (a +# hang or crash is contained), and compare per-namespace fail/error counts +# against the checked-in baseline test/chez/cts-known-failures.txt. +# +# The comparison is exact, like certify's allowlist: a namespace doing WORSE +# than the baseline fails the gate (regression), and one doing BETTER also +# fails (stale baseline — update the file in the same change that improved it). +# +# JOLT_CTS_JOBS=N parallel workers (default 4) +# JOLT_CTS_TIMEOUT=SECS per-namespace timeout (default 120) +# JOLT_CTS_WRITE_BASELINE=1 regenerate the baseline file instead of gating +# JOLT_CTS_NS=ns1,ns2 run only these namespaces, verbose, no gating +set -u +root="$(CDPATH= cd -- "$(dirname -- "$0")/../.." && pwd)" +cd "$root" + +suite="vendor/clojure-test-suite/test" +baseline="test/chez/cts-known-failures.txt" +app="$root/test/chez/cts-app" +jobs="${JOLT_CTS_JOBS:-4}" +tmo="${JOLT_CTS_TIMEOUT:-120}" + +if [ ! -d "$suite/clojure" ]; then + echo "cts: skipped (git submodule update --init vendor/clojure-test-suite)" + exit 0 +fi + +work="$(mktemp -d)" +trap 'rm -rf "$work"' EXIT + +# test namespaces from the .cljc files (portability is a helper, not a test ns) +find "$suite" -name '*.cljc' | sed "s|^$suite/||;s|\.cljc$||;s|/|.|g;s|_|-|g" \ + | grep -v '\.portability$' | sort > "$work/nses" +if [ -n "${JOLT_CTS_NS:-}" ]; then + echo "${JOLT_CTS_NS}" | tr ',' '\n' > "$work/nses" +fi + +# round-robin the namespaces over N sequential workers; each worker appends +# "ns pass fail error" lines (HUNG/CRASH in the pass column) to its own file. +awk -v j="$jobs" '{print > ("'"$work"'/chunk." (NR % j))}' "$work/nses" +run_chunk() { + chunk="$1"; out="$2" + while IFS= read -r ns; do + res=$(JOLT_PWD="$app" perl -e "alarm $tmo; exec @ARGV" -- "$root/bin/joltc" -M:cts "$ns" 2>&1 > "$out" + if [ -n "${JOLT_CTS_NS:-}" ]; then + echo "$res" | grep -E 'FAIL:|ERROR:|LOAD:' | sed 's/^/ /' >> "$out" + fi + elif [ $rc -ge 128 ]; then + echo "$ns HUNG 0 0" >> "$out" + else + echo "$ns CRASH 0 0" >> "$out" + fi + done < "$chunk" +} +for c in "$work"/chunk.*; do + run_chunk "$c" "$c.res" & +done +wait +cat "$work"/chunk.*.res 2>/dev/null | sort > "$work/results" + +if [ -n "${JOLT_CTS_NS:-}" ]; then + cat "$work/results" + exit 0 +fi + +summary=$(awk '$2!="HUNG" && $2!="CRASH" {p+=$2; f+=$3; e+=$4; c++} + $2=="HUNG" {h++} $2=="CRASH" {x++} + END {printf "%d namespaces: pass %d, fail %d, error %d, hung %d, crash %d", + c+h+x, p, f, e, h, x}' "$work/results") + +if [ "${JOLT_CTS_WRITE_BASELINE:-0}" = "1" ]; then + { + echo "# clojure-test-suite known failures: " + echo "# The gate fails on any per-namespace change, worse OR better; regenerate" + echo "# with: JOLT_CTS_WRITE_BASELINE=1 host/chez/cts.sh" + awk '$2=="HUNG" || $2=="CRASH" {print $1, $2, $2; next} + $3 != 0 || $4 != 0 {print $1, $3, $4}' "$work/results" + } > "$baseline" + echo "cts: $summary" + echo "cts: baseline written to $baseline ($(grep -cv '^#' "$baseline") namespaces)" + exit 0 +fi + +if [ ! -f "$baseline" ]; then + echo "cts: FAIL — no baseline; run JOLT_CTS_WRITE_BASELINE=1 host/chez/cts.sh" + exit 1 +fi + +status=0 +while read -r ns p f e; do + case "$p" in HUNG|CRASH) f="$p"; e="$p" ;; esac + bl=$(grep -v '^#' "$baseline" | awk -v n="$ns" '$1==n {print $2, $3; exit}') + if [ -n "$bl" ]; then bf="${bl%% *}"; be="${bl##* }"; else bf=0; be=0; fi + if [ "$f" = "$bf" ] && [ "$e" = "$be" ]; then + continue + elif [ "$f" = "HUNG" ] || [ "$f" = "CRASH" ] \ + || { [ "$bf" != "HUNG" ] && [ "$bf" != "CRASH" ] \ + && { [ "$f" -gt "$bf" ] || [ "$e" -gt "$be" ]; }; }; then + echo "cts: NEW regression in $ns — fail $f error $e (baseline $bf $be)" + status=1 + else + echo "cts: STALE baseline for $ns — now fail $f error $e (baseline $bf $be); update $baseline" + status=1 + fi +done < "$work/results" + +# a baseline entry whose namespace no longer reports is stale too +while read -r ns bf be; do + grep -q "^$ns " "$work/results" || { echo "cts: STALE baseline entry $ns (namespace gone)"; status=1; } +done < <(grep -v '^#' "$baseline") + +echo "cts: $summary" +if [ $status -eq 0 ]; then echo "cts: passed (matches baseline)"; else echo "cts: FAILED"; fi +exit $status diff --git a/test/chez/README.md b/test/chez/README.md index 46a8a23..b0418ef 100644 --- a/test/chez/README.md +++ b/test/chez/README.md @@ -32,6 +32,16 @@ answers with `regen-corpus.clj` and re-certify with `test/conformance/certify.cl - `selfcheck.sh` — self-host fixpoint: `bootstrap.ss` rebuild byte-equals the checked-in seed (`host/chez/seed/`). - `smoke.sh` — real `bin/joltc -e` CLI smoke. +- `cts.sh` — the vendored [jank-lang/clojure-test-suite](https://github.com/jank-lang/clojure-test-suite) + (`vendor/clojure-test-suite`, a per-core-fn clojure.test suite shared across + Clojure dialects), run one namespace per `joltc` process (a hang or crash is + contained) through the `test/chez/cts-app` project and `cts-run` runner. + Per-namespace fail/error counts must exactly match the checked-in baseline + `test/chez/cts-known-failures.txt` — a namespace doing worse fails the gate, + and one doing better fails as stale until the baseline is updated in the same + change. `make cts`; + `JOLT_CTS_NS=ns1,ns2` runs a subset verbosely, + `JOLT_CTS_WRITE_BASELINE=1` regenerates the baseline. ## Other Chez tests diff --git a/test/chez/cts-app/deps.edn b/test/chez/cts-app/deps.edn new file mode 100644 index 0000000..ed313cc --- /dev/null +++ b/test/chez/cts-app/deps.edn @@ -0,0 +1,2 @@ +{:paths ["src" "../../../vendor/clojure-test-suite/test"] + :aliases {:cts {:main-opts ["-m" "cts-run"]}}} diff --git a/test/chez/cts-app/src/cts_run.clj b/test/chez/cts-app/src/cts_run.clj new file mode 100644 index 0000000..d1360e0 --- /dev/null +++ b/test/chez/cts-app/src/cts_run.clj @@ -0,0 +1,18 @@ +(ns cts-run + "Runner for the vendored jank-lang/clojure-test-suite (vendor/clojure-test-suite): + requires each test namespace given on the command line and runs its clojure.test + tests, printing one machine-readable result line per namespace. Driven per-process + by host/chez/cts.sh so a hang or crash in one namespace can't take out the run." + (:require [clojure.test :as t])) + +(defn -main [& nses] + (doseq [n nses] + (let [ns-sym (symbol n)] + (try + (require ns-sym) + (let [r (t/run-tests ns-sym)] + (println "CTS-RESULT" n (:pass r 0) (:fail r 0) (:error r 0))) + (catch Throwable e + (println "CTS-RESULT" n 0 0 1 + (str "LOAD: " (.getName (class e)) ": " (.getMessage e))))))) + (System/exit 0)) diff --git a/test/chez/cts-known-failures.txt b/test/chez/cts-known-failures.txt new file mode 100644 index 0000000..a52ec59 --- /dev/null +++ b/test/chez/cts-known-failures.txt @@ -0,0 +1,91 @@ +# clojure-test-suite known failures: +# The gate fails on any per-namespace change, worse OR better; regenerate +# with: JOLT_CTS_WRITE_BASELINE=1 host/chez/cts.sh +clojure.core-test.abs 1 1 +clojure.core-test.add-watch 0 3 +clojure.core-test.ancestors 9 0 +clojure.core-test.atom 14 0 +clojure.core-test.bigint 6 0 +clojure.core-test.bit-set 1 0 +clojure.core-test.boolean-qmark 0 1 +clojure.core-test.byte 7 5 +clojure.core-test.char 1 0 +clojure.core-test.compare 1 0 +clojure.core-test.conj 1 0 +clojure.core-test.contains-qmark 3 0 +clojure.core-test.counted-qmark 1 0 +clojure.core-test.dec 1 0 +clojure.core-test.denominator 0 3 +clojure.core-test.derive 21 0 +clojure.core-test.descendants 4 0 +clojure.core-test.double 0 4 +clojure.core-test.double-qmark 3 0 +clojure.core-test.empty 1 0 +clojure.core-test.eq 2 0 +clojure.core-test.eval 0 3 +clojure.core-test.even-qmark 1 0 +clojure.core-test.float 4 4 +clojure.core-test.get 0 1 +clojure.core-test.gt 0 2 +clojure.core-test.gt-eq 0 2 +clojure.core-test.ifn-qmark 2 0 +clojure.core-test.inc 1 0 +clojure.core-test.int 4 5 +clojure.core-test.int-qmark 3 0 +clojure.core-test.intern 2 0 +clojure.core-test.keys 0 4 +clojure.core-test.lazy-seq 3 0 +clojure.core-test.long 2 5 +clojure.core-test.lt 0 2 +clojure.core-test.lt-eq 0 2 +clojure.core-test.max 3 1 +clojure.core-test.min 3 1 +clojure.core-test.minus 2 0 +clojure.core-test.mod 12 34 +clojure.core-test.neg-int-qmark 1 0 +clojure.core-test.not-eq 3 0 +clojure.core-test.nth 0 1 +clojure.core-test.num 2 1 +clojure.core-test.number-qmark 3 0 +clojure.core-test.numerator 0 3 +clojure.core-test.odd-qmark 1 0 +clojure.core-test.parents 10 0 +clojure.core-test.parse-uuid 3 0 +clojure.core-test.peek 2 0 +clojure.core-test.plus 11 0 +clojure.core-test.plus-squote 11 0 +clojure.core-test.pop 0 1 +clojure.core-test.pos-int-qmark 1 0 +clojure.core-test.quot 12 31 +clojure.core-test.rand-nth 0 1 +clojure.core-test.rational-qmark 3 0 +clojure.core-test.rationalize 10 0 +clojure.core-test.realized-qmark 3 0 +clojure.core-test.reduce 0 1 +clojure.core-test.rem 12 30 +clojure.core-test.remove-watch 0 1 +clojure.core-test.run-bang 1 0 +clojure.core-test.select-keys 2 0 +clojure.core-test.seqable-qmark 1 0 +clojure.core-test.short 7 5 +clojure.core-test.shuffle 1 0 +clojure.core-test.slash 2 48 +clojure.core-test.some-fn 3 0 +clojure.core-test.sort-by 2 0 +clojure.core-test.special-symbol-qmark 20 0 +clojure.core-test.star 22 0 +clojure.core-test.star-squote 19 0 +clojure.core-test.transient 23 0 +clojure.core-test.underive 7 0 +clojure.core-test.update 1 0 +clojure.core-test.vals 0 3 +clojure.core-test.vec 1 0 +clojure.core-test.when-let 1 0 +clojure.core-test.with-precision 17 0 +clojure.edn-test.read-string 0 1 +clojure.string-test.capitalize 0 4 +clojure.string-test.ends-with-qmark 1 4 +clojure.string-test.escape 1 0 +clojure.string-test.lower-case 0 4 +clojure.string-test.starts-with-qmark 1 10 +clojure.string-test.upper-case 0 4