Vendor clojure-test-suite as a standing gate (make cts)

jank-lang/clojure-test-suite (per-core-fn clojure.test suites shared across
Clojure dialects) joins the default gate as vendor/clojure-test-suite, run by
host/chez/cts.sh: one joltc process per test namespace (a hang or crash is
contained by a per-process timeout), through the test/chez/cts-app project and
its cts-run runner, parallel workers.

Gating is exact per namespace against test/chez/cts-known-failures.txt, like
certify's allowlist: a namespace doing worse than the baseline fails, and one
doing better also fails as stale until the baseline is updated in the same
change. JOLT_CTS_WRITE_BASELINE=1 regenerates it; JOLT_CTS_NS runs a subset
verbosely.

Current standing: 243 namespaces, 5302 assertions pass, 340 fail + 236 error
across 88 namespaces pinned in the baseline (dominant clusters: BigDecimal
arithmetic operands, derive/ancestors hierarchy, transients, special-symbol?,
clojure.string case fns, the accepted narrow-int and seq-type-model
divergences). Two consecutive full runs produce identical counts. Wired into
make ci; skips cleanly when the submodule isn't checked out.

Makefile

@@ -4,7 +4,7 @@
 # build step. `make test` is the full gate. `make remint` rebuilds the seed after a
 # source change.
 
-.PHONY: test ci values corpus unit smoke buildsmoke staticnativesmoke selfhost sci certify ffi transient infer wp devirt fieldread numwp fieldnum protoret narrow directlink numeric inline shakesmoke remint joltc joltc-release joltc-debug joltcsmoke
+.PHONY: test ci values corpus unit smoke buildsmoke staticnativesmoke selfhost sci cts certify ffi transient infer wp devirt fieldread numwp fieldnum protoret narrow directlink numeric inline shakesmoke remint joltc joltc-release joltc-debug joltcsmoke
 
 # Full gate (dev machine). Includes the self-host byte-fixpoint, which only holds
 # on the same Chez that minted the seed.
@@ -15,7 +15,7 @@ test: selfhost ci
 # lockfile) — it RUNS correctly on any Chez, but `selfhost` rebuilds it and a
 # different Chez version may emit byte-different (gensym/order) output, so the
 # byte-fixpoint is a dev-machine check, not a CI one (jolt-8479).
-ci: values corpus unit smoke buildsmoke staticnativesmoke sci ffi transient infer wp devirt fieldread numwp fieldnum protoret narrow directlink numeric inline certify
+ci: values corpus unit smoke buildsmoke staticnativesmoke sci cts ffi transient infer wp devirt fieldread numwp fieldnum protoret narrow directlink numeric inline certify
 	@echo "OK: CI gates passed"
 
 # Self-host fixpoint: bootstrap.ss rebuild == checked-in seed.
@@ -68,6 +68,12 @@ joltcsmoke:
 sci:
 	@chez --script host/chez/run-sci.ss
 
+# clojure-test-suite conformance: run the vendored jank-lang/clojure-test-suite
+# per-namespace under joltc, gated on the per-namespace baseline
+# (test/chez/cts-known-failures.txt).
+cts:
+	@bash host/chez/cts.sh
+
 # FFI: bind native functions (typed foreign-procedure), memory, and that a
 # :blocking call is collect-safe (a parked thread doesn't pin the collector).
 ffi:

host/chez/cts.sh

@@ -0,0 +1,120 @@
+#!/bin/bash
+# clojure-test-suite gate: run the vendored jank-lang/clojure-test-suite
+# (vendor/clojure-test-suite) against joltc, one process per test namespace (a
+# hang or crash is contained), and compare per-namespace fail/error counts
+# against the checked-in baseline test/chez/cts-known-failures.txt.
+#
+# The comparison is exact, like certify's allowlist: a namespace doing WORSE
+# than the baseline fails the gate (regression), and one doing BETTER also
+# fails (stale baseline — update the file in the same change that improved it).
+#
+#   JOLT_CTS_JOBS=N            parallel workers (default 4)
+#   JOLT_CTS_TIMEOUT=SECS      per-namespace timeout (default 120)
+#   JOLT_CTS_WRITE_BASELINE=1  regenerate the baseline file instead of gating
+#   JOLT_CTS_NS=ns1,ns2        run only these namespaces, verbose, no gating
+set -u
+root="$(CDPATH= cd -- "$(dirname -- "$0")/../.." && pwd)"
+cd "$root"
+
+suite="vendor/clojure-test-suite/test"
+baseline="test/chez/cts-known-failures.txt"
+app="$root/test/chez/cts-app"
+jobs="${JOLT_CTS_JOBS:-4}"
+tmo="${JOLT_CTS_TIMEOUT:-120}"
+
+if [ ! -d "$suite/clojure" ]; then
+  echo "cts: skipped (git submodule update --init vendor/clojure-test-suite)"
+  exit 0
+fi
+
+work="$(mktemp -d)"
+trap 'rm -rf "$work"' EXIT
+
+# test namespaces from the .cljc files (portability is a helper, not a test ns)
+find "$suite" -name '*.cljc' | sed "s|^$suite/||;s|\.cljc$||;s|/|.|g;s|_|-|g" \
+  | grep -v '\.portability$' | sort > "$work/nses"
+if [ -n "${JOLT_CTS_NS:-}" ]; then
+  echo "${JOLT_CTS_NS}" | tr ',' '\n' > "$work/nses"
+fi
+
+# round-robin the namespaces over N sequential workers; each worker appends
+# "ns pass fail error" lines (HUNG/CRASH in the pass column) to its own file.
+awk -v j="$jobs" '{print > ("'"$work"'/chunk." (NR % j))}' "$work/nses"
+run_chunk() {
+  chunk="$1"; out="$2"
+  while IFS= read -r ns; do
+    res=$(JOLT_PWD="$app" perl -e "alarm $tmo; exec @ARGV" -- "$root/bin/joltc" -M:cts "$ns" 2>&1 </dev/null)
+    rc=$?
+    line=$(echo "$res" | grep '^CTS-RESULT' | head -1)
+    if [ -n "$line" ]; then
+      echo "$line" | awk '{print $2, $3, $4, $5}' >> "$out"
+      if [ -n "${JOLT_CTS_NS:-}" ]; then
+        echo "$res" | grep -E 'FAIL:|ERROR:|LOAD:' | sed 's/^/    /' >> "$out"
+      fi
+    elif [ $rc -ge 128 ]; then
+      echo "$ns HUNG 0 0" >> "$out"
+    else
+      echo "$ns CRASH 0 0" >> "$out"
+    fi
+  done < "$chunk"
+}
+for c in "$work"/chunk.*; do
+  run_chunk "$c" "$c.res" &
+done
+wait
+cat "$work"/chunk.*.res 2>/dev/null | sort > "$work/results"
+
+if [ -n "${JOLT_CTS_NS:-}" ]; then
+  cat "$work/results"
+  exit 0
+fi
+
+summary=$(awk '$2!="HUNG" && $2!="CRASH" {p+=$2; f+=$3; e+=$4; c++}
+               $2=="HUNG" {h++} $2=="CRASH" {x++}
+               END {printf "%d namespaces: pass %d, fail %d, error %d, hung %d, crash %d",
+                    c+h+x, p, f, e, h, x}' "$work/results")
+
+if [ "${JOLT_CTS_WRITE_BASELINE:-0}" = "1" ]; then
+  {
+    echo "# clojure-test-suite known failures: <namespace> <fail> <error>"
+    echo "# The gate fails on any per-namespace change, worse OR better; regenerate"
+    echo "# with: JOLT_CTS_WRITE_BASELINE=1 host/chez/cts.sh"
+    awk '$2=="HUNG" || $2=="CRASH" {print $1, $2, $2; next}
+         $3 != 0 || $4 != 0 {print $1, $3, $4}' "$work/results"
+  } > "$baseline"
+  echo "cts: $summary"
+  echo "cts: baseline written to $baseline ($(grep -cv '^#' "$baseline") namespaces)"
+  exit 0
+fi
+
+if [ ! -f "$baseline" ]; then
+  echo "cts: FAIL — no baseline; run JOLT_CTS_WRITE_BASELINE=1 host/chez/cts.sh"
+  exit 1
+fi
+
+status=0
+while read -r ns p f e; do
+  case "$p" in HUNG|CRASH) f="$p"; e="$p" ;; esac
+  bl=$(grep -v '^#' "$baseline" | awk -v n="$ns" '$1==n {print $2, $3; exit}')
+  if [ -n "$bl" ]; then bf="${bl%% *}"; be="${bl##* }"; else bf=0; be=0; fi
+  if [ "$f" = "$bf" ] && [ "$e" = "$be" ]; then
+    continue
+  elif [ "$f" = "HUNG" ] || [ "$f" = "CRASH" ] \
+       || { [ "$bf" != "HUNG" ] && [ "$bf" != "CRASH" ] \
+            && { [ "$f" -gt "$bf" ] || [ "$e" -gt "$be" ]; }; }; then
+    echo "cts: NEW regression in $ns — fail $f error $e (baseline $bf $be)"
+    status=1
+  else
+    echo "cts: STALE baseline for $ns — now fail $f error $e (baseline $bf $be); update $baseline"
+    status=1
+  fi
+done < "$work/results"
+
+# a baseline entry whose namespace no longer reports is stale too
+while read -r ns bf be; do
+  grep -q "^$ns " "$work/results" || { echo "cts: STALE baseline entry $ns (namespace gone)"; status=1; }
+done < <(grep -v '^#' "$baseline")
+
+echo "cts: $summary"
+if [ $status -eq 0 ]; then echo "cts: passed (matches baseline)"; else echo "cts: FAILED"; fi
+exit $status

test/chez/README.md

@@ -32,6 +32,16 @@ answers with `regen-corpus.clj` and re-certify with `test/conformance/certify.cl
 - `selfcheck.sh` — self-host fixpoint: `bootstrap.ss` rebuild byte-equals the
   checked-in seed (`host/chez/seed/`).
 - `smoke.sh` — real `bin/joltc -e` CLI smoke.
+- `cts.sh` — the vendored [jank-lang/clojure-test-suite](https://github.com/jank-lang/clojure-test-suite)
+  (`vendor/clojure-test-suite`, a per-core-fn clojure.test suite shared across
+  Clojure dialects), run one namespace per `joltc` process (a hang or crash is
+  contained) through the `test/chez/cts-app` project and `cts-run` runner.
+  Per-namespace fail/error counts must exactly match the checked-in baseline
+  `test/chez/cts-known-failures.txt` — a namespace doing worse fails the gate,
+  and one doing better fails as stale until the baseline is updated in the same
+  change. `make cts`;
+  `JOLT_CTS_NS=ns1,ns2` runs a subset verbosely,
+  `JOLT_CTS_WRITE_BASELINE=1` regenerates the baseline.
 
 ## Other Chez tests
 

test/chez/cts-app/deps.edn

@@ -0,0 +1,2 @@
+{:paths ["src" "../../../vendor/clojure-test-suite/test"]
+ :aliases {:cts {:main-opts ["-m" "cts-run"]}}}

test/chez/cts-app/src/cts_run.clj

@@ -0,0 +1,18 @@
+(ns cts-run
+  "Runner for the vendored jank-lang/clojure-test-suite (vendor/clojure-test-suite):
+  requires each test namespace given on the command line and runs its clojure.test
+  tests, printing one machine-readable result line per namespace. Driven per-process
+  by host/chez/cts.sh so a hang or crash in one namespace can't take out the run."
+  (:require [clojure.test :as t]))
+
+(defn -main [& nses]
+  (doseq [n nses]
+    (let [ns-sym (symbol n)]
+      (try
+        (require ns-sym)
+        (let [r (t/run-tests ns-sym)]
+          (println "CTS-RESULT" n (:pass r 0) (:fail r 0) (:error r 0)))
+        (catch Throwable e
+          (println "CTS-RESULT" n 0 0 1
+                   (str "LOAD: " (.getName (class e)) ": " (.getMessage e)))))))
+  (System/exit 0))

test/chez/cts-known-failures.txt

@@ -0,0 +1,91 @@
+# clojure-test-suite known failures: <namespace> <fail> <error>
+# The gate fails on any per-namespace change, worse OR better; regenerate
+# with: JOLT_CTS_WRITE_BASELINE=1 host/chez/cts.sh
+clojure.core-test.abs 1 1
+clojure.core-test.add-watch 0 3
+clojure.core-test.ancestors 9 0
+clojure.core-test.atom 14 0
+clojure.core-test.bigint 6 0
+clojure.core-test.bit-set 1 0
+clojure.core-test.boolean-qmark 0 1
+clojure.core-test.byte 7 5
+clojure.core-test.char 1 0
+clojure.core-test.compare 1 0
+clojure.core-test.conj 1 0
+clojure.core-test.contains-qmark 3 0
+clojure.core-test.counted-qmark 1 0
+clojure.core-test.dec 1 0
+clojure.core-test.denominator 0 3
+clojure.core-test.derive 21 0
+clojure.core-test.descendants 4 0
+clojure.core-test.double 0 4
+clojure.core-test.double-qmark 3 0
+clojure.core-test.empty 1 0
+clojure.core-test.eq 2 0
+clojure.core-test.eval 0 3
+clojure.core-test.even-qmark 1 0
+clojure.core-test.float 4 4
+clojure.core-test.get 0 1
+clojure.core-test.gt 0 2
+clojure.core-test.gt-eq 0 2
+clojure.core-test.ifn-qmark 2 0
+clojure.core-test.inc 1 0
+clojure.core-test.int 4 5
+clojure.core-test.int-qmark 3 0
+clojure.core-test.intern 2 0
+clojure.core-test.keys 0 4
+clojure.core-test.lazy-seq 3 0
+clojure.core-test.long 2 5
+clojure.core-test.lt 0 2
+clojure.core-test.lt-eq 0 2
+clojure.core-test.max 3 1
+clojure.core-test.min 3 1
+clojure.core-test.minus 2 0
+clojure.core-test.mod 12 34
+clojure.core-test.neg-int-qmark 1 0
+clojure.core-test.not-eq 3 0
+clojure.core-test.nth 0 1
+clojure.core-test.num 2 1
+clojure.core-test.number-qmark 3 0
+clojure.core-test.numerator 0 3
+clojure.core-test.odd-qmark 1 0
+clojure.core-test.parents 10 0
+clojure.core-test.parse-uuid 3 0
+clojure.core-test.peek 2 0
+clojure.core-test.plus 11 0
+clojure.core-test.plus-squote 11 0
+clojure.core-test.pop 0 1
+clojure.core-test.pos-int-qmark 1 0
+clojure.core-test.quot 12 31
+clojure.core-test.rand-nth 0 1
+clojure.core-test.rational-qmark 3 0
+clojure.core-test.rationalize 10 0
+clojure.core-test.realized-qmark 3 0
+clojure.core-test.reduce 0 1
+clojure.core-test.rem 12 30
+clojure.core-test.remove-watch 0 1
+clojure.core-test.run-bang 1 0
+clojure.core-test.select-keys 2 0
+clojure.core-test.seqable-qmark 1 0
+clojure.core-test.short 7 5
+clojure.core-test.shuffle 1 0
+clojure.core-test.slash 2 48
+clojure.core-test.some-fn 3 0
+clojure.core-test.sort-by 2 0
+clojure.core-test.special-symbol-qmark 20 0
+clojure.core-test.star 22 0
+clojure.core-test.star-squote 19 0
+clojure.core-test.transient 23 0
+clojure.core-test.underive 7 0
+clojure.core-test.update 1 0
+clojure.core-test.vals 0 3
+clojure.core-test.vec 1 0
+clojure.core-test.when-let 1 0
+clojure.core-test.with-precision 17 0
+clojure.edn-test.read-string 0 1
+clojure.string-test.capitalize 0 4
+clojure.string-test.ends-with-qmark 1 4
+clojure.string-test.escape 1 0
+clojure.string-test.lower-case 0 4
+clojure.string-test.starts-with-qmark 1 10
+clojure.string-test.upper-case 0 4