diff --git a/.gitmodules b/.gitmodules index 77affa0..e2374ec 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,3 +4,6 @@ [submodule "vendor/sci"] path = vendor/sci url = https://github.com/borkdude/sci.git +[submodule "vendor/clojure-test-suite"] + path = vendor/clojure-test-suite + url = https://github.com/jank-lang/clojure-test-suite.git diff --git a/Makefile b/Makefile index 95b4260..84a569a 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ # build step. `make test` is the full gate. `make remint` rebuilds the seed after a # source change. -.PHONY: test ci values corpus unit smoke buildsmoke staticnativesmoke selfhost sci certify ffi transient infer wp devirt fieldread numwp fieldnum protoret narrow directlink numeric inline shakesmoke remint joltc joltc-release joltc-debug joltcsmoke +.PHONY: test ci values corpus unit smoke buildsmoke staticnativesmoke selfhost sci cts certify ffi transient infer wp devirt fieldread numwp fieldnum protoret narrow directlink numeric inline shakesmoke remint joltc joltc-release joltc-debug joltcsmoke # Full gate (dev machine). Includes the self-host byte-fixpoint, which only holds # on the same Chez that minted the seed. @@ -15,7 +15,7 @@ test: selfhost ci # lockfile) — it RUNS correctly on any Chez, but `selfhost` rebuilds it and a # different Chez version may emit byte-different (gensym/order) output, so the # byte-fixpoint is a dev-machine check, not a CI one (jolt-8479). -ci: values corpus unit smoke buildsmoke staticnativesmoke sci ffi transient infer wp devirt fieldread numwp fieldnum protoret narrow directlink numeric inline certify +ci: values corpus unit smoke buildsmoke staticnativesmoke sci cts ffi transient infer wp devirt fieldread numwp fieldnum protoret narrow directlink numeric inline certify @echo "OK: CI gates passed" # Self-host fixpoint: bootstrap.ss rebuild == checked-in seed. @@ -68,6 +68,12 @@ joltcsmoke: sci: @chez --script host/chez/run-sci.ss +# clojure-test-suite conformance: run the vendored jank-lang/clojure-test-suite +# per-namespace under joltc, gated on the per-namespace baseline +# (test/chez/cts-known-failures.txt). +cts: + @bash host/chez/cts.sh + # FFI: bind native functions (typed foreign-procedure), memory, and that a # :blocking call is collect-safe (a parked thread doesn't pin the collector). ffi: diff --git a/host/chez/cts.sh b/host/chez/cts.sh new file mode 100755 index 0000000..0c2f1cb --- /dev/null +++ b/host/chez/cts.sh @@ -0,0 +1,120 @@ +#!/bin/bash +# clojure-test-suite gate: run the vendored jank-lang/clojure-test-suite +# (vendor/clojure-test-suite) against joltc, one process per test namespace (a +# hang or crash is contained), and compare per-namespace fail/error counts +# against the checked-in baseline test/chez/cts-known-failures.txt. +# +# The comparison is exact, like certify's allowlist: a namespace doing WORSE +# than the baseline fails the gate (regression), and one doing BETTER also +# fails (stale baseline — update the file in the same change that improved it). +# +# JOLT_CTS_JOBS=N parallel workers (default 4) +# JOLT_CTS_TIMEOUT=SECS per-namespace timeout (default 120) +# JOLT_CTS_WRITE_BASELINE=1 regenerate the baseline file instead of gating +# JOLT_CTS_NS=ns1,ns2 run only these namespaces, verbose, no gating +set -u +root="$(CDPATH= cd -- "$(dirname -- "$0")/../.." && pwd)" +cd "$root" + +suite="vendor/clojure-test-suite/test" +baseline="test/chez/cts-known-failures.txt" +app="$root/test/chez/cts-app" +jobs="${JOLT_CTS_JOBS:-4}" +tmo="${JOLT_CTS_TIMEOUT:-120}" + +if [ ! -d "$suite/clojure" ]; then + echo "cts: skipped (git submodule update --init vendor/clojure-test-suite)" + exit 0 +fi + +work="$(mktemp -d)" +trap 'rm -rf "$work"' EXIT + +# test namespaces from the .cljc files (portability is a helper, not a test ns) +find "$suite" -name '*.cljc' | sed "s|^$suite/||;s|\.cljc$||;s|/|.|g;s|_|-|g" \ + | grep -v '\.portability$' | sort > "$work/nses" +if [ -n "${JOLT_CTS_NS:-}" ]; then + echo "${JOLT_CTS_NS}" | tr ',' '\n' > "$work/nses" +fi + +# round-robin the namespaces over N sequential workers; each worker appends +# "ns pass fail error" lines (HUNG/CRASH in the pass column) to its own file. +awk -v j="$jobs" '{print > ("'"$work"'/chunk." (NR % j))}' "$work/nses" +run_chunk() { + chunk="$1"; out="$2" + while IFS= read -r ns; do + res=$(JOLT_PWD="$app" perl -e "alarm $tmo; exec @ARGV" -- "$root/bin/joltc" -M:cts "$ns" 2>&1 > "$out" + if [ -n "${JOLT_CTS_NS:-}" ]; then + echo "$res" | grep -E 'FAIL:|ERROR:|LOAD:' | sed 's/^/ /' >> "$out" + fi + elif [ $rc -ge 128 ]; then + echo "$ns HUNG 0 0" >> "$out" + else + echo "$ns CRASH 0 0" >> "$out" + fi + done < "$chunk" +} +for c in "$work"/chunk.*; do + run_chunk "$c" "$c.res" & +done +wait +cat "$work"/chunk.*.res 2>/dev/null | sort > "$work/results" + +if [ -n "${JOLT_CTS_NS:-}" ]; then + cat "$work/results" + exit 0 +fi + +summary=$(awk '$2!="HUNG" && $2!="CRASH" {p+=$2; f+=$3; e+=$4; c++} + $2=="HUNG" {h++} $2=="CRASH" {x++} + END {printf "%d namespaces: pass %d, fail %d, error %d, hung %d, crash %d", + c+h+x, p, f, e, h, x}' "$work/results") + +if [ "${JOLT_CTS_WRITE_BASELINE:-0}" = "1" ]; then + { + echo "# clojure-test-suite known failures: " + echo "# The gate fails on any per-namespace change, worse OR better; regenerate" + echo "# with: JOLT_CTS_WRITE_BASELINE=1 host/chez/cts.sh" + awk '$2=="HUNG" || $2=="CRASH" {print $1, $2, $2; next} + $3 != 0 || $4 != 0 {print $1, $3, $4}' "$work/results" + } > "$baseline" + echo "cts: $summary" + echo "cts: baseline written to $baseline ($(grep -cv '^#' "$baseline") namespaces)" + exit 0 +fi + +if [ ! -f "$baseline" ]; then + echo "cts: FAIL — no baseline; run JOLT_CTS_WRITE_BASELINE=1 host/chez/cts.sh" + exit 1 +fi + +status=0 +while read -r ns p f e; do + case "$p" in HUNG|CRASH) f="$p"; e="$p" ;; esac + bl=$(grep -v '^#' "$baseline" | awk -v n="$ns" '$1==n {print $2, $3; exit}') + if [ -n "$bl" ]; then bf="${bl%% *}"; be="${bl##* }"; else bf=0; be=0; fi + if [ "$f" = "$bf" ] && [ "$e" = "$be" ]; then + continue + elif [ "$f" = "HUNG" ] || [ "$f" = "CRASH" ] \ + || { [ "$bf" != "HUNG" ] && [ "$bf" != "CRASH" ] \ + && { [ "$f" -gt "$bf" ] || [ "$e" -gt "$be" ]; }; }; then + echo "cts: NEW regression in $ns — fail $f error $e (baseline $bf $be)" + status=1 + else + echo "cts: STALE baseline for $ns — now fail $f error $e (baseline $bf $be); update $baseline" + status=1 + fi +done < "$work/results" + +# a baseline entry whose namespace no longer reports is stale too +while read -r ns bf be; do + grep -q "^$ns " "$work/results" || { echo "cts: STALE baseline entry $ns (namespace gone)"; status=1; } +done < <(grep -v '^#' "$baseline") + +echo "cts: $summary" +if [ $status -eq 0 ]; then echo "cts: passed (matches baseline)"; else echo "cts: FAILED"; fi +exit $status diff --git a/host/chez/post-prelude.ss b/host/chez/post-prelude.ss index 573e889..0f42ad6 100644 --- a/host/chez/post-prelude.ss +++ b/host/chez/post-prelude.ss @@ -63,6 +63,12 @@ ;; a lazy-seq carries its own realized? flag (lazy-bridge.ss). The overlay ;; realized? reads :jolt/type and throws on a jolt-lazyseq record. ((jolt-lazyseq? x) (jolt-lazyseq-realized? x)) + ;; a seq cell answers by its forced flag: the rest of a realized lazy + ;; chain is a cseq under jolt's seq model, and (realized? (rest s)) after + ;; a next must be true like the JVM's realized LazySeq — never a throw + ;; whose message renders the (possibly infinite) seq. + ((cseq? x) (if (cseq-forced? x) #t #f)) + ((empty-list-t? x) #t) (else (jolt-invoke overlay-realized? x)))))) ;; clojure.edn/read over a reader: drain the jhost reader, then read through the ;; overlay read-string so the opts map (:readers/:default/:eof) is honored. diff --git a/host/chez/seed/prelude.ss b/host/chez/seed/prelude.ss index ccab69a..56b77d3 100644 --- a/host/chez/seed/prelude.ss +++ b/host/chez/seed/prelude.ss @@ -319,7 +319,7 @@ (guard (e (#t #f)) (def-var! "clojure.core" "char?" (letrec ((char? (lambda (x) (let fnrec4943 ((x x)) (jolt= (jolt-get x (keyword "jolt" "type")) (keyword "jolt" "char")))))) char?))) (guard (e (#t #f)) - (def-var! "clojure.core" "realized?" (letrec ((realized? (lambda (x) (let fnrec4944 ((x x)) (if (jolt-truthy? (jolt-invoke (var-deref "clojure.core" "delay?") x)) (jolt-invoke (var-deref "clojure.core" "boolean") (jolt-get x (keyword #f "realized"))) (if (jolt-truthy? (jolt-invoke (var-deref "clojure.core" "future?") x)) (jolt-invoke (var-deref "clojure.core" "boolean") (jolt-get x (keyword #f "cached"))) (if (jolt= (keyword "jolt" "lazy-seq") (jolt-get x (keyword "jolt" "type"))) (jolt-invoke (var-deref "clojure.core" "boolean") (jolt-get x (keyword #f "realized"))) (if (jolt-truthy? (jolt-invoke (var-deref "clojure.core" "atom?") x)) #t (if (jolt-truthy? (keyword #f "else")) (jolt-throw (jolt-invoke (var-deref "clojure.core" "str") "realized? not supported on: " x)) jolt-nil))))))))) realized?))) + (def-var! "clojure.core" "realized?" (letrec ((realized? (lambda (x) (let fnrec4944 ((x x)) (if (jolt-truthy? (jolt-invoke (var-deref "clojure.core" "delay?") x)) (jolt-invoke (var-deref "clojure.core" "boolean") (jolt-get x (keyword #f "realized"))) (if (jolt-truthy? (jolt-invoke (var-deref "clojure.core" "future?") x)) (jolt-invoke (var-deref "clojure.core" "boolean") (jolt-get x (keyword #f "cached"))) (if (jolt= (keyword "jolt" "lazy-seq") (jolt-get x (keyword "jolt" "type"))) (jolt-invoke (var-deref "clojure.core" "boolean") (jolt-get x (keyword #f "realized"))) (if (jolt-truthy? (jolt-invoke (var-deref "clojure.core" "atom?") x)) #t (if (jolt-truthy? (keyword #f "else")) (jolt-throw (jolt-invoke (var-deref "clojure.core" "str") "realized? not supported on: " (jolt-invoke (var-deref "clojure.core" "class") x))) jolt-nil))))))))) realized?))) (guard (e (#t #f)) (def-var! "clojure.core" "force" (letrec ((force (lambda (x) (let fnrec4945 ((x x)) (if (jolt-truthy? (jolt-invoke (var-deref "clojure.core" "delay?") x)) (jolt-invoke (var-deref "clojure.core" "deref") x) x))))) force))) (guard (e (#t #f)) diff --git a/host/chez/seq.ss b/host/chez/seq.ss index ac9e7ed..3ae923f 100644 --- a/host/chez/seq.ss +++ b/host/chez/seq.ss @@ -257,12 +257,18 @@ => (lambda (m) (apply jolt-invoke m f args))) ((and (reified-methods f) (hashtable-ref (reified-methods f) "invoke" #f)) => (lambda (m) (apply jolt-invoke m f args))) - ;; calling a non-fn: a ClassCastException naming the operator, thrown via - ;; jolt-throw so it is catchable and carries the throw-site continuation for a - ;; stack trace. + ;; calling a non-fn: a ClassCastException naming the operator's CLASS (like + ;; the JVM's "class clojure.lang.LazySeq cannot be cast to ... IFn" — never + ;; the value, whose printed form may be unbounded: ((range)) must throw, not + ;; hang rendering an infinite seq). Thrown via jolt-throw so it is catchable + ;; and carries the throw-site continuation for a stack trace. (else (jolt-throw (jolt-host-throwable "java.lang.ClassCastException" - (string-append (guard (e (#t "value")) (jolt-pr-str f)) - " cannot be cast to clojure.lang.IFn")))))) + (string-append + "class " + (guard (e (#t "value")) + (let ((c (jolt-class-name f))) + (if (string? c) c (jolt-pr-str f)))) + " cannot be cast to class clojure.lang.IFn")))))) ;; ============================================================================ ;; chunked-seq accessors — the host side of the Clojure IChunkedSeq contract diff --git a/host/chez/values.ss b/host/chez/values.ss index 02a694a..e0640f2 100644 --- a/host/chez/values.ss +++ b/host/chez/values.ss @@ -96,10 +96,16 @@ ((and (jolt-coll? a) (jolt-coll? b)) (jolt-coll=? a b)) (else (eq? a b)))) (define (jolt=2 a b) - (let loop ((as jolt-eq-arms)) - (cond ((null? as) (jolt=2-base a b)) - (((caar as) a b) ((cdar as) a b)) - (else (loop (cdr as)))))) + ;; identity fast path, like Util.equiv's k1 == k2: the same object equals + ;; itself without a structural walk — (= s s) on an infinite lazy seq must not + ;; realize it. Numbers keep the exactness-aware arm (Chez may intern flonum + ;; literals, and (= ##NaN ##NaN) is false like the JVM's). + (if (and (eq? a b) (not (number? a))) + #t + (let loop ((as jolt-eq-arms)) + (cond ((null? as) (jolt=2-base a b)) + (((caar as) a b) ((cdar as) a b)) + (else (loop (cdr as))))))) (define (jolt= a . rest) (let loop ((a a) (rest rest)) (cond ((null? rest) #t) diff --git a/jolt-core/clojure/core/20-coll.clj b/jolt-core/clojure/core/20-coll.clj index b7c346c..6fd0297 100644 --- a/jolt-core/clojure/core/20-coll.clj +++ b/jolt-core/clojure/core/20-coll.clj @@ -426,7 +426,9 @@ (future? x) (boolean (get x :cached)) (= :jolt/lazy-seq (get x :jolt/type)) (boolean (get x :realized)) (atom? x) true - :else (throw (str "realized? not supported on: " x)))) + ;; name the class, never the value — an error message must not render an + ;; arbitrary (possibly infinite) argument. + :else (throw (str "realized? not supported on: " (class x))))) (defn force [x] (if (delay? x) (deref x) x)) diff --git a/stdlib/clojure/test.clj b/stdlib/clojure/test.clj index dabd851..3866b4a 100644 --- a/stdlib/clojure/test.clj +++ b/stdlib/clojure/test.clj @@ -19,8 +19,8 @@ (def jolt-report counters) ;; alias used by the suite harness (def ctx-stack (atom [])) (def registry (atom [])) ;; [{:name sym :fn thunk}] -(def once-fixtures (atom [])) -(def each-fixtures (atom [])) +(def once-fixtures (atom {})) ;; ns-sym -> [fixture-fns] +(def each-fixtures (atom {})) ;; ns-sym -> [fixture-fns] ;; clojure.test/*testing-vars* — the stack of vars under test. Real clojure.test ;; binds it around each test var; test.check's default reporter reads it, so a @@ -32,8 +32,8 @@ (reset! counters {:test 0 :pass 0 :fail 0 :error 0 :fails []}) (reset! ctx-stack []) (reset! registry []) - (reset! once-fixtures []) - (reset! each-fixtures [])) + (reset! once-fixtures {}) + (reset! each-fixtures {})) (defn- ctx-str [] (str/join " " @ctx-stack)) @@ -135,6 +135,15 @@ ([form] `(is ~form nil)) ([form msg] (cond + ;; a library-registered custom assertion (the assert-expr extension point) + ;; wins over every inline path, like clojure.test, where each `is` dispatches + ;; assert-expr on the exact head symbol and the built-ins are just + ;; pre-registered methods. In particular a registered alias-qualified + ;; `p/thrown?` must not be captured by the by-name thrown? path below. + (and (seq? form) (symbol? (first form)) + (contains? (methods clojure.test/assert-expr) (first form))) + (clojure.test/assert-expr msg form) + ;; (is (thrown? Class body...)) (thrown-form? form "thrown?") (let [klass-sym (second form) @@ -172,13 +181,6 @@ (clojure.test/inc-pass!) (clojure.test/fail! (str "expected throw of " ~klass " matching " ~re " but got " (clojure.core/class e#) ": " m#))))))) - ;; a library-registered custom assertion (the assert-expr extension point); - ;; only fires for a symbol with an explicitly registered method, so built-in - ;; predicate forms keep the inline path below. - (and (seq? form) (symbol? (first form)) - (contains? (methods clojure.test/assert-expr) (first form))) - (clojure.test/assert-expr msg form) - ;; a predicate call — (= a b), (< x y), (pred? v): evaluate the args so a ;; failure shows the actual values, like clojure.test's assert-predicate. (and (seq? form) (contains? clojure.test/reported-preds (first form))) @@ -209,7 +211,9 @@ (defmacro deftest [name & body] `(do (defn ~name [] ~@body) - (swap! clojure.test/registry conj {:name '~name :fn ~name}) + (swap! clojure.test/registry conj {:name '~name + :ns (clojure.core/ns-name clojure.core/*ns*) + :fn ~name}) (var ~name))) (defmacro are [argv expr & data] @@ -222,10 +226,15 @@ ;; --- fixtures + run -------------------------------------------------------- +;; Fixtures are per-namespace, like clojure.test (which stores them in ns +;; metadata): use-fixtures records them under the calling ns, and only that +;; ns's tests run through them — a suite loading many test namespaces into one +;; process doesn't cross-apply or clobber another ns's fixtures. (defn use-fixtures [kind & fns] - (cond - (= kind :once) (reset! once-fixtures (vec fns)) - (= kind :each) (reset! each-fixtures (vec fns)))) + (let [n (ns-name *ns*)] + (cond + (= kind :once) (swap! once-fixtures assoc n (vec fns)) + (= kind :each) (swap! each-fixtures assoc n (vec fns))))) (defn- wrap-fixtures [fixtures body-fn] (if (empty? fixtures) @@ -234,25 +243,46 @@ (defn- run-one [t] (swap! counters update :test inc) - (wrap-fixtures @each-fixtures + (wrap-fixtures (get @each-fixtures (:ns t) []) (fn [] (try ((:fn t)) (catch Throwable e (err! (str (:name t) " crashed: " (err-text e)))))))) -(defn run-registered [] - (doseq [t @registry] (run-one t)) +;; Run the registered tests grouped by namespace (registration order preserved +;; within each ns), each group wrapped in its ns's :once fixtures. ns-set nil +;; means all. +(defn- run-selected [ns-set] + (let [ts (if ns-set (filter (fn [t] (contains? ns-set (:ns t))) @registry) @registry)] + (doseq [n (distinct (map :ns ts))] + (wrap-fixtures (get @once-fixtures n []) + (fn [] (doseq [t ts :when (= n (:ns t))] (run-one t)))))) nil) -(defn run-tests [& _nses] - (wrap-fixtures @once-fixtures (fn [] (run-registered))) - (let [r @counters] - (println) - (println (str "Ran " (:test r) " tests. " - (:pass r) " assertions passed, " - (:fail r) " failures, " (:error r) " errors.")) - r)) +(defn run-registered [] (run-selected nil)) + +;; (run-tests 'ns1 'ns2 …) runs only those namespaces' tests, like clojure.test. +;; With no args it runs everything registered (a deliberate superset of the +;; JVM's current-ns default — jolt's harnesses load then run whole suites). +;; Prints and returns THIS call's summary; the global counters stay cumulative +;; for the n-pass/n-fail harness API. +(defn run-tests [& nses] + (let [before @counters + ns-set (when (seq nses) + (set (map (fn [n] (if (symbol? n) n (ns-name n))) nses)))] + (run-selected ns-set) + (let [r @counters + d {:type :summary + :test (- (:test r) (:test before)) + :pass (- (:pass r) (:pass before)) + :fail (- (:fail r) (:fail before)) + :error (- (:error r) (:error before))}] + (println) + (println (str "Ran " (:test d) " tests. " + (:pass d) " assertions passed, " + (:fail d) " failures, " (:error d) " errors.")) + d))) (defn run-test [& _] nil) (defn test-var [& _] nil) diff --git a/test/chez/README.md b/test/chez/README.md index 46a8a23..b0418ef 100644 --- a/test/chez/README.md +++ b/test/chez/README.md @@ -32,6 +32,16 @@ answers with `regen-corpus.clj` and re-certify with `test/conformance/certify.cl - `selfcheck.sh` — self-host fixpoint: `bootstrap.ss` rebuild byte-equals the checked-in seed (`host/chez/seed/`). - `smoke.sh` — real `bin/joltc -e` CLI smoke. +- `cts.sh` — the vendored [jank-lang/clojure-test-suite](https://github.com/jank-lang/clojure-test-suite) + (`vendor/clojure-test-suite`, a per-core-fn clojure.test suite shared across + Clojure dialects), run one namespace per `joltc` process (a hang or crash is + contained) through the `test/chez/cts-app` project and `cts-run` runner. + Per-namespace fail/error counts must exactly match the checked-in baseline + `test/chez/cts-known-failures.txt` — a namespace doing worse fails the gate, + and one doing better fails as stale until the baseline is updated in the same + change. `make cts`; + `JOLT_CTS_NS=ns1,ns2` runs a subset verbosely, + `JOLT_CTS_WRITE_BASELINE=1` regenerates the baseline. ## Other Chez tests diff --git a/test/chez/clojure-test.clj b/test/chez/clojure-test.clj index e222ae3..69db89c 100644 --- a/test/chez/clojure-test.clj +++ b/test/chez/clojure-test.clj @@ -11,6 +11,17 @@ (clojure.test/do-report {:type :pass}) (clojure.test/do-report {:type :fail :message ~msg :form '~form})))) +;; an ALIAS-QUALIFIED registered assertion whose simple name collides with the +;; built-in thrown? — the registered method must win over the by-name inline +;; path (clojure-test-suite's portability/thrown? registers exactly this shape). +(defmethod t/assert-expr 'p/thrown? [msg form] + `(try + (do ~@(rest form)) + (clojure.test/do-report {:type :fail :message ~msg :form '~form}) + (catch Throwable e# + (clojure.test/do-report {:type :pass}) + e#))) + ;; a custom report type (how test.check surfaces trial/shrink progress) (def trials (atom 0)) (defmethod t/report ::trial [_m] (swap! trials inc)) @@ -27,21 +38,28 @@ 6 (* 2 3)) (is (thrown? clojure.lang.ExceptionInfo (throw (ex-info "x" {})))) (is (thrown-with-msg? Exception #"bad" (throw (ex-info "bad" {})))) - (is (near? 1.0 1.005))) + (is (near? 1.0 1.005)) + (is (p/thrown? (throw (ex-info "boom" {}))))) (deftest expected-fail (is (= 1 2)) (is (near? 1.0 5.0))) -(run-tests) +;; run-tests returns THIS call's summary; with explicit nses it runs only their +;; tests (an unknown ns runs nothing). +(def r1 (run-tests)) +(def r2 (run-tests 'no.such.test-ns)) (t/do-report {:type ::trial}) (t/do-report {:type ::trial}) -;; 7 pass (= + vector? + 2 are rows + thrown? + thrown-with-msg? + near?), +;; 8 pass (= + vector? + 2 are rows + thrown? + thrown-with-msg? + near? + p/thrown?), ;; 2 fail (= 1 2, near? 1.0 5.0), 0 error, 2 fixture runs, 2 custom reports -(let [ok (and (= (t/n-pass) 7) (= (t/n-fail) 2) (= (t/n-error) 0) +(let [ok (and (= (t/n-pass) 8) (= (t/n-fail) 2) (= (t/n-error) 0) + (= 2 (:test r1)) (= 8 (:pass r1)) (= 2 (:fail r1)) + (= 0 (:test r2)) (= 0 (:pass r2)) (= @setups 2) (= @trials 2))] (println (if ok "CLOJURE-TEST OK" (str "CLOJURE-TEST FAIL pass=" (t/n-pass) " fail=" (t/n-fail) - " error=" (t/n-error) " setups=" @setups " trials=" @trials)))) + " error=" (t/n-error) " r1=" (pr-str r1) " r2=" (pr-str r2) + " setups=" @setups " trials=" @trials)))) diff --git a/test/chez/corpus.edn b/test/chez/corpus.edn index a99b970..b4aa774 100644 --- a/test/chez/corpus.edn +++ b/test/chez/corpus.edn @@ -3513,4 +3513,6 @@ {:suite "reduce / IReduceInit" :label "a deftype reduce honors reduced short-circuit" :expected "3" :actual "(do (deftype Rng [n] clojure.lang.IReduceInit (reduce [_ f init] (loop [i 0 acc init] (if (< i n) (let [a (f acc i)] (if (reduced? a) @a (recur (inc i) a))) acc)))) (reduce (fn [acc x] (if (= x 3) (reduced acc) (+ acc x))) 0 (->Rng 10)))"} {:suite "exceptions / typed throw" :label "a jolt-raised NumberFormatException reports its real class and message" :expected "[\"java.lang.NumberFormatException\" \"For input string: \\\"xyz\\\"\"]" :actual "(try (Long/parseLong \"xyz\") (catch Throwable e [(.getName (class e)) (.getMessage e)]))"} {:suite "exceptions / hierarchy catch" :label "a typed throwable matches its superclasses, not unrelated ones" :expected "[true true true false]" :actual "(let [e (try (Long/parseLong \"z\") (catch Throwable e e))] [(instance? NumberFormatException e) (instance? IllegalArgumentException e) (instance? Exception e) (instance? java.io.IOException e)])"} + {:suite "equality / identity" :label "= short-circuits on identity without realizing a lazy seq (Util.equiv k1 == k2)" :expected "0" :actual "(let [n (atom 0) s (map (fn [x] (swap! n inc) x) [1 2 3])] (= s s) @n)"} + {:suite "realized? / lazy seq" :label "realized? reads a lazy seq's realization flag" :expected "[false true]" :actual "(let [s (map inc [1 2 3])] [(realized? s) (do (doall s) (realized? s))])"} ] diff --git a/test/chez/cts-app/deps.edn b/test/chez/cts-app/deps.edn new file mode 100644 index 0000000..ed313cc --- /dev/null +++ b/test/chez/cts-app/deps.edn @@ -0,0 +1,2 @@ +{:paths ["src" "../../../vendor/clojure-test-suite/test"] + :aliases {:cts {:main-opts ["-m" "cts-run"]}}} diff --git a/test/chez/cts-app/src/cts_run.clj b/test/chez/cts-app/src/cts_run.clj new file mode 100644 index 0000000..d1360e0 --- /dev/null +++ b/test/chez/cts-app/src/cts_run.clj @@ -0,0 +1,18 @@ +(ns cts-run + "Runner for the vendored jank-lang/clojure-test-suite (vendor/clojure-test-suite): + requires each test namespace given on the command line and runs its clojure.test + tests, printing one machine-readable result line per namespace. Driven per-process + by host/chez/cts.sh so a hang or crash in one namespace can't take out the run." + (:require [clojure.test :as t])) + +(defn -main [& nses] + (doseq [n nses] + (let [ns-sym (symbol n)] + (try + (require ns-sym) + (let [r (t/run-tests ns-sym)] + (println "CTS-RESULT" n (:pass r 0) (:fail r 0) (:error r 0))) + (catch Throwable e + (println "CTS-RESULT" n 0 0 1 + (str "LOAD: " (.getName (class e)) ": " (.getMessage e))))))) + (System/exit 0)) diff --git a/test/chez/cts-known-failures.txt b/test/chez/cts-known-failures.txt new file mode 100644 index 0000000..a52ec59 --- /dev/null +++ b/test/chez/cts-known-failures.txt @@ -0,0 +1,91 @@ +# clojure-test-suite known failures: +# The gate fails on any per-namespace change, worse OR better; regenerate +# with: JOLT_CTS_WRITE_BASELINE=1 host/chez/cts.sh +clojure.core-test.abs 1 1 +clojure.core-test.add-watch 0 3 +clojure.core-test.ancestors 9 0 +clojure.core-test.atom 14 0 +clojure.core-test.bigint 6 0 +clojure.core-test.bit-set 1 0 +clojure.core-test.boolean-qmark 0 1 +clojure.core-test.byte 7 5 +clojure.core-test.char 1 0 +clojure.core-test.compare 1 0 +clojure.core-test.conj 1 0 +clojure.core-test.contains-qmark 3 0 +clojure.core-test.counted-qmark 1 0 +clojure.core-test.dec 1 0 +clojure.core-test.denominator 0 3 +clojure.core-test.derive 21 0 +clojure.core-test.descendants 4 0 +clojure.core-test.double 0 4 +clojure.core-test.double-qmark 3 0 +clojure.core-test.empty 1 0 +clojure.core-test.eq 2 0 +clojure.core-test.eval 0 3 +clojure.core-test.even-qmark 1 0 +clojure.core-test.float 4 4 +clojure.core-test.get 0 1 +clojure.core-test.gt 0 2 +clojure.core-test.gt-eq 0 2 +clojure.core-test.ifn-qmark 2 0 +clojure.core-test.inc 1 0 +clojure.core-test.int 4 5 +clojure.core-test.int-qmark 3 0 +clojure.core-test.intern 2 0 +clojure.core-test.keys 0 4 +clojure.core-test.lazy-seq 3 0 +clojure.core-test.long 2 5 +clojure.core-test.lt 0 2 +clojure.core-test.lt-eq 0 2 +clojure.core-test.max 3 1 +clojure.core-test.min 3 1 +clojure.core-test.minus 2 0 +clojure.core-test.mod 12 34 +clojure.core-test.neg-int-qmark 1 0 +clojure.core-test.not-eq 3 0 +clojure.core-test.nth 0 1 +clojure.core-test.num 2 1 +clojure.core-test.number-qmark 3 0 +clojure.core-test.numerator 0 3 +clojure.core-test.odd-qmark 1 0 +clojure.core-test.parents 10 0 +clojure.core-test.parse-uuid 3 0 +clojure.core-test.peek 2 0 +clojure.core-test.plus 11 0 +clojure.core-test.plus-squote 11 0 +clojure.core-test.pop 0 1 +clojure.core-test.pos-int-qmark 1 0 +clojure.core-test.quot 12 31 +clojure.core-test.rand-nth 0 1 +clojure.core-test.rational-qmark 3 0 +clojure.core-test.rationalize 10 0 +clojure.core-test.realized-qmark 3 0 +clojure.core-test.reduce 0 1 +clojure.core-test.rem 12 30 +clojure.core-test.remove-watch 0 1 +clojure.core-test.run-bang 1 0 +clojure.core-test.select-keys 2 0 +clojure.core-test.seqable-qmark 1 0 +clojure.core-test.short 7 5 +clojure.core-test.shuffle 1 0 +clojure.core-test.slash 2 48 +clojure.core-test.some-fn 3 0 +clojure.core-test.sort-by 2 0 +clojure.core-test.special-symbol-qmark 20 0 +clojure.core-test.star 22 0 +clojure.core-test.star-squote 19 0 +clojure.core-test.transient 23 0 +clojure.core-test.underive 7 0 +clojure.core-test.update 1 0 +clojure.core-test.vals 0 3 +clojure.core-test.vec 1 0 +clojure.core-test.when-let 1 0 +clojure.core-test.with-precision 17 0 +clojure.edn-test.read-string 0 1 +clojure.string-test.capitalize 0 4 +clojure.string-test.ends-with-qmark 1 4 +clojure.string-test.escape 1 0 +clojure.string-test.lower-case 0 4 +clojure.string-test.starts-with-qmark 1 10 +clojure.string-test.upper-case 0 4 diff --git a/vendor/clojure-test-suite b/vendor/clojure-test-suite new file mode 160000 index 0000000..489b674 --- /dev/null +++ b/vendor/clojure-test-suite @@ -0,0 +1 @@ +Subproject commit 489b6743e8421687ef96cec557830acf258d1886