From 0ca678a159d031ccd090ed83ea7a2d029fecda83 Mon Sep 17 00:00:00 2001 From: Yogthos Date: Fri, 5 Jun 2026 09:04:38 -0400 Subject: [PATCH] test: port clojure-test-suite as baseline-guarded integration battery MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Run the external cross-dialect clojure-test-suite (lread/clojure-test-suite, 240 per-fn .cljc files in clojure.test format) against Jolt: - test/support/clojure_test.clj: minimal clojure.test + portability shims (deftest/is/testing/are + when-var-exists/thrown?/big-int?/lazy-seq?) — just enough surface to load the suite and tally pass/fail/error. Pre-loaded so the suite's (require [clojure.test ...]) finds it already populated. - test/integration/suite-worker.janet: one-shot worker that loads the shim, the suite's number-range helper ns, and a single .cljc file, then prints 'pass fail error'. - test/integration/clojure-test-suite-test.janet: spawns a worker per file under an ev/with-deadline wall-clock budget (so infinite-seq tests that hang Jolt's eager evaluator are auto-contained, not a manual skip-list) and asserts pass/clean-file counts stay at/above a baseline. References ~/src/clojure-test-suite if present; skips cleanly when absent, like the jank battery. Current: 210 files run, 7 timed out, 2233 assertions -> 1683 pass / 350 fail / 200 error, 23 clean files. Remaining fails are genuine divergences (float/ratio/ bigint, lenient transients where Clojure throws), tracked separately. Fixes two real evaluator bugs the suite surfaced: - :refer now preserves a referred macro's :macro flag (was interned as a plain value, degrading referred macros to functions). - resolve-var now resolves ns aliases (like resolve-sym), so aliased macros (e.g. p/thrown? via :as p) dispatch as macros instead of being called as fns. --- .beads/issues.jsonl | 1 + README.md | 7 +- src/jolt/evaluator.janet | 13 +- .../integration/clojure-test-suite-test.janet | 107 ++++++++++++++++ test/integration/suite-worker.janet | 40 ++++++ test/support/clojure_test.clj | 116 ++++++++++++++++++ 6 files changed, 280 insertions(+), 4 deletions(-) create mode 100644 test/integration/clojure-test-suite-test.janet create mode 100644 test/integration/suite-worker.janet create mode 100644 test/support/clojure_test.clj diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl index 720f0db..d399bc4 100644 --- a/.beads/issues.jsonl +++ b/.beads/issues.jsonl @@ -5,6 +5,7 @@ {"_type":"issue","id":"jolt-x8s","title":"CRITICAL: collections not callable as IFn (vector/map/set)","status":"closed","priority":1,"issue_type":"bug","owner":"yogthos@gmail.com","created_at":"2026-06-04T17:46:14Z","created_by":"Yogthos","updated_at":"2026-06-04T17:50:54Z","closed_at":"2026-06-04T17:50:54Z","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"jolt-alz","title":"CRITICAL: self-referential lazy-seq/lazy-cat yields only literal prefix","status":"closed","priority":1,"issue_type":"bug","owner":"yogthos@gmail.com","created_at":"2026-06-04T17:46:13Z","created_by":"Yogthos","updated_at":"2026-06-04T18:09:15Z","closed_at":"2026-06-04T18:09:15Z","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"jolt-wec","title":"CRITICAL: multi-collection map returns unrealized thunk","status":"closed","priority":1,"issue_type":"bug","owner":"yogthos@gmail.com","created_at":"2026-06-04T17:46:12Z","created_by":"Yogthos","updated_at":"2026-06-04T18:06:02Z","closed_at":"2026-06-04T18:06:02Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"jolt-js6","title":"Port clojure-test-suite as integration battery","description":"Build a minimal clojure.test + portability shim so Jolt can load the external ~/src/clojure-test-suite (240 per-fn .cljc files in clojure.test format). Add a baseline-guarded integration runner following the jank-conformance pattern (skip if absent).","status":"in_progress","priority":2,"issue_type":"task","assignee":"Yogthos","owner":"yogthos@gmail.com","created_at":"2026-06-05T12:22:01Z","created_by":"Yogthos","updated_at":"2026-06-05T12:22:14Z","started_at":"2026-06-05T12:22:14Z","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"jolt-i3g","title":"Clear final 2 SCI bootstrap forms (Var/IVar collision + clojure.repl/template)","description":"namespaces.cljc forms 126/127 fail: (126) lang_stubs IVar collides with real vars.cljc IVar so unbind dispatch misses on sci.lang.Var; (127) clojure.repl + clojure.template namespaces missing. Fix: trim bootstrap stubs to let real modules own canonical protocols/types; port clojure.repl + clojure.template.","status":"closed","priority":2,"issue_type":"task","owner":"yogthos@gmail.com","created_at":"2026-06-05T03:21:40Z","created_by":"Yogthos","updated_at":"2026-06-05T03:35:43Z","closed_at":"2026-06-05T03:35:43Z","close_reason":"Done — SCI bootstrap now loads 422/422 with 0 failures. Fixed deftype protocol-method registration, with-meta on functions, select-keys over vectors, require :as+:refer, defmethod-on-fn; added clojure.set/join and host-module stubs. test-load-sci asserts 0 failures.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"jolt-ns4","title":"Persistent collections + build-time mutable/immutable toggle","description":"Add JOLT_MUTABLE build flag (config.janet). Fix list-mutation bug (conj mutated original). Round 2: structural-sharing persistent vector (32-way trie) hooked into all core ops. Round 3: persistent lists / HAMT maps.","notes":"R1 (build flag + list mutation fix) and R2 (structural-sharing persistent vectors + mutable toggle) done \u0026 pushed (1eb2843). Immutable: conformance 206/206, features 71/71, jank 119. R3 (persistent linked lists) pending user decision: lists are immutable Janet arrays — O(1) sharing cons, O(n) conj.","status":"closed","priority":2,"issue_type":"feature","owner":"yogthos@gmail.com","created_at":"2026-06-04T22:15:39Z","created_by":"Yogthos","updated_at":"2026-06-04T23:20:46Z","closed_at":"2026-06-04T23:20:46Z","close_reason":"Done across 3 rounds (pushed a0c9696): build-time mutable/immutable toggle, structural-sharing persistent vectors (32-way trie), and persistent O(1)-prepend linked lists. Plus bit-clear/get-method and a batch of missing core fns. conformance 206/206, features 78/78, jank 120.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"jolt-gxr","title":"HIGH: aliased namespace calls (require :as) don't resolve (s/join)","status":"closed","priority":2,"issue_type":"bug","owner":"yogthos@gmail.com","created_at":"2026-06-04T18:09:58Z","created_by":"Yogthos","updated_at":"2026-06-04T18:45:52Z","closed_at":"2026-06-04T18:45:52Z","dependency_count":0,"dependent_count":0,"comment_count":0} diff --git a/README.md b/README.md index 2628688..81df65b 100644 --- a/README.md +++ b/README.md @@ -99,8 +99,11 @@ Tests are organized in three layers: per public API area) that collectively pin down Jolt's defined behavior. This is the authoritative description of what Jolt promises. - **`test/integration/`** — cross-cutting and regression batteries: the Clojure - conformance suite, SCI bootstrap/runtime loading, jank conformance, compile-mode - tests, the library API, and a broad systematic-coverage net. + conformance suite, SCI bootstrap/runtime loading, jank conformance, the + cross-dialect [clojure-test-suite](https://github.com/lread/clojure-test-suite) + (run via a minimal `clojure.test` shim against `~/src/clojure-test-suite`, if + present, and baseline-guarded), compile-mode tests, the library API, and a + broad systematic-coverage net. - **`test/unit/`** — white-box tests for individual components (reader, evaluator, types, persistent collections, regex, compiler). diff --git a/src/jolt/evaluator.janet b/src/jolt/evaluator.janet index 2a7a958..783a6a4 100644 --- a/src/jolt/evaluator.janet +++ b/src/jolt/evaluator.janet @@ -149,7 +149,12 @@ [ctx bindings sym-s] (let [name (sym-s :name) ns (sym-s :ns)] (if (not (nil? ns)) - (let [target-ns (ctx-find-ns ctx ns)] (ns-find target-ns name)) + # Resolve ns aliases (e.g. `p/thrown?` where `p` is a require :as alias) + # so that aliased macros are recognized as macros, matching resolve-sym. + (let [current-ns (ctx-find-ns ctx (ctx-current-ns ctx)) + aliased-ns (ns-import-lookup current-ns ns) + target-ns (ctx-find-ns ctx (or aliased-ns ns))] + (ns-find target-ns name)) (if (get bindings name) nil (let [current-ns (ctx-current-ns ctx) ns (ctx-find-ns ctx current-ns) @@ -222,7 +227,11 @@ (each refer-sym refer-syms (let [name (if (struct? refer-sym) (refer-sym :name) refer-sym) v (ns-find source-ns name)] - (when v (ns-intern target-ns name (var-get v))))))) + (when v + # Preserve macro-ness: a referred macro must stay a macro, so copy + # the :macro flag onto the interned var (not just its value). + (let [nv (ns-intern target-ns name (var-get v))] + (when (get v :macro) (put nv :macro true)))))))) nil)) (defn- bind-put diff --git a/test/integration/clojure-test-suite-test.janet b/test/integration/clojure-test-suite-test.janet new file mode 100644 index 0000000..aefe856 --- /dev/null +++ b/test/integration/clojure-test-suite-test.janet @@ -0,0 +1,107 @@ +# clojure-test-suite conformance: runs the external, cross-dialect +# clojure-test-suite (https://github.com/lread/clojure-test-suite, EPL) against +# Jolt and asserts the number of passing per-function test files stays at/above +# a baseline. Like the jank battery, this does NOT vendor the suite — it +# references ~/src/clojure-test-suite if present and SKIPS cleanly when absent. +# +# Each suite file is a `clojure.test` namespace (one per clojure.core/string +# function). A minimal clojure.test + portability shim (test/support/clojure_test.clj) +# lets Jolt load them; `when-var-exists` auto-skips fns Jolt doesn't implement. +# +# Files are run in a one-shot worker subprocess (test/integration/suite-worker.janet) +# under a wall-clock deadline. Some suite tests build infinite sequences +# (cycle/range/transducers-over-infinite) that Jolt's eager evaluator can't +# truncate and so HANG rather than fail; the deadline contains them — a timed-out +# file is reported as :timeout and contributes nothing, no manual skip-list needed. + +(def suite-dir (string (os/getenv "HOME") "/src/clojure-test-suite/test/clojure")) + +# Baseline: assertions Jolt currently passes across the suite. Raise as Jolt +# improves so a regression (previously-passing assertion breaking) is caught. +(def baseline-pass 1650) +# A file is "clean" when it ran with zero failures AND zero errors. +(def baseline-clean-files 22) +# Per-file wall-clock budget (seconds). Normal files finish in well under 1s; +# this only fires on infinite-sequence hangs. +(def per-file-timeout 6) + +(defn- walk [dir acc] + (each e (os/dir dir) + (def p (string dir "/" e)) + (case ((os/stat p) :mode) + :directory (walk p acc) + :file (when (and (string/has-suffix? ".cljc" p) + (not (string/has-suffix? "portability.cljc" p))) + (array/push acc p)))) + acc) + +# Run one file in a worker subprocess; return its "pass fail error" stdout, or +# nil if it exceeded the deadline (hang) or crashed. +(defn- run-file [path] + (def proc (os/spawn ["janet" "test/integration/suite-worker.janet" path] :p {:out :pipe})) + (def out (proc :out)) + (var data nil) + (def ok + (try + (ev/with-deadline per-file-timeout + (set data (ev/read out 0x10000)) # workers print a single short line + (os/proc-wait proc) + true) + ([err] false))) + (when (not ok) + (protect (os/proc-kill proc true)) + (protect (ev/with-deadline 2 (os/proc-wait proc)))) + (protect (:close out)) + (if (and ok data) (string data) nil)) + +(defn- parse-counts [s] + # s is "pass fail error" + (def parts (string/split " " (string/trim s))) + (if (= 3 (length parts)) + [(scan-number (parts 0)) (scan-number (parts 1)) (scan-number (parts 2))] + nil)) + +(if (not (os/stat suite-dir)) + (print "clojure-test-suite: ~/src/clojure-test-suite not present — skipped") + (do + (def progress? (os/getenv "SUITE_PROGRESS")) + (def files (sort (walk suite-dir @[]))) + (var total-pass 0) + (var total-fail 0) + (var total-error 0) + (var clean-files 0) + (var ran-files 0) + (var timeouts 0) + (def worst @[]) + + (each path files + (def rel (string/slice path (+ 1 (length suite-dir)))) + (when progress? (eprintf " %s" rel) (eflush)) + (def out (run-file path)) + (def counts (and out (parse-counts out))) + (cond + (nil? out) (do (++ timeouts) (when progress? (eprint " TIMEOUT"))) + (nil? counts) (when progress? (eprint " (no counts)")) + (let [[pn fn* en] counts] + (++ ran-files) + (+= total-pass pn) + (+= total-fail fn*) + (+= total-error en) + (when (and (= 0 fn*) (= 0 en) (> pn 0)) (++ clean-files)) + (when (> (+ fn* en) 0) (array/push worst [(+ fn* en) rel pn fn* en]))))) + + (def total (+ total-pass total-fail total-error)) + (printf "\nclojure-test-suite: %d files ran (%d timed out), %d assertions — %d pass / %d fail / %d error" + ran-files timeouts total total-pass total-fail total-error) + (printf "\n clean files (0 fail/error, >0 pass): %d" clean-files) + (sort-by (fn [x] (- (x 0))) worst) + (when (> (length worst) 0) + (print " top files by fail+error:") + (each w (slice worst 0 (min 15 (length worst))) + (printf " %-40s pass=%d fail=%d err=%d" (w 1) (w 2) (w 3) (w 4)))) + + (assert (>= total-pass baseline-pass) + (string/format "regression: total-pass %d < baseline %d" total-pass baseline-pass)) + (assert (>= clean-files baseline-clean-files) + (string/format "regression: clean-files %d < baseline %d" clean-files baseline-clean-files)) + (printf "\nclojure-test-suite: OK (>= %d pass, >= %d clean files)\n" baseline-pass baseline-clean-files))) diff --git a/test/integration/suite-worker.janet b/test/integration/suite-worker.janet new file mode 100644 index 0000000..d22c942 --- /dev/null +++ b/test/integration/suite-worker.janet @@ -0,0 +1,40 @@ +# One-file worker for the clojure-test-suite battery. Loads the clojure.test +# shim, evaluates a single suite .cljc file, runs its deftests, and prints +# "pass fail error" to stdout. Used by the discovery pass to find files that +# hang under Jolt's eager evaluation (run under an external timeout). +(use ../../src/jolt/api) +(use ../../src/jolt/reader) +(use ../../src/jolt/evaluator) + +(defn- parse-forms [src] + (var s src) (def fs @[]) (var go true) + (while (and go (> (length (string/trim s)) 0)) + (def r (protect (parse-next s))) + (if (not (r 0)) (set go false) + (let [p (r 1)] (set s (p 1)) (when (not (nil? (p 0))) (array/push fs (p 0)))))) + fs) + +# A helper, not a standalone test: it needs a .cljc path argument. When `jpm +# test` runs it with no args, no-op cleanly so it doesn't count as a failure. +(def path (get (dyn :args) 1)) + +(when path + (def ctx (init)) + (each f (parse-forms (slurp "test/support/clojure_test.clj")) (eval-form ctx @{} f)) + + # Pre-load the suite's own clojure.core-test.number-range helper ns if present + # (35 files require it for r/max-int, r/max-double, … — its :default branches are + # plain numeric literals Jolt can read). Its `ns` form sets the namespace; the + # test file's own `ns` form switches back afterwards. + (let [dir (string/slice path 0 (- (length path) (length (last (string/split "/" path))))) + nr (string dir "number_range.cljc")] + (when (os/stat nr) + (each f (parse-forms (slurp nr)) (protect (eval-form ctx @{} f))))) + + (eval-string ctx "(clojure.test/reset-report!)") + (each form (parse-forms (slurp path)) (protect (eval-form ctx @{} form))) + (protect (eval-string ctx "(clojure.test/run-registered)")) + (def p (eval-string ctx "(clojure.test/n-pass)")) + (def f (eval-string ctx "(clojure.test/n-fail)")) + (def e (eval-string ctx "(clojure.test/n-error)")) + (printf "%d %d %d" (if (number? p) p 0) (if (number? f) f 0) (if (number? e) e 0))) diff --git a/test/support/clojure_test.clj b/test/support/clojure_test.clj new file mode 100644 index 0000000..c634321 --- /dev/null +++ b/test/support/clojure_test.clj @@ -0,0 +1,116 @@ +;; Minimal `clojure.test` + `clojure.core-test.portability` shims. +;; +;; These exist solely so Jolt can load the external clojure-test-suite +;; (https://github.com/lread/clojure-test-suite, EPL) — 240 per-function +;; `.cljc` files written against `clojure.test`. They are NOT a full +;; clojure.test: just enough surface (deftest/is/testing/are + the suite's +;; portability helpers) to run the suite and tally pass/fail/error. +;; +;; Loaded by test/integration/clojure-test-suite-test.janet, which pre-loads +;; this file so the suite's `(require [clojure.test ...])` finds it already +;; populated (Jolt's require is a no-op for already-loaded namespaces). + +(ns clojure.test) + +;; --- result accumulator (plain atoms; no dynamic vars needed) ------------- + +(def jolt-report (atom {:pass 0 :fail 0 :error 0 :fails []})) +(def jolt-ctx (atom [])) ;; stack of `testing` context strings +(def registry (atom [])) ;; deftest fns defined since last reset + +(defn reset-report! [] + (reset! jolt-report {:pass 0 :fail 0 :error 0 :fails []}) + (reset! jolt-ctx []) + (reset! registry [])) + +(defn- ctx-str [] (apply str (interpose " " @jolt-ctx))) + +(defn inc-pass! [] (swap! jolt-report update :pass inc)) + +(defn fail! [form] + (swap! jolt-report + (fn [r] (-> r (update :fail inc) + (update :fails conj (str (ctx-str) " FAIL: " form)))))) + +(defn err! [form] + (swap! jolt-report + (fn [r] (-> r (update :error inc) + (update :fails conj (str (ctx-str) " ERROR: " form)))))) + +(defn n-pass [] (:pass @jolt-report)) +(defn n-fail [] (:fail @jolt-report)) +(defn n-error [] (:error @jolt-report)) +(defn failures [] (:fails @jolt-report)) + +;; --- assertion macros ------------------------------------------------------ + +;; `(is form)` / `(is form msg)` — the optional msg is absorbed and ignored. +(defmacro is [form & _] + `(try + (if ~form + (clojure.test/inc-pass!) + (clojure.test/fail! (pr-str '~form))) + (catch :default e# + (clojure.test/err! (str (pr-str '~form) " threw"))))) + +(defmacro testing [s & body] + `(do + (swap! clojure.test/jolt-ctx conj ~s) + (try + (do ~@body) + (finally (swap! clojure.test/jolt-ctx pop))))) + +(defmacro deftest [name & body] + `(do + (def ~name (fn [] ~@body)) + (swap! clojure.test/registry conj ~name) + ~name)) + +;; `(are [bindings] expr & data)` — substitute each row of `data` into the +;; template and assert via `is`. Expands to a `do` of let+is forms. +(defmacro are [argv expr & data] + (let [n (count argv) + rows (partition n data)] + `(do ~@(map (fn [row] + `(let [~@(interleave argv row)] + (clojure.test/is ~expr))) + rows)))) + +;; Run every deftest registered since the last reset, isolating crashes. +(defn run-registered [] + (doseq [t @registry] + (try (t) (catch :default e (clojure.test/err! "deftest crashed")))) + nil) + +;; clojure.test entry points the suite may call — no-ops; the Janet runner +;; drives execution via run-registered instead. +(defn run-tests [& _] nil) +(defn run-test [& _] nil) +(defn test-var [& _] nil) + + +;; --- clojure.core-test.portability ---------------------------------------- + +(ns clojure.core-test.portability) + +;; Gate a test on whether its target var exists in this dialect. Jolt only +;; implements a subset of clojure.core, so unimplemented fns get skipped +;; cleanly rather than erroring. +(defmacro when-var-exists [var-sym & body] + (if (resolve var-sym) + `(do ~@body) + `(println "SKIP -" '~var-sym))) + +;; `(thrown? body)` — true iff evaluating body throws. The suite always uses +;; the single-arg (no exception-class) form via this portability helper. +(defmacro thrown? [& body] + `(try (do ~@body false) (catch :default e# true))) + +(defn big-int? [n] + (and (integer? n) (not (int? n)))) + +(defn lazy-seq? [x] + ;; Jolt has no public LazySeq type test; approximate with seq?-of-non-vector. + (and (seq? x) (not (vector? x)))) + +(defn sleep [ms] nil)