bb8b2d201c
str/subs/vec/keyword/symbol/compare/int/double/gensym — host-coupled seed natives the overlay assumes, now def-var!'d into clojure.core via host/chez/converters.ss (loaded last, str reuses jolt-pr-str). Semantics match the seed: str-render-one for str (nil->"", bare chars, regex source), the 3-way core-compare port, int truncates. The symbol no-ns sentinel is #f to match emit's quoted-symbol lowering (jolt-symbol #f "x"), so (= 'x (symbol "x")) holds — jolt= compares ns with strict equal?, and jolt-nil vs #f would otherwise diverge. Prelude parity 1220 -> 1326/2497, 0 new divergences. Floor raised to 1326; three newly-reached *ns*/var-rendering cases added to the allowlist (Phase 2). run-prelude and the per-case program file are now PID-unique so concurrent chez runs don't read each other's half-written files.
138 lines
6.6 KiB
Text
138 lines
6.6 KiB
Text
# Phase 1 (jolt-cf1q.2, inc 3j) — FULL parity against the -e-capable jolt-chez.
|
|
#
|
|
# run-corpus-chez.janet measures parity for the SUBSET the back end can compile
|
|
# without any clojure.core present (most cases are "out of subset" because they
|
|
# call core fns). This runner closes that gap: it assembles the ENTIRE non-macro
|
|
# clojure.core as a Scheme prelude (driver/emit-core-prelude — def-var! forms in
|
|
# tier dependency order), loads it before each case, and emits the case in
|
|
# prelude mode so every core ref resolves via var-deref. That is exactly the
|
|
# -e-capable jolt-chez the milestone calls for, measured in-process (one ctx,
|
|
# prelude assembled once) rather than via a spawned binary per case.
|
|
#
|
|
# With all of core available, "out of subset" collapses to genuine emit failures
|
|
# (host interop / unsupported IR). The new signal is RUNTIME parity: a case that
|
|
# emits but crashes (a missing/blank runtime prim — a lazy gap) or returns a
|
|
# wrong value (a divergence). The report buckets these so the gaps form a
|
|
# punch-list for the next increments.
|
|
# JOLT_CHEZ_PRELUDE_CORPUS=1 janet test/chez/run-corpus-prelude.janet
|
|
# JOLT_CORPUS_LIMIT=200 … (every-Nth stride, fast)
|
|
(import ../../host/chez/driver :as d)
|
|
|
|
(unless (os/getenv "JOLT_CHEZ_PRELUDE_CORPUS")
|
|
(print "skip: set JOLT_CHEZ_PRELUDE_CORPUS=1 to run the prelude parity gate")
|
|
(os/exit 0))
|
|
(unless (d/chez-available?)
|
|
(print "skip: chez not on PATH")
|
|
(os/exit 0))
|
|
|
|
(def corpus (parse (slurp "test/chez/corpus.edn")))
|
|
(def cases
|
|
(if-let [n (os/getenv "JOLT_CORPUS_LIMIT")]
|
|
(let [stride (max 1 (math/floor (/ (length corpus) (scan-number n))))]
|
|
(seq [i :range [0 (length corpus) stride]] (in corpus i)))
|
|
corpus))
|
|
|
|
# Known divergences: cases that emit + run on Chez but yield a non-canonical
|
|
# value because of a gap beyond this increment. The gate fails only on a NEW
|
|
# (un-allowlisted) divergence — a real Chez correctness regression. Each here is
|
|
# a deferred Phase-2 / dynamic-var / eval-order gap, NOT a wrong shim:
|
|
# - class names ((class x), .getName) — no Chez class system yet (Phase 2)
|
|
# - *ns* / *clojure-version* / *unchecked-math* — dynamic vars not bound on Chez
|
|
# - eval-order probes — assert left-to-right side-effect order via host state
|
|
# the emitted Scheme doesn't yet reproduce
|
|
# - close on throw — with-open/finally resource close semantics
|
|
(def known-divergences
|
|
{"class name evaluates to canonical string" true
|
|
"dispatch-only class name" true
|
|
"inside class" true
|
|
"values evaluate in source order" true
|
|
"keys evaluate before their values, pairwise" true
|
|
"source order with a nil value (phm form)" true
|
|
"close on throw" true
|
|
"ns-name of *ns*" true
|
|
"str of *ns*" true
|
|
"*ns* user" true
|
|
"str of a var" true
|
|
"*clojure-version* major" true
|
|
"*unchecked-math*" true})
|
|
|
|
(def ctx (d/make-ctx))
|
|
|
|
# Assemble the prelude once and write it to a file the per-case programs `load`.
|
|
(def t0 (os/clock))
|
|
(def [prelude-scm emitted total] (d/emit-core-prelude ctx))
|
|
(def prelude-path (string "/tmp/jolt-chez-prelude-" (os/getpid) ".ss"))
|
|
(spit prelude-path prelude-scm)
|
|
(printf "prelude: %d/%d non-macro core forms emitted (%.1fs, %d bytes) -> %s"
|
|
emitted total (- (os/clock) t0) (length prelude-scm) prelude-path)
|
|
(flush)
|
|
|
|
(var pass 0)
|
|
(def emit-errs @[]) # user form can't emit (host interop / unsupported IR)
|
|
(def crashes @[]) # emitted, chez exited non-zero (lazy runtime gap or bug)
|
|
(def diverged @[]) # emitted + ran, NEW wrong value (fails the gate)
|
|
(def known-hit @[]) # emitted + ran, allowlisted wrong value (tolerated)
|
|
(def crash-keys @{}) # grouped crash reason -> count
|
|
(def emit-keys @{}) # grouped emit-failure reason -> count
|
|
|
|
(defn- bucket [tbl k] (put tbl k (+ 1 (or (get tbl k) 0))))
|
|
(defn- crash-reason [err]
|
|
(def e (string err))
|
|
(if-let [i (string/find "Exception" e)]
|
|
(string/slice e i (min (length e) (+ i 70)))
|
|
(string/slice e 0 (min 60 (length e)))))
|
|
(defn- emit-reason [msg]
|
|
(def m (string msg))
|
|
(cond
|
|
(string/find "out of subset" m) (let [i (string/find "`" m)]
|
|
(if i (string "core fn: " (string/slice m (inc i) (or (string/find "`" m (inc i)) (length m)))) "out-of-subset"))
|
|
(string/find "host" m) "host-interop"
|
|
(string/find "unhandled op" m) (string/slice m (max 0 (- (length m) 28)))
|
|
(string/slice m 0 (min 48 (length m)))))
|
|
|
|
(def t1 (os/clock))
|
|
(each row cases
|
|
(def {:expected e :actual a :label l} row)
|
|
(if (= e :throws)
|
|
nil # :throws error-semantics aren't modeled here; skip (counted out of run)
|
|
(let [src (string "(= " e " " a ")")
|
|
res (d/eval-e-with-prelude ctx src prelude-path)]
|
|
(cond
|
|
(= (get res 0) :emit-err)
|
|
(let [k (emit-reason (get res 1))] (bucket emit-keys k) (array/push emit-errs [l k]))
|
|
(not= (get res 0) 0)
|
|
(let [k (crash-reason (get res 2))] (bucket crash-keys k) (array/push crashes [l k]))
|
|
(= (get res 1) "true") (++ pass)
|
|
(known-divergences l) (array/push known-hit l)
|
|
(array/push diverged [l (string "got " (get res 1))])))))
|
|
|
|
(def n-eval (+ pass (length emit-errs) (length crashes) (length diverged) (length known-hit)))
|
|
(printf "\nPrelude parity: %d/%d evaluated cases pass (%.1fs)" pass n-eval (- (os/clock) t1))
|
|
(printf " emit-fail (out of subset): %d runtime crash: %d NEW divergence: %d known divergence: %d"
|
|
(length emit-errs) (length crashes) (length diverged) (length known-hit))
|
|
|
|
(defn- report [title tbl]
|
|
(when (> (length tbl) 0)
|
|
(printf "\n%s:" title)
|
|
(each k (sort-by (fn [k] (- (get tbl k))) (keys tbl))
|
|
(printf " %4d x %s" (get tbl k) k))))
|
|
(report "emit-failure reasons" emit-keys)
|
|
(report "runtime-crash reasons" crash-keys)
|
|
(when (> (length diverged) 0)
|
|
(printf "\nNEW divergences (emit+ran, wrong value) — gate FAILS:")
|
|
(each [l m] (slice diverged 0 (min 30 (length diverged)))
|
|
(printf " [%s] %s" l m)))
|
|
(when (> (length known-hit) 0)
|
|
(printf "\n%d known (allowlisted) divergences tolerated." (length known-hit)))
|
|
(flush)
|
|
|
|
# Regression floor (inc 3j baseline): raise as runtime gaps close, like the probe
|
|
# reach-floor and the suite baseline. The gate fails if parity drops below it, or
|
|
# on any NEW (un-allowlisted) divergence — a real Chez correctness regression.
|
|
# Full-corpus baseline: inc 3j 1220/2497; inc 3k (converters) 1326. Strided runs
|
|
# scale the floor down.
|
|
(def base-floor (scan-number (or (os/getenv "JOLT_CHEZ_PRELUDE_FLOOR") "1326")))
|
|
(def floor (if (os/getenv "JOLT_CORPUS_LIMIT") 0 base-floor))
|
|
(when (or (> (length diverged) 0) (< pass floor))
|
|
(printf "REGRESSION: pass %d < floor %d or %d new divergence(s)" pass floor (length diverged)))
|
|
(os/exit (if (or (> (length diverged) 0) (< pass floor)) 1 0))
|