Retire the Janet-host corpus gates
run-corpus.janet drove a target binary (default build/jolt, the Janet host) through the corpus and run-corpus-chez.janet was the all-flonum subset probe. Both compare against corpus.edn, which is now JVM-sourced, so the all-flonum hosts diverge on every numeric/host case. The zero-Janet spine gate (run-corpus-zero-janet.janet) plus certify.clj are the oracles; drop these and the stale test/chez/known-divergences.edn allowlist they used.
This commit is contained in:
Yogthos
parent
da775802d6
commit
8d4f83e0f7
3 changed files with 0 additions and 137 deletions
|
|
@ -1,14 +0,0 @@
|
|||
["max non-number throws"
|
||||
"min-key keys nonnum"
|
||||
"neg? keyword"
|
||||
"< nil"
|
||||
"> with nil"
|
||||
"max non-number"
|
||||
"neg? throws"
|
||||
"vector index"
|
||||
"map key as fn"
|
||||
"map key default"
|
||||
"set membership"
|
||||
"set miss default"
|
||||
"collection key"
|
||||
"xml-seq walks"]
|
||||
|
|
@ -1,70 +0,0 @@
|
|||
# Phase 1 (jolt-cf1q.2) — FIRST parity number for the Chez back end.
|
||||
#
|
||||
# The full 0b gate (test/chez/run-corpus.janet) drives an `-e`-capable jolt
|
||||
# binary; that needs all of clojure.core bootstrapped onto Chez, which is Phase 2.
|
||||
# Until then, this probe reports parity for the subset the back end can ALREADY
|
||||
# compile: each corpus case `(= expected actual)` is run through the live
|
||||
# analyzer -> Scheme emitter -> Chez. Cases that reference unimplemented core fns
|
||||
# fail to EMIT (a clean compile-time signal) and are counted "out of subset",
|
||||
# not as divergences. The number to watch is parity WITHIN the compiled subset.
|
||||
# janet test/chez/run-corpus-chez.janet
|
||||
# JOLT_CORPUS_LIMIT=400 … (every-Nth stride, fast)
|
||||
(import ../../host/chez/driver :as d)
|
||||
|
||||
# Slow reporting tool (~20s: a Chez subprocess per compiled case), not a pass/fail
|
||||
# unit test — gate it out of the default suite like the benches (JOLT_BENCH).
|
||||
(unless (os/getenv "JOLT_CHEZ_CORPUS")
|
||||
(print "skip: set JOLT_CHEZ_CORPUS=1 to run the Chez subset parity probe")
|
||||
(os/exit 0))
|
||||
(unless (d/chez-available?)
|
||||
(print "skip: chez not on PATH")
|
||||
(os/exit 0))
|
||||
|
||||
(def corpus (parse (slurp "test/chez/corpus.edn")))
|
||||
(def cases
|
||||
(if-let [n (os/getenv "JOLT_CORPUS_LIMIT")]
|
||||
(let [stride (max 1 (math/floor (/ (length corpus) (scan-number n))))]
|
||||
(seq [i :range [0 (length corpus) stride]] (in corpus i)))
|
||||
corpus))
|
||||
|
||||
# Known subset divergences: cases that compile but need a feature beyond the
|
||||
# current increment. None as of inc 3b — dynamic IFn dispatch (a keyword/vector
|
||||
# held in a LOCAL then called as a fn, (let [k :a] (k m))) now routes through the
|
||||
# jolt-invoke fallback, and the seq tier closed the rest. Add a label here if a
|
||||
# future increment surfaces a case that compiles but needs deferred semantics;
|
||||
# the gate fails only on a NEW (un-allowlisted) divergence.
|
||||
(def known-divergences
|
||||
{})
|
||||
|
||||
(def ctx (d/make-ctx))
|
||||
(var compiled 0) (var pass 0) (var out-of-subset 0)
|
||||
(def diverged @[])
|
||||
(def known-hit @[])
|
||||
|
||||
(each row cases
|
||||
(def {:expected e :actual a :label l} row)
|
||||
# :throws cases need error-semantics we don't model yet — skip.
|
||||
(if (= e :throws)
|
||||
(++ out-of-subset)
|
||||
(let [src (string "(= " e " " a ")")
|
||||
# compile-program can throw (unsupported op/core fn) or the analyzer can
|
||||
# punt; either way the case is outside the compilable subset.
|
||||
res (try (d/run-on-chez ctx src) ([err] :uncompilable))]
|
||||
(if (= res :uncompilable)
|
||||
(++ out-of-subset)
|
||||
(let [[code out] res]
|
||||
(++ compiled)
|
||||
(defn record-div [m] (if (known-divergences l) (array/push known-hit l) (array/push diverged [l m])))
|
||||
(cond
|
||||
(not= code 0) (record-div (string "exit " code))
|
||||
(= out "true") (++ pass)
|
||||
(record-div (string "got " out))))))))
|
||||
|
||||
(printf "\nChez subset parity: %d/%d compiled cases pass (%d/%d out of subset, %d known divergences)"
|
||||
pass compiled out-of-subset (length cases) (length known-hit))
|
||||
(when (> (length diverged) 0)
|
||||
(printf "%d NEW divergence(s) within the compiled subset:" (length diverged))
|
||||
(each [l m] (slice diverged 0 (min 25 (length diverged)))
|
||||
(printf " [%s] %s" l m)))
|
||||
(flush)
|
||||
(os/exit (if (> (length diverged) 0) 1 0))
|
||||
|
|
@ -1,53 +0,0 @@
|
|||
# Phase 0b — host-neutral parity runner.
|
||||
#
|
||||
# Feeds each corpus case to a TARGET jolt binary as a fresh subprocess (fresh
|
||||
# ctx = the per-case isolation the in-process harness gives), checking
|
||||
# (= expected actual) prints `true`, or that a :throws case exits non-zero. The
|
||||
# target is pluggable so the SAME corpus gates every host:
|
||||
# JOLT_BIN=build/jolt janet test/chez/run-corpus.janet # Janet ref
|
||||
# JOLT_BIN=build/jolt-chez ... (Phase 1+) # Chez host
|
||||
# Env: JOLT_CORPUS_LIMIT=N caps the run (every-Nth stride) for fast iteration.
|
||||
(def corpus (parse (slurp "test/chez/corpus.edn")))
|
||||
(def jolt-bin (or (os/getenv "JOLT_BIN") "build/jolt"))
|
||||
(def known (let [k @{}] (each l (parse (slurp "test/chez/known-divergences.edn")) (put k l true)) k))
|
||||
|
||||
(def cases
|
||||
(if-let [n (os/getenv "JOLT_CORPUS_LIMIT")]
|
||||
(let [stride (max 1 (math/floor (/ (length corpus) (scan-number n))))]
|
||||
(seq [i :range [0 (length corpus) stride]] (in corpus i)))
|
||||
corpus))
|
||||
|
||||
(defn run-capture [args]
|
||||
(def proc (os/spawn [jolt-bin ;args] :p {:out :pipe :err :pipe}))
|
||||
(def out (ev/read (proc :out) 0x100000))
|
||||
(ev/read (proc :err) 0x100000)
|
||||
(def code (os/proc-wait proc))
|
||||
# take the LAST non-empty line: a case may print side effects before its value
|
||||
(def lines (filter (fn [l] (not (empty? l))) (string/split "\n" (string/trim (if out (string out) "")))))
|
||||
[code (if (empty? lines) "" (last lines))])
|
||||
|
||||
(var pass 0)
|
||||
(def fails @[]) # NEW divergences — these fail the gate
|
||||
(def known-hit @[]) # expected (allowlisted) divergences
|
||||
(defn record-fail [l m]
|
||||
(if (known l) (array/push known-hit l) (array/push fails [l m])))
|
||||
(each row cases
|
||||
(def {:expected e :actual a :label l} row)
|
||||
(if (= e :throws)
|
||||
(let [[code _] (run-capture ["-e" a])]
|
||||
(if (not= code 0) (++ pass)
|
||||
(record-fail l "expected an error, exited 0")))
|
||||
(let [[code out] (run-capture ["-e" (string "(= " e " " a ")")])]
|
||||
(cond
|
||||
(not= code 0) (record-fail l (string "errored (exit " code ")"))
|
||||
(= out "true") (++ pass)
|
||||
(record-fail l (string "want true, got " out))))))
|
||||
|
||||
(printf "\ncorpus parity [%s]: %d/%d passed (%d known divergences)"
|
||||
jolt-bin pass (length cases) (length known-hit))
|
||||
(when (> (length fails) 0)
|
||||
(printf "%d NEW divergence(s) — gate FAILED:" (length fails))
|
||||
(each [l m] (slice fails 0 (min 20 (length fails)))
|
||||
(printf " FAIL [%s] %s" l m)))
|
||||
(flush)
|
||||
(os/exit (if (> (length fails) 0) 1 0))
|
||||
Loading…
Add table
Add a link
Reference in a new issue