Conformance inc3: promote the corpus to a documented portable spec
Makes the host-neutral corpus a first-class language specification with conformance levels, not just a regression suite. - [suite label] is now a unique, stable case id (extract-corpus disambiguates duplicate labels with ' (N)' — one collision existed). - certify.clj --profile emits test/conformance/profile.edn: every non-portable case classified by the host feature it requires (numerics/double-only, concurrency/snapshot, host/jvm-interop, host/arrays, host/janet, async/core-async, runtime/eval, reader/jolt, printer/jolt, strictness/jolt, impl/representation, bug). 2670 of 2919 cases are portable (pass on any faithful Clojure); 249 are feature-gated. - SPEC.md documents the contract: row schema, the JVM oracle, conformance levels, the feature vocabulary, and a worked new-runtime harness — so hosting jolt elsewhere and proving it correct is read-one-file mechanical. Janet gate 155 files 0 failed; certify + zero-janet gates green.
This commit is contained in:
parent
635bbbcc27
commit
f54e99cc08
6 changed files with 1223 additions and 3 deletions
|
|
@ -130,6 +130,51 @@
|
|||
:detail (str "jolt-expected=" (pr-str (second e))
|
||||
" JVM-result=" (pr-str (second a)))})))))
|
||||
|
||||
(def profile-out
|
||||
(let [args (vec *command-line-args*)
|
||||
i (.indexOf args "--profile")]
|
||||
(when (and (>= i 0) (< (inc i) (count args))) (nth args (inc i)))))
|
||||
|
||||
;; Allowlist category -> conformance feature (for divergent / throws-mismatch rows
|
||||
;; whose nature a human classified in known-divergences.edn).
|
||||
(def category->feature
|
||||
{:numeric-model :numerics/double-only
|
||||
:concurrency-model :concurrency/snapshot
|
||||
:reader-model :reader/jolt
|
||||
:printer-model :printer/jolt
|
||||
:strictness :strictness/jolt
|
||||
:impl-detail :impl/representation
|
||||
:host-model :host/jvm-interop
|
||||
:bug :bug})
|
||||
(def allow-category
|
||||
(into {} (map (fn [e] [[(:suite e) (:label e)] (:category e)]) allowlist-entries)))
|
||||
|
||||
;; Coarse feature for a row the JVM couldn't certify (jvm-error) — by scanning the
|
||||
;; source for what host capability it exercises. Defaults to generic host interop.
|
||||
(defn jvm-error-feature [actual]
|
||||
(let [a (str/lower-case actual)]
|
||||
(cond
|
||||
(re-find #"janet" a) :host/janet
|
||||
(re-find #"chan|>!|<!|go-loop|\(go |alts!|core\.async" a) :async/core-async
|
||||
(re-find #"load-string|\beval\b|read-string.*eval" a) :runtime/eval
|
||||
(re-find #"-array|into-array|to-array|aget|aset|aclone|alength" a) :host/arrays
|
||||
(re-find #"proxy|reify|gen-class|definterface|deftype.*:|\.\.|bean" a) :host/jvm-interop
|
||||
(re-find #"class|forname|instance\?|cast|\.getclass" a) :host/jvm-interop
|
||||
:else :host/jvm-interop)))
|
||||
|
||||
;; The full conformance feature(s) a row requires (empty = portable). Certified
|
||||
;; rows are portable; everything else gets a feature so a runtime knows what host
|
||||
;; capability the case assumes.
|
||||
(defn row-features [bucket suite label actual]
|
||||
(case bucket
|
||||
(:certified :certified-throws) []
|
||||
(:divergent :throws-mismatch) [(category->feature (allow-category [suite label] :host/jvm-interop)
|
||||
:host/jvm-interop)]
|
||||
:read-error [:reader/jolt]
|
||||
:timeout [:perf/unbounded]
|
||||
:jvm-error [(jvm-error-feature actual)]
|
||||
[:host/jvm-interop]))
|
||||
|
||||
(defn -main [& _]
|
||||
(let [corpus (edn/read-string (slurp corpus-path))
|
||||
results (mapv (fn [row] (assoc (classify row) :row row)) corpus)
|
||||
|
|
@ -170,6 +215,34 @@
|
|||
(def new-divergences news)
|
||||
(def stale-entries stale))
|
||||
|
||||
;; Conformance profile: every NON-portable case keyed by [suite label] -> its
|
||||
;; feature(s) + bucket. Certified (portable) cases are omitted — they are the
|
||||
;; baseline every runtime must pass. A runtime computes its conformance LEVEL by
|
||||
;; subtracting the features it doesn't implement. Written when --profile is given.
|
||||
(when profile-out
|
||||
(let [entries (->> results
|
||||
(remove #(#{:certified :certified-throws} (:bucket %)))
|
||||
(map (fn [{:keys [bucket row]}]
|
||||
(let [{:keys [suite label actual]} row]
|
||||
{:suite suite :label label :bucket bucket
|
||||
:features (row-features bucket suite label actual)})))
|
||||
(sort-by (juxt :suite :label)) vec)
|
||||
feat-counts (->> entries (mapcat :features) frequencies (into (sorted-map)))]
|
||||
(spit profile-out
|
||||
(with-out-str
|
||||
(pp/pprint
|
||||
{:doc (str "Conformance profile for test/chez/corpus.edn, generated by certify.clj. "
|
||||
"Each entry is a NON-portable case (keyed by [suite label]) and the host "
|
||||
"feature(s) it requires. Cases NOT listed are portable — they pass on any "
|
||||
"faithful Clojure. A runtime's conformance LEVEL = portable + the feature "
|
||||
"families it implements. See SPEC.md.")
|
||||
:clojure-version (clojure-version)
|
||||
:portable-count (+ (cnt :certified) (cnt :certified-throws))
|
||||
:non-portable-count (count entries)
|
||||
:feature-counts feat-counts
|
||||
:entries entries})))
|
||||
(println (format "\nwrote conformance profile (%d non-portable cases) to %s" (count entries) profile-out))))
|
||||
|
||||
;; Full per-row divergence detail goes to the --edn report (for triage); the
|
||||
;; console stays quiet about KNOWN divergences (the NEW/STALE sections above are
|
||||
;; what matters for the gate).
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue